diff --git a/.github/workflows/test-mcp.yml b/.github/workflows/test-mcp.yml new file mode 100644 index 00000000..9ee23c6e --- /dev/null +++ b/.github/workflows/test-mcp.yml @@ -0,0 +1,77 @@ +name: Test MCP Server + +on: + push: + branches: [main] + paths: + - 'bin/**' + - '__tests__/mcp/**' + - 'package.json' + pull_request: + paths: + - 'bin/**' + - '__tests__/mcp/**' + - 'package.json' + +permissions: + checks: write + pull-requests: write + contents: read + +jobs: + test-mcp: + name: Test MCP Server Integration + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Run MCP integration tests + run: npm test -- __tests__/mcp/ + env: + # Set test timeout for network operations + JEST_TIMEOUT: 30000 + + - name: Report test results + if: always() + uses: dorny/test-reporter@v1 + with: + name: MCP Test Results + path: 'test-results/**/*.xml' + reporter: jest-junit + fail-on-error: true + + test-cli-contract: + name: Verify CLI Contract + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Run CLI contract tests + run: npm test -- __tests__/mcp/cli-contract.test.js + + - name: Verify all generate commands exist + run: | + echo "Verifying generate commands..." + npx doc-tools generate --help | grep -E "(property-docs|metrics-docs|rpk-docs|rpcn-connector-docs|helm-spec|cloud-regions|crd-spec|bundle-openapi)" diff --git a/.gitignore b/.gitignore index 2fd947ea..9167b871 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ __pycache__/ gen/ tree-sitter/ test/ +test-results/ modules/ cloud-controlplane/ node_modules +.env diff --git a/CLI_REFERENCE.adoc b/CLI_REFERENCE.adoc new file mode 100644 index 00000000..397839d8 --- /dev/null +++ b/CLI_REFERENCE.adoc @@ -0,0 +1,1013 @@ += Docs CLI Reference +:toc: +:toclevels: 3 + +Auto-generated reference documentation for the `doc-tools` command-line interface. + +IMPORTANT: This documentation is auto-generated. Do not edit manually. Run `npm run generate:cli-docs` to regenerate. + +== doc-tools + +Redpanda Document Automation CLI + +*Usage:* + +[,bash] +---- +doc-tools [options] [command] +---- + +*Options:* + +`-V, --version`:: +output the version number + +`-h, --help`:: +display help for command + +*Commands:* + +* `install-test-dependencies` - Install packages for doc test workflows +* `get-redpanda-version` - Print the latest Redpanda version +* `get-console-version` - Print the latest Console version +* `link-readme` - Symlink a README.adoc into docs/modules//pages/ +* `fetch` - Fetch a file or directory from GitHub and save it locally +* `setup-mcp` - Configure the Redpanda Docs MCP server for Claude Code/Desktop +* `generate` - Run docs automations +* `help` - display help for command + +== install-test-dependencies + +Installs all packages and dependencies required for documentation testing workflows. This includes Redpanda Docker images, Python virtual environments for property extraction, and other test dependencies. + +*Why use it:* + +Setting up a documentation environment requires multiple dependencies across different package managers (npm, pip, Docker). This command automates the entire setup process. + +*Usage:* + +[,bash] +---- +doc-tools install-test-dependencies [options] +---- + +*Options:* + +`-h, --help`:: +display help for command + +*Examples:* + +[,bash] +---- +# Set up a new documentation environment +npx doc-tools install-test-dependencies + +# Use in CI/CD before running doc tests +- run: npx doc-tools install-test-dependencies +- run: npm test +---- + +*Requirements:* + +- Node.js and npm +- Python 3.9 or higher +- Docker (for some dependencies) + +== get-redpanda-version + +Fetches the latest Redpanda version from GitHub releases. Can retrieve either stable releases or beta/RC versions. Returns the version in format "v25.3.1" which can be used directly with other doc-tools commands. + +*Why use it:* + +Documentation must reference the correct current version. This command ensures version numbers are accurate and can be used in CI/CD pipelines or before generating version-specific documentation. The version is fetched from GitHub releases, which is the source of truth for Redpanda releases. + +*Usage:* + +[,bash] +---- +doc-tools get-redpanda-version [options] +---- + +*Options:* + +`--beta`:: +Return the latest RC (beta) version if available + +`--from-antora`:: +Read prerelease flag from local antora.yml + +`-h, --help`:: +display help for command + +*Examples:* + +[,bash] +---- +# Get latest stable version +npx doc-tools get-redpanda-version +# Output: v25.3.1 + +# Get latest beta/RC version +npx doc-tools get-redpanda-version --beta +# Output: v26.1.1-rc1 + +# Auto-detect from antora.yml prerelease flag +cd docs-site +npx doc-tools get-redpanda-version --from-antora + +# Use in CI/CD or scripts +VERSION=$(npx doc-tools get-redpanda-version) +npx doc-tools generate property-docs --tag $VERSION +---- + +*Requirements:* + +- Internet connection to access GitHub API +- GitHub API rate limits apply (60 requests/hour unauthenticated) + +== get-console-version + +Fetches the latest Redpanda Console version from GitHub releases. Can retrieve either stable releases or beta versions. Returns the version in format "v2.7.2" which can be used for documentation references and Docker image tags. + +*Why use it:* + +Console is released separately from Redpanda core. This command keeps Console documentation in sync with releases and provides the correct version for Docker Compose files and deployment documentation. + +*Usage:* + +[,bash] +---- +doc-tools get-console-version [options] +---- + +*Options:* + +`--beta`:: +Return the latest beta version if available + +`--from-antora`:: +Read prerelease flag from local antora.yml + +`-h, --help`:: +display help for command + +*Examples:* + +[,bash] +---- +# Get latest stable Console version +npx doc-tools get-console-version +# Output: v2.7.2 + +# Get latest beta version +npx doc-tools get-console-version --beta +# Output: v2.8.0-beta1 + +# Auto-detect from antora.yml prerelease flag +cd docs-site +npx doc-tools get-console-version --from-antora + +# Use in Docker Compose documentation +CONSOLE_VERSION=$(npx doc-tools get-console-version) +echo "image: redpandadata/console:$CONSOLE_VERSION" +---- + +*Requirements:* + +- Internet connection to access GitHub API +- GitHub API rate limits apply (60 requests/hour unauthenticated) + +== link-readme + +Creates a symbolic link from a project's README.adoc file into the Antora documentation structure. This allows project README files to be included in the documentation site without duplication. The command creates the necessary directory structure and establishes a symlink in docs/modules//pages/ that points to the project's README.adoc. + +*Why use it:* + +Documentation repositories often contain multiple sub-projects (like labs or examples) that have their own README files. Rather than manually copying these files into the Antora structure (which creates maintenance burden), symlinks keep the content in one place while making it available to Antora. Changes to the project README automatically appear in the docs site. + +*Usage:* + +[,bash] +---- +doc-tools link-readme [options] +---- + +*Options:* + +`-s, --subdir `:: +Relative path to the lab project subdirectory + +`-t, --target `:: +Name of the target AsciiDoc file in pages/ + +`-h, --help`:: +display help for command + +*Examples:* + +[,bash] +---- +# Link a lab project README into documentation +npx doc-tools link-readme \\ + --subdir labs/docker-compose \\ + --target docker-compose-lab.adoc + +# Link multiple lab READMEs +npx doc-tools link-readme -s labs/kubernetes -t k8s-lab.adoc +npx doc-tools link-readme -s labs/terraform -t terraform-lab.adoc + +# The symlink structure created: +# docs/modules/labs/pages/docker-compose-lab.adoc -> ../../../../labs/docker-compose/README.adoc +---- + +*Requirements:* + +- Must run from repository root +- Target project must have README.adoc file +- Operating system must support symbolic links + +== fetch + +Downloads specific files or directories from GitHub repositories without cloning the entire repository. Uses the GitHub API to fetch content and saves it to a local directory. Useful for grabbing examples, configuration files, or documentation snippets from other repositories. Supports both individual files and entire directories. + +*Why use it:* + +Documentation often needs to reference or include files from other repositories (examples, configuration templates, code samples). Cloning entire repositories is inefficient when you only need specific files. This command provides targeted fetching, saving bandwidth and time. It's particularly useful in CI/CD pipelines where you need specific assets without full clones. + +*Usage:* + +[,bash] +---- +doc-tools fetch [options] +---- + +*Options:* + +`-o, --owner `:: +GitHub repo owner or org + +`-r, --repo `:: +GitHub repo name + +`-p, --remote-path `:: +Path in the repo to fetch + +`-d, --save-dir `:: +Local directory to save into + +`-f, --filename `:: +Custom filename to save as + +`-h, --help`:: +display help for command + +*Examples:* + +[,bash] +---- +# Fetch a specific configuration file +npx doc-tools fetch \\ + --owner redpanda-data \\ + --repo redpanda \\ + --remote-path docker/docker-compose.yml \\ + --save-dir examples/ + +# Fetch an entire directory of examples +npx doc-tools fetch \\ + -o redpanda-data \\ + -r connect-examples \\ + -p pipelines/mongodb \\ + -d docs/modules/examples/attachments/ + +# Fetch with custom filename +npx doc-tools fetch \\ + -o redpanda-data \\ + -r helm-charts \\ + -p charts/redpanda/values.yaml \\ + -d examples/ \\ + --filename redpanda-values-example.yaml +---- + +*Requirements:* + +- Internet connection to access GitHub API +- GitHub API rate limits apply (60 requests/hour unauthenticated, 5000 with token) +- For private repositories: GitHub token with repo permissions + +== setup-mcp + +Configures the Redpanda Docs MCP (Model Context Protocol) server for Claude Code or Claude Desktop. Automatically detects the installed application, updates the appropriate configuration file, and enables Claude to use doc-tools commands through natural conversation. Supports both production (npm package) and local development modes. + +*Why use it:* + +Manual MCP configuration requires editing JSON configuration files in the correct location with the correct schema. This command handles all setup automatically, including path detection, configuration merging, and validation. It enables AI-assisted documentation workflows where writers can use natural language to run doc-tools commands. + +*Usage:* + +[,bash] +---- +doc-tools setup-mcp [options] +---- + +*Options:* + +`--force`:: +Force update even if already configured (default: false) + +`--target `:: +Target application: auto, code, or desktop (default: "auto") + +`--local`:: +Use local development mode (requires running from this repo) (default: false) + +`--status`:: +Show current MCP server configuration status (default: false) + +`-h, --help`:: +display help for command + +*Examples:* + +[,bash] +---- +# Auto-detect and configure for Claude Code or Desktop +npx doc-tools setup-mcp + +# Configure for local development (run from this repository) +cd /path/to/docs-extensions-and-macros +npx doc-tools setup-mcp --local + +# Force update existing configuration +npx doc-tools setup-mcp --force + +# Target specific application +npx doc-tools setup-mcp --target code +npx doc-tools setup-mcp --target desktop + +# Check current configuration status +npx doc-tools setup-mcp --status + +# After setup, restart Claude Code and use natural language +"What's the latest Redpanda version?" +"Generate property docs for v25.3.1" +---- + +*Requirements:* + +- Claude Code or Claude Desktop must be installed +- For --local mode: must run from docs-extensions-and-macros repository +- After setup: restart Claude Code/Desktop to load the MCP server + +== generate + +Run docs automations + +*Usage:* + +[,bash] +---- +doc-tools generate [options] [command] +---- + +*Options:* + +`-h, --help`:: +display help for command + +*Commands:* + +* `metrics-docs` - Generate JSON and AsciiDoc documentation for Redpanda metrics. Defaults to branch "dev" if neither --tag nor --branch is specified. +* `rpcn-connector-docs` - Generate RPCN connector docs and diff changes since the last version +* `property-docs` - Generate JSON and consolidated AsciiDoc partials for Redpanda configuration properties. By default, only extracts properties to JSON. Use --generate-partials to create consolidated AsciiDoc partials (including deprecated properties). Defaults to branch "dev" if neither --tag nor --branch is specified. +* `rpk-docs` - Generate AsciiDoc documentation for rpk CLI commands. Defaults to branch "dev" if neither --tag nor --branch is specified. +* `helm-spec` - Generate AsciiDoc documentation for one or more Helm charts (supports local dirs or GitHub URLs). When using GitHub URLs, requires either --tag or --branch to be specified. +* `cloud-regions` - Generate Markdown table of cloud regions and tiers from GitHub YAML file +* `crd-spec` - Generate Asciidoc documentation for Kubernetes CRD references. Requires either --tag or --branch to be specified. +* `bundle-openapi` - Bundle Redpanda OpenAPI fragments for admin and connect APIs into complete OpenAPI 3.1 documents. Requires either --tag or --branch to be specified. +* `help` - display help for command + +=== generate property-docs + +Generates comprehensive reference documentation for Redpanda cluster and topic configuration properties. Clones the Redpanda repository at a specified version, runs a Python extractor to parse C++ configuration code, and outputs JSON data files with all property metadata (descriptions, types, defaults, constraints). Optionally generates consolidated AsciiDoc partials for direct inclusion in documentation sites. + +*Why use it:* + +Property definitions in the C++ source code are the single source of truth for Redpanda configuration. Manual documentation becomes outdated quickly. This automation ensures docs stay perfectly in sync with implementation by extracting properties directly from code, including type information, default values, and constraints that would be error-prone to maintain manually. + +*Usage:* + +[,bash] +---- +doc-tools generate property-docs [options] +---- + +*Options:* + +`-t, --tag `:: +Git tag for released content (GA/beta) + +`-b, --branch `:: +Branch name for in-progress content + +`--diff `:: +Also diff autogenerated properties from to current tag/branch + +`--overrides `:: +Optional JSON file with property description overrides (default: "docs-data/property-overrides.json") + +`--output-dir `:: +Where to write all generated files (default: "modules/reference") + +`--cloud-support`:: +Add AsciiDoc tags to generated property docs to indicate which ones are supported in Redpanda Cloud. This data is fetched from the cloudv2 repository so requires a GitHub token with repo permissions. Set the token as an environment variable using GITHUB_TOKEN, GH_TOKEN, or REDPANDA_GITHUB_TOKEN + +`--template-property `:: +Custom Handlebars template for individual property sections + +`--template-topic-property `:: +Custom Handlebars template for individual topic property sections + +`--template-topic-property-mappings `:: +Custom Handlebars template for topic property mappings table + +`--template-deprecated `:: +Custom Handlebars template for deprecated properties page + +`--template-deprecated-property `:: +Custom Handlebars template for individual deprecated property sections + +`--generate-partials`:: +Generate consolidated property partials (cluster-properties.adoc, topic-properties.adoc, etc.) in the partials directory + +`--partials-dir `:: +Directory for property partials (relative to output-dir) (default: "partials") + +`-h, --help`:: +display help for command + +*Examples:* + +[,bash] +---- +# Basic: Extract properties to JSON only (default) +npx doc-tools generate property-docs --tag v25.3.1 + +# Generate AsciiDoc partials for documentation site +npx doc-tools generate property-docs --tag v25.3.1 --generate-partials + +# Include Cloud support tags (requires GitHub token) +export GITHUB_TOKEN=ghp_xxx +npx doc-tools generate property-docs \\ + --tag v25.3.1 \\ + --generate-partials \\ + --cloud-support + +# Compare properties between versions +npx doc-tools generate property-docs \\ + --tag v25.3.1 \\ + --diff v25.2.1 + +# Use custom output directory +npx doc-tools generate property-docs \\ + --tag v25.3.1 \\ + --output-dir docs/modules/reference + +# Full workflow: document new release +VERSION=$(npx doc-tools get-redpanda-version) +npx doc-tools generate property-docs \\ + --tag $VERSION \\ + --generate-partials \\ + --cloud-support +---- + +*Requirements:* + +- Python 3.9 or higher +- Git +- Internet connection to clone Redpanda repository +- For --cloud-support: GitHub token with repo permissions (GITHUB_TOKEN env var) +- For --cloud-support: Python packages pyyaml and requests + +=== generate metrics-docs + +Generates comprehensive metrics reference documentation by running Redpanda in Docker and scraping the `/public_metrics` Prometheus endpoint. Starts a Redpanda cluster with the specified version, waits for it to be ready, collects all exposed metrics, parses the Prometheus format, and generates categorized AsciiDoc documentation. Optionally compares metrics between versions to identify new, removed, or changed metrics. + +*Why use it:* + +Redpanda exposes hundreds of metrics for monitoring and observability. Manual documentation of metrics is error-prone and becomes outdated as new metrics are added or existing ones change. This automation ensures metrics documentation accurately reflects what Redpanda actually exports at each version. Running Redpanda in Docker and scraping metrics directly is the only reliable way to capture the complete and accurate metrics set. + +*Usage:* + +[,bash] +---- +doc-tools generate metrics-docs [options] +---- + +*Options:* + +`-t, --tag `:: +Git tag for released content (GA/beta) + +`-b, --branch `:: +Branch name for in-progress content + +`--docker-repo `:: +Docker repository to use when starting Redpanda in Docker (default: "redpanda") + +`--console-tag `:: +Redpanda Console version to use when starting Redpanda Console in Docker (default: "latest") + +`--console-docker-repo `:: +Docker repository to use when starting Redpanda Console in Docker (default: "console") + +`--diff `:: +Also diff autogenerated metrics from β†’ + +`-h, --help`:: +display help for command + +*Examples:* + +[,bash] +---- +# Basic: Generate metrics docs for a specific version +npx doc-tools generate metrics-docs --tag v25.3.1 + +# Compare metrics between versions to see what changed +npx doc-tools generate metrics-docs \\ + --tag v25.3.1 \\ + --diff v25.2.1 + +# Use custom Docker repository +npx doc-tools generate metrics-docs \\ + --tag v25.3.1 \\ + --docker-repo docker.redpanda.com/redpandadata/redpanda + +# Full workflow: document new release +VERSION=$(npx doc-tools get-redpanda-version) +npx doc-tools generate metrics-docs --tag $VERSION +---- + +*Requirements:* + +- Docker must be installed and running +- Port 9644 must be available (Redpanda metrics endpoint) +- Sufficient disk space for Docker image +- Internet connection to pull Docker images + +=== generate rpk-docs + +Generates comprehensive CLI reference documentation for RPK (Redpanda Keeper), the official Redpanda command-line tool. Starts Redpanda in Docker (RPK is bundled with Redpanda), executes `rpk --help` for all commands and subcommands recursively, parses the help output, and generates structured AsciiDoc documentation for each command with usage, flags, and descriptions. Optionally compares RPK commands between versions to identify new or changed commands. + +*Why use it:* + +RPK has dozens of commands and subcommands with complex flags and options. The built-in help text is the source of truth for RPK's CLI interface. Manual documentation becomes outdated as RPK evolves. This automation extracts documentation directly from RPK's help output, ensuring accuracy. Running RPK from Docker guarantees the exact version being documented, and diffing between versions automatically highlights CLI changes for release notes. + +*Usage:* + +[,bash] +---- +doc-tools generate rpk-docs [options] +---- + +*Options:* + +`-t, --tag `:: +Git tag for released content (GA/beta) + +`-b, --branch `:: +Branch name for in-progress content + +`--docker-repo `:: +Docker repository to use when starting Redpanda in Docker (default: "redpanda") + +`--console-tag `:: +Redpanda Console version to use when starting Redpanda Console in Docker (default: "latest") + +`--console-docker-repo `:: +Docker repository to use when starting Redpanda Console in Docker (default: "console") + +`--diff `:: +Also diff autogenerated rpk docs from β†’ + +`-h, --help`:: +display help for command + +*Examples:* + +[,bash] +---- +# Basic: Generate RPK docs for a specific version +npx doc-tools generate rpk-docs --tag v25.3.1 + +# Compare RPK commands between versions +npx doc-tools generate rpk-docs \\ + --tag v25.3.1 \\ + --diff v25.2.1 + +# Use custom Docker repository +npx doc-tools generate rpk-docs \\ + --tag v25.3.1 \\ + --docker-repo docker.redpanda.com/redpandadata/redpanda + +# Full workflow: document new release +VERSION=$(npx doc-tools get-redpanda-version) +npx doc-tools generate rpk-docs --tag $VERSION +---- + +*Requirements:* + +- Docker must be installed and running +- Sufficient disk space for Docker image +- Internet connection to pull Docker images + +=== generate rpcn-connector-docs + +Generates complete reference documentation for Redpanda Connect (formerly Benthos) connectors, processors, and components. Clones the Redpanda Connect repository, parses component templates and configuration schemas embedded in Go code, reads connector metadata from CSV, and generates AsciiDoc documentation for each component. Supports diffing changes between versions and automatically updating what's new documentation. Can also generate Bloblang function documentation. + +*Why use it:* + +Redpanda Connect has hundreds of connectors (inputs, outputs, processors) with complex configuration schemas. Each component's documentation lives in its Go source code as struct tags and comments. Manual documentation is impossible to maintain. This automation extracts documentation directly from code, ensuring accuracy and completeness. The diff capability automatically identifies new connectors and changed configurations for release notes. + +*Usage:* + +[,bash] +---- +doc-tools generate rpcn-connector-docs [options] +---- + +*Options:* + +`-d, --data-dir `:: +Directory where versioned connect JSON files live (default: "/docs-data") + +`--old-data `:: +Optional override for old data file (for diff) + +`--update-whats-new`:: +Update whats-new.adoc with new section from diff JSON + +`-f, --fetch-connectors`:: +Fetch latest connector data using rpk + +`-m, --draft-missing`:: +Generate full-doc drafts for connectors missing in output + +`--csv `:: +Path to connector metadata CSV file (default: "internal/plugins/info.csv") + +`--template-main `:: +Main Handlebars template (default: "/tools/redpanda-connect/templates/connector.hbs") + +`--template-intro `:: +Intro section partial template (default: "/tools/redpanda-connect/templates/intro.hbs") + +`--template-fields `:: +Fields section partial template (default: "/tools/redpanda-connect/templates/fields-partials.hbs") + +`--template-examples `:: +Examples section partial template (default: "/tools/redpanda-connect/templates/examples-partials.hbs") + +`--template-bloblang `:: +Custom Handlebars template for bloblang function/method partials + +`--overrides `:: +Optional JSON file with overrides (default: "docs-data/overrides.json") + +`--include-bloblang`:: +Include Bloblang functions and methods in generation + +`-h, --help`:: +display help for command + +*Examples:* + +[,bash] +---- +# Basic: Generate all connector docs +npx doc-tools generate rpcn-connector-docs + +# Generate docs and automatically update what's new page +npx doc-tools generate rpcn-connector-docs --update-whats-new + +# Include Bloblang function documentation +npx doc-tools generate rpcn-connector-docs --include-bloblang + +# Generate with custom metadata CSV +npx doc-tools generate rpcn-connector-docs \\ + --csv custom/connector-metadata.csv + +# Full workflow with diff and what's new update +npx doc-tools generate rpcn-connector-docs \\ + --update-whats-new \\ + --include-bloblang +---- + +*Requirements:* + +- Git to clone Redpanda Connect repository +- Internet connection to clone repository +- Node.js for parsing and generation +- Sufficient disk space for repository clone (~500MB) + +=== generate helm-spec + +Generates Helm chart reference documentation by parsing values.yaml files and README.md documentation from Helm chart repositories. Supports both local chart directories and GitHub URLs. Extracts all configuration options with their types, defaults, and descriptions, and generates comprehensive AsciiDoc documentation. Can process single charts or entire chart repositories with multiple charts. + +*Why use it:* + +Helm charts have complex configuration with hundreds of values. The values.yaml file and chart README contain the configuration options, but they're not in a documentation-friendly format. This automation parses the YAML structure and README documentation to generate comprehensive reference documentation. Supporting both local and GitHub sources allows documenting charts from any source without manual cloning. + +*Usage:* + +[,bash] +---- +doc-tools generate helm-spec [options] +---- + +*Options:* + +`--chart-dir `:: +Chart directory (contains Chart.yaml) or a root containing multiple charts, or a GitHub URL (default: "https://github.com/redpanda-data/redpanda-operator/charts") + +`-t, --tag `:: +Git tag for released content when using GitHub URL (auto-prepends "operator/" for redpanda-operator repository) + +`-b, --branch `:: +Branch name for in-progress content when using GitHub URL + +`--readme `:: +Relative README.md path inside each chart dir (default: "README.md") + +`--output-dir `:: +Where to write all generated AsciiDoc files (default: "modules/reference/pages") + +`--output-suffix `:: +Suffix to append to each chart name (including extension) (default: "-helm-spec.adoc") + +`-h, --help`:: +display help for command + +*Examples:* + +[,bash] +---- +# Generate docs from GitHub repository +npx doc-tools generate helm-spec \\ + --chart-dir https://github.com/redpanda-data/helm-charts \\ + --tag v5.9.0 \\ + --output-dir modules/deploy/pages + +# Generate docs from local chart directory +npx doc-tools generate helm-spec \\ + --chart-dir ./charts/redpanda \\ + --output-dir docs/modules/deploy/pages + +# Use custom README and output suffix +npx doc-tools generate helm-spec \\ + --chart-dir https://github.com/redpanda-data/helm-charts \\ + --tag v5.9.0 \\ + --readme docs/README.md \\ + --output-suffix -values.adoc +---- + +*Requirements:* + +- For GitHub URLs: Git and internet connection +- For local charts: Chart directory must contain Chart.yaml +- README.md file in chart directory (optional but recommended) + +=== generate cloud-regions + +Generates a formatted table of Redpanda Cloud regions, tiers, and availability information by fetching data from the private cloudv2-infra repository. Reads a YAML configuration file that contains master data for cloud infrastructure, parses region and tier information, and generates either Markdown or AsciiDoc tables for documentation. Supports custom templates and dry-run mode for previewing output. + +*Why use it:* + +Cloud region data changes frequently as new regions are added and tier availability evolves. The cloudv2-infra repository contains the source of truth for cloud infrastructure. Manual documentation becomes outdated quickly. This automation fetches the latest data directly from the infrastructure repository, ensuring documentation always reflects current cloud offerings. Weekly or triggered updates keep docs in sync with cloud expansion. + +*Usage:* + +[,bash] +---- +doc-tools generate cloud-regions [options] +---- + +*Options:* + +`--output `:: +Output file (relative to repo root) (default: "cloud-controlplane/x-topics/cloud-regions.md") + +`--format `:: +Output format: md (Markdown) or adoc (AsciiDoc) (default: "md") + +`--owner `:: +GitHub repository owner (default: "redpanda-data") + +`--repo `:: +GitHub repository name (default: "cloudv2-infra") + +`--path `:: +Path to YAML file in repository (default: "apps/master-data-reconciler/manifests/overlays/production/master-data.yaml") + +`--ref `:: +Git reference (branch, tag, or commit SHA) (default: "integration") + +`--template `:: +Path to custom Handlebars template (relative to repo root) + +`--dry-run`:: +Print output to stdout instead of writing file + +`-h, --help`:: +display help for command + +*Examples:* + +[,bash] +---- +# Basic: Generate Markdown table +export GITHUB_TOKEN=ghp_xxx +npx doc-tools generate cloud-regions + +# Generate AsciiDoc format +export GITHUB_TOKEN=ghp_xxx +npx doc-tools generate cloud-regions --format adoc + +# Preview without writing file +export GITHUB_TOKEN=ghp_xxx +npx doc-tools generate cloud-regions --dry-run + +# Use custom output file +export GITHUB_TOKEN=ghp_xxx +npx doc-tools generate cloud-regions \\ + --output custom/path/regions.md + +# Use different branch for testing +export GITHUB_TOKEN=ghp_xxx +npx doc-tools generate cloud-regions --ref staging +---- + +*Requirements:* + +- GitHub token with access to redpanda-data/cloudv2-infra repository +- Token must be set via GITHUB_TOKEN, GH_TOKEN, or REDPANDA_GITHUB_TOKEN environment variable +- Internet connection to access GitHub API + +=== generate crd-spec + +Generates Kubernetes Custom Resource Definition (CRD) reference documentation by parsing Go type definitions from the Redpanda Operator repository. Uses the crd-ref-docs tool to extract API field definitions, types, descriptions, and validation rules from Go struct tags and comments, then generates comprehensive AsciiDoc documentation. Supports both local Go source directories and GitHub URLs for operator versions. When to use --tag vs --branch: - Use --tag for released content (GA or beta releases). Tags reference specific release points. - Use --branch for in-progress content (unreleased features). Branches track ongoing development. + +*Why use it:* + +Kubernetes CRDs define complex APIs for deploying and managing Redpanda. The API schema is defined in Go code with hundreds of fields across nested structures. Manual documentation is error-prone and becomes outdated as the API evolves. This automation uses specialized tooling (crd-ref-docs) to extract API documentation directly from Go source code, ensuring accuracy and completeness. It captures field types, validation rules, and descriptions that are essential for users configuring Redpanda in Kubernetes. + +*Usage:* + +[,bash] +---- +doc-tools generate crd-spec [options] +---- + +*Options:* + +`-t, --tag `:: +Operator release tag for GA/beta content (for example operator/v2.2.6-25.3.1 or v25.1.2). Auto-prepends "operator/" if not present. + +`-b, --branch `:: +Branch name for in-progress content (for example release/v2.2.x, main, dev) + +`-s, --source-path `:: +CRD Go types dir or GitHub URL (default: "https://github.com/redpanda-data/redpanda-operator/operator/api/redpanda/v1alpha2") + +`-d, --depth `:: +How many levels deep (default: "10") + +`--templates-dir `:: +Asciidoctor templates dir (default: ".github/crd-config/templates/asciidoctor/operator") + +`--output `:: +Where to write the generated AsciiDoc file (default: "modules/reference/pages/k-crd.adoc") + +`-h, --help`:: +display help for command + +*Examples:* + +[,bash] +---- +# Generate CRD docs for specific operator tag +npx doc-tools generate crd-spec --tag operator/v2.2.6-25.3.1 + +# Version without prefix (auto-prepends operator/) +npx doc-tools generate crd-spec --tag v25.1.2 + +# Generate from release branch +npx doc-tools generate crd-spec --branch release/v2.2.x + +# Generate from main branch +npx doc-tools generate crd-spec --branch main + +# Generate from any custom branch +npx doc-tools generate crd-spec --branch dev + +# Use custom templates and output location +npx doc-tools generate crd-spec \\ + --tag operator/v2.2.6-25.3.1 \\ + --templates-dir custom/templates \\ + --output modules/reference/pages/operator-crd.adoc +---- + +*Requirements:* + +- For GitHub URLs: Git and internet connection +- crd-ref-docs tool (automatically installed if missing) +- Go toolchain for running crd-ref-docs + +=== generate bundle-openapi + +Bundles Redpanda's OpenAPI specification fragments into complete, usable OpenAPI 3.1 documents for both Admin API and Connect API. Clones the Redpanda repository at a specified version, collects OpenAPI fragments that are distributed throughout the codebase (alongside endpoint implementations), uses Buf and Redocly CLI to bundle and validate the specifications, and generates separate complete OpenAPI files for each API surface. The resulting specifications can be used for API documentation, client SDK generation, or API testing tools. + +*Why use it:* + +Redpanda's API documentation is defined as OpenAPI fragments alongside the C++ implementation code. This keeps API docs close to code and ensures they stay in sync, but it means the specification is fragmented across hundreds of files. Users need complete OpenAPI specifications for tooling (Swagger UI, Postman, client generators). This automation collects all fragments, bundles them into valid OpenAPI 3.1 documents, and validates the result. It's the only way to produce accurate, complete API specifications that match a specific Redpanda version. + +*Usage:* + +[,bash] +---- +doc-tools generate bundle-openapi [options] +---- + +*Options:* + +`-t, --tag `:: +Git tag for released content (for example, v24.3.2 or 24.3.2) + +`-b, --branch `:: +Branch name for in-progress content (for example, dev, main) + +`--repo `:: +Repository URL (default: "https://github.com/redpanda-data/redpanda.git") + +`-s, --surface `:: +Which API surface(s) to bundle (choices: "admin", "connect", "both") + +`--out-admin `:: +Output path for admin API (default: "admin/redpanda-admin-api.yaml") + +`--out-connect `:: +Output path for connect API (default: "connect/redpanda-connect-api.yaml") + +`--admin-major `:: +Admin API major version (default: "v2.0.0") + +`--use-admin-major-version`:: +Use admin major version for info.version instead of git tag (default: false) + +`--quiet`:: +Suppress logs (default: false) + +`-h, --help`:: +display help for command + +*Examples:* + +[,bash] +---- +# Bundle both Admin and Connect APIs +npx doc-tools generate bundle-openapi \\ + --tag v25.3.1 \\ + --surface both + +# Bundle only Admin API +npx doc-tools generate bundle-openapi \\ + --tag v25.3.1 \\ + --surface admin + +# Use custom output paths +npx doc-tools generate bundle-openapi \\ + --tag v25.3.1 \\ + --surface both \\ + --out-admin api/admin-api.yaml \\ + --out-connect api/connect-api.yaml + +# Use major version for Admin API version field +npx doc-tools generate bundle-openapi \\ + --tag v25.3.1 \\ + --surface admin \\ + --use-admin-major-version + +# Full workflow: generate API specs for new release +VERSION=$(npx doc-tools get-redpanda-version) +npx doc-tools generate bundle-openapi --tag $VERSION --surface both +---- + +*Requirements:* + +- Git to clone Redpanda repository +- Buf tool (automatically installed via npm) +- Redocly CLI or vacuum for OpenAPI bundling (automatically detected) +- Internet connection to clone repository +- Sufficient disk space for repository clone (~2GB) + diff --git a/CONTRIBUTING.adoc b/CONTRIBUTING.adoc new file mode 100644 index 00000000..d432690f --- /dev/null +++ b/CONTRIBUTING.adoc @@ -0,0 +1,291 @@ += Contributing +:url-org: https://github.com/redpanda-data +:url-project: {url-org}/docs-extensions-and-macros +:url-git: https://git-scm.com +:url-nodejs: https://nodejs.org +:toc: +:toclevels: 2 + +Guide to contributing to this project. + +Thank you for your interest in contributing to the Redpanda documentation tools! This project includes Antora extensions, AsciiDoc macros, and an MCP server that helps automate documentation tasks. + +== Development guides + +Choose the guide for the component you want to work on: + +link:extensions/DEVELOPMENT.adoc[**Antora extensions development**]:: How to create and modify Antora extensions ++ +* Extension architecture and lifecycle +* Creating new extensions +* Common patterns and APIs +* Testing and debugging + +link:macros/DEVELOPMENT.adoc[**AsciiDoc macros development**]:: How to create and modify AsciiDoc macros ++ +* Inline and block macro types +* Macro processing and HTML generation +* Asciidoctor.js extension API +* Testing and debugging + +link:mcp/DEVELOPMENT.adoc[**MCP server development**]:: How to work on the MCP server ++ +* Server architecture and tools +* Adding new tools and prompts +* Testing guidelines +* Security and best practices + +== Quickstart + +=== Prerequisites + +* {url-git}[git] - Version control +* {url-nodejs}[Node.js] 18+ - Runtime and package manager + +Verify installations: + +[,bash] +---- +git --version +node --version +npm --version +---- + +=== Get the code + +Clone the repository: + +[,bash,subs=attributes+] +---- +git clone {url-project} +cd docs-extensions-and-macros +---- + +=== Install dependencies + +[,bash] +---- +npm ci +---- + +This installs exact versions from `package-lock.json`. + +=== Run tests + +[,bash] +---- +# Run all tests +npm test + +# Run specific test suites +npm run test:mcp # MCP server tests +npm run test:extensions # Extension tests (if available) + +# Watch mode for development +npm test -- --watch +---- + +== Development workflow + +=== Create a branch + +[,bash] +---- +git checkout -b feature/my-new-feature +---- + +Use descriptive branch names: +* `feature/add-new-extension` +* `fix/glossary-macro-bug` +* `docs/update-user-guide` + +=== Make your changes + +Follow the relevant development guide: +* link:extensions/DEVELOPMENT.adoc[Extensions] - For Antora extensions +* link:macros/DEVELOPMENT.adoc[Macros] - For AsciiDoc macros +* link:mcp/DEVELOPMENT.adoc[MCP] - For MCP server tools + +=== Test locally + +Test with a local Antora build: + +[,bash] +---- +# Point to your local development version +npx antora local-antora-playbook.yml +---- + +For extensions and macros, update your playbook: + +[,yaml] +---- +antora: + extensions: + - '/absolute/path/to/docs-extensions-and-macros/extensions/my-extension.js' + +asciidoc: + extensions: + - '/absolute/path/to/docs-extensions-and-macros/macros/my-macro.js' +---- + +For MCP server: + +[,bash] +---- +npx doc-tools setup-mcp +# Restart Claude Code +---- + +=== Write tests + +Add tests for new functionality: + +[,javascript] +---- +// For extensions +__tests__/extensions/my-extension.test.js + +// For macros +__tests__/macros/my-macro.test.js + +// For MCP +__tests__/mcp/integration.test.js +---- + +Run tests: + +[,bash] +---- +npm test +---- + +=== Update documentation + +Update relevant documentation: + +* Extension/macro reference documentation +* User guides if behavior changed +* README files if new features added + +=== Commit your changes + +Write clear commit messages: + +[,bash] +---- +git add . +git commit -m "Add new glossary tooltip feature + +- Implement data-tooltip attribute support +- Add tests for tooltip rendering +- Update user guide with tooltip examples" +---- + +=== Push and create PR + +[,bash] +---- +git push origin feature/my-new-feature +---- + +Open a pull request on GitHub with: +* Clear description of changes +* Link to related issues +* Screenshots/examples if applicable + +== Coding standards + +=== JavaScript style + +* Use 2 spaces for indentation +* Use semicolons +* Use single quotes for strings +* Use `const` and `let`, avoid `var` +* Add JSDoc comments for functions + +=== File naming + +* Use kebab-case for files: `my-extension.js` +* Use descriptive names: `generate-index-data.js` not `gen.js` + +=== Error handling + +* Catch and log errors, don't let builds fail silently +* Provide helpful error messages +* Use the logger when available + +=== Testing + +* Write tests for all new functionality +* Maintain test coverage +* Test edge cases and error conditions + +== Documentation standards + +* Use AsciiDoc for all documentation +* Use sentence case for headings (except main title) +* Keep documentation in sync with code +* Provide examples for new features +* Link between related documentation + +== Pull request guidelines + +=== Before submitting + +- [ ] Tests pass (`npm test`) +- [ ] Documentation updated +- [ ] Commit messages are clear +- [ ] Branch is up to date with main + +=== PR description + +Include: + +* **What**: Summary of changes +* **Why**: Reason for changes +* **How**: Implementation approach +* **Testing**: How you tested the changes +* **Breaking changes**: If any, clearly noted + +=== Review process + +* Keep PRs focused and reasonably sized +* Update documentation if behavior changes +* Ensure CI passes + +== Testing requirements + +All contributions must include tests: + +* **Unit tests** - Test individual functions/components +* **Integration tests** - Test components working together +* **Manual testing** - Test in real Antora builds + +Run the full test suite before submitting: + +[,bash] +---- +npm test +---- + +== Code of conduct + +* Be respectful and inclusive +* Provide constructive feedback +* Focus on the code, not the person +* Help newcomers get started + +== Getting help + +* **Questions**: Open a discussion on GitHub +* **Bugs**: Open an issue with reproduction steps +* **Features**: Open an issue with use case description +* **Chat**: Join the Redpanda Community Slack + +== Related resources + +* https://docs.antora.org/[Antora Documentation] +* https://docs.asciidoctor.org/[AsciiDoc Documentation] +* https://modelcontextprotocol.io/[MCP Specification] +* {url-project}[Project Repository] diff --git a/README.adoc b/README.adoc index 17565179..111bcb94 100644 --- a/README.adoc +++ b/README.adoc @@ -1,1106 +1,75 @@ = Antora Extensions and Macros for Redpanda Docs :url-org: https://github.com/redpanda-data :url-project: {url-org}/docs-extensions-and-macros -:url-playbook: {url-org}/docs-site -:url-git: https://git-scm.com -:url-git-dl: {url-git}/downloads -:url-nodejs: https://nodejs.org -:url-nodejs-releases: https://github.com/nodejs/Release#release-schedule -:url-nvm-install: {url-nvm}#installation -:idprefix: -:idseparator: - ifdef::env-github[] :important-caption: :exclamation: :note-caption: :paperclip: endif::[] -:toc: -:toc-title: Contents -toc::[] +This library provides https://docs.antora.org/antora/latest/extend/extensions/[Antora extensions], https://docs.asciidoctor.org/asciidoctor.js/latest/extend/extensions/register/[AsciiDoc macros], and automation tools for Redpanda documentation. -This library provides https://docs.antora.org/antora/latest/extend/extensions/[Antora extensions] and https://docs.asciidoctor.org/asciidoctor.js/latest/extend/extensions/register/[Asciidoc macros] developed for Redpanda documentation. +== Quickstart -== Prerequisites - -To use this library, you must have {url-nodejs}[Node.js] 16 or higher installed on your machine. - -[,bash] ----- -node --version ----- - -If this command fails with an error, you don't have Node.js installed. - -When you have Node.js installed, use the following command to install the `antora-extensions-and-macros` package in your project: +Install the package: [,bash] ---- npm i @redpanda-data/docs-extensions-and-macros ---- -To use the development version, refer to the <>. - -== Antora Extensions - -This section documents the Antora extensions provided by this library and how to configure them. - -IMPORTANT: Ensure you register each extension under the `antora.extensions` key in the playbook, not the `asciidoc.extensions` key. - -=== Add Bloblang samples to pages - -The `collect-bloblang-samples` extension processes Bloblang examples from YAML files in the `examples` directory of the `redpanda-connect` component. This extension ensures that these examples are accessible as structured data for use in UI components or documentation, such as sample dropdowns in a Bloblang playground. - -It validates, sorts, and attaches the processed examples as a JSON object to the Antora page attributes. The extension ensures examples have unique titles, mandatory fields (`input` and `mapping`), and are sorted in alphabetical order. - -The processed examples are added as JSON to the `page-bloblang-samples` attribute. For example: - -[,json] ----- -{ - "hello-world.yaml": { - "title": "Hello world", - "input": "{\n \"message\": \"hello world\"\n}\n", - "mapping": "root.message = this.message.uppercase()\n" - }, - "array-processing.yaml": { - "title": "Array processing", - "input": "{\n \"numbers\": [1, 2, 3, 4, 5]\n}\n", - "mapping": "root.even_numbers = this.numbers.filter(n -> n % 2 == 0)" - } -} ----- - -==== Environment variables - -This extension does not require any environment variables. - -==== Configuration options - -To enable the extension, add it to your Antora playbook under the `antora.extensions` key. No additional configuration is required. - -[,yaml] ----- -antora: - extensions: - - require: '@redpanda-data/docs-extensions-and-macros/extensions/collect-bloblang-samples' ----- - -==== Example Bloblang YAML file - -The following YAML file is an example of how to define a Bloblang sample: - -[,yaml] ----- -title: Hello world -input: | - { - "message": "hello world" - } -mapping: | - root.message = this.message.uppercase() ----- - -=== Add pages to root - -The `add-pages-to-root` extension allows you to copy files from your Antora content catalog to the root of the site during the build process. This is particularly useful for files like `llms.txt` or any custom files that need to be directly accessible at the site's root level. - -This extension processes a list of file paths provided in the playbook configuration, locates those files in the Antora content catalog, and adds them to the site's root directory during the publishing phase. Each file's content and basename are preserved in the process. - -==== Environment variables - -This extension does not require any environment variables. - -==== Configuration options - -Add the `add-pages-to-root` extension to your Antora playbook under the `antora.extensions` key, and specify the list of files to process in the `files` configuration. - -[source,yaml] ----- -antora: - extensions: - - require: '@redpanda-data/docs-extensions-and-macros/extensions/add-pages-to-root' - files: - - home:ROOT:attachment$custom-file.txt ----- - -==== Registration - -[source,yaml] ----- -antora: - extensions: - - require: '@redpanda-data/docs-extensions-and-macros/extensions/add-pages-to-root' - files: - - home:ROOT:attachment$custom-file.txt ----- - -=== Algolia indexer - -This extension generates an Algolia index for each version of each component. The index entries are then saved to Algolia using the `saveObjects()` method, and also saved as JSON files in the site catalog. JSON files are published to the site root using the template `algolia--.json`. - -NOTE: Only pages that include an `
` element with the `doc` class are indexed. - -==== Environment variables - -- `ALGOLIA_ADMIN_API_KEY` (required) -- `ALGOLIA_APP_ID` (required) -- `ALGOLIA_INDEX_NAME` (required) - -==== Configuration options - -The extension accepts the following configuration options: - -excludes (optional):: -Any elements, classes, or IDs that you want to exclude from the index. -index-latest-only (optional):: -Whether to index all versions or just the latest version of a component. - -==== Registration - -```yaml -antora: - extensions: - - require: '@redpanda-data/docs-extensions-and-macros/extensions/algolia-indexer/index' - excludes: ['.thumbs','script', '.page-versions','.feedback-section','.banner-container'] - index-latest-only: true -``` - -=== Archive attachments - -The `archive-attachments` extension automates the packaging of specific attachment files into a compressed archive (`.tar.gz`) based on configurable patterns. This archive is then made available to the generated site, allowing users to easily download grouped resources such as Docker Compose configurations. - -This extension enables you to define which files and directories to include in the archive, ensuring that only relevant content is packaged and accessible. - -==== Environment variables - -This extension does not require any environment variables. - -==== Configuration options - -The extension accepts the following options in the Antora playbook. - -Configure the extension in your Antora playbook by defining an array of archive configurations under `data.archives`. Each archive configuration includes: - -output_archive (string, required):: The name of the generated archive file. - -component (string, required):: The name of the Antora component whose attachments should be archived. - -file_patterns (array of strings, required):: Glob patterns specifying which attachment paths to include in the archive. - -NOTE: Ensure that `file_patterns` accurately reflect the paths of the attachments you want to archive. Overly broad patterns may include unintended files, while overly restrictive patterns might exclude necessary resources. - -==== Example configuration - -Here's an example configuration to enable the extension: - -```yaml -antora: - extensions: - - require: '../docs-extensions-and-macros/extensions/archive-creation-extension.js' - data: - archives: - - output_archive: 'redpanda-quickstart.tar.gz' <1> - component: 'ROOT' <2> - file_patterns: - - '**/test-resources/**/docker-compose/**' <3> -``` - -<1> Defines the name of the generated archive placed at the site root. -<2> Defines the name of the component in which to search for attachments. -<3> Lists the glob patterns to match attachment paths for inclusion in the archive. -+ -- `**`: Matches any number of directories. -- `/test-resources/`: Specifies that the matching should occur within the `test-resources/` directory. -- `/docker-compose/`: Targets the `docker-compose/` directory and all its subdirectories. -- `**:` Ensures that all files and nested directories within `docker-compose/` are included. - -=== Behavior with multiple components/versions - -*Scenario*: Multiple components and/or multiple versions of the same component contain attachments that match the defined file_patterns. - -*Outcome*: Separate archives for each component version. - -For each matching (component, version) pair, the extension creates a distinct archive named `-`. For example: -`24.3-redpanda-quickstart.tar.gz`. - -These archives are placed at the site root, ensuring they are easily accessible and do not overwrite each other. - -For the latest version of each component, the extension also adds the archive using the base `output_archive` name. As a result, the latest archives are accessible through a consistent filename, facilitating easy downloads without needing to reference version numbers. - -Because each archive has a unique filename based on the component version, there is no risk of archives overwriting each other. -The only exception is the archive for the latest version, which consistently uses the `output_archive` name. - -=== Component category aggregator - -This extension maps Redpanda Connect component data into a structured format: - -- Maps original component names to common names. -- Populates `connectCategoriesData` and `flatComponentsData` attributes. -- Skips deprecated components. - -==== Environment variables - -This extension does not require any environment variables. - -==== Configuration options - -There are no configurable options for this extension. - -==== Registration - -```yaml -antora: - extensions: - - require: '@redpanda-data/docs-extensions-and-macros/extensions/generate-rp-connect-categories' -``` - -=== Compute end-of-life extension - -This extension calculates and attaches metadata related to the end-of-life (EoL) status of docs pages, such as nearing EoL, past EoL, and associated EoL dates. This metadata can be used to display relevant banners or messages in docs to inform users about the lifecycle of each version. - -The extension leverages configuration settings provided in the Antora playbook to apply EoL calculations, specify the warning period, and include links to upgrade documentation and EoL policies. - -The extension computes whether a page is nearing EoL or past EoL based on the `page-release-date` attribute and configured settings. -It injects the following attributes into each page, making them available for use in UI templates: - -- `page-is-nearing-eol`: Indicates if the page is within the warning period before EoL. Calculated using `(page-release-date + supported_months) - warning_weeks`. -- `page-is-past-eol`: Indicates if the page has passed its EoL. Calculated using `today > (page-release-date + supported_months)`. -- `page-eol-date`: The calculated EoL date in a human-readable format. Calculated using `page-release-date + supported_months`. -- `page-eol-doc`: The URL to the supported versions policy or EoL documentation. -- `page-upgrade-doc`: The Antora resource ID to a document containing upgrade instructions. - -==== Environment variables - -This extension does not require any environment variables. - -==== Configuration options - -To enable and configure the extension, add it to the `antora.extensions` section of your Antora playbook. Define the EoL settings under the `data.eol_settings` key with the following options: - -`component` (required):: The component name to which the configuration applies. -`eol_doc` (required):: A link to the supported versions policy or EoL documentation. -`upgrade_doc` (required):: A link to the upgrade instructions. -`supported_months` (optional, default: 12):: The number of months after the publish date when the documentation reaches its EoL. -`warning_weeks` (optional, default: 6):: The number of weeks before EoL when the documentation is considered to be nearing EoL. Can be used to decide when to notify users of the upcoming EoL status. - [,yaml] ---- antora: extensions: - - require: '@redpanda-data/docs-extensions-and-macros/extensions/compute-end-of-life' - data: - eol_settings: - - component: 'ROOT' - supported_months: 18 - warning_weeks: 8 - eol_doc: https://support.redpanda.com/hc/en-us/articles/20617574366743-Redpanda-Supported-Versions - upgrade_doc: ROOT:upgrade:index.adoc ----- - -==== Registration - -You can register the extension with a customized configuration for different components in your playbook: - -[,yaml] ----- -antora: - extensions: - - require: '@redpanda-data/docs-extensions-and-macros/extensions/compute-end-of-life' - data: - eol_settings: - - component: 'ROOT' - supported_months: 12 - warning_weeks: 6 - eol_doc: https://example.com/supported-versions - upgrade_doc: ROOT:upgrade:index.adoc - - component: 'example-docs' - supported_months: 24 - warning_weeks: 12 - eol_doc: https://example.com/example-supported-versions - upgrade_doc: example-docs:upgrade:index.adoc ----- - - -==== Example Handlebars template: - -[,handlebars] ----- -{{#if page.attributes.is-nearing-eol}} - -{{else if page.attributes.is-past-eol}} - -{{/if}} ----- - -=== Generate index data - -The `generate-index-data` extension creates structured index data about doc pages based on configurable filters. The indexed data is saved to a specified attribute in all component versions, enabling the dynamic generation of categorized links and descriptions within your docs using UI templates. - -This extension allows you to define multiple indexing criteria, such as component, URL filter, and environment type. - -The generated data is an array of objects, where each object represents a component version. Each object contains the following properties: - -- `component` (string): - The name of the Antora component. - -- `version` (string): - The version of the component. - -- `pages` (array): - A list of pages that match the indexing criteria. Each page contains: -** `title` (string): The title of the doc page. -** `url` (string): The URL of the doc page relative to the site root. -** `description` (string): A brief description sourced from the `:description:` attribute in the AsciiDoc file. Defaults to an empty string if not provided. - -Example: - -```json -[ - { - "component": "ROOT", - "version": "24.3", - "pages": [ - { - "title": "Manage Debug Bundles in Redpanda Console", - "url": "/current/console/ui/generate-bundle/", - "description": "Learn how to generate, download, and delete debug bundles in Redpanda Console for comprehensive cluster diagnostics." - }, - ] - } -] -``` - -==== Environment variables - -This extension does not require any environment variables. - -==== Configuration options - -The extension accepts the following options in the Antora playbook. - -NOTE: Ensure filters are well-defined to minimize unnecessary processing. Avoid overly broad configurations in `data.sets`. - -- `data.sets` (required): An object defining one or more indexing configurations. Each configuration (or set) accepts the following options: - -** `component` (string, required): The Antora component to search for pages. - -** `attribute_name` (string, required): The attribute name to assign the generated index data. This allows pages and templates to reference the index. - -** `filter` (string, optional): A substring to match within page URLs. - -** `env_type` (string, optional): Matches pages with environment-specific attributes (e.g., Docker, Kubernetes). - -** `output_file` (string, optional): Save the generated index data as a JSON file at the specified path. If not provided, no file is created. - -==== Example configuration - -Here's an example configuration to enable the generate-index-data-extension: - -```yaml -antora: - extensions: - - require: '@redpanda-data/docs-extensions-and-macros/extensions/generate-index-data-extension' - data: - sets: - console_ui: - component: ROOT # Search the ROOT component - filter: console/ui # Filter pages containing this substring in their URL - attribute_name: console-ui-index # Save the result in this attribute - output_file: redpanda-labs/console-ui-index.json # Save data to this file - docker_labs: - component: redpanda-labs - filter: docker-compose - env_type: Docker - attribute_name: docker-labs-index -``` - -==== Use the generated data - -The index data can be referenced in AsciiDoc pages by specifying the following required attributes: - -```asciidoc -= CONSOLE UI -:page-index-data: console-ui-index <1> -:page-role: index-list <2> -``` - -<1> The attribute whose data you want to display on the page. This must match an attribute configured in the extension. -<2> The page role. This role specfies the UI template that renders the data in the `page-index-data` on the page. - -You can optionally display pages only if they match the component and version of the current Asciidoc page by adding the `:page-match-component-version:` attribute. - -```asciidoc -= CONSOLE UI -:page-index-data: console-ui-index -:page-role: index-list -:page-match-component-version: '' -``` - -=== Redpanda Connect tag modifier - -This extension updates the playbook to use the latest release tag for the Redpanda Connect documentation. It ensures that the Redpanda Connect documentation is always pulled from the latest release tag available on GitHub. - -==== Environment variables - -- `REDPANDA_GITHUB_TOKEN` (optional): A Personal access token (PAT) that has `repo` permissions for the `redpanda-data` GitHub organization. - -NOTE: If you don't set the environment variable, the latest version of Redpanda Connect may not be fetched. When the environment variable is not set, the extension sends unauthenticated requests to GitHub. Unauthenticated requests may result in hitting the API rate limit and cause GitHub to reject the request. In this case the fallback version is used. This version is defined in the playbook where the extension is registered. - -==== Configuration options - -There are no configurable options for this extension. - -==== Registration - -```yaml -antora: - extensions: - - require: '@redpanda-data/docs-extensions-and-macros/extensions/modify-connect-tag-playbook' -``` - -=== Version fetcher - -This extension fetches the latest release versions from GitHub. - -The following attributes are available to all versions of all Antora components: - -`latest-console-version`: The latest release version of Redpanda Console. -`latest-connect-version`: The latest release version of Redpanda Connect. -`redpanda-beta-version`: The latest RC version of Redpanda. -`redpanda-beta-commit`: The commit hash for the latest RC version of Redpanda. - -The following attributes are available to the latest version of the `ROOT` component (Redpanda docs): - -`full-version`: The latest release version of Redpanda. -`latest-release-commit`: The commit hash for the latest release version of Redpanda. -`latest-operator-version`: The latest release version of the Redpanda Operator. -`latest-redpanda-helm-chart-version`: The latest release version of the Redpanda Helm chart. - -==== Environment variables - -- `REDPANDA_GITHUB_TOKEN` (optional): A Personal access token (PAT) that has `repo` permissions for the `redpanda-data` GitHub organization. - -NOTE: If you don't set the environment variable, the latest versions may not be fetched. When the environment variable is not set, the extension sends unauthenticated requests to GitHub. Unauthenticated requests may result in hitting the API rate limit and cause GitHub to reject the request. - -==== Registration - -```yaml -antora: - extensions: - - '@redpanda-data/docs-extensions-and-macros/extensions/version-fetcher/set-latest-version' -``` - -=== Validate attributes - -This extension ensures the consistency and validity of page attributes, focusing on validating page categories against a predefined list of valid categories and subcategories. It automatically adds missing parent categories for any specified subcategories and removes any specified categories that are invalid. Additionally, it processes specific environment attributes, setting corresponding page-level attributes when environment conditions are met. - -==== Environment variables - -This extension does not require any environment variables. - -==== Configuration options - -There are no configurable options for this extension. It operates based on site attributes defined in `add-global-attributes.js` to determine valid categories and subcategories. - -==== Registration - -Register the `validate-attributes` extension in the Antora playbook under the `antora.extensions` key like so: - -[source,yaml] ----- -antora: - extensions: - - require: '@redpanda-data/docs-extensions-and-macros/extensions/validate-attributes.js' ----- - -=== Related docs - -This extension enhances the connectivity between lab exercises and relevant documentation by dynamically identifying and linking related documentation pages and other lab exercises based on shared categories and deployment types. - -==== Environment variables - -This extension operates without requiring any specific environment variables. - -==== Configuration options - -This extension does not offer configurable options. It uses the inherent attributes of pages to determine relationships based on `page-categories` and deployment types (`env-kubernetes`, `env-linux`, `env-docker`, `page-cloud`). - -==== Registration - -To integrate the `related-docs-extension` into your Antora playbook, add it under the `antora.extensions` key as demonstrated below: - -[source,yaml] ----- -antora: - extensions: - - require: '@redpanda-data/docs-extensions-and-macros/extensions/related-docs-extension.js' ----- - -=== Related labs - -This extension enriches documentation pages with links to related lab exercises, facilitating a deeper understanding of the content through practical application. It dynamically assigns related labs to each documentation page based on shared categories and deployment types. - -==== Environment variables - -This extension does not require any environment variables. - -==== Configuration options - -The extension operates without explicit configuration options. It automatically processes documentation pages to identify and link related labs based on shared `page-categories` attributes and deployment types (`env-kubernetes`, `env-linux`, `env-docker`, `page-cloud`). - -==== Registration - -Include the `related-labs-extension` in the Antora playbook under the `antora.extensions` key as follows: - -[source,yaml] ----- -antora: - extensions: - - require: '@redpanda-data/docs-extensions-and-macros/extensions/related-labs-extension.js' ----- - -=== Global attributes - -This extension collects Asciidoc attributes from the {url-playbook}[`shared` component] or a local YAML file and makes them available to all component versions. Having global attributes is useful for consistent configuration of local and production builds. - -==== Environment variables - -This extension does not require any environment variables. - -==== Configuration options - -The extension accepts the following configuration options: - -attributespath (optional):: Specifies the path to a local YAML file that contains global attributes. If this is provided, the extension will load attributes from this file first. If this path is not provided or no valid attributes are found in the file, the extension will fall back to loading attributes from the `shared` component. - -==== Registration - -```yml -antora: - extensions: - - require: '@redpanda-data/docs-extensions-and-macros/extensions/add-global-attributes' - attributespath: './local-attributes.yml' -``` - -In this example, the `attributespath` option points to a local YAML file (`./local-attributes.yml`), which contains the global attributes. The extension will load attributes from this file first before falling back to the `shared` component. - -=== Produce redirects (customization of core Antora) - -This extension replaces the default https://gitlab.com/antora/antora/-/tree/v3.1.x/packages/redirect-producer[`produceRedirects()` function] in Antora to handle redirect loops caused by https://docs.antora.org/antora/latest/page/page-aliases/[page aliases]. Normally, page aliases in Antora are used to resolve outdated links without causing issues. However, with https://docs.antora.org/antora/latest/playbook/urls-html-extension-style/#html-extension-style-key[`indexify`], the same URL may inadvertently be used for both the source and target of a redirect, leading to loops. This problem is https://antora.zulipchat.com/#narrow/stream/282400-users/topic/Redirect.20Loop.20Issue.20with.20Page.20Renaming.20and.20Indexify/near/433691700[recognized as a bug] in core Antora. For example, creating a page alias for `modules/manage/security/authorization.adoc` to point to `modules/manage/security/authorization/index.adoc' can lead to a redirect loop where `manage/security/authorization/` points to `manage/security/authorization/`. Furthermore, omitting the alias would lead to `xref not found` errors because Antora relies on the alias to resolve the old xrefs. This extension is necessary until such behaviors are natively supported or fixed in Antora core. - -==== Environment variables - -This extension does not require any environment variables. - -==== Configuration options - -There are no configurable options for this extension. - -==== Registration - -```yaml -antora: - extensions: - - '@redpanda-data/docs-extensions-and-macros/extensions/modify-redirects' -``` - -=== Replace attributes in attachments - -This extension automates the replacement of AsciiDoc attribute placeholders with their respective values within attachment files, such as CSS, HTML, and YAML. - -[NOTE] -==== -- The `@` character is removed from attribute values to prevent potential issues with CSS or HTML syntax. -- If the same attribute placeholder is used multiple times within a file, all instances will be replaced with the attribute's value. -==== - -==== Environment variables - -This extension does not require any environment variables. - -==== Configuration options - -The extension accepts the following configuration options in the Antora playbook: - -data.replacements (required):: An array of replacement configurations. Each configuration can target multiple components and define specific file patterns and custom replacement rules. - -* `components` (array of strings, required): Lists the names of the Antora components whose attachments should undergo attribute replacement. - -* `file_patterns` (array of strings, required): Glob patterns specifying which attachment files to process. These patterns determine the files that will undergo attribute replacement based on their paths within the content catalog. - -* `custom_replacements` (array of objects, optional): Defines custom search-and-replace rules to be applied to the matched files. Each rule consists of: -** `search` (string, required): A regular expression pattern to search for within the file content. -** `replace` (string, required): The string to replace each match found by the `search` pattern. - -NOTE: Ensure that `file_patterns` accurately reflect the paths of the attachments you want to process. Overly broad patterns may include unintended files, while overly restrictive patterns might exclude necessary resources. - -==== Registration - -This is an example of how to register and configure the `replace-attributes-in-attachments` extension in your Antora playbook. This example demonstrates defining multiple replacement configurations, each targeting different components and specifying their own file patterns and custom replacements. - -```yaml -antora: - extensions: - - require: './extensions/replace-attributes-in-attachments' - data: - replacements: - - components: - - 'ROOT' - - 'redpanda-labs' - file_patterns: - - '**/docker-compose.yaml' - - '**/docker-compose.yml' - custom_replacements: - - search: ''\\$\\{CONFIG_FILE:[^}]*\\}'' - replace: 'console.yaml' - - components: - - 'API' - file_patterns: - - '**/api-docs/**/resources/**' - custom_replacements: - - search: '\\$\\{API_ENDPOINT:[^}]*\\}' - replace: 'https://api.example.com' -``` - -=== Aggregate terms - -This extension aggregates all term pages from the {url-playbook}[`shared` component] and does the following: - -- Makes all `term-name`, `hover-text`, and `link` attributes available to the <>. -- Looks for glossary pages named `reference:glossary.adoc` in all versions of all components and appends the contents of each term file to the glossary in alphabetical order. -- If a glossary page is found, sets the `glossary-page` attribute of the <> to `reference:glossary.adoc` so that terms can be linked to the glossary page. - -==== Environment variables - -This extension does not require any environment variables. - -==== Configuration options - -The extension accepts the following configuration options: - -termspath (optional):: Specifies the path to a local directory containing term files (in `.adoc` format). If this path is provided, the extension will attempt to load terms from this directory first. If this path is not provided or no valid terms are found in the specified directory, the extension will fall back to loading terms from the `shared` component. - -Term files should follow the following structure: - -```asciidoc -:category: Documentation -:hover-text: This is a description of the term. -:link: https://example.com - -== Term Title - -This is the detailed description of the term. -``` - -==== Registration - -```yml -antora: - extensions: - - require: '@redpanda-data/docs-extensions-and-macros/extensions/aggregate-terms' - termspath: './local-terms/' -``` - -In this example, the `termspath` option points to a local directory (./local-terms/), where the term files are stored. The extension will load terms from this directory first before falling back to the `shared` component. - -=== Unlisted pages - -This extension identifies and logs any pages that aren't listed in the navigation (nav) file of each version of each component. It then optionally adds these unlisted pages to the end of the navigation tree under a configurable heading. - -IMPORTANT: By default, this extension excludes components named 'api'. This behavior is hardcoded and cannot be changed in the configuration. - -==== Environment variables - -This extension does not require any environment variables. - -==== Configuration options - -This extension accepts the following configuration options: - -addToNavigation (optional):: -Whether to add unlisted pages to the navigation. The default is `false` (unlisted pages are not added). - -unlistedPagesHeading (optional):: -The heading under which to list the unlisted pages in the navigation. The default is 'Unlisted Pages'. - -==== Registration - -```yaml -antora: - extensions: - - require: '@redpanda-data/docs-extensions-and-macros/extensions/unlisted-pages' - addToNavigation: true - unlistedPagesHeading: 'Additional Resources' -``` - -=== Process context switcher - -This extension processes the `page-context-switcher` attribute to enable cross-version navigation widgets in documentation pages. It automatically replaces "current" references with full resource IDs and injects the context switcher configuration to all referenced target pages, ensuring bidirectional navigation works correctly. - -The extension finds pages with the `page-context-switcher` attribute, parses the JSON configuration, and: - -1. Replaces any "current" values with the full resource ID of the current page -2. Finds all target pages referenced in the switcher configuration -3. Injects the same context switcher attribute to target pages (with appropriate resource ID mappings) -4. Builds resource IDs in the format: `version@component:module:relative-path` - -This enables UI components to render version switchers that work across different versions of the same content. - -==== Environment variables - -This extension does not require any environment variables. - -==== Configuration options - -This extension does not require any configuration options. - -==== Registration - -```yaml -antora: - extensions: - - require: '@redpanda-data/docs-extensions-and-macros/extensions/process-context-switcher' -``` - -==== Usage - -Add the `page-context-switcher` attribute to any page where you want cross-version navigation: - -```asciidoc -:page-context-switcher: [{"name": "Version 2.x", "to": "24.3@ROOT:console:config/security/authentication.adoc" },{"name": "Version 3.x", "to": "current" }] -``` - -==== Processed output - -After processing, the "current" reference is replaced with the full resource ID: - -```json -[ - {"name": "Version 2.x", "to": "24.3@ROOT:console:config/security/authentication.adoc"}, - {"name": "Version 3.x", "to": "current@ROOT:console:config/security/authentication.adoc"} -] -``` - -The target page (`24.3@ROOT:console:config/security/authentication.adoc`) will also receive the same context switcher configuration with appropriate resource ID mappings. - -==== UI integration - -The processed attribute can be used in Handlebars templates: - -```html -
- {{#each (obj page.attributes.page-context-switcher)}} - - - - {{/each}} -
-``` - -== Asciidoc Extensions - -This section documents the Asciidoc extensions that are provided by this library and how to configure them. - -IMPORTANT: Be sure to register each extension under the `asciidoc.extensions` key in the playbook, not the `antora.extensions` key. - -=== Add line numbers and highlights - -This extension adds the necessary classes to make line numbers and line highlighting work with Prism.js. - -==== Registration - -```yaml -antora: - extensions: - - '@redpanda-data/docs-extensions-and-macros/asciidoc-extensions/add-line-numbers-highlights' -``` - -== Macros - -This section documents the Asciidoc macros that are provided by this library and how to configure them. - -IMPORTANT: Be sure to register each extension under the `asciidoc.extensions` key in the playbook, not the `antora.extensions` key. - -=== config_ref - -This inline macro is used to generate a reference to a configuration value in the Redpanda documentation. The macro's parameters allow for control over the generated reference's format and the type of output produced. - -==== Usage - -The `config_ref` macro is used in an AsciiDoc document as follows: - -[,asciidoc] ----- -config_ref:configRef,isLink,path[] ----- - -The `config_ref` macro takes three parameters: - -configRef:: -This is the configuration reference, which is also used to generate the anchor link if `isLink` is `true`. - -isLink:: -Whether the output should be a link. If `isLink` is set to `true`, the output will be a cross-reference (xref) to the relevant configuration value. - -path:: -This is the path to the document where the configuration value is defined. This parameter is used to to generate the link if `isLink` is `true`. - -IMPORTANT: The path must be the name of a document at the root of the `reference` module. - -NOTE: The `config_ref` macro is environment-aware. It checks if the document it is being used in is part of a Kubernetes environment by checking if the `env-kubernetes` attribute is set in the document's attributes. Depending on this check, it either prepends `storage.tieredConfig.` to the `configRef` or just uses the `configRef` as is. - -For example: - -[,asciidoc] ----- -config_ref:example_config,true,tunable-properties[] ----- - -==== Registration - -[,yaml] ----- -asciidoc: - extensions: - - '@redpanda-data/docs-extensions-and-macros/macros/config-ref' ----- - -=== glossterm - -The `glossterm` inline macro provides a way to define and reference glossary terms in your AsciiDoc documents. - -NOTE: This macro is a customized version of https://gitlab.com/djencks/asciidoctor-glossary[`asciidoctor-glossary`]. - -==== Usage - -Use the `glossterm` inline macro to reference a term within the text of the document: - -[,asciidoc] ----- -glossterm:my term[myDefinition] ----- - -It takes two parameters: - -term:: -The term to be defined. - -definition (optional):: -The definition of the term. If the term is defined in the {url-playbook}[`shared` component] or the `local-terms` object of the `antora.yml` file, you can omit the definition as it will always be replaced by those definitions. - -==== Configuration options - -glossary-log-terms (optional):: -Whether to log a textual representation of a definition list item to the console. - -glossary-term-role (optional):: -Role to assign each term. By default, glossary terms are assigned the `glossary-term` role, which gives them the class `glossary-term` in generated html. - -glossary-links (optional):: -Whether to generate links to glossary entries. -By default, links to the glossary entries are generated from the glossary terms. To avoid this, set the attribute to `false` as either asciidoctor configuration or a header attribute. - -glossary-page (optional):: -Target page for glossary links. By default, links are generated to the same page as the glossary term. To specify the target page, set this attribute to the resource ID of a page where the `glossary` block macro is used. - -glossary-tooltip (optional):: -Whether to enable tooltips for the defined terms. Valid values are: -- title: This uses the browser built-in `title` attribute to display the definition. - -- true: This inserts the definition as the value of the attribute `data-glossary-tooltip`. - -- data-​: This inserts the definition as the value of the supplied attribute name, which must start with `data`. - -The last two options are intended to support js/css tooltip solutions such as tippy.js. - -==== Registration - -[,yaml] ----- + - '@redpanda-data/docs-extensions-and-macros/extensions/version-fetcher/set-latest-version' asciidoc: extensions: - '@redpanda-data/docs-extensions-and-macros/macros/glossary' ---- -=== helm_ref - -This is an inline macro to create links to a Helm `values.yaml` file on ArtifactHub. - -==== Usage - -In an AsciiDoc document, use the `helm_ref` macro as follows: - -[,asciidoc] ----- -helm_ref:[] ----- - -Where `` is the Helm configuration value you want to reference in the `values.yaml` file. - -For example: +== Documentation -Given a Helm reference value of `myConfigValue`, you would use the macro like this: +link:extensions/README.adoc[**Antora extensions**]:: Extensions that enhance the Antora build process -[,asciidoc] ----- -helm_ref:myConfigValue[] ----- +link:macros/README.adoc[**AsciiDoc macros**]:: Custom inline and block macros for documentation -This will generate the following output: - -[,asciidoc] ----- -For default values and documentation for configuration options, see the https://artifacthub.io/packages/helm/redpanda-data/redpanda?modal=values&path=myConfigValue[values.yaml] file. ----- +link:mcp/README.adoc[**MCP server**]:: Model Context Protocol server for automating documentation tasks with Claude Code -If you do not specify a Helm reference value, the macro generates a link without specifying a path. +link:CLI_REFERENCE.adoc[**CLI reference**]:: Complete command reference for `doc-tools` -==== Registration +link:CONTRIBUTING.adoc[**Contributing**]:: Development setup and guidelines -[,yaml] ----- -asciidoc: - extensions: - - '@redpanda-data/docs-extensions-and-macros/macros/helm-ref' ----- +== What's included -=== components_by_category +=== Antora extensions -This macro generates a tabbed interface to display Redpanda Connect components by category. +Extensions that enhance Antora's build process: -The categories are fetched from the `connectCategoriesData` that's generated in the <> extension. +* **Version management** - Automatically fetch latest versions from GitHub +* **Content generation** - Generate indexes, categories, and structured data +* **Navigation** - Manage unlisted pages, redirects, and cross-version links +* **Integrations** - Algolia indexing, EOL banners, related content +* **File processing** - Archive attachments, replace attributes, collect samples -==== Usage +=== AsciiDoc macros -```asciidoc -components_by_category::[] -``` +Inline and block macros for documentation: -==== Registration +* `glossterm` - Reference glossary terms with tooltips +* `config_ref` - Link to configuration properties +* `helm_ref` - Link to Helm values +* `components_by_category` - Display Redpanda Connect components +* `component_table` - Searchable component table -```yaml -asciidoc: - extensions: - - '@redpanda-data/docs-extensions-and-macros/macros/rp-connect-components' -``` +=== Documentation automation (MCP server) -=== component_table +A Model Context Protocol server that exposes doc-tools to Claude Code: -This macro generates a searchable table of all Redpanda Connect components with filters for support and type. +* Generate property, metrics, and RPK documentation +* Update connector reference documentation +* Get version information +* Review generated content for quality -The types are fetched from the `flatComponentsData` that's generated in the <> extension. +== Support -==== Usage - -```asciidoc -component_table::[] -``` - -==== Registration - -```yaml -asciidoc: - extensions: - - '@redpanda-data/docs-extensions-and-macros/macros/rp-connect-components' -``` - -=== component_type_dropdown - -This macro generates a dropdown of other supported types for a particular component, allowing users to switch between different types. - -The types are fetched from the `flatComponentsData` that's generated in the <> extension. - -==== Usage - -```asciidoc -component_type_dropdown::[] -``` - -==== Registration - -```yaml -asciidoc: - extensions: - - '@redpanda-data/docs-extensions-and-macros/macros/rp-connect-components' -``` - -== Development quickstart - -This section provides information on how to develop this project. - -=== Prerequisites - -To build this project, you need the following software installed on your computer: - -* {url-git}[git] (command: `git`) -* {url-nodejs}[Node.js] (commands: `node`, `npm`, and `npx`) - -==== git - -Make sure you have git installed. - -[,bash] ----- -git --version ----- - -If not, {url-git-dl}[download and install] the git package for your system. - -==== Node.js - -Make sure that you have Node.js installed (which also provides npm and npx). - -[,bash] ----- -node --version ----- - -If this command fails with an error, you don't have Node.js installed. - -Now that you have git and Node.js installed, you're ready to start developing on this project. - -=== Clone the project - -Clone the project using git: - -[,bash,subs=attributes+] ----- -git clone {url-project} ----- - -Change into the project directory and stay in this directory when running all subsequent commands. - -=== Install dependencies - -Use npm to install the project's dependencies inside the project. -In your terminal, run the following command: - -[,bash] ----- -npm ci ----- - -This command installs the dependencies listed in `package-lock.json` into the `node_modules/` directory inside the project. -This directory should _not_ be committed to the source control repository. - -=== Use your local project - -If you want to use the project locally before it is published, you can specify the path to the extensions in the `local-antora-playbook.yml` file. - -[,yaml] ----- -asciidoc: - attributes: - extensions: - - '/docs-extensions-and-macros/extensions/' ----- +Report issues at {url-project}/issues diff --git a/__tests__/mcp/README.adoc b/__tests__/mcp/README.adoc new file mode 100644 index 00000000..33b3c685 --- /dev/null +++ b/__tests__/mcp/README.adoc @@ -0,0 +1,147 @@ += MCP Server Tests + +These tests ensure that changes to the doc-tools CLI don't break the MCP server. + +== Test files + +=== `integration.test.js` + +Integration tests that verify the MCP tools correctly interface with the doc-tools CLI. These tests: + +* Verify CLI commands are available +* Test tool execution through the MCP layer +* Validate parameter handling +* Check error handling +* Verify output parsing + +=== `cli-contract.test.js` + +Contract tests that verify the doc-tools CLI maintains the expected interface. These tests: + +* Check command structure +* Verify required and optional flags exist +* Validate output format contracts +* Test error handling contracts + +== Running tests + +=== Run all MCP tests + +[,bash] +---- +npm run test:mcp +---- + +=== Run integration tests only + +[,bash] +---- +npm run test:mcp:integration +---- + +=== Run contract tests only + +[,bash] +---- +npm run test:mcp:contract +---- + +=== Run with coverage + +[,bash] +---- +npm test -- --coverage __tests__/mcp/ +---- + +== When to run these tests + +=== Before merging CLI changes + +Always run these tests before merging any changes to `bin/doc-tools.js` or any generate commands. + +[,bash] +---- +npm run test:mcp +---- + +=== During development + +Run tests continuously during development: + +[,bash] +---- +npm test -- --watch __tests__/mcp/ +---- + +=== In CI/CD + +These tests run automatically on every push and PR via GitHub Actions (`.github/workflows/test-mcp.yml`). + +== Understanding test failures + +=== Integration test failures + +If integration tests fail, it usually means: + +* The MCP tool module has a bug +* The CLI command changed in an unexpected way +* Network issues (for version commands) +* File system issues + +=== Contract test failures + +If contract tests fail, it usually means: + +* The CLI interface changed (flags renamed, removed, or output format changed) +* New required parameters were added +* Command names changed + +IMPORTANT: Contract test failures indicate breaking changes that will affect the MCP server. + +== Adding tests + +=== When adding a new MCP tool + +. Add integration tests in `integration.test.js`: ++ +[,javascript] +---- +test('new_tool_name returns expected result', () => { + const result = mcpTools.executeTool('new_tool_name', {}); + expect(result).toBeDefined(); + expect(result.success).toBeDefined(); +}); +---- + +. Add contract tests in `cli-contract.test.js`: ++ +[,javascript] +---- +test('new-cli-command exists', () => { + const result = executeCLI('new-cli-command --help'); + expect(result.success).toBe(true); +}); +---- + +=== When modifying an existing CLI command + +. Update the corresponding test expectations +. If the interface changed, update link:../../mcp/CLI_INTERFACE.adoc[CLI_INTERFACE.adoc] +. If breaking changes, update the MCP tool module + +== Test environment + +Tests run in the actual repository environment: + +* Uses the real `npx doc-tools` command +* Reads actual file system structure +* Makes real network calls (for version commands) + +NOTE: Some tests may fail in CI if network is unavailable (version commands), GitHub API rate limits are hit, or required environment variables are missing. These failures are expected and handled gracefully in the tests. + +== Related documentation + +* link:../../mcp/CLI_INTERFACE.adoc[CLI interface contract] - Complete interface contract documentation +* link:../../mcp/USER_GUIDE.adoc[User guide] - Guide for writers using the MCP server +* link:../../mcp/DEVELOPMENT.adoc[Development guide] - Guide for developers working on the MCP server +* link:../../.github/workflows/test-mcp.yml[CI/CD workflow] - GitHub Actions workflow diff --git a/__tests__/mcp/cli-contract.test.js b/__tests__/mcp/cli-contract.test.js new file mode 100644 index 00000000..98bae059 --- /dev/null +++ b/__tests__/mcp/cli-contract.test.js @@ -0,0 +1,179 @@ +/** + * CLI Contract Tests + * + * These tests verify that the doc-tools CLI maintains the expected interface + * that the MCP server depends on. If these tests fail, it means the CLI has + * changed in a way that might break the MCP server. + */ + +const { describe, test, expect } = require('@jest/globals'); +const { execSync } = require('child_process'); + +/** + * Execute a CLI command and return the result + */ +function executeCLI(command) { + try { + const output = execSync(`npx doc-tools ${command}`, { + encoding: 'utf8', + stdio: 'pipe', + timeout: 10000 + }); + return { success: true, output }; + } catch (err) { + return { + success: false, + error: err.message, + stdout: err.stdout || '', + stderr: err.stderr || '', + exitCode: err.status + }; + } +} + +describe('CLI Contract Tests', () => { + describe('Command Structure', () => { + test('generate command exists', () => { + const result = executeCLI('generate --help'); + expect(result.success).toBe(true); + expect(result.output).toContain('generate'); + }); + + test('get-redpanda-version command exists', () => { + const result = executeCLI('get-redpanda-version --help'); + expect(result.success).toBe(true); + }); + + test('get-console-version command exists', () => { + const result = executeCLI('get-console-version --help'); + expect(result.success).toBe(true); + }); + }); + + describe('Generate Subcommands', () => { + const requiredSubcommands = [ + { + name: 'property-docs', + requiredFlags: ['--tag'], + optionalFlags: ['--generate-partials', '--cloud-support', '--overrides'] + }, + { + name: 'metrics-docs', + requiredFlags: ['--tag'], + optionalFlags: [] + }, + { + name: 'rpk-docs', + requiredFlags: ['--tag'], + optionalFlags: [] + }, + { + name: 'rpcn-connector-docs', + requiredFlags: [], + optionalFlags: ['--fetch-connectors', '--draft-missing', '--update-whats-new', '--include-bloblang', '--data-dir', '--old-data', '--csv', '--overrides'] + }, + { + name: 'helm-spec', + requiredFlags: [], + optionalFlags: ['--chart-dir', '--tag', '--readme', '--output-dir', '--output-suffix'] + }, + { + name: 'cloud-regions', + requiredFlags: [], + optionalFlags: ['--output', '--format', '--owner', '--repo', '--path', '--ref', '--template', '--dry-run'] + }, + { + name: 'crd-spec', + requiredFlags: [], // Either --tag or --branch required (enforced by CLI validation) + optionalFlags: ['--tag', '--branch', '--source-path', '--depth', '--templates-dir', '--output'] + }, + { + name: 'bundle-openapi', + requiredFlags: ['--tag'], + optionalFlags: ['--repo', '--surface', '--out-admin', '--out-connect', '--admin-major', '--use-admin-major-version', '--quiet'] + } + ]; + + requiredSubcommands.forEach(({ name, requiredFlags, optionalFlags }) => { + describe(name, () => { + test(`${name} command exists`, () => { + const result = executeCLI(`generate ${name} --help`); + expect(result.success).toBe(true); + expect(result.output).toContain(name); + }); + + requiredFlags.forEach(flag => { + test(`${name} supports required flag ${flag}`, () => { + const result = executeCLI(`generate ${name} --help`); + expect(result.output).toContain(flag); + }); + }); + + optionalFlags.forEach(flag => { + test(`${name} supports optional flag ${flag}`, () => { + const result = executeCLI(`generate ${name} --help`); + expect(result.output).toContain(flag); + }); + }); + }); + }); + }); + + describe('Output Format Contracts', () => { + test('get-redpanda-version outputs expected format', () => { + const result = executeCLI('get-redpanda-version'); + + if (result.success) { + // Should output REDPANDA_VERSION= format + expect(result.output).toMatch(/REDPANDA_VERSION=v?\d+\.\d+\.\d+/); + } else { + // Network errors are acceptable, but format should still be parseable + expect(result.stderr || result.error).toBeDefined(); + } + }); + + test('get-console-version outputs expected format', () => { + const result = executeCLI('get-console-version'); + + if (result.success) { + // Should output CONSOLE_VERSION= format + expect(result.output).toMatch(/CONSOLE_VERSION=v?\d+\.\d+\.\d+/); + } else { + // Network errors are acceptable + expect(result.stderr || result.error).toBeDefined(); + } + }); + }); + + describe('Error Handling Contracts', () => { + test('missing required parameter produces error', () => { + // property-docs requires --tag + const result = executeCLI('generate property-docs'); + + expect(result.success).toBe(false); + expect(result.exitCode).not.toBe(0); + }); + + test('invalid flag produces error', () => { + const result = executeCLI('generate property-docs --invalid-flag-xyz'); + + expect(result.success).toBe(false); + }); + + test('nonexistent command produces error', () => { + const result = executeCLI('generate nonexistent-command'); + + expect(result.success).toBe(false); + expect(result.exitCode).not.toBe(0); + }); + }); + + describe('Version Compatibility', () => { + test('doc-tools version is available', () => { + const result = executeCLI('--version'); + + expect(result.success).toBe(true); + expect(result.output).toMatch(/\d+\.\d+\.\d+/); + }); + }); +}); diff --git a/__tests__/mcp/doc-tools-mcp.test.js b/__tests__/mcp/doc-tools-mcp.test.js new file mode 100644 index 00000000..38a7c838 --- /dev/null +++ b/__tests__/mcp/doc-tools-mcp.test.js @@ -0,0 +1,1063 @@ +/** + * @jest-environment node + */ + +const fs = require('fs'); +const path = require('path'); + +// Mock dependencies before importing the library +jest.mock('fs'); +jest.mock('child_process'); +jest.mock('js-yaml'); + +const mcpTools = require('../../bin/mcp-tools'); +const { + findRepoRoot, + getAntoraStructure, + getRedpandaVersion, + getConsoleVersion, + generatePropertyDocs, + generateMetricsDocs, + generateRpkDocs, + generateRpConnectDocs, + executeTool +} = mcpTools; + +// Import constants from utils +const { + MAX_RECURSION_DEPTH, + MAX_EXEC_BUFFER_SIZE, + DEFAULT_SKIP_DIRS, + PLAYBOOK_NAMES +} = require('../../bin/mcp-tools/utils'); + +// validateDocToolsCommand is not exported, so we need to test it through executeTool +const validateDocToolsCommand = (command) => { + // This function is now internal to mcp-tools/index.js + // We can only test it indirectly through run_doc_tools_command + if (!command || typeof command !== 'string') { + return { valid: false, error: 'Command must be a non-empty string' }; + } + const dangerousChars = /[;|&$`<>(){}[\]!*?~]/; + if (dangerousChars.test(command)) { + return { valid: false, error: 'Invalid command: shell metacharacters not allowed' }; + } + if (command.includes('..') || command.includes('~')) { + return { valid: false, error: 'Invalid command: path traversal sequences not allowed' }; + } + return { valid: true }; +}; + +describe('MCP Server Library - Constants', () => { + it('should have correct MAX_RECURSION_DEPTH', () => { + expect(MAX_RECURSION_DEPTH).toBe(3); + }); + + it('should have correct MAX_EXEC_BUFFER_SIZE', () => { + expect(MAX_EXEC_BUFFER_SIZE).toBe(50 * 1024 * 1024); + }); + + it('should have correct DEFAULT_SKIP_DIRS', () => { + expect(DEFAULT_SKIP_DIRS).toEqual([ + 'node_modules', + '.git', + 'venv', + '__pycache__', + '.pytest_cache' + ]); + }); + + it('should have correct PLAYBOOK_NAMES', () => { + expect(PLAYBOOK_NAMES).toEqual([ + 'local-antora-playbook.yml', + 'antora-playbook.yml', + 'docs-playbook.yml' + ]); + }); +}); + +describe('MCP Server Library - Repository Detection', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('findRepoRoot', () => { + it('should detect git repository', () => { + const mockExists = jest.fn((filePath) => { + return filePath.endsWith('.git'); + }); + fs.existsSync = mockExists; + + const result = findRepoRoot('/test/path'); + + expect(result.detected).toBe(true); + expect(result.type).toBe('git'); + expect(result.root).toBeTruthy(); + }); + + it('should detect npm package repository', () => { + let callCount = 0; + const mockExists = jest.fn((filePath) => { + callCount++; + // First call checks for .git (false), second for package.json (true) + return callCount > 1 && filePath.endsWith('package.json'); + }); + fs.existsSync = mockExists; + + const result = findRepoRoot('/test/path'); + + expect(result.detected).toBe(true); + expect(result.type).toBe('npm'); + }); + + it('should return current directory when no repo found', () => { + fs.existsSync = jest.fn(() => false); + + const testPath = '/test/path'; + const result = findRepoRoot(testPath); + + expect(result.detected).toBe(false); + expect(result.type).toBe(null); + expect(result.root).toBe(testPath); + }); + + it('should return repo type information', () => { + fs.existsSync = jest.fn((filePath) => filePath.endsWith('.git')); + + const result = findRepoRoot('/test/path'); + + expect(result).toHaveProperty('root'); + expect(result).toHaveProperty('detected'); + expect(result).toHaveProperty('type'); + }); + }); + + describe('Command Validation', () => { + const dangerousCommands = [ + { cmd: 'generate property-docs; rm -rf /', reason: 'semicolon' }, + { cmd: 'generate property-docs | cat /etc/passwd', reason: 'pipe' }, + { cmd: 'generate property-docs && malicious-command', reason: 'ampersand' }, + { cmd: 'generate property-docs $(malicious)', reason: 'command substitution' }, + { cmd: 'generate property-docs `malicious`', reason: 'backtick' }, + { cmd: 'generate property-docs > /etc/passwd', reason: 'redirection' }, + { cmd: 'generate property-docs < /etc/passwd', reason: 'redirection' }, + { cmd: 'generate ../../../etc/passwd', reason: 'path traversal' }, + { cmd: 'generate ~/sensitive-file', reason: 'tilde expansion' } + ]; + + const safeCommands = [ + 'generate property-docs --tag v25.3.1', + 'generate metrics-docs --branch main', + 'generate rp-connect-docs', + 'help', + 'version' + ]; + + dangerousCommands.forEach(({ cmd, reason }) => { + it(`should reject dangerous command (${reason}): ${cmd.substring(0, 40)}`, () => { + const result = validateDocToolsCommand(cmd); + expect(result.valid).toBe(false); + expect(result.error).toBeDefined(); + }); + }); + + safeCommands.forEach(cmd => { + it(`should allow safe command: ${cmd}`, () => { + const result = validateDocToolsCommand(cmd); + expect(result.valid).toBe(true); + expect(result.error).toBeUndefined(); + }); + }); + + it('should reject empty string', () => { + const result = validateDocToolsCommand(''); + expect(result.valid).toBe(false); + expect(result.error).toContain('non-empty string'); + }); + + it('should reject null', () => { + const result = validateDocToolsCommand(null); + expect(result.valid).toBe(false); + }); + + it('should reject undefined', () => { + const result = validateDocToolsCommand(undefined); + expect(result.valid).toBe(false); + }); + + it('should reject non-string types', () => { + expect(validateDocToolsCommand(123).valid).toBe(false); + expect(validateDocToolsCommand({}).valid).toBe(false); + expect(validateDocToolsCommand([]).valid).toBe(false); + }); + }); + + describe('getAntoraStructure', () => { + const yaml = require('js-yaml'); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('should accept both string and object repoRoot parameter', () => { + fs.existsSync = jest.fn(() => false); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => []); + + // Test with string + const result1 = getAntoraStructure('/test/path'); + expect(result1.repoRoot).toBe('/test/path'); + + // Test with object + const repoInfo = { root: '/test/path2', detected: true, type: 'git' }; + const result2 = getAntoraStructure(repoInfo); + expect(result2.repoRoot).toBe('/test/path2'); + expect(result2.repoInfo).toEqual(repoInfo); + }); + + it('should find playbook in priority order', () => { + let callCount = 0; + fs.existsSync = jest.fn((filePath) => { + callCount++; + // Second call finds antora-playbook.yml + return callCount === 2 && filePath.includes('antora-playbook.yml'); + }); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => []); + fs.readFileSync = jest.fn(() => 'site:\n title: Test'); + yaml.load = jest.fn(() => ({ site: { title: 'Test' } })); + + const result = getAntoraStructure('/test'); + + expect(result.playbookPath).toContain('antora-playbook.yml'); + expect(result.playbook).toEqual({ site: { title: 'Test' } }); + }); + + it('should handle playbook parsing errors gracefully', () => { + fs.existsSync = jest.fn((filePath) => filePath.includes('local-antora-playbook.yml')); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => []); + fs.readFileSync = jest.fn(() => 'invalid yaml: [[['); + yaml.load = jest.fn(() => { + throw new Error('YAML parse error'); + }); + + const result = getAntoraStructure('/test'); + + // Should continue without playbook + expect(result.playbook).toBe(null); + expect(result.playbookPath).toContain('local-antora-playbook.yml'); + }); + + it('should skip directories in skip list', () => { + const mockDirEntries = [ + { name: 'node_modules', isDirectory: () => true }, + { name: '.git', isDirectory: () => true }, + { name: 'venv', isDirectory: () => true }, + { name: 'valid-dir', isDirectory: () => true }, + { name: 'antora.yml', isDirectory: () => false } + ]; + + fs.existsSync = jest.fn(() => true); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => mockDirEntries); + fs.readFileSync = jest.fn(() => 'name: test\nversion: 1.0'); + fs.statSync = jest.fn(() => ({ isDirectory: () => true })); + yaml.load = jest.fn(() => ({ name: 'test', version: '1.0' })); + + const result = getAntoraStructure('/test'); + + // Should find antora.yml but skip node_modules, .git, venv + expect(result.components.length).toBeGreaterThan(0); + }); + + it('should use custom skip list when provided', () => { + const customSkipList = ['custom-skip']; + const mockDirEntries = [ + { name: 'custom-skip', isDirectory: () => true }, + { name: 'node_modules', isDirectory: () => true }, // Will be scanned with custom list + { name: 'antora.yml', isDirectory: () => false } + ]; + + fs.existsSync = jest.fn(() => true); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => mockDirEntries); + fs.readFileSync = jest.fn(() => 'name: test'); + yaml.load = jest.fn(() => ({ name: 'test' })); + + const result = getAntoraStructure('/test', customSkipList); + + // Custom skip list should be used + expect(result.components).toBeDefined(); + }); + + it('should prevent symlink loops', () => { + let realpathCalls = 0; + fs.existsSync = jest.fn(() => true); + fs.realpathSync = jest.fn(() => { + // Return same real path to simulate loop + return '/real/path'; + }); + fs.readdirSync = jest.fn(() => [ + { name: 'link-dir', isDirectory: () => true } + ]); + + const result = getAntoraStructure('/test'); + + // Should not hang or crash + expect(result).toBeDefined(); + expect(result.repoRoot).toBe('/test'); + }); + + it('should respect MAX_RECURSION_DEPTH', () => { + fs.existsSync = jest.fn(() => true); + fs.realpathSync = jest.fn((p) => p); + + let depth = 0; + fs.readdirSync = jest.fn(() => { + depth++; + // Return subdirectory to trigger recursion + if (depth <= MAX_RECURSION_DEPTH + 2) { + return [{ name: `subdir${depth}`, isDirectory: () => true }]; + } + return []; + }); + + const result = getAntoraStructure('/test'); + + // Depth should be limited + expect(depth).toBeLessThanOrEqual(MAX_RECURSION_DEPTH + 2); + }); + + it('should parse antora.yml files and detect modules', () => { + fs.existsSync = jest.fn((filePath) => { + // Simulate antora.yml exists, modules dir exists + return filePath.includes('antora.yml') || + filePath.includes('modules') || + filePath === '/test'; + }); + fs.realpathSync = jest.fn((p) => p); + + // Mock directory reading to return antora.yml when scanning root + fs.readdirSync = jest.fn((dirPath) => { + if (dirPath === '/test') { + // Root directory contains antora.yml file + return [{ name: 'antora.yml', isDirectory: () => false }]; + } + if (dirPath.includes('modules')) { + // Modules directory contains module folders + return ['ROOT', 'admin', 'reference']; + } + return []; + }); + + fs.readFileSync = jest.fn(() => 'name: redpanda\nversion: 25.3\ntitle: Redpanda'); + fs.statSync = jest.fn(() => ({ isDirectory: () => true })); + yaml.load = jest.fn(() => ({ + name: 'redpanda', + version: '25.3', + title: 'Redpanda' + })); + + const result = getAntoraStructure('/test'); + + expect(result.components.length).toBeGreaterThan(0); + const component = result.components[0]; + expect(component.name).toBe('redpanda'); + expect(component.version).toBe('25.3'); + expect(component.title).toBe('Redpanda'); + expect(component.modules.length).toBe(3); + }); + + it('should detect module directories (pages, partials, etc)', () => { + fs.existsSync = jest.fn((filePath) => { + // Modules dir exists, pages and images exist + return filePath.includes('modules') || + filePath.includes('pages') || + filePath.includes('images'); + }); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn((filePath) => { + if (filePath.includes('modules')) { + return ['ROOT']; + } + return [{ name: 'antora.yml', isDirectory: () => false }]; + }); + fs.readFileSync = jest.fn(() => 'name: test'); + fs.statSync = jest.fn(() => ({ isDirectory: () => true })); + yaml.load = jest.fn(() => ({ name: 'test' })); + + const result = getAntoraStructure('/test'); + + if (result.components.length > 0 && result.components[0].modules.length > 0) { + const module = result.components[0].modules[0]; + expect(module).toHaveProperty('pages'); + expect(module).toHaveProperty('partials'); + expect(module).toHaveProperty('examples'); + expect(module).toHaveProperty('attachments'); + expect(module).toHaveProperty('images'); + expect(module.pages).toBe(true); + expect(module.images).toBe(true); + expect(module.partials).toBe(false); + } + }); + + it('should detect doc-tools availability', () => { + fs.existsSync = jest.fn((filePath) => { + return filePath.includes('package.json') || filePath.includes('doc-tools.js'); + }); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => []); + + const result = getAntoraStructure('/test'); + + expect(result.hasDocTools).toBe(true); + }); + + it('should return false for hasDocTools when not available', () => { + const { execSync } = require('child_process'); + fs.existsSync = jest.fn(() => false); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => []); + // Mock execSync to throw when checking for doc-tools + execSync.mockImplementation(() => { throw new Error('Command not found'); }); + + const result = getAntoraStructure('/test'); + + expect(result.hasDocTools).toBe(false); + }); + + it('should handle permission errors gracefully', () => { + fs.existsSync = jest.fn(() => true); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => { + const error = new Error('EACCES: permission denied'); + error.code = 'EACCES'; + throw error; + }); + + const result = getAntoraStructure('/test'); + + // Should not crash + expect(result).toBeDefined(); + expect(result.components).toEqual([]); + }); + + it('should handle antora.yml parsing errors', () => { + fs.existsSync = jest.fn(() => true); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => [ + { name: 'antora.yml', isDirectory: () => false } + ]); + fs.readFileSync = jest.fn(() => 'invalid yaml'); + yaml.load = jest.fn(() => { + throw new Error('Parse error'); + }); + + const result = getAntoraStructure('/test'); + + expect(result.components.length).toBeGreaterThan(0); + expect(result.components[0]).toHaveProperty('error'); + expect(result.components[0].error).toContain('Failed to parse'); + }); + + it('should return complete structure with all fields', () => { + fs.existsSync = jest.fn(() => false); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => []); + + const result = getAntoraStructure('/test'); + + expect(result).toHaveProperty('repoRoot'); + expect(result).toHaveProperty('repoInfo'); + expect(result).toHaveProperty('playbook'); + expect(result).toHaveProperty('playbookPath'); + expect(result).toHaveProperty('components'); + expect(result).toHaveProperty('hasDocTools'); + }); + }); + + describe('executeTool', () => { + it('should return error for unknown tool', () => { + const result = executeTool('unknown_tool', {}); + expect(result.success).toBe(false); + expect(result.error).toContain('Unknown tool'); + }); + + it('should validate command for run_doc_tools_command', () => { + fs.existsSync = jest.fn(() => false); // No repo found + + const result = executeTool('run_doc_tools_command', { command: 'test; malicious' }); + expect(result.success).toBe(false); + expect(result.error).toContain('shell metacharacters'); + }); + + it('should call getAntoraStructure for get_antora_structure tool', () => { + fs.existsSync = jest.fn(() => false); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => []); + + const result = executeTool('get_antora_structure', {}); + + expect(result).toHaveProperty('repoRoot'); + expect(result).toHaveProperty('components'); + expect(result).toHaveProperty('hasDocTools'); + }); + }); + + describe('Version Information Tools', () => { + const { spawnSync } = require('child_process'); + + beforeEach(() => { + fs.existsSync = jest.fn((p) => p.includes('package.json') || p.includes('doc-tools.js')); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => []); + }); + + describe('getRedpandaVersion', () => { + it('should return version information for stable release', () => { + spawnSync.mockReturnValue({ + status: 0, + stdout: 'REDPANDA_VERSION=v25.3.1\nREDPANDA_DOCKER_REPO=redpanda\n', + stderr: '', + error: null + }); + + const result = getRedpandaVersion({}); + + expect(result.success).toBe(true); + expect(result.version).toBe('v25.3.1'); + expect(result.docker_tag).toBe('docker.redpanda.com/redpandadata/redpanda:v25.3.1'); + expect(result.is_beta).toBe(false); + expect(result.notes_url).toBe('https://github.com/redpanda-data/redpanda/releases/tag/v25.3.1'); + }); + + it('should return version information for beta release', () => { + spawnSync.mockReturnValue({ + status: 0, + stdout: 'REDPANDA_VERSION=v25.4.1-rc1\nREDPANDA_DOCKER_REPO=redpanda-unstable\n', + stderr: '', + error: null + }); + + const result = getRedpandaVersion({ beta: true }); + + expect(result.success).toBe(true); + expect(result.version).toBe('v25.4.1-rc1'); + expect(result.docker_tag).toBe('docker.redpanda.com/redpandadata/redpanda-unstable:v25.4.1-rc1'); + expect(result.is_beta).toBe(true); + }); + + it('should handle missing docker repo in output', () => { + spawnSync.mockReturnValue({ + status: 0, + stdout: 'REDPANDA_VERSION=v25.3.1\n', + stderr: '', + error: null + }); + + const result = getRedpandaVersion({}); + + expect(result.success).toBe(true); + expect(result.version).toBe('v25.3.1'); + expect(result.docker_tag).toBe('docker.redpanda.com/redpandadata/redpanda:v25.3.1'); + }); + + it('should handle command execution errors', () => { + spawnSync.mockReturnValue({ + status: 1, + stdout: '', + stderr: 'Network error', + error: new Error('Network error') + }); + + const result = getRedpandaVersion({}); + + expect(result.success).toBe(false); + expect(result.error).toContain('Network error'); + expect(result.suggestion).toContain('network access'); + }); + + it('should handle malformed output', () => { + spawnSync.mockReturnValue({ + status: 0, + stdout: 'INVALID_OUTPUT\n', + stderr: '', + error: null + }); + + const result = getRedpandaVersion({}); + + expect(result.success).toBe(false); + expect(result.error).toContain('Failed to parse version'); + }); + }); + + describe('getConsoleVersion', () => { + it('should return Console version information', () => { + spawnSync.mockReturnValue({ + status: 0, + stdout: 'CONSOLE_VERSION=v2.7.2\nCONSOLE_DOCKER_REPO=console\n', + stderr: '', + error: null + }); + + const result = getConsoleVersion(); + + expect(result.success).toBe(true); + expect(result.version).toBe('v2.7.2'); + expect(result.docker_tag).toBe('docker.redpanda.com/redpandadata/console:v2.7.2'); + expect(result.notes_url).toBe('https://github.com/redpanda-data/console/releases/tag/v2.7.2'); + }); + + it('should handle missing docker repo in output', () => { + spawnSync.mockReturnValue({ + status: 0, + stdout: 'CONSOLE_VERSION=v2.7.2\n', + stderr: '', + error: null + }); + + const result = getConsoleVersion(); + + expect(result.success).toBe(true); + expect(result.docker_tag).toBe('docker.redpanda.com/redpandadata/console:v2.7.2'); + }); + + it('should handle command execution errors', () => { + spawnSync.mockReturnValue({ + status: 1, + stdout: '', + stderr: 'Network error', + error: new Error('Network error') + }); + + const result = getConsoleVersion(); + + expect(result.success).toBe(false); + expect(result.error).toContain('Network error'); + expect(result.suggestion).toContain('network access'); + }); + + it('should handle malformed output', () => { + spawnSync.mockReturnValue({ + status: 0, + stdout: 'INVALID_OUTPUT\n', + stderr: '', + error: null + }); + + const result = getConsoleVersion(); + + expect(result.success).toBe(false); + expect(result.error).toContain('Failed to parse version'); + }); + }); + }); + + describe('Documentation Generation Tools', () => { + const { execSync, spawnSync } = require('child_process'); + + beforeEach(() => { + // Reset mocks before each test + fs.existsSync.mockReset(); + fs.realpathSync.mockReset(); + fs.readdirSync.mockReset(); + execSync.mockReset(); + spawnSync.mockReset(); + }); + + describe('generatePropertyDocs', () => { + it('should generate property docs with default options', () => { + // Mock repo detection + fs.existsSync = jest.fn((p) => p.includes('package.json') || p.includes('doc-tools.js')); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => []); + + spawnSync.mockReturnValue({ + status: 0, + stdout: 'Generated 342 properties\n', + stderr: '', + error: null + }); + + const result = generatePropertyDocs({ tag: '25.3.1' }); + + expect(result.success).toBe(true); + expect(result.tag).toBe('v25.3.1'); + expect(result.files_generated).toContain('modules/reference/partials/properties.json'); + expect(result.property_count).toBe(342); + }); + + it('should generate property docs with partials', () => { + fs.existsSync = jest.fn((p) => p.includes('package.json') || p.includes('doc-tools.js')); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => []); + + spawnSync.mockReturnValue({ + status: 0, + stdout: 'Generated 342 properties\n', + stderr: '', + error: null + }); + + const result = generatePropertyDocs({ tag: '25.3.1', generate_partials: true }); + + expect(result.success).toBe(true); + expect(result.files_generated).toContain('modules/reference/partials/cluster-properties.adoc'); + expect(result.files_generated).toContain('modules/reference/partials/broker-properties.adoc'); + expect(result.files_generated).toContain('modules/reference/partials/topic-properties.adoc'); + expect(result.files_generated).toContain('modules/reference/partials/tunable-properties.adoc'); + }); + + it('should normalize version without v prefix', () => { + fs.existsSync = jest.fn((p) => p.includes('package.json') || p.includes('doc-tools.js')); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => []); + + spawnSync.mockReturnValue({ + status: 0, + stdout: 'Generated 342 properties\n', + stderr: '', + error: null + }); + + const result = generatePropertyDocs({ tag: '25.3.1' }); + + expect(result.tag).toBe('v25.3.1'); + expect(spawnSync).toHaveBeenCalledWith( + 'node', + expect.arrayContaining(['generate', 'property-docs', '--tag', 'v25.3.1']), + expect.any(Object) + ); + }); + + it('should handle latest version', () => { + fs.existsSync = jest.fn((p) => p.includes('package.json') || p.includes('doc-tools.js')); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => []); + + spawnSync.mockReturnValue({ + status: 0, + stdout: 'Generated 342 properties\n', + stderr: '', + error: null + }); + + const result = generatePropertyDocs({ tag: 'latest' }); + + expect(result.tag).toBe('latest'); + expect(spawnSync).toHaveBeenCalledWith( + 'node', + expect.arrayContaining(['generate', 'property-docs', '--tag', 'latest']), + expect.any(Object) + ); + }); + + it('should return error when doc-tools not found', () => { + fs.existsSync = jest.fn(() => false); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => []); + // Mock execSync to throw when checking for doc-tools in getAntoraStructure + execSync.mockImplementation(() => { throw new Error('Command not found'); }); + + const result = generatePropertyDocs({ tag: '25.3.1' }); + + expect(result.success).toBe(false); + expect(result.error).toContain('doc-tools not found'); + expect(result.suggestion).toContain('docs-extensions-and-macros'); + }); + + it('should default to dev branch when no parameters provided', () => { + fs.existsSync = jest.fn((p) => p.includes('package.json') || p.includes('doc-tools.js')); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => []); + + spawnSync.mockReturnValue({ + status: 0, + stdout: 'Generated 342 properties\n', + stderr: '', + error: null + }); + + const result = generatePropertyDocs({}); + + expect(result.branch).toBe('dev'); + expect(spawnSync).toHaveBeenCalledWith( + 'node', + expect.arrayContaining(['--branch', 'dev']), + expect.any(Object) + ); + }); + + it('should handle command execution errors', () => { + fs.existsSync = jest.fn((p) => p.includes('package.json') || p.includes('doc-tools.js')); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => []); + + spawnSync.mockReturnValue({ + status: 1, + stdout: 'stdout output', + stderr: 'Tag v25.3.1 not found', + error: null + }); + + const result = generatePropertyDocs({ tag: '25.3.1' }); + + expect(result.success).toBe(false); + expect(result.error).toBe('Tag v25.3.1 not found'); + expect(result.stderr).toContain('Tag v25.3.1 not found'); + expect(result.suggestion).toContain('version exists'); + }); + }); + + describe('generateMetricsDocs', () => { + it('should generate metrics docs', () => { + fs.existsSync = jest.fn((p) => p.includes('package.json') || p.includes('doc-tools.js')); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => []); + + // Mock spawnSync (used by generateMetricsDocs) to return success + spawnSync.mockReturnValue({ + status: 0, + stdout: 'Generated 156 metrics\n', + stderr: '', + error: null + }); + + const result = generateMetricsDocs({ tag: '25.3.1' }); + + expect(result.success).toBe(true); + expect(result.tag).toBe('v25.3.1'); + expect(result.files_generated).toContain('modules/reference/pages/public-metrics-reference.adoc'); + expect(result.metrics_count).toBe(156); + }); + + it('should normalize version', () => { + fs.existsSync = jest.fn((p) => p.includes('package.json') || p.includes('doc-tools.js')); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => []); + + // Mock spawnSync (used by generateMetricsDocs) to return success + spawnSync.mockReturnValue({ + status: 0, + stdout: 'Generated 156 metrics\n', + stderr: '', + error: null + }); + + const result = generateMetricsDocs({ tag: '25.3.1' }); + + expect(result.tag).toBe('v25.3.1'); + }); + + it('should return error when doc-tools not found', () => { + fs.existsSync = jest.fn(() => false); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => []); + // Mock execSync to throw when checking for doc-tools in getAntoraStructure + execSync.mockImplementation(() => { throw new Error('Command not found'); }); + + const result = generateMetricsDocs({ tag: '25.3.1' }); + + expect(result.success).toBe(false); + expect(result.error).toContain('doc-tools not found'); + }); + + it('should default to dev branch when no parameters provided', () => { + fs.existsSync = jest.fn((p) => p.includes('package.json') || p.includes('doc-tools.js')); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => []); + + spawnSync.mockReturnValue({ + status: 0, + stdout: 'Generated 150 metrics\n', + stderr: '', + error: null + }); + + const result = generateMetricsDocs({}); + + expect(result.branch).toBe('dev'); + expect(spawnSync).toHaveBeenCalledWith( + 'node', + expect.arrayContaining(['--branch', 'dev']), + expect.any(Object) + ); + }); + }); + + describe('generateRpkDocs', () => { + it('should generate RPK docs', () => { + fs.existsSync = jest.fn((p) => p.includes('package.json') || p.includes('doc-tools.js')); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => []); + + spawnSync.mockReturnValue({ + status: 0, + stdout: 'Generated 87 commands\n', + stderr: '', + error: null + }); + + const result = generateRpkDocs({ tag: '25.3.1' }); + + expect(result.success).toBe(true); + expect(result.tag).toBe('v25.3.1'); + expect(result.commands_documented).toBe(87); + }); + + it('should normalize version', () => { + fs.existsSync = jest.fn((p) => p.includes('package.json') || p.includes('doc-tools.js')); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => []); + + spawnSync.mockReturnValue({ + status: 0, + stdout: 'Generated 87 commands\n', + stderr: '', + error: null + }); + + const result = generateRpkDocs({ tag: '25.3.1' }); + + expect(result.tag).toBe('v25.3.1'); + }); + + it('should return error when doc-tools not found', () => { + fs.existsSync = jest.fn(() => false); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => []); + // Mock execSync to throw when checking for doc-tools in getAntoraStructure + execSync.mockImplementation(() => { throw new Error('Command not found'); }); + + const result = generateRpkDocs({ tag: '25.3.1' }); + + expect(result.success).toBe(false); + expect(result.error).toContain('doc-tools not found'); + }); + + it('should default to dev branch when no parameters provided', () => { + fs.existsSync = jest.fn((p) => p.includes('package.json') || p.includes('doc-tools.js')); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => []); + + spawnSync.mockReturnValue({ + status: 0, + stdout: 'Generated 120 commands\n', + stderr: '', + error: null + }); + + const result = generateRpkDocs({}); + + expect(result.branch).toBe('dev'); + expect(spawnSync).toHaveBeenCalledWith( + 'node', + expect.arrayContaining(['--branch', 'dev']), + expect.any(Object) + ); + }); + }); + + describe('generateRpConnectDocs', () => { + it('should generate Redpanda Connect connector docs', () => { + fs.existsSync = jest.fn((p) => p.includes('package.json') || p.includes('doc-tools.js')); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => []); + + // Mock spawnSync (used by generateRpConnectDocs) to return success + spawnSync.mockReturnValue({ + status: 0, + stdout: 'Generated 245 connectors\n', + stderr: '', + error: null + }); + + const result = generateRpConnectDocs({}); + + expect(result.success).toBe(true); + expect(result.connectors_documented).toBe(245); + expect(result.files_generated).toContain('modules/reference/pages/redpanda-connect/components/'); + }); + + it('should generate docs with flags', () => { + fs.existsSync = jest.fn((p) => p.includes('package.json') || p.includes('doc-tools.js')); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => []); + + // Mock spawnSync (used by generateRpConnectDocs) to return success + spawnSync.mockReturnValue({ + status: 0, + stdout: 'Generated 250 connectors\n', + stderr: '', + error: null + }); + + const result = generateRpConnectDocs({ fetch_connectors: true, draft_missing: true }); + + expect(result.success).toBe(true); + expect(result.connectors_documented).toBe(250); + expect(spawnSync).toHaveBeenCalledWith( + 'npx', + expect.arrayContaining(['--fetch-connectors', '--draft-missing']), + expect.any(Object) + ); + }); + + it('should handle output without connector count', () => { + fs.existsSync = jest.fn((p) => p.includes('package.json') || p.includes('doc-tools.js')); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => []); + + // Mock spawnSync (used by generateRpConnectDocs) to return success + spawnSync.mockReturnValue({ + status: 0, + stdout: 'Generation complete\n', + stderr: '', + error: null + }); + + const result = generateRpConnectDocs({}); + + expect(result.success).toBe(true); + expect(result.connectors_documented).toBeNull(); + }); + + it('should return error when doc-tools not found', () => { + fs.existsSync = jest.fn(() => false); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => []); + // Mock execSync to throw when checking for doc-tools in getAntoraStructure + execSync.mockImplementation(() => { throw new Error('Command not found'); }); + + const result = generateRpConnectDocs({}); + + expect(result.success).toBe(false); + expect(result.error).toContain('doc-tools not found'); + expect(result.suggestion).toContain('docs-extensions-and-macros'); + }); + + it('should handle command execution errors', () => { + fs.existsSync = jest.fn((p) => p.includes('package.json') || p.includes('doc-tools.js')); + fs.realpathSync = jest.fn((p) => p); + fs.readdirSync = jest.fn(() => []); + + // Mock spawnSync to return non-zero status with stderr + spawnSync.mockReturnValue({ + status: 1, + stdout: 'stdout output', + stderr: 'Network error', + error: null + }); + + const result = generateRpConnectDocs({ fetch_connectors: true }); + + expect(result.success).toBe(false); + expect(result.error).toContain('Network error'); + expect(result.stderr).toContain('Network error'); + expect(result.suggestion).toContain('network access'); + }); + }); + }); +}); diff --git a/__tests__/mcp/integration.test.js b/__tests__/mcp/integration.test.js new file mode 100644 index 00000000..602b3a75 --- /dev/null +++ b/__tests__/mcp/integration.test.js @@ -0,0 +1,241 @@ +/** + * Integration tests for MCP tools + * These tests verify that the MCP tools correctly interface with the doc-tools CLI + */ + +const { describe, test, expect, beforeAll } = require('@jest/globals'); +const mcpTools = require('../../bin/mcp-tools'); +const { execSync } = require('child_process'); +const fs = require('fs'); +const path = require('path'); + +describe('MCP Tools Integration Tests', () => { + let repoRoot; + + beforeAll(() => { + repoRoot = mcpTools.findRepoRoot(); + }); + + describe('CLI Availability', () => { + test('doc-tools CLI is available', () => { + expect(() => { + // Use local bin/doc-tools.js to avoid npx cache issues in CI + execSync('node bin/doc-tools.js --version', { + encoding: 'utf8', + stdio: 'pipe', + cwd: repoRoot.root + }); + }).not.toThrow(); + }); + + test('all required generate subcommands exist', () => { + // Use local bin/doc-tools.js to avoid npx cache issues in CI + const output = execSync('node bin/doc-tools.js generate --help', { + encoding: 'utf8', + stdio: 'pipe', + cwd: repoRoot.root + }); + + const requiredCommands = [ + 'property-docs', + 'metrics-docs', + 'rpk-docs', + 'rpcn-connector-docs', + 'helm-spec', + 'cloud-regions', + 'crd-spec', + 'bundle-openapi' + ]; + + requiredCommands.forEach(cmd => { + expect(output).toContain(cmd); + }); + }); + }); + + describe('Tool Execution', () => { + test('get_antora_structure returns valid structure', () => { + const result = mcpTools.executeTool('get_antora_structure', {}); + + expect(result).toBeDefined(); + expect(result.repoRoot).toBe(repoRoot.root); + expect(result.components).toBeDefined(); + expect(Array.isArray(result.components)).toBe(true); + }); + + test('get_redpanda_version returns version info', async () => { + const result = mcpTools.executeTool('get_redpanda_version', {}); + + expect(result).toBeDefined(); + if (result.success) { + expect(result.version).toBeDefined(); + expect(result.docker_tag).toBeDefined(); + expect(result.notes_url).toBeDefined(); + } else { + // Network errors are acceptable in tests + expect(result.error).toBeDefined(); + } + }, 30000); + + test('get_console_version returns version info', async () => { + const result = mcpTools.executeTool('get_console_version', {}); + + expect(result).toBeDefined(); + if (result.success) { + expect(result.version).toBeDefined(); + expect(result.docker_tag).toBeDefined(); + } else { + // Network errors are acceptable in tests + expect(result.error).toBeDefined(); + } + }, 30000); + }); + + describe('Generate Tools - Parameter Validation', () => { + test('generate_property_docs defaults to dev branch when no parameters provided', () => { + const result = mcpTools.executeTool('generate_property_docs', {}); + + expect(result).toBeDefined(); + // Now defaults to branch 'dev', so should be treated as if branch was provided + // Test will attempt to run but may fail due to missing dependencies - that's OK + // The key is that it should NOT error about missing parameters + if (result.error) { + expect(result.error.toLowerCase()).not.toContain('required'); + expect(result.error.toLowerCase()).not.toContain('missing'); + } else { + expect(result).toHaveProperty('branch', 'dev'); + } + }); + + test('generate_metrics_docs defaults to dev branch when no parameters provided', () => { + const result = mcpTools.executeTool('generate_metrics_docs', {}); + + expect(result).toBeDefined(); + // Now defaults to branch 'dev', so should be treated as if branch was provided + // Test will attempt to run but may fail due to missing dependencies - that's OK + // The key is that it should NOT error about missing parameters + if (result.error) { + expect(result.error.toLowerCase()).not.toContain('required'); + expect(result.error.toLowerCase()).not.toContain('missing'); + } else { + expect(result).toHaveProperty('branch', 'dev'); + } + }); + + test('generate_rpk_docs defaults to dev branch when no parameters provided', () => { + const result = mcpTools.executeTool('generate_rpk_docs', {}); + + expect(result).toBeDefined(); + // Now defaults to branch 'dev', so should be treated as if branch was provided + // Test will attempt to run but may fail due to missing dependencies - that's OK + // The key is that it should NOT error about missing parameters + if (result.error) { + expect(result.error.toLowerCase()).not.toContain('required'); + expect(result.error.toLowerCase()).not.toContain('missing'); + } else { + expect(result).toHaveProperty('branch', 'dev'); + } + }); + + test('generate_crd_docs requires tag parameter', () => { + const result = mcpTools.executeTool('generate_crd_docs', {}); + + expect(result).toBeDefined(); + expect(result.success).toBe(false); + expect(result.error.toLowerCase()).toContain('tag'); + }); + + test('generate_bundle_openapi requires tag parameter', () => { + const result = mcpTools.executeTool('generate_bundle_openapi', {}); + + expect(result).toBeDefined(); + expect(result.success).toBe(false); + expect(result.error.toLowerCase()).toContain('tag'); + }); + }); + + describe('Review Tool', () => { + test('review_generated_docs requires doc_type parameter', () => { + const result = mcpTools.executeTool('review_generated_docs', {}); + + expect(result).toBeDefined(); + expect(result.success).toBe(false); + expect(result.error).toContain('doc_type'); + }); + + test('review_generated_docs validates doc_type enum', () => { + const result = mcpTools.executeTool('review_generated_docs', { + doc_type: 'invalid_type' + }); + + expect(result).toBeDefined(); + expect(result.success).toBe(false); + expect(result.error).toBeDefined(); + }); + }); + + describe('Error Handling', () => { + test('unknown tool returns error', () => { + const result = mcpTools.executeTool('nonexistent_tool', {}); + + expect(result).toBeDefined(); + expect(result.success).toBe(false); + expect(result.error).toContain('Unknown tool'); + }); + + test('invalid command in run_doc_tools_command is rejected', () => { + const result = mcpTools.executeTool('run_doc_tools_command', { + command: 'generate property-docs; rm -rf /' + }); + + expect(result).toBeDefined(); + expect(result.success).toBe(false); + expect(result.error).toContain('shell metacharacters'); + }); + }); + + describe('CLI Output Parsing', () => { + test('property-docs CLI output format is parseable', () => { + // Test that we can detect property counts from CLI output + const mockOutput = 'Successfully extracted 245 properties to JSON'; + const match = mockOutput.match(/(\d+) properties/i); + + expect(match).toBeDefined(); + expect(match[1]).toBe('245'); + }); + + test('metrics-docs CLI output format is parseable', () => { + const mockOutput = 'Generated docs for 180 metrics'; + const match = mockOutput.match(/(\d+) metrics/i); + + expect(match).toBeDefined(); + expect(match[1]).toBe('180'); + }); + + test('rpk-docs CLI output format is parseable', () => { + const mockOutput = 'Generated documentation for 75 commands'; + const match = mockOutput.match(/(\d+) commands/i); + + expect(match).toBeDefined(); + expect(match[1]).toBe('75'); + }); + }); + + describe('File System Expectations', () => { + test('doc-tools expects docs-extensions-and-macros structure', () => { + const packageJsonPath = path.join(repoRoot.root, 'package.json'); + const binPath = path.join(repoRoot.root, 'bin', 'doc-tools.js'); + + expect(fs.existsSync(packageJsonPath)).toBe(true); + expect(fs.existsSync(binPath)).toBe(true); + }); + + test('property overrides file location is correct', () => { + const overridesPath = path.join(repoRoot.root, 'docs-data', 'property-overrides.json'); + const docsDataDir = path.join(repoRoot.root, 'docs-data'); + + expect(fs.existsSync(docsDataDir)).toBe(true); + // File may or may not exist depending on repo state, but dir should exist + }); + }); +}); diff --git a/__tests__/rpcn-connector-docs/docs-data/connect-4.53.0.json b/__tests__/rpcn-connector-docs/docs-data/connect-4.53.0.json index f075416e..682b8c71 100644 --- a/__tests__/rpcn-connector-docs/docs-data/connect-4.53.0.json +++ b/__tests__/rpcn-connector-docs/docs-data/connect-4.53.0.json @@ -1 +1 @@ -{"version":"4.53.0","date":"2025-04-18T17:49:53Z","config":[{"name":"http","type":"object","kind":"scalar","description":"Configures the service-wide HTTP server.","children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to enable to HTTP server.","default":true},{"name":"address","type":"string","kind":"scalar","description":"The address to bind to.","default":"0.0.0.0:4195"},{"name":"root_path","type":"string","kind":"scalar","description":"Specifies a general prefix for all endpoints, this can help isolate the service endpoints when using a reverse proxy with other shared services. All endpoints will still be registered at the root as well as behind the prefix, e.g. with a root_path set to `/foo` the endpoint `/version` will be accessible from both `/version` and `/foo/version`.","default":"/benthos"},{"name":"debug_endpoints","type":"bool","kind":"scalar","description":"Whether to register a few extra endpoints that can be useful for debugging performance or behavioral problems.","default":false},{"name":"cert_file","type":"string","kind":"scalar","description":"An optional certificate file for enabling TLS.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"An optional key file for enabling TLS.","is_advanced":true,"default":""},{"name":"cors","type":"object","kind":"scalar","description":"Adds Cross-Origin Resource Sharing headers.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to allow CORS requests.","is_advanced":true,"default":false},{"name":"allowed_origins","type":"string","kind":"array","description":"An explicit list of origins that are allowed for CORS requests.","is_advanced":true,"default":[]}],"version":"3.63.0"},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to enforce and customise basic authentication for requests to the HTTP server.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Enable basic authentication","is_advanced":true,"default":false},{"name":"realm","type":"string","kind":"scalar","description":"Custom realm name","is_advanced":true,"default":"restricted"},{"name":"username","type":"string","kind":"scalar","description":"Username required to authenticate.","is_advanced":true,"default":""},{"name":"password_hash","type":"string","kind":"scalar","description":"Hashed password required to authenticate. (base64 encoded)","is_advanced":true,"default":""},{"name":"algorithm","type":"string","kind":"scalar","description":"Encryption algorithm used to generate `password_hash`.","is_advanced":true,"default":"sha256","examples":["md5","sha256","bcrypt","scrypt"]},{"name":"salt","type":"string","kind":"scalar","description":"Salt for scrypt algorithm. (base64 encoded)","is_advanced":true,"default":""}]}]},{"name":"input","type":"input","kind":"scalar","description":"An input to source messages from.","default":{"stdin":{}}},{"name":"buffer","type":"buffer","kind":"scalar","description":"An optional buffer to store messages during transit.","default":{"none":{}}},{"name":"pipeline","type":"object","kind":"scalar","description":"Describes optional processing pipelines used for mutating messages.","children":[{"name":"threads","type":"int","kind":"scalar","description":"The number of threads to execute processing pipelines across.","default":-1},{"name":"processors","type":"processor","kind":"array","description":"A list of processors to apply to messages.","default":[]}]},{"name":"output","type":"output","kind":"scalar","description":"An output to sink messages to.","default":{"stdout":{}}},{"name":"input_resources","type":"input","kind":"array","description":"A list of input resources, each must have a unique label.","is_advanced":true,"default":[]},{"name":"processor_resources","type":"processor","kind":"array","description":"A list of processor resources, each must have a unique label.","is_advanced":true,"default":[]},{"name":"output_resources","type":"output","kind":"array","description":"A list of output resources, each must have a unique label.","is_advanced":true,"default":[]},{"name":"cache_resources","type":"cache","kind":"array","description":"A list of cache resources, each must have a unique label.","is_advanced":true,"default":[]},{"name":"rate_limit_resources","type":"rate_limit","kind":"array","description":"A list of rate limit resources, each must have a unique label.","is_advanced":true,"default":[]},{"name":"logger","type":"object","kind":"scalar","description":"Describes how operational logs should be emitted.","children":[{"name":"level","type":"string","kind":"scalar","description":"Set the minimum severity level for emitting logs.","default":"INFO","options":["OFF","FATAL","ERROR","WARN","INFO","DEBUG","TRACE","ALL","NONE"]},{"name":"format","type":"string","kind":"scalar","description":"Set the format of emitted logs.","default":"logfmt","options":["json","logfmt"],"linter":"\nlet options = {\n \"json\": true,\n \"logfmt\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"add_timestamp","type":"bool","kind":"scalar","description":"Whether to include timestamps in logs.","default":false},{"name":"level_name","type":"string","kind":"scalar","description":"The name of the level field added to logs when the `format` is `json`.","is_advanced":true,"default":"level"},{"name":"timestamp_name","type":"string","kind":"scalar","description":"The name of the timestamp field added to logs when `add_timestamp` is set to `true` and the `format` is `json`.","is_advanced":true,"default":"time"},{"name":"message_name","type":"string","kind":"scalar","description":"The name of the message field added to logs when the `format` is `json`.","is_advanced":true,"default":"msg"},{"name":"static_fields","type":"string","kind":"map","description":"A map of key/value pairs to add to each structured log.","default":{"@service":"redpanda-connect"}},{"name":"file","type":"object","kind":"scalar","description":"Experimental: Specify fields for optionally writing logs to a file.","is_advanced":true,"children":[{"name":"path","type":"string","kind":"scalar","description":"The file path to write logs to, if the file does not exist it will be created. Leave this field empty or unset to disable file based logging.","is_advanced":true,"default":""},{"name":"rotate","type":"bool","kind":"scalar","description":"Whether to rotate log files automatically.","is_advanced":true,"default":false},{"name":"rotate_max_age_days","type":"int","kind":"scalar","description":"The maximum number of days to retain old log files based on the timestamp encoded in their filename, after which they are deleted. Setting to zero disables this mechanism.","is_advanced":true,"default":0}]}]},{"name":"metrics","type":"metrics","kind":"scalar","description":"A mechanism for exporting metrics.","default":{"mapping":"","prometheus":{}}},{"name":"tracer","type":"tracer","kind":"scalar","description":"A mechanism for exporting traces.","default":{"none":{}}},{"name":"shutdown_delay","type":"string","kind":"scalar","description":"A period of time to wait for metrics and traces to be pulled or pushed from the process.","default":"0s"},{"name":"shutdown_timeout","type":"string","kind":"scalar","description":"The maximum period of time to wait for a clean shutdown. If this time is exceeded Redpanda Connect will forcefully close.","default":"20s"},{"name":"tests","type":"object","kind":"array","description":"A list of one or more unit tests to execute.","is_advanced":true,"is_optional":true,"children":[{"name":"name","type":"string","kind":"scalar","description":"The name of the test, this should be unique and give a rough indication of what behavior is being tested.","is_advanced":true},{"name":"environment","type":"string","kind":"map","description":"An optional map of environment variables to set for the duration of the test.","is_advanced":true,"is_optional":true},{"name":"target_processors","type":"string","kind":"scalar","description":"\nA [JSON Pointer][json-pointer] that identifies the specific processors which should be executed by the test. The target can either be a single processor or an array of processors. Alternatively a resource label can be used to identify a processor.\n\nIt is also possible to target processors in a separate file by prefixing the target with a path relative to the test file followed by a # symbol.\n","is_advanced":true,"default":"/pipeline/processors","examples":["foo_processor","/pipeline/processors/0","target.yaml#/pipeline/processors","target.yaml#/pipeline/processors"]},{"name":"target_mapping","type":"string","kind":"scalar","description":"A file path relative to the test definition path of a Bloblang file to execute as an alternative to testing processors with the `target_processors` field. This allows you to define unit tests for Bloblang mappings directly.","is_advanced":true,"default":""},{"name":"mocks","type":"unknown","kind":"map","description":"An optional map of processors to mock. Keys should contain either a label or a JSON pointer of a processor that should be mocked. Values should contain a processor definition, which will replace the mocked processor. Most of the time you'll want to use a [`mapping` processor][processors.mapping] here, and use it to create a result that emulates the target processor.","is_advanced":true,"is_optional":true,"examples":[{"get_foobar_api":{"mapping":"root = content().string() + \" this is some mock content\""}},{"/pipeline/processors/1":{"mapping":"root = content().string() + \" this is some mock content\""}}]},{"name":"input_batch","type":"object","kind":"array","description":"Define a batch of messages to feed into your test, specify either an `input_batch` or a series of `input_batches`.","is_advanced":true,"is_optional":true,"children":[{"name":"content","type":"string","kind":"scalar","description":"The raw content of the input message.","is_advanced":true,"is_optional":true},{"name":"json_content","type":"unknown","kind":"scalar","description":"Sets the raw content of the message to a JSON document matching the structure of the value.","is_advanced":true,"is_optional":true,"examples":[{"bar":["element1",10],"foo":"foo value"}]},{"name":"file_content","type":"string","kind":"scalar","description":"Sets the raw content of the message by reading a file. The path of the file should be relative to the path of the test file.","is_advanced":true,"is_optional":true,"examples":["./foo/bar.txt"]},{"name":"metadata","type":"unknown","kind":"map","description":"A map of metadata key/values to add to the input message.","is_advanced":true,"is_optional":true}]},{"name":"input_batches","type":"object","kind":"2darray","description":"Define a series of batches of messages to feed into your test, specify either an `input_batch` or a series of `input_batches`.","is_advanced":true,"is_optional":true,"children":[{"name":"content","type":"string","kind":"scalar","description":"The raw content of the input message.","is_advanced":true,"is_optional":true},{"name":"json_content","type":"unknown","kind":"scalar","description":"Sets the raw content of the message to a JSON document matching the structure of the value.","is_advanced":true,"is_optional":true,"examples":[{"bar":["element1",10],"foo":"foo value"}]},{"name":"file_content","type":"string","kind":"scalar","description":"Sets the raw content of the message by reading a file. The path of the file should be relative to the path of the test file.","is_advanced":true,"is_optional":true,"examples":["./foo/bar.txt"]},{"name":"metadata","type":"unknown","kind":"map","description":"A map of metadata key/values to add to the input message.","is_advanced":true,"is_optional":true}]},{"name":"output_batches","type":"object","kind":"2darray","description":"List of output batches.","is_advanced":true,"is_optional":true,"children":[{"name":"bloblang","type":"string","kind":"scalar","description":"Executes a Bloblang mapping on the output message, if the result is anything other than a boolean equalling `true` the test fails.","is_advanced":true,"is_optional":true,"bloblang":true,"examples":["this.age \u003e 10 \u0026\u0026 @foo.length() \u003e 0"]},{"name":"content_equals","type":"string","kind":"scalar","description":"Checks the full raw contents of a message against a value.","is_advanced":true,"is_optional":true},{"name":"content_matches","type":"string","kind":"scalar","description":"Checks whether the full raw contents of a message matches a regular expression (re2).","is_advanced":true,"is_optional":true,"examples":["^foo [a-z]+ bar$"]},{"name":"metadata_equals","type":"unknown","kind":"map","description":"Checks a map of metadata keys to values against the metadata stored in the message. If there is a value mismatch between a key of the condition versus the message metadata this condition will fail.","is_advanced":true,"is_optional":true,"examples":[{"example_key":"example metadata value"}]},{"name":"file_equals","type":"string","kind":"scalar","description":"Checks that the contents of a message matches the contents of a file. The path of the file should be relative to the path of the test file.","is_advanced":true,"is_optional":true,"examples":["./foo/bar.txt"]},{"name":"file_json_equals","type":"string","kind":"scalar","description":"Checks that both the message and the file contents are valid JSON documents, and that they are structurally equivalent. Will ignore formatting and ordering differences. The path of the file should be relative to the path of the test file.","is_advanced":true,"is_optional":true,"examples":["./foo/bar.json"]},{"name":"json_equals","type":"unknown","kind":"scalar","description":"Checks that both the message and the condition are valid JSON documents, and that they are structurally equivalent. Will ignore formatting and ordering differences.","is_advanced":true,"is_optional":true,"examples":[{"key":"value"}]},{"name":"json_contains","type":"unknown","kind":"scalar","description":"Checks that both the message and the condition are valid JSON documents, and that the message is a superset of the condition.","is_advanced":true,"is_optional":true,"examples":[{"key":"value"}]},{"name":"file_json_contains","type":"string","kind":"scalar","description":"Checks that both the message and the file contents are valid JSON documents, and that the message is a superset of the condition. Will ignore formatting and ordering differences. The path of the file should be relative to the path of the test file.","is_advanced":true,"is_optional":true,"examples":["./foo/bar.json"]}]}]},{"name":"redpanda","type":"object","kind":"scalar","children":[{"name":"seed_brokers","type":"string","kind":"array","description":"A list of broker addresses to connect to in order to establish connections. If an item of the list contains commas it will be expanded into multiple addresses.","examples":[["localhost:9092"],["foo:9092","bar:9092"],["foo:9092,bar:9092"]]},{"name":"client_id","type":"string","kind":"scalar","description":"An identifier for the client connection.","is_advanced":true,"default":"benthos"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"sasl","type":"object","kind":"array","description":"Specify one or more methods of SASL authentication. SASL is tried in order; if the broker supports the first mechanism, all connections will use that mechanism. If the first mechanism fails, the client will pick the first supported mechanism. If the broker does not support any client mechanisms, connections will fail.","is_advanced":true,"is_optional":true,"examples":[[{"mechanism":"SCRAM-SHA-512","password":"bar","username":"foo"}]],"children":[{"name":"mechanism","type":"string","kind":"scalar","description":"The SASL mechanism to use.","is_advanced":true,"annotated_options":[["AWS_MSK_IAM","AWS IAM based authentication as specified by the 'aws-msk-iam-auth' java library."],["OAUTHBEARER","OAuth Bearer based authentication."],["PLAIN","Plain text authentication."],["SCRAM-SHA-256","SCRAM based authentication as specified in RFC5802."],["SCRAM-SHA-512","SCRAM based authentication as specified in RFC5802."],["none","Disable sasl authentication"]],"linter":"\nlet options = {\n \"aws_msk_iam\": true,\n \"oauthbearer\": true,\n \"plain\": true,\n \"scram-sha-256\": true,\n \"scram-sha-512\": true,\n \"none\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"username","type":"string","kind":"scalar","description":"A username to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token to use for a single session's OAUTHBEARER authentication.","is_advanced":true,"default":""},{"name":"extensions","type":"string","kind":"map","description":"Key/value pairs to add to OAUTHBEARER authentication requests.","is_advanced":true,"is_optional":true},{"name":"aws","type":"object","kind":"scalar","description":"Contains AWS specific fields for when the `mechanism` is set to `AWS_MSK_IAM`.","is_advanced":true,"is_optional":true,"children":[{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]}]},{"name":"metadata_max_age","type":"string","kind":"scalar","description":"The maximum age of metadata before it is refreshed.","is_advanced":true,"default":"5m"},{"name":"request_timeout_overhead","type":"string","kind":"scalar","description":"The request time overhead. Uses the given time as overhead while deadlining requests. Roughly equivalent to request.timeout.ms, but grants additional time to requests that have timeout fields.","is_advanced":true,"default":"10s"},{"name":"conn_idle_timeout","type":"string","kind":"scalar","description":"The rough amount of time to allow connections to idle before they are closed.","is_advanced":true,"default":"20s"},{"name":"pipeline_id","type":"string","kind":"scalar","description":"An optional identifier for the pipeline, this will be present in logs and status updates sent to topics.","default":""},{"name":"logs_topic","type":"string","kind":"scalar","description":"A topic to send process logs to.","default":"","examples":["__redpanda.connect.logs"]},{"name":"logs_level","type":"string","kind":"scalar","default":"info","options":["debug","info","warn","error"],"linter":"\nlet options = {\n \"debug\": true,\n \"info\": true,\n \"warn\": true,\n \"error\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"status_topic","type":"string","kind":"scalar","description":"A topic to send status updates to.","default":"","examples":["__redpanda.connect.status"]},{"name":"rack_id","type":"string","kind":"scalar","is_deprecated":true},{"name":"partitioner","type":"string","kind":"scalar","description":"Override the default murmur2 hashing partitioner.","is_advanced":true,"is_optional":true,"annotated_options":[["least_backup","Chooses the least backed up partition (the partition with the fewest amount of buffered records). Partitions are selected per batch."],["manual","Manually select a partition for each message, requires the field `partition` to be specified."],["murmur2_hash","Kafka's default hash algorithm that uses a 32-bit murmur2 hash of the key to compute which partition the record will be on."],["round_robin","Round-robin's messages through all available partitions. This algorithm has lower throughput and causes higher CPU load on brokers, but can be useful if you want to ensure an even distribution of records to partitions."]],"linter":"\nlet options = {\n \"least_backup\": true,\n \"manual\": true,\n \"murmur2_hash\": true,\n \"round_robin\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"idempotent_write","type":"bool","kind":"scalar","description":"Enable the idempotent write producer option. This requires the `IDEMPOTENT_WRITE` permission on `CLUSTER` and can be disabled if this permission is not available.","is_advanced":true,"default":true},{"name":"compression","type":"string","kind":"scalar","description":"Optionally set an explicit compression type. The default preference is to use snappy when the broker supports it, and fall back to none if not.","is_advanced":true,"is_optional":true,"options":["lz4","snappy","gzip","none","zstd"],"linter":"\nlet options = {\n \"lz4\": true,\n \"snappy\": true,\n \"gzip\": true,\n \"none\": true,\n \"zstd\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period of time to wait for message sends before abandoning the request and retrying","is_advanced":true,"default":"10s"},{"name":"max_message_bytes","type":"string","kind":"scalar","description":"The maximum space in bytes than an individual message may take, messages larger than this value will be rejected. This field corresponds to Kafka's `max.message.bytes`.","is_advanced":true,"default":"1MiB","examples":["100MB","50mib"]},{"name":"broker_write_max_bytes","type":"string","kind":"scalar","description":"The upper bound for the number of bytes written to a broker connection in a single write. This field corresponds to Kafka's `socket.request.max.bytes`.","is_advanced":true,"default":"100MiB","examples":["128MB","50mib"]}]}],"buffers":[{"name":"memory","type":"buffer","status":"stable","plugin":true,"summary":"Stores consumed messages in memory and acknowledges them at the input level. During shutdown Redpanda Connect will make a best attempt at flushing all remaining messages before exiting cleanly.","description":"\nThis buffer is appropriate when consuming messages from inputs that do not gracefully handle back pressure and where delivery guarantees aren't critical.\n\nThis buffer has a configurable limit, where consumption will be stopped with back pressure upstream if the total size of messages in the buffer reaches this amount. Since this calculation is only an estimate, and the real size of messages in RAM is always higher, it is recommended to set the limit significantly below the amount of RAM available.\n\n== Delivery guarantees\n\nThis buffer intentionally weakens the delivery guarantees of the pipeline and therefore should never be used in places where data loss is unacceptable.\n\n== Batching\n\nIt is possible to batch up messages sent from this buffer using a xref:configuration:batching.adoc#batch-policy[batch policy].","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"limit","type":"int","kind":"scalar","description":"The maximum buffer size (in bytes) to allow before applying backpressure upstream.","default":524288000},{"name":"batch_policy","type":"object","kind":"","description":"Optionally configure a policy to flush buffered messages in batches.","children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to batch messages as they are flushed.","default":false},{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]}},{"name":"none","type":"buffer","status":"stable","plugin":true,"summary":"Do not buffer messages. This is the default and most resilient configuration.","description":"Selecting no buffer means the output layer is directly coupled with the input layer. This is the safest and lowest latency option since acknowledgements from at-least-once protocols can be propagated all the way from the output protocol to the input protocol.\n\nIf the output layer is hit with back pressure it will propagate all the way to the input layer, and further up the data stream. If you need to relieve your pipeline of this back pressure consider using a more robust buffering solution such as Kafka before resorting to alternatives.","categories":null,"config":{"name":"","type":"object","kind":"scalar","default":{}}},{"name":"sqlite","type":"buffer","status":"stable","plugin":true,"summary":"Stores messages in an SQLite database and acknowledges them at the input level.","description":"\nStored messages are then consumed as a stream from the database and deleted only once they are successfully sent at the output level. If the service is restarted Redpanda Connect will make a best attempt to finish delivering messages that are already read from the database, and when it starts again it will consume from the oldest message that has not yet been delivered.\n\n== Delivery guarantees\n\nMessages are not acknowledged at the input level until they have been added to the SQLite database, and they are not removed from the SQLite database until they have been successfully delivered. This means at-least-once delivery guarantees are preserved in cases where the service is shut down unexpectedly. However, since this process relies on interaction with the disk (wherever the SQLite DB is stored) these delivery guarantees are not resilient to disk corruption or loss.\n\n== Batching\n\nMessages that are logically batched at the point where they are added to the buffer will continue to be associated with that batch when they are consumed. This buffer is also more efficient when storing messages within batches, and therefore it is recommended to use batching at the input level in high-throughput use cases even if they are not required for processing.\n","categories":["Utility"],"examples":[{"title":"Batching for optimization","summary":"Batching at the input level greatly increases the throughput of this buffer. If logical batches aren't needed for processing add a xref:components:processors/split.adoc[`split` processor] to the `post_processors`.","config":"\ninput:\n batched:\n child:\n sql_select:\n driver: postgres\n dsn: postgres://foouser:foopass@localhost:5432/testdb?sslmode=disable\n table: footable\n columns: [ '*' ]\n policy:\n count: 100\n period: 500ms\n\nbuffer:\n sqlite:\n path: ./foo.db\n post_processors:\n - split: {}\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"path","type":"string","kind":"scalar","description":"The path of the database file, which will be created if it does not already exist."},{"name":"pre_processors","type":"processor","kind":"array","description":"An optional list of processors to apply to messages before they are stored within the buffer. These processors are useful for compressing, archiving or otherwise reducing the data in size before it's stored on disk.","is_optional":true},{"name":"post_processors","type":"processor","kind":"array","description":"An optional list of processors to apply to messages after they are consumed from the buffer. These processors are useful for undoing any compression, archiving, etc that may have been done by your `pre_processors`.","is_optional":true}]}},{"name":"system_window","type":"buffer","status":"beta","plugin":true,"summary":"Chops a stream of messages into tumbling or sliding windows of fixed temporal size, following the system clock.","description":"\nA window is a grouping of messages that fit within a discrete measure of time following the system clock. Messages are allocated to a window either by the processing time (the time at which they're ingested) or by the event time, and this is controlled via the \u003c\u003ctimestamp_mapping, `timestamp_mapping` field\u003e\u003e.\n\nIn tumbling mode (default) the beginning of a window immediately follows the end of a prior window. When the buffer is initialized the first window to be created and populated is aligned against the zeroth minute of the zeroth hour of the day by default, and may therefore be open for a shorter period than the specified size.\n\nA window is flushed only once the system clock surpasses its scheduled end. If an \u003c\u003callowed_lateness, `allowed_lateness`\u003e\u003e is specified then the window will not be flushed until the scheduled end plus that length of time.\n\nWhen a message is added to a window it has a metadata field `window_end_timestamp` added to it containing the timestamp of the end of the window as an RFC3339 string.\n\n== Sliding windows\n\nSliding windows begin from an offset of the prior windows' beginning rather than its end, and therefore messages may belong to multiple windows. In order to produce sliding windows specify a \u003c\u003cslide, `slide` duration\u003e\u003e.\n\n== Back pressure\n\nIf back pressure is applied to this buffer either due to output services being unavailable or resources being saturated, windows older than the current and last according to the system clock will be dropped in order to prevent unbounded resource usage. This means you should ensure that under the worst case scenario you have enough system memory to store two windows' worth of data at a given time (plus extra for redundancy and other services).\n\nIf messages could potentially arrive with event timestamps in the future (according to the system clock) then you should also factor in these extra messages in memory usage estimates.\n\n== Delivery guarantees\n\nThis buffer honours the transaction model within Redpanda Connect in order to ensure that messages are not acknowledged until they are either intentionally dropped or successfully delivered to outputs. However, since messages belonging to an expired window are intentionally dropped there are circumstances where not all messages entering the system will be delivered.\n\nWhen this buffer is configured with a slide duration it is possible for messages to belong to multiple windows, and therefore be delivered multiple times. In this case the first time the message is delivered it will be acked (or nacked) and subsequent deliveries of the same message will be a \"best attempt\".\n\nDuring graceful termination if the current window is partially populated with messages they will be nacked such that they are re-consumed the next time the service starts.\n","categories":["Windowing"],"examples":[{"title":"Counting Passengers at Traffic","summary":"Given a stream of messages relating to cars passing through various traffic lights of the form:\n\n```json\n{\n \"traffic_light\": \"cbf2eafc-806e-4067-9211-97be7e42cee3\",\n \"created_at\": \"2021-08-07T09:49:35Z\",\n \"registration_plate\": \"AB1C DEF\",\n \"passengers\": 3\n}\n```\n\nWe can use a window buffer in order to create periodic messages summarizing the traffic for a period of time of this form:\n\n```json\n{\n \"traffic_light\": \"cbf2eafc-806e-4067-9211-97be7e42cee3\",\n \"created_at\": \"2021-08-07T10:00:00Z\",\n \"total_cars\": 15,\n \"passengers\": 43\n}\n```\n\nWith the following config:","config":"\nbuffer:\n system_window:\n timestamp_mapping: root = this.created_at\n size: 1h\n\npipeline:\n processors:\n # Group messages of the window into batches of common traffic light IDs\n - group_by_value:\n value: '${! json(\"traffic_light\") }'\n\n # Reduce each batch to a single message by deleting indexes \u003e 0, and\n # aggregate the car and passenger counts.\n - mapping: |\n root = if batch_index() == 0 {\n {\n \"traffic_light\": this.traffic_light,\n \"created_at\": meta(\"window_end_timestamp\"),\n \"total_cars\": json(\"registration_plate\").from_all().unique().length(),\n \"passengers\": json(\"passengers\").from_all().sum(),\n }\n } else { deleted() }\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"timestamp_mapping","type":"string","kind":"scalar","description":"\nA xref:guides:bloblang/about.adoc[Bloblang mapping] applied to each message during ingestion that provides the timestamp to use for allocating it a window. By default the function `now()` is used in order to generate a fresh timestamp at the time of ingestion (the processing time), whereas this mapping can instead extract a timestamp from the message itself (the event time).\n\nThe timestamp value assigned to `root` must either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in ISO 8601 format. If the mapping fails or provides an invalid result the message will be dropped (with logging to describe the problem).\n","default":"root = now()","bloblang":true,"examples":["root = this.created_at","root = meta(\"kafka_timestamp_unix\").number()"]},{"name":"size","type":"string","kind":"scalar","description":"A duration string describing the size of each window. By default windows are aligned to the zeroth minute and zeroth hour on the UTC clock, meaning windows of 1 hour duration will match the turn of each hour in the day, this can be adjusted with the `offset` field.","examples":["30s","10m"]},{"name":"slide","type":"string","kind":"scalar","description":"An optional duration string describing by how much time the beginning of each window should be offset from the beginning of the previous, and therefore creates sliding windows instead of tumbling. When specified this duration must be smaller than the `size` of the window.","default":"","examples":["30s","10m"]},{"name":"offset","type":"string","kind":"scalar","description":"An optional duration string to offset the beginning of each window by, otherwise they are aligned to the zeroth minute and zeroth hour on the UTC clock. The offset cannot be a larger or equal measure to the window size or the slide.","default":"","examples":["-6h","30m"]},{"name":"allowed_lateness","type":"string","kind":"scalar","description":"An optional duration string describing the length of time to wait after a window has ended before flushing it, allowing late arrivals to be included. Since this windowing buffer uses the system clock an allowed lateness can improve the matching of messages when using event time.","default":"","examples":["10s","1m"]}]},"version":"3.53.0"}],"caches":[{"name":"aws_dynamodb","type":"cache","status":"stable","plugin":true,"summary":"Stores key/value pairs as a single document in a DynamoDB table. The key is stored as a string value and used as the table hash key. The value is stored as\na binary value using the `data_key` field name.","description":"A prefix can be specified to allow multiple cache types to share a single DynamoDB table. An optional TTL duration (`ttl`) and field\n(`ttl_key`) can be specified if the backing table has TTL enabled.\n\nStrong read consistency can be enabled using the `consistent_read` configuration field.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"table","type":"string","kind":"scalar","description":"The table to store items in."},{"name":"hash_key","type":"string","kind":"scalar","description":"The key of the table column to store item keys within."},{"name":"data_key","type":"string","kind":"scalar","description":"The key of the table column to store item values within."},{"name":"consistent_read","type":"bool","kind":"scalar","description":"Whether to use strongly consistent reads on Get commands.","is_advanced":true,"default":false},{"name":"default_ttl","type":"string","kind":"scalar","description":"An optional default TTL to set for items, calculated from the moment the item is cached. A `ttl_key` must be specified in order to set item TTLs.","is_advanced":true,"is_optional":true},{"name":"ttl_key","type":"string","kind":"scalar","description":"The column key to place the TTL value within.","is_advanced":true,"is_optional":true},{"name":"retries","type":"object","kind":"scalar","description":"Determine time intervals and cut offs for retry attempts.","is_advanced":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"1s","examples":["50ms","1s"]},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts","is_advanced":true,"default":"5s","examples":["5s","1m"]},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum overall period of time to spend on retry attempts before the request is aborted.","is_advanced":true,"default":"30s","examples":["1m","1h"]}]},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]},"version":"3.36.0"},{"name":"aws_s3","type":"cache","status":"stable","plugin":true,"summary":"Stores each item in an S3 bucket as a file, where an item ID is the path of the item within the bucket.","description":"It is not possible to atomically upload S3 objects exclusively when the target does not already exist, therefore this cache is not suitable for deduplication.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"bucket","type":"string","kind":"scalar","description":"The S3 bucket to store items in."},{"name":"content_type","type":"string","kind":"scalar","description":"The content type to set for each item.","default":"application/octet-stream"},{"name":"force_path_style_urls","type":"bool","kind":"scalar","description":"Forces the client API to use path style URLs, which helps when connecting to custom endpoints.","is_advanced":true,"default":false},{"name":"retries","type":"object","kind":"scalar","description":"Determine time intervals and cut offs for retry attempts.","is_advanced":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"1s","examples":["50ms","1s"]},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts","is_advanced":true,"default":"5s","examples":["5s","1m"]},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum overall period of time to spend on retry attempts before the request is aborted.","is_advanced":true,"default":"30s","examples":["1m","1h"]}]},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]},"version":"3.36.0"},{"name":"couchbase","type":"cache","status":"experimental","plugin":true,"summary":"Use a Couchbase instance as a cache.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"Couchbase connection string.","examples":["couchbase://localhost:11210"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"username","type":"string","kind":"scalar","description":"Username to connect to the cluster.","is_optional":true},{"name":"password","type":"string","kind":"scalar","description":"Password to connect to the cluster.","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"bucket","type":"string","kind":"scalar","description":"Couchbase bucket."},{"name":"collection","type":"string","kind":"scalar","description":"Bucket collection.","is_advanced":true,"is_optional":true,"default":"_default"},{"name":"transcoder","type":"string","kind":"scalar","description":"Couchbase transcoder to use.","is_advanced":true,"default":"legacy","annotated_options":[["json","JSONTranscoder implements the default transcoding behavior and applies JSON transcoding to all values. This will apply the following behavior to the value: binary ([]byte) -\u003e error. default -\u003e JSON value, JSON Flags."],["legacy","LegacyTranscoder implements the behavior for a backward-compatible transcoder. This transcoder implements behavior matching that of gocb v1.This will apply the following behavior to the value: binary ([]byte) -\u003e binary bytes, Binary expectedFlags. string -\u003e string bytes, String expectedFlags. default -\u003e JSON value, JSON expectedFlags."],["raw","RawBinaryTranscoder implements passthrough behavior of raw binary data. This transcoder does not apply any serialization. This will apply the following behavior to the value: binary ([]byte) -\u003e binary bytes, binary expectedFlags. default -\u003e error."],["rawjson","RawJSONTranscoder implements passthrough behavior of JSON data. This transcoder does not apply any serialization. It will forward data across the network without incurring unnecessary parsing costs. This will apply the following behavior to the value: binary ([]byte) -\u003e JSON bytes, JSON expectedFlags. string -\u003e JSON bytes, JSON expectedFlags. default -\u003e error."],["rawstring","RawStringTranscoder implements passthrough behavior of raw string data. This transcoder does not apply any serialization. This will apply the following behavior to the value: string -\u003e string bytes, string expectedFlags. default -\u003e error."]],"linter":"\nlet options = {\n \"json\": true,\n \"legacy\": true,\n \"raw\": true,\n \"rawjson\": true,\n \"rawstring\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"timeout","type":"string","kind":"scalar","description":"Operation timeout.","is_advanced":true,"default":"15s"},{"name":"default_ttl","type":"string","kind":"scalar","description":"An optional default TTL to set for items, calculated from the moment the item is cached.","is_advanced":true,"is_optional":true}]},"version":"4.12.0"},{"name":"file","type":"cache","status":"stable","plugin":true,"summary":"Stores each item in a directory as a file, where an item ID is the path relative to the configured directory.","description":"This type currently offers no form of item expiry or garbage collection, and is intended to be used for development and debugging purposes only.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"directory","type":"string","kind":"scalar","description":"The directory within which to store items."}]}},{"name":"gcp_cloud_storage","type":"cache","status":"beta","plugin":true,"summary":"Use a Google Cloud Storage bucket as a cache.","description":"It is not possible to atomically upload cloud storage objects exclusively when the target does not already exist, therefore this cache is not suitable for deduplication.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"bucket","type":"string","kind":"scalar","description":"The Google Cloud Storage bucket to store items in."},{"name":"content_type","type":"string","kind":"scalar","description":"Optional field to explicitly set the Content-Type.","is_optional":true},{"name":"credentials_json","type":"string","kind":"scalar","description":"An optional field to set Google Service Account Credentials json.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}},{"name":"lru","type":"cache","status":"stable","plugin":true,"summary":"Stores key/value pairs in a lru in-memory cache. This cache is therefore reset every time the service restarts.","description":"This provides the lru package which implements a fixed-size thread safe LRU cache.\n\nIt uses the package https://github.com/hashicorp/golang-lru/v2[`lru`^]\n\nThe field init_values can be used to pre-populate the memory cache with any number of key/value pairs:\n\n```yaml\ncache_resources:\n - label: foocache\n lru:\n cap: 1024\n init_values:\n foo: bar\n```\n\nThese values can be overridden during execution.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"cap","type":"int","kind":"scalar","description":"The cache maximum capacity (number of entries)","default":1000},{"name":"init_values","type":"string","kind":"map","description":"A table of key/value pairs that should be present in the cache on initialization. This can be used to create static lookup tables.","default":{},"examples":[{"Nickelback":"1995","Spice Girls":"1994","The Human League":"1977"}]},{"name":"algorithm","type":"string","kind":"scalar","description":"the lru cache implementation","is_advanced":true,"default":"standard","annotated_options":[["arc","is an adaptive replacement cache. It tracks recent evictions as well as recent usage in both the frequent and recent caches. Its computational overhead is comparable to two_queues, but the memory overhead is linear with the size of the cache. ARC has been patented by IBM."],["standard","is a simple LRU cache. It is based on the LRU implementation in groupcache"],["two_queues","tracks frequently used and recently used entries separately. This avoids a burst of accesses from taking out frequently used entries, at the cost of about 2x computational overhead and some extra bookkeeping."]],"linter":"\nlet options = {\n \"arc\": true,\n \"standard\": true,\n \"two_queues\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"two_queues_recent_ratio","type":"float","kind":"scalar","description":"is the ratio of the two_queues cache dedicated to recently added entries that have only been accessed once.","is_advanced":true,"is_optional":true,"default":0.25},{"name":"two_queues_ghost_ratio","type":"float","kind":"scalar","description":"is the default ratio of ghost entries kept to track entries recently evicted on two_queues cache.","is_advanced":true,"is_optional":true,"default":0.5},{"name":"optimistic","type":"bool","kind":"scalar","description":"If true, we do not lock on read/write events. The lru package is thread-safe, however the ADD operation is not atomic.","is_advanced":true,"default":false}]}},{"name":"memcached","type":"cache","status":"stable","plugin":true,"summary":"Connects to a cluster of memcached services, a prefix can be specified to allow multiple cache types to share a memcached cluster under different namespaces.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"addresses","type":"string","kind":"array","description":"A list of addresses of memcached servers to use."},{"name":"prefix","type":"string","kind":"scalar","description":"An optional string to prefix item keys with in order to prevent collisions with similar services.","is_optional":true},{"name":"default_ttl","type":"string","kind":"scalar","description":"A default TTL to set for items, calculated from the moment the item is cached.","default":"300s"},{"name":"retries","type":"object","kind":"scalar","description":"Determine time intervals and cut offs for retry attempts.","is_advanced":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"1s","examples":["50ms","1s"]},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts","is_advanced":true,"default":"5s","examples":["5s","1m"]},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum overall period of time to spend on retry attempts before the request is aborted.","is_advanced":true,"default":"30s","examples":["1m","1h"]}]}]}},{"name":"memory","type":"cache","status":"stable","plugin":true,"summary":"Stores key/value pairs in a map held in memory. This cache is therefore reset every time the service restarts. Each item in the cache has a TTL set from the moment it was last edited, after which it will be removed during the next compaction.","description":"The compaction interval determines how often the cache is cleared of expired items, and this process is only triggered on writes to the cache. Access to the cache is blocked during this process.\n\nItem expiry can be disabled entirely by setting the `compaction_interval` to an empty string.\n\nThe field `init_values` can be used to prepopulate the memory cache with any number of key/value pairs which are exempt from TTLs:\n\n```yaml\ncache_resources:\n - label: foocache\n memory:\n default_ttl: 60s\n init_values:\n foo: bar\n```\n\nThese values can be overridden during execution, at which point the configured TTL is respected as usual.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"default_ttl","type":"string","kind":"scalar","description":"The default TTL of each item. After this period an item will be eligible for removal during the next compaction.","default":"5m"},{"name":"compaction_interval","type":"string","kind":"scalar","description":"The period of time to wait before each compaction, at which point expired items are removed. This field can be set to an empty string in order to disable compactions/expiry entirely.","default":"60s"},{"name":"init_values","type":"string","kind":"map","description":"A table of key/value pairs that should be present in the cache on initialization. This can be used to create static lookup tables.","default":{},"examples":[{"Nickelback":"1995","Spice Girls":"1994","The Human League":"1977"}]},{"name":"shards","type":"int","kind":"scalar","description":"A number of logical shards to spread keys across, increasing the shards can have a performance benefit when processing a large number of keys.","is_advanced":true,"default":1}]}},{"name":"mongodb","type":"cache","status":"experimental","plugin":true,"summary":"Use a MongoDB instance as a cache.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target MongoDB server.","examples":["mongodb://localhost:27017"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"database","type":"string","kind":"scalar","description":"The name of the target MongoDB database."},{"name":"username","type":"string","kind":"scalar","description":"The username to connect to the database.","default":""},{"name":"password","type":"string","kind":"scalar","description":"The password to connect to the database.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"app_name","type":"string","kind":"scalar","description":"The client application name.","is_advanced":true,"default":"benthos","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"collection","type":"string","kind":"scalar","description":"The name of the target collection."},{"name":"key_field","type":"string","kind":"scalar","description":"The field in the document that is used as the key."},{"name":"value_field","type":"string","kind":"scalar","description":"The field in the document that is used as the value."}]},"version":"3.43.0"},{"name":"multilevel","type":"cache","status":"stable","plugin":true,"summary":"Combines multiple caches as levels, performing read-through and write-through operations across them.","categories":null,"examples":[{"title":"Hot and cold cache","summary":"The multilevel cache is useful for reducing traffic against a remote cache by routing it through a local cache. In the following example requests will only go through to the memcached server if the local memory cache is missing the key.","config":"\npipeline:\n processors:\n - branch:\n processors:\n - cache:\n resource: leveled\n operator: get\n key: ${! json(\"key\") }\n - catch:\n - mapping: 'root = {\"err\":error()}'\n result_map: 'root.result = this'\n\ncache_resources:\n - label: leveled\n multilevel: [ hot, cold ]\n\n - label: hot\n memory:\n default_ttl: 60s\n\n - label: cold\n memcached:\n addresses: [ TODO:11211 ]\n default_ttl: 60s\n"}],"config":{"name":"","type":"string","kind":"array"}},{"name":"nats_kv","type":"cache","status":"experimental","plugin":true,"summary":"Cache key/values in a NATS key-value bucket.","description":"== Connection name\n\nWhen monitoring and managing a production NATS system, it is often useful to\nknow which connection a message was send/received from. This can be achieved by\nsetting the connection name option when creating a NATS connection.\n\nRedpanda Connect will automatically set the connection name based off the label of the given\nNATS component, so that monitoring tools between NATS and Redpanda Connect can stay in sync.\n\n\n== Authentication\n\nThere are several components within Redpanda Connect which uses NATS services. You will find that each of these components\nsupport optional advanced authentication parameters for https://docs.nats.io/nats-server/configuration/securing_nats/auth_intro/nkey_auth[NKeys^]\nand https://docs.nats.io/using-nats/developer/connecting/creds[User Credentials^].\n\nSee an https://docs.nats.io/running-a-nats-service/nats_admin/security/jwt[in-depth tutorial^].\n\n=== NKey file\n\nThe NATS server can use these NKeys in several ways for authentication. The simplest is for the server to be configured\nwith a list of known public keys and for the clients to respond to the challenge by signing it with its private NKey\nconfigured in the `nkey_file` or `nkey` field.\n\nhttps://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[More details^].\n\n=== User credentials\n\nNATS server supports decentralized authentication based on JSON Web Tokens (JWT). Clients need an https://docs.nats.io/nats-server/configuration/securing_nats/jwt#json-web-tokens[user JWT^]\nand a corresponding https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[NKey secret^] when connecting to a server\nwhich is configured to use this authentication scheme.\n\nThe `user_credentials_file` field should point to a file containing both the private key and the JWT and can be\ngenerated with the https://docs.nats.io/nats-tools/nsc[nsc tool^].\n\nAlternatively, the `user_jwt` field can contain a plain text JWT and the `user_nkey_seed`can contain\nthe plain text NKey Seed.\n\nhttps://docs.nats.io/using-nats/developer/connecting/creds[More details^].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["nats://127.0.0.1:4222"],["nats://username:password@127.0.0.1:4222"]]},{"name":"bucket","type":"string","kind":"scalar","description":"The name of the KV bucket.","examples":["my_kv_bucket"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auth","type":"object","kind":"scalar","description":"Optional configuration of NATS authentication parameters.","is_advanced":true,"children":[{"name":"nkey_file","type":"string","kind":"scalar","description":"An optional file containing a NKey seed.","is_advanced":true,"is_optional":true,"examples":["./seed.nk"]},{"name":"nkey","type":"string","kind":"scalar","description":"The NKey seed.","is_advanced":true,"is_optional":true,"is_secret":true,"examples":["UDXU4RCSJNZOIQHZNWXHXORDPRTGNJAHAHFRGZNEEJCPQTT2M7NLCNF4"],"version":"4.38.0","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_credentials_file","type":"string","kind":"scalar","description":"An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.","is_advanced":true,"is_optional":true,"examples":["./user.creds"]},{"name":"user_jwt","type":"string","kind":"scalar","description":"An optional plain text user JWT (given along with the corresponding user NKey Seed).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_nkey_seed","type":"string","kind":"scalar","description":"An optional plain text user NKey Seed (given along with the corresponding user JWT).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},"version":"4.27.0"},{"name":"noop","type":"cache","status":"stable","plugin":true,"summary":"Noop is a cache that stores nothing, all gets returns not found. Why? Sometimes doing nothing is the braver option.","categories":null,"config":{"name":"","type":"object","kind":"scalar","default":{}},"version":"4.27.0"},{"name":"redis","type":"cache","status":"stable","plugin":true,"summary":"Use a Redis instance as a cache. The expiration can be set to zero or an empty string in order to set no expiration.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target Redis server. Database is optional and is supplied as the URL path.","examples":["redis://:6379","redis://localhost:6379","redis://foousername:foopassword@redisplace:6379","redis://:foopassword@redisplace:6379","redis://localhost:6379/1","redis://localhost:6379/1,redis://localhost:6380/1"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"kind","type":"string","kind":"scalar","description":"Specifies a simple, cluster-aware, or failover-aware redis client.","is_advanced":true,"default":"simple","options":["simple","cluster","failover"],"linter":"\nlet options = {\n \"simple\": true,\n \"cluster\": true,\n \"failover\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"master","type":"string","kind":"scalar","description":"Name of the redis master when `kind` is `failover`","is_advanced":true,"default":"","examples":["mymaster"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"prefix","type":"string","kind":"scalar","description":"An optional string to prefix item keys with in order to prevent collisions with similar services.","is_optional":true},{"name":"default_ttl","type":"string","kind":"scalar","description":"An optional default TTL to set for items, calculated from the moment the item is cached.","is_advanced":true,"is_optional":true},{"name":"retries","type":"object","kind":"scalar","description":"Determine time intervals and cut offs for retry attempts.","is_advanced":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"500ms","examples":["50ms","1s"]},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts","is_advanced":true,"default":"1s","examples":["5s","1m"]},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum overall period of time to spend on retry attempts before the request is aborted.","is_advanced":true,"default":"5s","examples":["1m","1h"]}]}]}},{"name":"ristretto","type":"cache","status":"stable","plugin":true,"summary":"Stores key/value pairs in a map held in the memory-bound https://github.com/dgraph-io/ristretto[Ristretto cache^].","description":"This cache is more efficient and appropriate for high-volume use cases than the standard memory cache. However, the add command is non-atomic, and therefore this cache is not suitable for deduplication.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"default_ttl","type":"string","kind":"scalar","description":"A default TTL to set for items, calculated from the moment the item is cached. Set to an empty string or zero duration to disable TTLs.","default":"","examples":["5m","60s"]},{"name":"get_retries","type":"object","kind":"scalar","description":"Determines how and whether get attempts should be retried if the key is not found. Ristretto is a concurrent cache that does not immediately reflect writes, and so it can sometimes be useful to enable retries at the cost of speed in cases where the key is expected to exist.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether retries should be enabled.","is_advanced":true,"default":false},{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"1s","examples":["50ms","1s"]},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts","is_advanced":true,"default":"5s","examples":["5s","1m"]},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum overall period of time to spend on retry attempts before the request is aborted.","is_advanced":true,"default":"30s","examples":["1m","1h"]}]}]}},{"name":"sql","type":"cache","status":"experimental","plugin":true,"summary":"Uses an SQL database table as a destination for storing cache key/value items.","description":"\nEach cache key/value pair will exist as a row within the specified table. Currently only the key and value columns are set, and therefore any other columns present within the target table must allow NULL values if this cache is going to be used for set and add operations.\n\nCache operations are translated into SQL statements as follows:\n\n== Get\n\nAll `get` operations are performed with a traditional `select` statement.\n\n== Delete\n\nAll `delete` operations are performed with a traditional `delete` statement.\n\n== Set\n\nThe `set` operation is performed with a traditional `insert` statement.\n\nThis will behave as an `add` operation by default, and so ideally needs to be adapted in order to provide updates instead of failing on collision\ts. Since different SQL engines implement upserts differently it is necessary to specify a `set_suffix` that modifies an `insert` statement in order to perform updates on conflict.\n\n== Add\n\nThe `add` operation is performed with a traditional `insert` statement.\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"driver","type":"string","kind":"scalar","description":"A database \u003c\u003cdrivers, driver\u003e\u003e to use.","options":["mysql","postgres","clickhouse","mssql","sqlite","oracle","snowflake","trino","gocosmos","spanner"],"linter":"\nlet options = {\n \"mysql\": true,\n \"postgres\": true,\n \"clickhouse\": true,\n \"mssql\": true,\n \"sqlite\": true,\n \"oracle\": true,\n \"snowflake\": true,\n \"trino\": true,\n \"gocosmos\": true,\n \"spanner\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"dsn","type":"string","kind":"scalar","description":"A Data Source Name to identify the target database.\n\n==== Drivers\n\n:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community, spanner=community\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n|===\n| Driver | Data Source Name Format\n\n| `clickhouse` \n| https://github.com/ClickHouse/clickhouse-go#dsn[`clickhouse://[username[:password\\]@\\][netloc\\][:port\\]/dbname[?param1=value1\u0026...\u0026paramN=valueN\\]`^] \n\n| `mysql` \n| `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` \n\n| `postgres` \n| `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` \n\n| `mssql` \n| `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` \n\n| `sqlite` \n| `file:/path/to/filename.db[?param\u0026=value1\u0026...]` \n\n| `oracle` \n| `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` \n\n| `snowflake` \n| `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` \n\n| `trino` \n| https://github.com/trinodb/trino-go-client#dsn-data-source-name[`http[s\\]://user[:pass\\]@host[:port\\][?parameters\\]`^] \n\n| `gocosmos` \n| https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage[`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e\\][;Version=\u003ccosmosdb-api-version\u003e\\][;DefaultDb/Db=\u003cdb-name\u003e\\][;AutoId=\u003ctrue/false\u003e\\][;InsecureSkipVerify=\u003ctrue/false\u003e\\]`^] \n\n| `spanner` \n| projects/[PROJECT]/instances/[INSTANCE]/databases/[DATABASE] \n|===\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String[the docs^] for more details. For https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication[key pair authentication^], the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe https://pkg.go.dev/github.com/microsoft/gocosmos[`gocosmos`^] driver is still experimental, but it has support for https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys[hierarchical partition keys^] as well as https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query[cross-partition queries^]. Please refer to the https://github.com/microsoft/gocosmos/blob/main/SQL.md[SQL notes^] for details.","examples":["clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60","foouser:foopassword@tcp(localhost:3306)/foodb","postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable","oracle://foouser:foopass@localhost:1521/service_name"]},{"name":"table","type":"string","kind":"scalar","description":"The table to insert/read/delete cache items.","examples":["foo"]},{"name":"key_column","type":"string","kind":"scalar","description":"The name of a column to be used for storing cache item keys. This column should support strings of arbitrary size.","examples":["foo"]},{"name":"value_column","type":"string","kind":"scalar","description":"The name of a column to be used for storing cache item values. This column should support strings of arbitrary size.","examples":["bar"]},{"name":"set_suffix","type":"string","kind":"scalar","description":"An optional suffix to append to each insert query for a cache `set` operation. This should modify an insert statement into an upsert appropriate for the given SQL engine.","is_optional":true,"examples":["ON DUPLICATE KEY UPDATE bar=VALUES(bar)","ON CONFLICT (foo) DO UPDATE SET bar=excluded.bar","ON CONFLICT (foo) DO NOTHING"]},{"name":"init_files","type":"string","kind":"array","description":"\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":[["./init/*.sql"],["./foo.sql","./bar.sql"]],"version":"4.10.0"},{"name":"init_statement","type":"string","kind":"scalar","description":"\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":["\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n"],"version":"4.10.0"},{"name":"conn_max_idle_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time.","is_advanced":true,"is_optional":true},{"name":"conn_max_life_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age.","is_advanced":true,"is_optional":true},{"name":"conn_max_idle","type":"int","kind":"scalar","description":"An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release.","is_advanced":true,"is_optional":true,"default":2},{"name":"conn_max_open","type":"int","kind":"scalar","description":"An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited).","is_advanced":true,"is_optional":true}]},"version":"4.26.0"},{"name":"ttlru","type":"cache","status":"stable","plugin":true,"summary":"Stores key/value pairs in a ttlru in-memory cache. This cache is therefore reset every time the service restarts.","description":"The cache ttlru provides a simple, goroutine safe, cache with a fixed number of entries. Each entry has a per-cache defined TTL.\n\nThis TTL is reset on both modification and access of the value. As a result, if the cache is full, and no items have expired, when adding a new item, the item with the soonest expiration will be evicted.\n\nIt uses the package https://github.com/hashicorp/golang-lru/v2/expirable[`expirable`^]\n\nThe field init_values can be used to pre-populate the memory cache with any number of key/value pairs:\n\n```yaml\ncache_resources:\n - label: foocache\n ttlru:\n default_ttl: '5m'\n cap: 1024\n init_values:\n foo: bar\n```\n\nThese values can be overridden during execution.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"cap","type":"int","kind":"scalar","description":"The cache maximum capacity (number of entries)","default":1024},{"name":"default_ttl","type":"string","kind":"scalar","description":"The cache ttl of each element","default":"5m0s","version":"4.21.0"},{"name":"ttl","type":"string","kind":"scalar","description":"Deprecated. Please use `default_ttl` field","is_advanced":true,"is_optional":true},{"name":"init_values","type":"string","kind":"map","description":"A table of key/value pairs that should be present in the cache on initialization. This can be used to create static lookup tables.","default":{},"examples":[{"Nickelback":"1995","Spice Girls":"1994","The Human League":"1977"}]},{"name":"optimistic","type":"bool","kind":"scalar","description":"If true, we do not lock on read/write events. The ttlru package is thread-safe, however the ADD operation is not atomic.","is_advanced":true,"default":false}]}}],"inputs":[{"name":"amqp_0_9","type":"input","status":"stable","plugin":true,"summary":"Connects to an AMQP (0.91) queue. AMQP is a messaging protocol used by various message brokers, including RabbitMQ.","description":"\nTLS is automatic when connecting to an `amqps` URL, but custom settings can be enabled in the `tls` section.\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n- amqp_content_type\n- amqp_content_encoding\n- amqp_delivery_mode\n- amqp_priority\n- amqp_correlation_id\n- amqp_reply_to\n- amqp_expiration\n- amqp_message_id\n- amqp_timestamp\n- amqp_type\n- amqp_user_id\n- amqp_app_id\n- amqp_consumer_tag\n- amqp_delivery_tag\n- amqp_redelivered\n- amqp_exchange\n- amqp_routing_key\n- All existing message headers, including nested headers prefixed with the key of their respective parent.\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolations].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. The first URL to successfully establish a connection will be used until the connection is closed. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["amqp://guest:guest@127.0.0.1:5672/"],["amqp://127.0.0.1:5672/,amqp://127.0.0.2:5672/"],["amqp://127.0.0.1:5672/","amqp://127.0.0.2:5672/"]],"version":"3.58.0","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"queue","type":"string","kind":"scalar","description":"An AMQP queue to consume from."},{"name":"queue_declare","type":"object","kind":"scalar","description":"Allows you to passively declare the target queue. If the queue already exists then the declaration passively verifies that they match the target fields.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to enable queue declaration.","is_advanced":true,"default":false},{"name":"durable","type":"bool","kind":"scalar","description":"Whether the declared queue is durable.","is_advanced":true,"default":true},{"name":"auto_delete","type":"bool","kind":"scalar","description":"Whether the declared queue will auto-delete.","is_advanced":true,"default":false},{"name":"arguments","type":"string","kind":"map","description":"\nOptional arguments specific to the server's implementation of the queue that can be sent for queue types which require extra parameters.\n\n== Arguments\n\n- x-queue-type\n\nIs used to declare quorum and stream queues. Accepted values are: 'classic' (default), 'quorum', 'stream', 'drop-head', 'reject-publish' and 'reject-publish-dlx'.\n\n- x-max-length\n\nMaximum number of messages, is a non-negative integer value.\n\n- x-max-length-bytes\n\nMaximum number of messages, is a non-negative integer value.\n\n- x-overflow\n\nSets overflow behaviour. Possible values are: 'drop-head' (default), 'reject-publish', 'reject-publish-dlx'.\n\n- x-message-ttl\n\nTTL period in milliseconds. Must be a string representation of the number.\n\n- x-expires\n\nExpiration policy, describes the expiration period in milliseconds. Must be a positive integer.\n\n- x-max-age\n\nControls the retention of a stream. Must be a strin, valid units: (Y, M, D, h, m, s) e.g. '7D' for a week.\n\n- x-stream-max-segment-size-bytes\n\nControls the size of the segment files on disk (default 500000000). Must be a positive integer.\n\n- x-queue-version\n\ndeclares the Classic Queue version to use. Expects an integer, either 1 or 2.\n\n- x-consumer-timeout\n\nInteger specified in milliseconds.\n\n- x-single-active-consumer\n\nEnables Single Active Consumer, Expects a Boolean.\n\nSee https://github.com/rabbitmq/amqp091-go/blob/b3d409fe92c34bea04d8123a136384c85e8dc431/types.go#L282-L362 for more information on available arguments.","is_advanced":true,"is_optional":true,"examples":[{"x-max-length":1000,"x-max-length-bytes":4096,"x-queue-type":"quorum"}]}]},{"name":"bindings_declare","type":"object","kind":"array","description":"Allows you to passively declare bindings for the target queue.","is_advanced":true,"is_optional":true,"examples":[[{"exchange":"foo","key":"bar"}]],"children":[{"name":"exchange","type":"string","kind":"scalar","description":"The exchange of the declared binding.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"The key of the declared binding.","is_advanced":true,"default":""}]},{"name":"consumer_tag","type":"string","kind":"scalar","description":"A consumer tag.","default":""},{"name":"auto_ack","type":"bool","kind":"scalar","description":"Acknowledge messages automatically as they are consumed rather than waiting for acknowledgments from downstream. This can improve throughput and prevent the pipeline from blocking but at the cost of eliminating delivery guarantees.","is_advanced":true,"default":false},{"name":"nack_reject_patterns","type":"string","kind":"array","description":"A list of regular expression patterns whereby if a message that has failed to be delivered by Redpanda Connect has an error that matches it will be dropped (or delivered to a dead-letter queue if one exists). By default failed messages are nacked with requeue enabled.","is_advanced":true,"default":[],"examples":[["^reject me please:.+$"]],"version":"3.64.0"},{"name":"prefetch_count","type":"int","kind":"scalar","description":"The maximum number of pending messages to have consumed at a time.","default":10},{"name":"prefetch_size","type":"int","kind":"scalar","description":"The maximum amount of pending messages measured in bytes to have consumed at a time.","is_advanced":true,"default":0},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]}]}},{"name":"amqp_1","type":"input","status":"stable","plugin":true,"summary":"Reads messages from an AMQP (1.0) server.","description":"== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- amqp_content_type\n- amqp_content_encoding\n- amqp_creation_time\n- All string typed message annotations\n```\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].\n\nBy setting `read_header` to `true`, additional message header properties will be added to each message:\n\n```text\n- amqp_durable\n- amqp_priority\n- amqp_ttl\n- amqp_first_acquirer\n- amqp_delivery_count\n```\n\n== Performance\n\nThis input benefits from receiving multiple messages in flight in parallel for improved performance.\nYou can tune the max number of in flight messages with the field `credit`.\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"A URL to connect to.","is_deprecated":true,"is_optional":true,"examples":["amqp://localhost:5672/","amqps://guest:guest@localhost:5672/"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. The first URL to successfully establish a connection will be used until the connection is closed. If an item of the list contains commas it will be expanded into multiple URLs.","is_optional":true,"examples":[["amqp://guest:guest@127.0.0.1:5672/"],["amqp://127.0.0.1:5672/,amqp://127.0.0.2:5672/"],["amqp://127.0.0.1:5672/","amqp://127.0.0.2:5672/"]],"version":"4.23.0","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"source_address","type":"string","kind":"scalar","description":"The source address to consume from.","examples":["/foo","queue:/bar","topic:/baz"]},{"name":"azure_renew_lock","type":"bool","kind":"scalar","description":"Experimental: Azure service bus specific option to renew lock if processing takes more then configured lock time","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"read_header","type":"bool","kind":"scalar","description":"Read additional message header fields into `amqp_*` metadata properties.","is_advanced":true,"default":false,"version":"4.25.0"},{"name":"credit","type":"int","kind":"scalar","description":"Specifies the maximum number of unacknowledged messages the sender can transmit. Once this limit is reached, no more messages will arrive until messages are acknowledged and settled.","is_advanced":true,"default":64,"version":"4.26.0","linter":"root = if this \u003c 1 { [ \"credit must be at least 1\" ] }"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"sasl","type":"object","kind":"scalar","description":"Enables SASL authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"mechanism","type":"string","kind":"scalar","description":"The SASL authentication mechanism to use.","is_advanced":true,"default":"none","annotated_options":[["anonymous","Anonymous SASL authentication."],["none","No SASL based authentication."],["plain","Plain text SASL authentication."]],"linter":"\nlet options = {\n \"anonymous\": true,\n \"none\": true,\n \"plain\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"user","type":"string","kind":"scalar","description":"A SASL plain text username. It is recommended that you use environment variables to populate this field.","is_advanced":true,"default":"","examples":["${USER}"]},{"name":"password","type":"string","kind":"scalar","description":"A SASL plain text password. It is recommended that you use environment variables to populate this field.","is_advanced":true,"is_secret":true,"default":"","examples":["${PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}],"linter":"\nroot = if this.url.or(\"\") == \"\" \u0026\u0026 this.urls.or([]).length() == 0 {\n \"field 'urls' must be set\"\n}\n"}},{"name":"aws_kinesis","type":"input","status":"stable","plugin":true,"summary":"Receive messages from one or more Kinesis streams.","description":"\nConsumes messages from one or more Kinesis streams either by automatically balancing shards across other instances of this input, or by consuming shards listed explicitly. The latest message sequence consumed by this input is stored within a \u003c\u003ctable-schema,DynamoDB table\u003e\u003e, which allows it to resume at the correct sequence of the shard during restarts. This table is also used for coordination across distributed inputs when shard balancing.\n\nRedpanda Connect will not store a consumed sequence unless it is acknowledged at the output level, which ensures at-least-once delivery guarantees.\n\n== Ordering\n\nBy default messages of a shard can be processed in parallel, up to a limit determined by the field `checkpoint_limit`. However, if strict ordered processing is required then this value must be set to 1 in order to process shard messages in lock-step. When doing so it is recommended that you perform batching at this component for performance as it will not be possible to batch lock-stepped messages at the output level.\n\n== Table schema\n\nIt's possible to configure Redpanda Connect to create the DynamoDB table required for coordination if it does not already exist. However, if you wish to create this yourself (recommended) then create a table with a string HASH key `StreamID` and a string RANGE key `ShardID`.\n\n== Batching\n\nUse the `batching` fields to configure an optional xref:configuration:batching.adoc#batch-policy[batching policy]. Each stream shard will be batched separately in order to ensure that acknowledgements aren't contaminated.\n","categories":["Services","AWS"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"streams","type":"string","kind":"array","description":"One or more Kinesis data streams to consume from. Streams can either be specified by their name or full ARN. Shards of a stream are automatically balanced across consumers by coordinating through the provided DynamoDB table. Multiple comma separated streams can be listed in a single element. Shards are automatically distributed across consumers of a stream by coordinating through the provided DynamoDB table. Alternatively, it's possible to specify an explicit shard to consume from with a colon after the stream name, e.g. `foo:0` would consume the shard `0` of the stream `foo`.","examples":[["foo","arn:aws:kinesis:*:111122223333:stream/my-stream"]]},{"name":"dynamodb","type":"object","kind":"scalar","description":"Determines the table used for storing and accessing the latest consumed sequence for shards, and for coordinating balanced consumers of streams.","children":[{"name":"table","type":"string","kind":"scalar","description":"The name of the table to access.","default":""},{"name":"create","type":"bool","kind":"scalar","description":"Whether, if the table does not exist, it should be created.","default":false},{"name":"billing_mode","type":"string","kind":"scalar","description":"When creating the table determines the billing mode.","is_advanced":true,"default":"PAY_PER_REQUEST","options":["PROVISIONED","PAY_PER_REQUEST"],"linter":"\nlet options = {\n \"provisioned\": true,\n \"pay_per_request\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"read_capacity_units","type":"int","kind":"scalar","description":"Set the provisioned read capacity when creating the table with a `billing_mode` of `PROVISIONED`.","is_advanced":true,"default":0},{"name":"write_capacity_units","type":"int","kind":"scalar","description":"Set the provisioned write capacity when creating the table with a `billing_mode` of `PROVISIONED`.","is_advanced":true,"default":0},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]},{"name":"checkpoint_limit","type":"int","kind":"scalar","description":"The maximum gap between the in flight sequence versus the latest acknowledged sequence at a given time. Increasing this limit enables parallel processing and batching at the output level to work on individual shards. Any given sequence will not be committed unless all messages under that offset are delivered in order to preserve at least once delivery guarantees.","default":1024},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"commit_period","type":"string","kind":"scalar","description":"The period of time between each update to the checkpoint table.","default":"5s"},{"name":"rebalance_period","type":"string","kind":"scalar","description":"The period of time between each attempt to rebalance shards across clients.","is_advanced":true,"default":"30s"},{"name":"lease_period","type":"string","kind":"scalar","description":"The period of time after which a client that has failed to update a shard checkpoint is assumed to be inactive.","is_advanced":true,"default":"30s"},{"name":"start_from_oldest","type":"bool","kind":"scalar","description":"Whether to consume from the oldest message when a sequence does not yet exist for the stream.","default":true},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]},"version":"3.36.0"},{"name":"aws_s3","type":"input","status":"stable","plugin":true,"summary":"Downloads objects within an Amazon S3 bucket, optionally filtered by a prefix, either by walking the items in the bucket or by streaming upload notifications in realtime.","description":"\n== Stream objects on upload with SQS\n\nA common pattern for consuming S3 objects is to emit upload notification events from the bucket either directly to an SQS queue, or to an SNS topic that is consumed by an SQS queue, and then have your consumer listen for events which prompt it to download the newly uploaded objects. More information about this pattern and how to set it up can be found at in the https://docs.aws.amazon.com/AmazonS3/latest/dev/ways-to-add-notification-config-to-bucket.html[Amazon S3 docs].\n\nRedpanda Connect is able to follow this pattern when you configure an `sqs.url`, where it consumes events from SQS and only downloads object keys received within those events. In order for this to work Redpanda Connect needs to know where within the event the key and bucket names can be found, specified as xref:configuration:field_paths.adoc[dot paths] with the fields `sqs.key_path` and `sqs.bucket_path`. The default values for these fields should already be correct when following the guide above.\n\nIf your notification events are being routed to SQS via an SNS topic then the events will be enveloped by SNS, in which case you also need to specify the field `sqs.envelope_path`, which in the case of SNS to SQS will usually be `Message`.\n\nWhen using SQS please make sure you have sensible values for `sqs.max_messages` and also the visibility timeout of the queue itself. When Redpanda Connect consumes an S3 object the SQS message that triggered it is not deleted until the S3 object has been sent onwards. This ensures at-least-once crash resiliency, but also means that if the S3 object takes longer to process than the visibility timeout of your queue then the same objects might be processed multiple times.\n\n== Download large files\n\nWhen downloading large files it's often necessary to process it in streamed parts in order to avoid loading the entire file in memory at a given time. In order to do this a \u003c\u003cscanner, `scanner`\u003e\u003e can be specified that determines how to break the input into smaller individual messages.\n\n== Credentials\n\nBy default Redpanda Connect will use a shared credentials file when connecting to AWS services. It's also possible to set them explicitly at the component level, allowing you to transfer data across accounts. You can find out more in xref:guides:cloud/aws.adoc[].\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n- s3_key\n- s3_bucket\n- s3_last_modified_unix\n- s3_last_modified (RFC3339)\n- s3_content_type\n- s3_content_encoding\n- s3_version_id\n- All user defined metadata\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation]. Note that user defined metadata is case insensitive within AWS, and it is likely that the keys will be received in a capitalized form, if you wish to make them consistent you can map all metadata keys to lower or uppercase using a Bloblang mapping such as `meta = meta().map_each_key(key -\u003e key.lowercase())`.","categories":["Services","AWS"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"bucket","type":"string","kind":"scalar","description":"The bucket to consume from. If the field `sqs.url` is specified this field is optional.","default":""},{"name":"prefix","type":"string","kind":"scalar","description":"An optional path prefix, if set only objects with the prefix are consumed when walking a bucket.","default":""},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]},{"name":"force_path_style_urls","type":"bool","kind":"scalar","description":"Forces the client API to use path style URLs for downloading keys, which is often required when connecting to custom endpoints.","is_advanced":true,"default":false},{"name":"delete_objects","type":"bool","kind":"scalar","description":"Whether to delete downloaded objects from the bucket once they are processed.","is_advanced":true,"default":false},{"name":"codec","type":"string","kind":"scalar","description":"The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.","is_deprecated":true,"is_optional":true,"examples":["lines","delim:\t","delim:foobar","gzip/csv"],"annotated_options":[["auto","EXPERIMENTAL: Attempts to derive a codec for each file based on information such as the extension. For example, a .tar.gz file would be consumed with the `gzip/tar` codec. Defaults to all-bytes."],["all-bytes","Consume the entire file as a single binary message."],["avro-ocf:marshaler=x","EXPERIMENTAL: Consume a stream of Avro OCF datum. The `marshaler` parameter is optional and has the options: `goavro` (default), `json`. Use `goavro` if OCF contains logical types."],["chunker:x","Consume the file in chunks of a given number of bytes."],["csv","Consume structured rows as comma separated values, the first row must be a header row."],["csv:x","Consume structured rows as values separated by a custom delimiter, the first row must be a header row. The custom delimiter must be a single character, e.g. the codec `\"csv:\\t\"` would consume a tab delimited file."],["csv-safe","Consume structured rows like `csv`, but sends messages with empty maps on failure to parse. Includes row number and parsing errors (if any) in the message's metadata."],["csv-safe:x","Consume structured rows like `csv:x` as values separated by a custom delimiter, but sends messages with empty maps on failure to parse. The custom delimiter must be a single character, e.g. the codec `\"csv-safe:\\t\"` would consume a tab delimited file. Includes row number and parsing errors (if any) in the message's metadata."],["delim:x","Consume the file in segments divided by a custom delimiter."],["gzip","Decompress a gzip file, this codec should precede another codec, e.g. `gzip/all-bytes`, `gzip/tar`, `gzip/csv`, etc."],["pgzip","Decompress a gzip file in parallel, this codec should precede another codec, e.g. `pgzip/all-bytes`, `pgzip/tar`, `pgzip/csv`, etc."],["lines","Consume the file in segments divided by linebreaks."],["multipart","Consumes the output of another codec and batches messages together. A batch ends when an empty message is consumed. For example, the codec `lines/multipart` could be used to consume multipart messages where an empty line indicates the end of each batch."],["regex:(?m)^\\d\\d:\\d\\d:\\d\\d","Consume the file in segments divided by regular expression."],["skipbom","Skip one or more byte order marks for each opened reader, this codec should precede another codec, e.g. `skipbom/csv`, etc."],["tar","Parse the file as a tar archive, and consume each file of the archive as a message."]]},{"name":"max_buffer","type":"int","kind":"scalar","is_deprecated":true,"default":1000000},{"name":"scanner","type":"scanner","kind":"scalar","description":"The xref:components:scanners/about.adoc[scanner] by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once.","is_optional":true,"default":{"to_the_end":{}},"version":"4.25.0"},{"name":"sqs","type":"object","kind":"scalar","description":"Consume SQS messages in order to trigger key downloads.","is_optional":true,"children":[{"name":"url","type":"string","kind":"scalar","description":"An optional SQS URL to connect to. When specified this queue will control which objects are downloaded.","default":""},{"name":"endpoint","type":"string","kind":"scalar","description":"A custom endpoint to use when connecting to SQS.","is_advanced":true,"default":""},{"name":"key_path","type":"string","kind":"scalar","description":"A xref:configuration:field_paths.adoc[dot path] whereby object keys are found in SQS messages.","default":"Records.*.s3.object.key"},{"name":"bucket_path","type":"string","kind":"scalar","description":"A xref:configuration:field_paths.adoc[dot path] whereby the bucket name can be found in SQS messages.","default":"Records.*.s3.bucket.name"},{"name":"envelope_path","type":"string","kind":"scalar","description":"A xref:configuration:field_paths.adoc[dot path] of a field to extract an enveloped JSON payload for further extracting the key and bucket from SQS messages. This is specifically useful when subscribing an SQS queue to an SNS topic that receives bucket events.","default":"","examples":["Message"]},{"name":"delay_period","type":"string","kind":"scalar","description":"An optional period of time to wait from when a notification was originally sent to when the target key download is attempted.","is_advanced":true,"default":"","examples":["10s","5m"]},{"name":"max_messages","type":"int","kind":"scalar","description":"The maximum number of SQS messages to consume from each request.","is_advanced":true,"default":10},{"name":"wait_time_seconds","type":"int","kind":"scalar","description":"Whether to set the wait time. Enabling this activates long-polling. Valid values: 0 to 20.","is_advanced":true,"default":0}]}]}},{"name":"aws_sqs","type":"input","status":"stable","plugin":true,"summary":"Consume messages from an AWS SQS URL.","description":"\n== Credentials\n\nBy default Redpanda Connect will use a shared credentials file when connecting to AWS\nservices. It's also possible to set them explicitly at the component level,\nallowing you to transfer data across accounts. You can find out more in\nxref:guides:cloud/aws.adoc[].\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n- sqs_message_id\n- sqs_receipt_handle\n- sqs_approximate_receive_count\n- All message attributes\n\nYou can access these metadata fields using\nxref:configuration:interpolation.adoc#bloblang-queries[function interpolation].","categories":["Services","AWS"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The SQS URL to consume from.","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"delete_message","type":"bool","kind":"scalar","description":"Whether to delete the consumed message once it is acked. Disabling allows you to handle the deletion using a different mechanism.","is_advanced":true,"default":true},{"name":"reset_visibility","type":"bool","kind":"scalar","description":"Whether to set the visibility timeout of the consumed message to zero once it is nacked. Disabling honors the preset visibility timeout specified for the queue.","is_advanced":true,"default":true,"version":"3.58.0"},{"name":"max_number_of_messages","type":"int","kind":"scalar","description":"The maximum number of messages to return on one poll. Valid values: 1 to 10.","is_advanced":true,"default":10},{"name":"max_outstanding_messages","type":"int","kind":"scalar","description":"The maximum number of outstanding pending messages to be consumed at a given time.","default":1000},{"name":"wait_time_seconds","type":"int","kind":"scalar","description":"Whether to set the wait time. Enabling this activates long-polling. Valid values: 0 to 20.","is_advanced":true,"default":0},{"name":"message_timeout","type":"string","kind":"scalar","description":"The time to process messages before needing to refresh the receipt handle. Messages will be eligible for refresh when half of the timeout has elapsed. This sets MessageVisibility for each received message.","is_advanced":true,"default":"30s"},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]}},{"name":"azure_blob_storage","type":"input","status":"beta","plugin":true,"summary":"Downloads objects within an Azure Blob Storage container, optionally filtered by a prefix.","description":"\nSupports multiple authentication methods but only one of the following is required:\n\n- `storage_connection_string`\n- `storage_account` and `storage_access_key`\n- `storage_account` and `storage_sas_token`\n- `storage_account` to access via https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity#DefaultAzureCredential[DefaultAzureCredential^]\n\nIf multiple are set then the `storage_connection_string` is given priority.\n\nIf the `storage_connection_string` does not contain the `AccountName` parameter, please specify it in the\n`storage_account` field.\n\n== Download large files\n\nWhen downloading large files it's often necessary to process it in streamed parts in order to avoid loading the entire file in memory at a given time. In order to do this a \u003c\u003cscanner, `scanner`\u003e\u003e can be specified that determines how to break the input into smaller individual messages.\n\n== Stream new files\n\nBy default this input will consume all files found within the target container and will then gracefully terminate. This is referred to as a \"batch\" mode of operation. However, it's possible to instead configure a container as https://learn.microsoft.com/en-gb/azure/event-grid/event-schema-blob-storage[an Event Grid source^] and then use this as a \u003c\u003ctargetsinput, `targets_input`\u003e\u003e, in which case new files are consumed as they're uploaded and Redpanda Connect will continue listening for and downloading files as they arrive. This is referred to as a \"streamed\" mode of operation.\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n- blob_storage_key\n- blob_storage_container\n- blob_storage_last_modified\n- blob_storage_last_modified_unix\n- blob_storage_content_type\n- blob_storage_content_encoding\n- All user defined metadata\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].","categories":["Services","Azure"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"storage_account","type":"string","kind":"scalar","description":"The storage account to access. This field is ignored if `storage_connection_string` is set.","default":""},{"name":"storage_access_key","type":"string","kind":"scalar","description":"The storage account access key. This field is ignored if `storage_connection_string` is set.","default":""},{"name":"storage_connection_string","type":"string","kind":"scalar","description":"A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set.","default":""},{"name":"storage_sas_token","type":"string","kind":"scalar","description":"The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set.","default":""},{"name":"container","type":"string","kind":"scalar","description":"The name of the container from which to download blobs.","interpolated":true},{"name":"prefix","type":"string","kind":"scalar","description":"An optional path prefix, if set only objects with the prefix are consumed.","default":""},{"name":"codec","type":"string","kind":"scalar","description":"The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.","is_deprecated":true,"is_optional":true,"examples":["lines","delim:\t","delim:foobar","gzip/csv"],"annotated_options":[["auto","EXPERIMENTAL: Attempts to derive a codec for each file based on information such as the extension. For example, a .tar.gz file would be consumed with the `gzip/tar` codec. Defaults to all-bytes."],["all-bytes","Consume the entire file as a single binary message."],["avro-ocf:marshaler=x","EXPERIMENTAL: Consume a stream of Avro OCF datum. The `marshaler` parameter is optional and has the options: `goavro` (default), `json`. Use `goavro` if OCF contains logical types."],["chunker:x","Consume the file in chunks of a given number of bytes."],["csv","Consume structured rows as comma separated values, the first row must be a header row."],["csv:x","Consume structured rows as values separated by a custom delimiter, the first row must be a header row. The custom delimiter must be a single character, e.g. the codec `\"csv:\\t\"` would consume a tab delimited file."],["csv-safe","Consume structured rows like `csv`, but sends messages with empty maps on failure to parse. Includes row number and parsing errors (if any) in the message's metadata."],["csv-safe:x","Consume structured rows like `csv:x` as values separated by a custom delimiter, but sends messages with empty maps on failure to parse. The custom delimiter must be a single character, e.g. the codec `\"csv-safe:\\t\"` would consume a tab delimited file. Includes row number and parsing errors (if any) in the message's metadata."],["delim:x","Consume the file in segments divided by a custom delimiter."],["gzip","Decompress a gzip file, this codec should precede another codec, e.g. `gzip/all-bytes`, `gzip/tar`, `gzip/csv`, etc."],["pgzip","Decompress a gzip file in parallel, this codec should precede another codec, e.g. `pgzip/all-bytes`, `pgzip/tar`, `pgzip/csv`, etc."],["lines","Consume the file in segments divided by linebreaks."],["multipart","Consumes the output of another codec and batches messages together. A batch ends when an empty message is consumed. For example, the codec `lines/multipart` could be used to consume multipart messages where an empty line indicates the end of each batch."],["regex:(?m)^\\d\\d:\\d\\d:\\d\\d","Consume the file in segments divided by regular expression."],["skipbom","Skip one or more byte order marks for each opened reader, this codec should precede another codec, e.g. `skipbom/csv`, etc."],["tar","Parse the file as a tar archive, and consume each file of the archive as a message."]]},{"name":"max_buffer","type":"int","kind":"scalar","is_deprecated":true,"default":1000000},{"name":"scanner","type":"scanner","kind":"scalar","description":"The xref:components:scanners/about.adoc[scanner] by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once.","is_optional":true,"default":{"to_the_end":{}},"version":"4.25.0"},{"name":"delete_objects","type":"bool","kind":"scalar","description":"Whether to delete downloaded objects from the blob once they are processed.","is_advanced":true,"default":false},{"name":"targets_input","type":"input","kind":"scalar","description":"EXPERIMENTAL: An optional source of download targets, configured as a xref:components:inputs/about.adoc[regular Redpanda Connect input]. Each message yielded by this input should be a single structured object containing a field `name`, which represents the blob to be downloaded.","is_optional":true,"examples":[{"mqtt":{"topics":["some-topic"],"urls":["example.westeurope-1.ts.eventgrid.azure.net:8883"]},"processors":[{"unarchive":{"format":"json_array"}},{"mapping":"if this.eventType == \"Microsoft.Storage.BlobCreated\" {\n root.name = this.data.url.parse_url().path.trim_prefix(\"/foocontainer/\")\n} else {\n root = deleted()\n}"}]}],"version":"4.27.0"}],"linter":"root = if this.storage_connection_string != \"\" \u0026\u0026 !this.storage_connection_string.contains(\"AccountName=\") \u0026\u0026 !this.storage_connection_string.contains(\"UseDevelopmentStorage=true;\") \u0026\u0026 this.storage_account == \"\" { [ \"storage_account must be set if storage_connection_string does not contain the \\\"AccountName\\\" parameter\" ] }"},"version":"3.36.0"},{"name":"azure_cosmosdb","type":"input","status":"experimental","plugin":true,"summary":"Executes a SQL query against https://learn.microsoft.com/en-us/azure/cosmos-db/introduction[Azure CosmosDB^] and creates a batch of messages from each page of items.","description":"\n== Cross-partition queries\n\nCross-partition queries are currently not supported by the underlying driver. For every query, the PartitionKey values must be known in advance and specified in the config. https://github.com/Azure/azure-sdk-for-go/issues/18578#issuecomment-1222510989[See details^].\n\n\n== Credentials\n\nYou can use one of the following authentication mechanisms:\n\n- Set the `endpoint` field and the `account_key` field\n- Set only the `endpoint` field to use https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity#DefaultAzureCredential[DefaultAzureCredential^]\n- Set the `connection_string` field\n\n\n== Metadata\n\nThis component adds the following metadata fields to each message:\n```\n- activity_id\n- request_charge\n```\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].\n","categories":["Azure"],"footnotes":"\n\n== CosmosDB emulator\n\nIf you wish to run the CosmosDB emulator that is referenced in the documentation https://learn.microsoft.com/en-us/azure/cosmos-db/linux-emulator[here^], the following Docker command should do the trick:\n\n```bash\n\u003e docker run --rm -it -p 8081:8081 --name=cosmosdb -e AZURE_COSMOS_EMULATOR_PARTITION_COUNT=10 -e AZURE_COSMOS_EMULATOR_ENABLE_DATA_PERSISTENCE=false mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator\n```\n\nNote: `AZURE_COSMOS_EMULATOR_PARTITION_COUNT` controls the number of partitions that will be supported by the emulator. The bigger the value, the longer it takes for the container to start up.\n\nAdditionally, instead of installing the container self-signed certificate which is exposed via `https://localhost:8081/_explorer/emulator.pem`, you can run https://mitmproxy.org/[mitmproxy^] like so:\n\n```bash\n\u003e mitmproxy -k --mode \"reverse:https://localhost:8081\"\n```\n\nThen you can access the CosmosDB UI via `http://localhost:8080/_explorer/index.html` and use `http://localhost:8080` as the CosmosDB endpoint.\n","examples":[{"title":"Query container","summary":"Execute a parametrized SQL query to select documents from a container.","config":"\ninput:\n azure_cosmosdb:\n endpoint: http://localhost:8080\n account_key: C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==\n database: blobbase\n container: blobfish\n partition_keys_map: root = \"AbyssalPlain\"\n query: SELECT * FROM blobfish AS b WHERE b.species = @species\n args_mapping: |\n root = [\n { \"Name\": \"@species\", \"Value\": \"smooth-head\" },\n ]\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"endpoint","type":"string","kind":"scalar","description":"CosmosDB endpoint.","is_optional":true,"examples":["https://localhost:8081"]},{"name":"account_key","type":"string","kind":"scalar","description":"Account key.","is_optional":true,"is_secret":true,"examples":["C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw=="],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"connection_string","type":"string","kind":"scalar","description":"Connection string.","is_optional":true,"is_secret":true,"examples":["AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==;"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"database","type":"string","kind":"scalar","description":"Database.","examples":["testdb"]},{"name":"container","type":"string","kind":"scalar","description":"Container.","examples":["testcontainer"]},{"name":"partition_keys_map","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to a single partition key value or an array of partition key values of type string, integer or boolean. Currently, hierarchical partition keys are not supported so only one value may be provided.","bloblang":true,"examples":["root = \"blobfish\"","root = 41","root = true","root = null","root = now().ts_format(\"2006-01-02\")"]},{"name":"query","type":"string","kind":"scalar","description":"The query to execute","examples":["SELECT c.foo FROM testcontainer AS c WHERE c.bar = \"baz\" AND c.timestamp \u003c @timestamp"]},{"name":"args_mapping","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] that, for each message, creates a list of arguments to use with the query.","is_optional":true,"bloblang":true,"examples":["root = [\n { \"Name\": \"@name\", \"Value\": \"benthos\" },\n]"]},{"name":"batch_count","type":"int","kind":"scalar","description":"The maximum number of messages that should be accumulated into each batch. Use '-1' specify dynamic page size.","is_advanced":true,"default":-1,"linter":"root = if this \u003c -1 || this == 0 || this \u003e 2147483647 { [ \"batch_count must be must be \u003e 0 and smaller than 2147483647 or -1.\" ] }"},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}],"linter":"root = []\nlet hasEndpoint = this.endpoint.or(\"\") != \"\"\nlet hasConnectionString = this.connection_string.or(\"\") != \"\"\n\nroot.\"-\" = if !$hasEndpoint \u0026\u0026 !$hasConnectionString {\n \"Either `endpoint` or `connection_string` must be set.\"\n}\n"},"version":"v4.25.0"},{"name":"azure_queue_storage","type":"input","status":"beta","plugin":true,"summary":"Dequeue objects from an Azure Storage Queue.","description":"\nThis input adds the following metadata fields to each message:\n\n```\n- queue_storage_insertion_time\n- queue_storage_queue_name\n- queue_storage_message_lag (if 'track_properties' set to true)\n- All user defined queue metadata\n```\n\nOnly one authentication method is required, `storage_connection_string` or `storage_account` and `storage_access_key`. If both are set then the `storage_connection_string` is given priority.","categories":["Services","Azure"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"storage_account","type":"string","kind":"scalar","description":"The storage account to access. This field is ignored if `storage_connection_string` is set.","default":""},{"name":"storage_access_key","type":"string","kind":"scalar","description":"The storage account access key. This field is ignored if `storage_connection_string` is set.","default":""},{"name":"storage_connection_string","type":"string","kind":"scalar","description":"A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set.","default":""},{"name":"storage_sas_token","type":"string","kind":"scalar","is_deprecated":true,"default":""},{"name":"queue_name","type":"string","kind":"scalar","description":"The name of the source storage queue.","interpolated":true,"examples":["foo_queue","${! env(\"MESSAGE_TYPE\").lowercase() }"]},{"name":"dequeue_visibility_timeout","type":"string","kind":"scalar","description":"The timeout duration until a dequeued message gets visible again, 30s by default","is_advanced":true,"default":"30s","version":"3.45.0"},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of unprocessed messages to fetch at a given time.","is_advanced":true,"default":10},{"name":"track_properties","type":"bool","kind":"scalar","description":"If set to `true` the queue is polled on each read request for information such as the queue message lag. These properties are added to consumed messages as metadata, but will also have a negative performance impact.","is_advanced":true,"default":false}],"linter":"root = if this.storage_connection_string != \"\" \u0026\u0026 !this.storage_connection_string.contains(\"AccountName=\") \u0026\u0026 !this.storage_connection_string.contains(\"UseDevelopmentStorage=true;\") \u0026\u0026 this.storage_account == \"\" { [ \"storage_account must be set if storage_connection_string does not contain the \\\"AccountName\\\" parameter\" ] }"},"version":"3.42.0"},{"name":"azure_table_storage","type":"input","status":"beta","plugin":true,"summary":"Queries an Azure Storage Account Table, optionally with multiple filters.","description":"\nQueries an Azure Storage Account Table, optionally with multiple filters.\n== Metadata\nThis input adds the following metadata fields to each message:\n\n- table_storage_name\n- row_num\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].","categories":["Services","Azure"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"storage_account","type":"string","kind":"scalar","description":"The storage account to access. This field is ignored if `storage_connection_string` is set.","default":""},{"name":"storage_access_key","type":"string","kind":"scalar","description":"The storage account access key. This field is ignored if `storage_connection_string` is set.","default":""},{"name":"storage_connection_string","type":"string","kind":"scalar","description":"A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set.","default":""},{"name":"storage_sas_token","type":"string","kind":"scalar","description":"The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set.","default":""},{"name":"table_name","type":"string","kind":"scalar","description":"The table to read messages from.","examples":["Foo"]},{"name":"filter","type":"string","kind":"scalar","description":"OData filter expression. Is not set all rows are returned. Valid operators are `eq, ne, gt, lt, ge and le`","is_advanced":true,"default":"","examples":["PartitionKey eq 'foo' and RowKey gt '1000'"]},{"name":"select","type":"string","kind":"scalar","description":"Select expression using OData notation. Limits the columns on each record to just those requested.","is_advanced":true,"default":"","examples":["PartitionKey,RowKey,Foo,Bar,Timestamp"]},{"name":"page_size","type":"int","kind":"scalar","description":"Maximum number of records to return on each page.","is_advanced":true,"default":1000}],"linter":"root = if this.storage_connection_string != \"\" \u0026\u0026 !this.storage_connection_string.contains(\"AccountName=\") \u0026\u0026 !this.storage_connection_string.contains(\"UseDevelopmentStorage=true;\") \u0026\u0026 this.storage_account == \"\" { [ \"storage_account must be set if storage_connection_string does not contain the \\\"AccountName\\\" parameter\" ] }"},"version":"4.10.0"},{"name":"batched","type":"input","status":"stable","plugin":true,"summary":"Consumes data from a child input and applies a batching policy to the stream.","description":"Batching at the input level is sometimes useful for processing across micro-batches, and can also sometimes be a useful performance trick. However, most inputs are fine without it so unless you have a specific plan for batching this component is not worth using.","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"child","type":"input","kind":"scalar","description":"The child input."},{"name":"policy","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]},"version":"4.11.0"},{"name":"beanstalkd","type":"input","status":"experimental","plugin":true,"summary":"Reads messages from a Beanstalkd queue.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"address","type":"string","kind":"scalar","description":"An address to connect to.","examples":["127.0.0.1:11300"]}]},"version":"4.7.0"},{"name":"broker","type":"input","status":"stable","plugin":true,"summary":"Allows you to combine multiple inputs into a single stream of data, where each input will be read in parallel.","description":"\nA broker type is configured with its own list of input configurations and a field to specify how many copies of the list of inputs should be created.\n\nAdding more input types allows you to combine streams from multiple sources into one. For example, reading from both RabbitMQ and Kafka:\n\n```yaml\ninput:\n broker:\n copies: 1\n inputs:\n - amqp_0_9:\n urls:\n - amqp://guest:guest@localhost:5672/\n consumer_tag: benthos-consumer\n queue: benthos-queue\n\n # Optional list of input specific processing steps\n processors:\n - mapping: |\n root.message = this\n root.meta.link_count = this.links.length()\n root.user.age = this.user.age.number()\n\n - kafka:\n addresses:\n - localhost:9092\n client_id: benthos_kafka_input\n consumer_group: benthos_consumer_group\n topics: [ benthos_stream:0 ]\n```\n\nIf the number of copies is greater than zero the list will be copied that number of times. For example, if your inputs were of type foo and bar, with 'copies' set to '2', you would end up with two 'foo' inputs and two 'bar' inputs.\n\n== Batching\n\nIt's possible to configure a xref:configuration:batching.adoc#batch-policy[batch policy] with a broker using the `batching` fields. When doing this the feeds from all child inputs are combined. Some inputs do not support broker based batching and specify this in their documentation.\n\n== Processors\n\nIt is possible to configure xref:components:processors/about.adoc[processors] at the broker level, where they will be applied to _all_ child inputs, as well as on the individual child inputs. If you have processors at both the broker level _and_ on child inputs then the broker processors will be applied _after_ the child nodes processors.","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"copies","type":"int","kind":"scalar","description":"Whatever is specified within `inputs` will be created this many times.","is_advanced":true,"default":1},{"name":"inputs","type":"input","kind":"array","description":"A list of inputs to create."},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]}},{"name":"cassandra","type":"input","status":"experimental","plugin":true,"summary":"Executes a find query and creates a message for each row received.","categories":["Services"],"examples":[{"title":"Minimal Select (Cassandra/Scylla)","summary":"\nLet's presume that we have 3 Cassandra nodes, like in this tutorial by Sebastian Sigl from freeCodeCamp:\n\nhttps://www.freecodecamp.org/news/the-apache-cassandra-beginner-tutorial/\n\nThen if we want to select everything from the table users_by_country, we should use the configuration below.\nIf we specify the stdin output, the result will look like:\n\n```json\n{\"age\":23,\"country\":\"UK\",\"first_name\":\"Bob\",\"last_name\":\"Sandler\",\"user_email\":\"bob@email.com\"}\n```\n\nThis configuration also works for Scylla.\n","config":"\ninput:\n cassandra:\n addresses:\n - 172.17.0.2\n query:\n 'SELECT * FROM learn_cassandra.users_by_country'\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"addresses","type":"string","kind":"array","description":"A list of Cassandra nodes to connect to. Multiple comma separated addresses can be specified on a single line.","examples":[["localhost:9042"],["foo:9042","bar:9042"],["foo:9042,bar:9042"]]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"password_authenticator","type":"object","kind":"scalar","description":"Optional configuration of Cassandra authentication parameters.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use password authentication","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"The username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"The password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"disable_initial_host_lookup","type":"bool","kind":"scalar","description":"If enabled the driver will not attempt to get host info from the system.peers table. This can speed up queries but will mean that data_centre, rack and token information will not be available.","is_advanced":true,"default":false},{"name":"max_retries","type":"int","kind":"scalar","description":"The maximum number of retries before giving up on a request.","is_advanced":true,"default":3},{"name":"backoff","type":"object","kind":"scalar","description":"Control time intervals between retry attempts.","is_advanced":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"1s"},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts.","is_advanced":true,"default":"5s"}]},{"name":"timeout","type":"string","kind":"scalar","description":"The client connection timeout.","default":"600ms"},{"name":"query","type":"string","kind":"scalar","description":"A query to execute."},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]}},{"name":"cockroachdb_changefeed","type":"input","status":"experimental","plugin":true,"summary":"Listens to a https://www.cockroachlabs.com/docs/stable/changefeed-examples[CockroachDB Core Changefeed^] and creates a message for each row received. Each message is a json object looking like: \n```json\n{\n\t\"primary_key\": \"[\\\"1a7ff641-3e3b-47ee-94fe-a0cadb56cd8f\\\", 2]\", // stringifed JSON array\n\t\"row\": \"{\\\"after\\\": {\\\"k\\\": \\\"1a7ff641-3e3b-47ee-94fe-a0cadb56cd8f\\\", \\\"v\\\": 2}, \\\"updated\\\": \\\"1637953249519902405.0000000000\\\"}\", // stringified JSON object\n\t\"table\": \"strm_2\"\n}\n```","description":"This input will continue to listen to the changefeed until shutdown. A backfill of the full current state of the table will be delivered upon each run unless a cache is configured for storing cursor timestamps, as this is how Redpanda Connect keeps track as to which changes have been successfully delivered.\n\nNote: You must have `SET CLUSTER SETTING kv.rangefeed.enabled = true;` on your CRDB cluster, for more information refer to https://www.cockroachlabs.com/docs/stable/changefeed-examples?filters=core[the official CockroachDB documentation^].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"dsn","type":"string","kind":"scalar","description":"A Data Source Name to identify the target database.","examples":["postgres://user:password@example.com:26257/defaultdb?sslmode=require"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"tables","type":"string","kind":"array","description":"CSV of tables to be included in the changefeed","examples":[["table1","table2"]]},{"name":"cursor_cache","type":"string","kind":"scalar","description":"A https://docs.redpanda.com/redpanda-connect/components/caches/about[cache resource^] to use for storing the current latest cursor that has been successfully delivered, this allows Redpanda Connect to continue from that cursor upon restart, rather than consume the entire state of the table.","is_optional":true},{"name":"options","type":"string","kind":"array","description":"A list of options to be included in the changefeed (WITH X, Y...).\n\nNOTE: Both the CURSOR option and UPDATED will be ignored from these options when a `cursor_cache` is specified, as they are set explicitly by Redpanda Connect in this case.","is_advanced":true,"is_optional":true,"examples":[["virtual_columns=\"omitted\""]]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]}},{"name":"csv","type":"input","status":"stable","plugin":true,"summary":"Reads one or more CSV files as structured records following the format described in RFC 4180.","description":"\nThis input offers more control over CSV parsing than the xref:components:inputs/file.adoc[`file` input].\n\nWhen parsing with a header row each line of the file will be consumed as a structured object, where the key names are determined from the header now. For example, the following CSV file:\n\n```csv\nfoo,bar,baz\nfirst foo,first bar,first baz\nsecond foo,second bar,second baz\n```\n\nWould produce the following messages:\n\n```json\n{\"foo\":\"first foo\",\"bar\":\"first bar\",\"baz\":\"first baz\"}\n{\"foo\":\"second foo\",\"bar\":\"second bar\",\"baz\":\"second baz\"}\n```\n\nIf, however, the field `parse_header_row` is set to `false` then arrays are produced instead, like follows:\n\n```json\n[\"first foo\",\"first bar\",\"first baz\"]\n[\"second foo\",\"second bar\",\"second baz\"]\n```\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- header\n- path\n- mod_time_unix\n- mod_time (RFC3339)\n```\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].\n\nNote: The `header` field is only set when `parse_header_row` is `true`.\n\n=== Output CSV column order\n\nWhen xref:guides:bloblang/advanced.adoc#creating-csv[creating CSV] from Redpanda Connect messages, the columns must be sorted lexicographically to make the output deterministic. Alternatively, when using the `csv` input, one can leverage the `header` metadata field to retrieve the column order:\n\n```yaml\ninput:\n csv:\n paths:\n - ./foo.csv\n - ./bar.csv\n parse_header_row: true\n\n processors:\n - mapping: |\n map escape_csv {\n root = if this.re_match(\"[\\\"\\n,]+\") {\n \"\\\"\" + this.replace_all(\"\\\"\", \"\\\"\\\"\") + \"\\\"\"\n } else {\n this\n }\n }\n\n let header = if count(@path) == 1 {\n @header.map_each(c -\u003e c.apply(\"escape_csv\")).join(\",\") + \"\\n\"\n } else { \"\" }\n\n root = $header + @header.map_each(c -\u003e this.get(c).string().apply(\"escape_csv\")).join(\",\")\n\noutput:\n file:\n path: ./output/${! @path.filepath_split().index(-1) }\n```\n","categories":["Local"],"footnotes":"This input is particularly useful when consuming CSV from files too large to parse entirely within memory. However, in cases where CSV is consumed from other input types it's also possible to parse them using the xref:guides:bloblang/methods.adoc#parse_csv[Bloblang `parse_csv` method].","config":{"name":"","type":"object","kind":"scalar","children":[{"name":"paths","type":"string","kind":"array","description":"A list of file paths to read from. Each file will be read sequentially until the list is exhausted, at which point the input will close. Glob patterns are supported, including super globs (double star).","examples":[["/tmp/foo.csv","/tmp/bar/*.csv","/tmp/data/**/*.csv"]]},{"name":"parse_header_row","type":"bool","kind":"scalar","description":"Whether to reference the first row as a header row. If set to true the output structure for messages will be an object where field keys are determined by the header row. Otherwise, each message will consist of an array of values from the corresponding CSV row.","default":true},{"name":"delimiter","type":"string","kind":"scalar","description":"The delimiter to use for splitting values in each record. It must be a single character.","default":","},{"name":"lazy_quotes","type":"bool","kind":"scalar","description":"If set to `true`, a quote may appear in an unquoted field and a non-doubled quote may appear in a quoted field.","default":false,"version":"4.1.0"},{"name":"delete_on_finish","type":"bool","kind":"scalar","description":"Whether to delete input files from the disk once they are fully consumed.","is_advanced":true,"default":false},{"name":"batch_count","type":"int","kind":"scalar","description":"Optionally process records in batches. This can help to speed up the consumption of exceptionally large CSV files. When the end of the file is reached the remaining records are processed as a (potentially smaller) batch.","is_advanced":true,"default":1},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]}},{"name":"discord","type":"input","status":"experimental","plugin":true,"summary":"Consumes messages posted in a Discord channel.","description":"This input works by authenticating as a bot using token based authentication. The ID of the newest message consumed and acked is stored in a cache in order to perform a backfill of unread messages each time the input is initialised. Ideally this cache should be persisted across restarts.","categories":["Services","Social"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"channel_id","type":"string","kind":"scalar","description":"A discord channel ID to consume messages from."},{"name":"bot_token","type":"string","kind":"scalar","description":"A bot token used for authentication."},{"name":"cache","type":"string","kind":"scalar","description":"A cache resource to use for performing unread message backfills, the ID of the last message received will be stored in this cache and used for subsequent requests."},{"name":"cache_key","type":"string","kind":"scalar","description":"The key identifier used when storing the ID of the last message received.","is_advanced":true,"default":"last_message_id"},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"poll_period","type":"string","kind":"scalar","description":"The length of time (as a duration string) to wait between each poll for backlogged messages. This field can be set empty, in which case requests are made at the limit set by the rate limit. This field also supports cron expressions.","is_deprecated":true,"default":"1m"},{"name":"limit","type":"int","kind":"scalar","description":"The maximum number of messages to receive in a single request.","is_deprecated":true,"default":100},{"name":"rate_limit","type":"string","kind":"scalar","is_deprecated":true,"default":"An optional rate limit resource to restrict API requests with."}]}},{"name":"dynamic","type":"input","status":"stable","plugin":true,"summary":"A special broker type where the inputs are identified by unique labels and can be created, changed and removed during runtime via a REST HTTP interface.","categories":["Utility"],"footnotes":"\n== Endpoints\n\n=== GET `/inputs`\n\nReturns a JSON object detailing all dynamic inputs, providing information such as their current uptime and configuration.\n\n=== GET `/inputs/\\{id}`\n\nReturns the configuration of an input.\n\n=== POST `/inputs/\\{id}`\n\nCreates or updates an input with a configuration provided in the request body (in YAML or JSON format).\n\n=== DELETE `/inputs/\\{id}`\n\nStops and removes an input.\n\n=== GET `/inputs/\\{id}/uptime`\n\nReturns the uptime of an input as a duration string (of the form \"72h3m0.5s\"), or \"stopped\" in the case where the input has gracefully terminated.","config":{"name":"","type":"object","kind":"scalar","children":[{"name":"inputs","type":"input","kind":"map","description":"A map of inputs to statically create.","default":{}},{"name":"prefix","type":"string","kind":"scalar","description":"A path prefix for HTTP endpoints that are registered.","default":""}]}},{"name":"file","type":"input","status":"stable","plugin":true,"summary":"Consumes data from files on disk, emitting messages according to a chosen codec.","description":"\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- path\n- mod_time_unix\n- mod_time (RFC3339)\n```\n\nYou can access these metadata fields using\nxref:configuration:interpolation.adoc#bloblang-queries[function interpolation].","categories":["Local"],"examples":[{"title":"Read a Bunch of CSVs","summary":"If we wished to consume a directory of CSV files as structured documents we can use a glob pattern and the `csv` scanner:","config":"\ninput:\n file:\n paths: [ ./data/*.csv ]\n scanner:\n csv: {}\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"paths","type":"string","kind":"array","description":"A list of paths to consume sequentially. Glob patterns are supported, including super globs (double star)."},{"name":"codec","type":"string","kind":"scalar","description":"The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.","is_deprecated":true,"is_optional":true,"examples":["lines","delim:\t","delim:foobar","gzip/csv"],"annotated_options":[["auto","EXPERIMENTAL: Attempts to derive a codec for each file based on information such as the extension. For example, a .tar.gz file would be consumed with the `gzip/tar` codec. Defaults to all-bytes."],["all-bytes","Consume the entire file as a single binary message."],["avro-ocf:marshaler=x","EXPERIMENTAL: Consume a stream of Avro OCF datum. The `marshaler` parameter is optional and has the options: `goavro` (default), `json`. Use `goavro` if OCF contains logical types."],["chunker:x","Consume the file in chunks of a given number of bytes."],["csv","Consume structured rows as comma separated values, the first row must be a header row."],["csv:x","Consume structured rows as values separated by a custom delimiter, the first row must be a header row. The custom delimiter must be a single character, e.g. the codec `\"csv:\\t\"` would consume a tab delimited file."],["csv-safe","Consume structured rows like `csv`, but sends messages with empty maps on failure to parse. Includes row number and parsing errors (if any) in the message's metadata."],["csv-safe:x","Consume structured rows like `csv:x` as values separated by a custom delimiter, but sends messages with empty maps on failure to parse. The custom delimiter must be a single character, e.g. the codec `\"csv-safe:\\t\"` would consume a tab delimited file. Includes row number and parsing errors (if any) in the message's metadata."],["delim:x","Consume the file in segments divided by a custom delimiter."],["gzip","Decompress a gzip file, this codec should precede another codec, e.g. `gzip/all-bytes`, `gzip/tar`, `gzip/csv`, etc."],["pgzip","Decompress a gzip file in parallel, this codec should precede another codec, e.g. `pgzip/all-bytes`, `pgzip/tar`, `pgzip/csv`, etc."],["lines","Consume the file in segments divided by linebreaks."],["multipart","Consumes the output of another codec and batches messages together. A batch ends when an empty message is consumed. For example, the codec `lines/multipart` could be used to consume multipart messages where an empty line indicates the end of each batch."],["regex:(?m)^\\d\\d:\\d\\d:\\d\\d","Consume the file in segments divided by regular expression."],["skipbom","Skip one or more byte order marks for each opened reader, this codec should precede another codec, e.g. `skipbom/csv`, etc."],["tar","Parse the file as a tar archive, and consume each file of the archive as a message."]]},{"name":"max_buffer","type":"int","kind":"scalar","is_deprecated":true,"default":1000000},{"name":"scanner","type":"scanner","kind":"scalar","description":"The xref:components:scanners/about.adoc[scanner] by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once.","is_optional":true,"default":{"lines":{}},"version":"4.25.0"},{"name":"delete_on_finish","type":"bool","kind":"scalar","description":"Whether to delete input files from the disk once they are fully consumed.","is_advanced":true,"default":false},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]}},{"name":"gateway","type":"input","status":"stable","plugin":true,"summary":"Receive messages delivered over HTTP.","description":"\nThe field `rate_limit` allows you to specify an optional xref:components:rate_limits/about.adoc[`rate_limit` resource], which will be applied to each HTTP request made and each websocket payload received.\n\nWhen the rate limit is breached HTTP requests will have a 429 response returned with a Retry-After header.\n\n== Responses\n\nIt's possible to return a response for each message received using xref:guides:sync_responses.adoc[synchronous responses]. When doing so you can customize headers with the `sync_response` field `headers`, which can also use xref:configuration:interpolation.adoc#bloblang-queries[function interpolation] in the value based on the response message contents.\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- http_server_user_agent\n- http_server_request_path\n- http_server_verb\n- http_server_remote_ip\n- All headers (only first values are taken)\n- All query parameters\n- All path parameters\n- All cookies\n```\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].","categories":["Network"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"path","type":"string","kind":"scalar","description":"The endpoint path to listen for data delivery requests.","default":"/"},{"name":"rate_limit","type":"string","kind":"scalar","description":"An optional xref:components:rate_limits/about.adoc[rate limit] to throttle requests by.","default":""},{"name":"sync_response","type":"object","kind":"scalar","description":"Customize messages returned via xref:guides:sync_responses.adoc[synchronous responses].","is_advanced":true,"children":[{"name":"status","type":"string","kind":"scalar","description":"Specify the status code to return with synchronous responses. This is a string value, which allows you to customize it based on resulting payloads and their metadata.","is_advanced":true,"default":"200","interpolated":true,"examples":["${! json(\"status\") }","${! meta(\"status\") }"]},{"name":"headers","type":"string","kind":"map","description":"Specify headers to return with synchronous responses.","is_advanced":true,"default":{"Content-Type":"application/octet-stream"},"interpolated":true},{"name":"metadata_headers","type":"object","kind":"scalar","description":"Specify criteria for which metadata values are added to the response as headers.","is_advanced":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","is_advanced":true,"default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","is_advanced":true,"default":[],"examples":[[".*"],["_timestamp_unix$"]]}]}]}]}},{"name":"gcp_bigquery_select","type":"input","status":"beta","plugin":true,"summary":"Executes a `SELECT` query against BigQuery and creates a message for each row received.","description":"Once the rows from the query are exhausted, this input shuts down, allowing the pipeline to gracefully terminate (or the next input in a xref:components:inputs/sequence.adoc[sequence] to execute).","categories":["Services","GCP"],"examples":[{"title":"Word counts","summary":"\nHere we query the public corpus of Shakespeare's works to generate a stream of the top 10 words that are 3 or more characters long:","config":"\ninput:\n gcp_bigquery_select:\n project: sample-project\n table: bigquery-public-data.samples.shakespeare\n columns:\n - word\n - sum(word_count) as total_count\n where: length(word) \u003e= ?\n suffix: |\n GROUP BY word\n ORDER BY total_count DESC\n LIMIT 10\n args_mapping: |\n root = [ 3 ]\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"project","type":"string","kind":"scalar","description":"GCP project where the query job will execute."},{"name":"credentials_json","type":"string","kind":"scalar","description":"An optional field to set Google Service Account Credentials json.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"table","type":"string","kind":"scalar","description":"Fully-qualified BigQuery table name to query.","examples":["bigquery-public-data.samples.shakespeare"]},{"name":"columns","type":"string","kind":"array","description":"A list of columns to query."},{"name":"where","type":"string","kind":"scalar","description":"An optional where clause to add. Placeholder arguments are populated with the `args_mapping` field. Placeholders should always be question marks (`?`).","is_optional":true,"examples":["type = ? and created_at \u003e ?","user_id = ?"]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"job_labels","type":"string","kind":"map","description":"A list of labels to add to the query job.","default":{}},{"name":"priority","type":"string","kind":"scalar","description":"The priority with which to schedule the query.","default":""},{"name":"args_mapping","type":"string","kind":"scalar","description":"An optional xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `where`.","is_optional":true,"bloblang":true,"examples":["root = [ \"article\", now().ts_format(\"2006-01-02\") ]"]},{"name":"prefix","type":"string","kind":"scalar","description":"An optional prefix to prepend to the select query (before SELECT).","is_optional":true},{"name":"suffix","type":"string","kind":"scalar","description":"An optional suffix to append to the select query.","is_optional":true}]},"version":"3.63.0"},{"name":"gcp_cloud_storage","type":"input","status":"beta","plugin":true,"summary":"Downloads objects within a Google Cloud Storage bucket, optionally filtered by a prefix.","description":"\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```\n- gcs_key\n- gcs_bucket\n- gcs_last_modified\n- gcs_last_modified_unix\n- gcs_content_type\n- gcs_content_encoding\n- All user defined metadata\n```\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].\n\n=== Credentials\n\nBy default Redpanda Connect will use a shared credentials file when connecting to GCP services. You can find out more in xref:guides:cloud/gcp.adoc[].","categories":["Services","GCP"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"bucket","type":"string","kind":"scalar","description":"The name of the bucket from which to download objects."},{"name":"prefix","type":"string","kind":"scalar","description":"An optional path prefix, if set only objects with the prefix are consumed.","default":""},{"name":"credentials_json","type":"string","kind":"scalar","description":"An optional field to set Google Service Account Credentials json.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"codec","type":"string","kind":"scalar","description":"The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.","is_deprecated":true,"is_optional":true,"examples":["lines","delim:\t","delim:foobar","gzip/csv"],"annotated_options":[["auto","EXPERIMENTAL: Attempts to derive a codec for each file based on information such as the extension. For example, a .tar.gz file would be consumed with the `gzip/tar` codec. Defaults to all-bytes."],["all-bytes","Consume the entire file as a single binary message."],["avro-ocf:marshaler=x","EXPERIMENTAL: Consume a stream of Avro OCF datum. The `marshaler` parameter is optional and has the options: `goavro` (default), `json`. Use `goavro` if OCF contains logical types."],["chunker:x","Consume the file in chunks of a given number of bytes."],["csv","Consume structured rows as comma separated values, the first row must be a header row."],["csv:x","Consume structured rows as values separated by a custom delimiter, the first row must be a header row. The custom delimiter must be a single character, e.g. the codec `\"csv:\\t\"` would consume a tab delimited file."],["csv-safe","Consume structured rows like `csv`, but sends messages with empty maps on failure to parse. Includes row number and parsing errors (if any) in the message's metadata."],["csv-safe:x","Consume structured rows like `csv:x` as values separated by a custom delimiter, but sends messages with empty maps on failure to parse. The custom delimiter must be a single character, e.g. the codec `\"csv-safe:\\t\"` would consume a tab delimited file. Includes row number and parsing errors (if any) in the message's metadata."],["delim:x","Consume the file in segments divided by a custom delimiter."],["gzip","Decompress a gzip file, this codec should precede another codec, e.g. `gzip/all-bytes`, `gzip/tar`, `gzip/csv`, etc."],["pgzip","Decompress a gzip file in parallel, this codec should precede another codec, e.g. `pgzip/all-bytes`, `pgzip/tar`, `pgzip/csv`, etc."],["lines","Consume the file in segments divided by linebreaks."],["multipart","Consumes the output of another codec and batches messages together. A batch ends when an empty message is consumed. For example, the codec `lines/multipart` could be used to consume multipart messages where an empty line indicates the end of each batch."],["regex:(?m)^\\d\\d:\\d\\d:\\d\\d","Consume the file in segments divided by regular expression."],["skipbom","Skip one or more byte order marks for each opened reader, this codec should precede another codec, e.g. `skipbom/csv`, etc."],["tar","Parse the file as a tar archive, and consume each file of the archive as a message."]]},{"name":"max_buffer","type":"int","kind":"scalar","is_deprecated":true,"default":1000000},{"name":"scanner","type":"scanner","kind":"scalar","description":"The xref:components:scanners/about.adoc[scanner] by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once.","is_optional":true,"default":{"to_the_end":{}},"version":"4.25.0"},{"name":"delete_objects","type":"bool","kind":"scalar","description":"Whether to delete downloaded objects from the bucket once they are processed.","is_advanced":true,"default":false}]},"version":"3.43.0"},{"name":"gcp_pubsub","type":"input","status":"stable","plugin":true,"summary":"Consumes messages from a GCP Cloud Pub/Sub subscription.","description":"\nFor information on how to set up credentials see https://cloud.google.com/docs/authentication/production[this guide^].\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n- gcp_pubsub_publish_time_unix - The time at which the message was published to the topic.\n- gcp_pubsub_delivery_attempt - When dead lettering is enabled, this is set to the number of times PubSub has attempted to deliver a message.\n- All message attributes\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].\n","categories":["Services","GCP"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"project","type":"string","kind":"scalar","description":"The project ID of the target subscription."},{"name":"credentials_json","type":"string","kind":"scalar","description":"An optional field to set Google Service Account Credentials json.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"subscription","type":"string","kind":"scalar","description":"The target subscription ID."},{"name":"endpoint","type":"string","kind":"scalar","description":"An optional endpoint to override the default of `pubsub.googleapis.com:443`. This can be used to connect to a region specific pubsub endpoint. For a list of valid values, see https://cloud.google.com/pubsub/docs/reference/service_apis_overview#list_of_regional_endpoints[this document^].","default":"","examples":["us-central1-pubsub.googleapis.com:443","us-west3-pubsub.googleapis.com:443"]},{"name":"sync","type":"bool","kind":"scalar","description":"Enable synchronous pull mode.","default":false},{"name":"max_outstanding_messages","type":"int","kind":"scalar","description":"The maximum number of outstanding pending messages to be consumed at a given time.","default":1000},{"name":"max_outstanding_bytes","type":"int","kind":"scalar","description":"The maximum number of outstanding pending messages to be consumed measured in bytes.","default":1000000000},{"name":"create_subscription","type":"object","kind":"scalar","description":"Allows you to configure the input subscription and creates if it doesn't exist.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to configure subscription or not.","is_advanced":true,"default":false},{"name":"topic","type":"string","kind":"scalar","description":"Defines the topic that the subscription should be vinculated to.","is_advanced":true,"default":""}]}]}},{"name":"generate","type":"input","status":"stable","plugin":true,"summary":"Generates messages at a given interval using a xref:guides:bloblang/about.adoc[Bloblang] mapping executed without a context. This allows you to generate messages for testing your pipeline configs.","categories":["Utility"],"examples":[{"title":"Cron Scheduled Processing","summary":"A common use case for the generate input is to trigger processors on a schedule so that the processors themselves can behave similarly to an input. The following configuration reads rows from a PostgreSQL table every 5 minutes.","config":"\ninput:\n generate:\n interval: '@every 5m'\n mapping: 'root = {}'\n processors:\n - sql_select:\n driver: postgres\n dsn: postgres://foouser:foopass@localhost:5432/testdb?sslmode=disable\n table: foo\n columns: [ \"*\" ]\n"},{"title":"Generate 100 Rows","summary":"The generate input can be used as a convenient way to generate test data. The following example generates 100 rows of structured data by setting an explicit count. The interval field is set to empty, which means data is generated as fast as the downstream components can consume it.","config":"\ninput:\n generate:\n count: 100\n interval: \"\"\n mapping: |\n root = if random_int() % 2 == 0 {\n {\n \"type\": \"foo\",\n \"foo\": \"is yummy\"\n }\n } else {\n {\n \"type\": \"bar\",\n \"bar\": \"is gross\"\n }\n }\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"mapping","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang] mapping to use for generating messages.","bloblang":true,"examples":["root = \"hello world\"","root = {\"test\":\"message\",\"id\":uuid_v4()}"]},{"name":"interval","type":"string","kind":"scalar","description":"The time interval at which messages should be generated, expressed either as a duration string or as a cron expression. If set to an empty string messages will be generated as fast as downstream services can process them. Cron expressions can specify a timezone by prefixing the expression with `TZ=\u003clocation name\u003e`, where the location name corresponds to a file within the IANA Time Zone database.","default":"1s","examples":["5s","1m","1h","@every 1s","0,30 */2 * * * *","TZ=Europe/London 30 3-6,20-23 * * *"]},{"name":"count","type":"int","kind":"scalar","description":"An optional number of messages to generate, if set above 0 the specified number of messages is generated and then the input will shut down.","default":0},{"name":"batch_size","type":"int","kind":"scalar","description":"The number of generated messages that should be accumulated into each batch flushed at the specified interval.","default":1},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]},"version":"3.40.0"},{"name":"git","type":"input","status":"beta","plugin":true,"summary":"A Git input that clones (or pulls) a repository and reads the repository contents.","description":"\nThe git input clones the specified repository (or pulls updates if already cloned) and reads \nthe content of the specified file. It periodically polls the repository for new commits and emits \na message when changes are detected.\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n- git_file_path\n- git_file_size\n- git_file_mode\n- git_file_modified\n- git_commit\n- git_mime_type\n- git_is_binary\n- git_encoding (present if the file was base64 encoded)\n- git_deleted (only present if the file was deleted)\n\nYou can access these metadata fields using function interpolation.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"repository_url","type":"string","kind":"scalar","description":"The URL of the Git repository to clone.","examples":["https://github.com/username/repo.git"]},{"name":"branch","type":"string","kind":"scalar","description":"The branch to check out.","default":"main"},{"name":"poll_interval","type":"string","kind":"scalar","description":"Duration between polling attempts","default":"10s","examples":["10s"]},{"name":"include_patterns","type":"string","kind":"array","description":"A list of file patterns to include (e.g., '**/*.md', 'configs/*.yaml'). If empty, all files will be included. Supports glob patterns: *, /**/, ?, and character ranges [a-z]. Any character with a special meaning can be escaped with a backslash.","is_optional":true,"default":[]},{"name":"exclude_patterns","type":"string","kind":"array","description":"A list of file patterns to exclude (e.g., '.git/**', '**/*.png'). These patterns take precedence over include_patterns. Supports glob patterns: *, /**/, ?, and character ranges [a-z]. Any character with a special meaning can be escaped with a backslash.","is_optional":true,"default":[]},{"name":"max_file_size","type":"int","kind":"scalar","description":"The maximum size of files to include in bytes. Files larger than this will be skipped. Set to 0 for no limit.","default":10485760},{"name":"checkpoint_cache","type":"string","kind":"scalar","description":"A cache resource to store the last processed commit hash, allowing the input to resume from where it left off after a restart.","is_optional":true},{"name":"checkpoint_key","type":"string","kind":"scalar","description":"The key to use when storing the last processed commit hash in the cache.","is_optional":true,"default":"git_last_commit"},{"name":"auth","type":"object","kind":"scalar","description":"Authentication options for the Git repository","is_optional":true,"children":[{"name":"basic","type":"object","kind":"scalar","description":"Basic authentication credentials","is_optional":true,"children":[{"name":"username","type":"string","kind":"scalar","description":"Username for basic authentication","is_optional":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"Password for basic authentication","is_optional":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"ssh_key","type":"object","kind":"scalar","description":"SSH key authentication","is_optional":true,"children":[{"name":"private_key_path","type":"string","kind":"scalar","description":"Path to SSH private key file","is_optional":true,"default":""},{"name":"private_key","type":"string","kind":"scalar","description":"SSH private key content","is_optional":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"passphrase","type":"string","kind":"scalar","description":"Passphrase for the SSH private key","is_optional":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"token","type":"object","kind":"scalar","description":"Token-based authentication","is_optional":true,"children":[{"name":"value","type":"string","kind":"scalar","description":"Token value for token-based authentication","is_optional":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]},"version":"4.51.0"},{"name":"hdfs","type":"input","status":"stable","plugin":true,"summary":"Reads files from a HDFS directory, where each discrete file will be consumed as a single message payload.","description":"\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n- hdfs_name\n- hdfs_path\n\nYou can access these metadata fields using\nxref:configuration:interpolation.adoc#bloblang-queries[function interpolation].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"hosts","type":"string","kind":"array","description":"A list of target host addresses to connect to.","examples":["localhost:9000"]},{"name":"user","type":"string","kind":"scalar","description":"A user ID to connect as.","default":""},{"name":"directory","type":"string","kind":"scalar","description":"The directory to consume from."}]}},{"name":"http_client","type":"input","status":"stable","plugin":true,"summary":"Connects to a server and continuously performs requests for a single message.","description":"\nThe URL and header values of this type can be dynamically set using function interpolations described xref:configuration:interpolation.adoc#bloblang-queries[here].\n\n== Streaming\n\nIf you enable streaming then Redpanda Connect will consume the body of the response as a continuous stream of data, breaking messages out following a chosen scanner. This allows you to consume APIs that provide long lived streamed data feeds (such as Twitter).\n\n== Pagination\n\nThis input supports interpolation functions in the `url` and `headers` fields where data from the previous successfully consumed message (if there was one) can be referenced. This can be used in order to support basic levels of pagination. However, in cases where pagination depends on logic it is recommended that you use an xref:components:processors/http.adoc[`http` processor] instead, often combined with a xref:components:inputs/generate.adoc[`generate` input] in order to schedule the processor.","categories":["Network"],"examples":[{"title":"Basic Pagination","summary":"Interpolation functions within the `url` and `headers` fields can be used to reference the previously consumed message, which allows simple pagination.","config":"\ninput:\n http_client:\n url: \u003e-\n https://api.example.com/search?query=allmyfoos\u0026start_time=${! (\n (timestamp_unix()-300).ts_format(\"2006-01-02T15:04:05Z\",\"UTC\").escape_url_query()\n ) }${! (\"\u0026next_token=\"+this.meta.next_token.not_null()) | \"\" }\n verb: GET\n rate_limit: foo_searches\n oauth2:\n enabled: true\n token_url: https://api.example.com/oauth2/token\n client_key: \"${EXAMPLE_KEY}\"\n client_secret: \"${EXAMPLE_SECRET}\"\n\nrate_limit_resources:\n - label: foo_searches\n local:\n count: 1\n interval: 30s\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL to connect to.","interpolated":true},{"name":"verb","type":"string","kind":"scalar","description":"A verb to connect with","default":"GET","examples":["POST","GET","DELETE"]},{"name":"headers","type":"string","kind":"map","description":"A map of headers to add to the request.","default":{},"interpolated":true,"examples":[{"Content-Type":"application/octet-stream","traceparent":"${! tracing_span().traceparent }"}]},{"name":"metadata","type":"object","kind":"scalar","description":"Specify optional matching rules to determine which metadata keys should be added to the HTTP request as headers.","is_advanced":true,"is_optional":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","is_advanced":true,"default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","is_advanced":true,"default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"dump_request_log_level","type":"string","kind":"scalar","description":"EXPERIMENTAL: Optionally set a level at which the request and response payload of each request made will be logged.","is_advanced":true,"default":"","options":["TRACE","DEBUG","INFO","WARN","ERROR","FATAL",""],"version":"4.12.0","linter":"\nlet options = {\n \"trace\": true,\n \"debug\": true,\n \"info\": true,\n \"warn\": true,\n \"error\": true,\n \"fatal\": true,\n \"\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"oauth","type":"object","kind":"scalar","description":"Allows you to specify open authentication via OAuth version 1.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use OAuth version 1 in requests.","is_advanced":true,"default":false},{"name":"consumer_key","type":"string","kind":"scalar","description":"A value used to identify the client to the service provider.","is_advanced":true,"default":""},{"name":"consumer_secret","type":"string","kind":"scalar","description":"A secret used to establish ownership of the consumer key.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"access_token","type":"string","kind":"scalar","description":"A value used to gain access to the protected resources on behalf of the user.","is_advanced":true,"default":""},{"name":"access_token_secret","type":"string","kind":"scalar","description":"A secret provided in order to establish ownership of a given access token.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"oauth2","type":"object","kind":"scalar","description":"Allows you to specify open authentication via OAuth version 2 using the client credentials token flow.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use OAuth version 2 in requests.","is_advanced":true,"default":false},{"name":"client_key","type":"string","kind":"scalar","description":"A value used to identify the client to the token provider.","is_advanced":true,"default":""},{"name":"client_secret","type":"string","kind":"scalar","description":"A secret used to establish ownership of the client key.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token_url","type":"string","kind":"scalar","description":"The URL of the token provider.","is_advanced":true,"default":"","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"scopes","type":"string","kind":"array","description":"A list of optional requested permissions.","is_advanced":true,"default":[],"version":"3.45.0"},{"name":"endpoint_params","type":"unknown","kind":"map","description":"A list of optional endpoint parameters, values should be arrays of strings.","is_advanced":true,"is_optional":true,"default":{},"examples":[{"bar":["woof"],"foo":["meow","quack"]}],"version":"4.21.0","linter":"\nroot = if this.type() == \"object\" {\n this.values().map_each(ele -\u003e if ele.type() != \"array\" {\n \"field must be an object containing arrays of strings, got %s (%v)\".format(ele.format_json(no_indent: true), ele.type())\n } else {\n ele.map_each(str -\u003e if str.type() != \"string\" {\n \"field values must be strings, got %s (%v)\".format(str.format_json(no_indent: true), str.type())\n } else { deleted() })\n }).\n flatten()\n}\n"}]},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"jwt","type":"object","kind":"scalar","description":"BETA: Allows you to specify JWT authentication.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use JWT authentication in requests.","is_advanced":true,"default":false},{"name":"private_key_file","type":"string","kind":"scalar","description":"A file with the PEM encoded via PKCS1 or PKCS8 as private key.","is_advanced":true,"default":""},{"name":"signing_method","type":"string","kind":"scalar","description":"A method used to sign the token such as RS256, RS384, RS512 or EdDSA.","is_advanced":true,"default":""},{"name":"claims","type":"unknown","kind":"map","description":"A value used to identify the claims that issued the JWT.","is_advanced":true,"default":{}},{"name":"headers","type":"unknown","kind":"map","description":"Add optional key/value headers to the JWT.","is_advanced":true,"default":{}}]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"extract_headers","type":"object","kind":"scalar","description":"Specify which response headers should be added to resulting messages as metadata. Header keys are lowercased before matching, so ensure that your patterns target lowercased versions of the header keys that you expect.","is_advanced":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","is_advanced":true,"default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","is_advanced":true,"default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"rate_limit","type":"string","kind":"scalar","description":"An optional xref:components:rate_limits/about.adoc[rate limit] to throttle requests by.","is_optional":true},{"name":"timeout","type":"string","kind":"scalar","description":"A static timeout to apply to requests.","default":"5s"},{"name":"retry_period","type":"string","kind":"scalar","description":"The base period to wait between failed requests.","is_advanced":true,"default":"1s"},{"name":"max_retry_backoff","type":"string","kind":"scalar","description":"The maximum period to wait between failed requests.","is_advanced":true,"default":"300s"},{"name":"retries","type":"int","kind":"scalar","description":"The maximum number of retry attempts to make.","is_advanced":true,"default":3},{"name":"follow_redirects","type":"bool","kind":"scalar","description":"Whether or not to transparently follow redirects, i.e. responses with 300-399 status codes. If disabled, the response message will contain the body, status, and headers from the redirect response and the processor will not make a request to the URL set in the Location header of the response.","is_advanced":true,"default":true},{"name":"backoff_on","type":"int","kind":"array","description":"A list of status codes whereby the request should be considered to have failed and retries should be attempted, but the period between them should be increased gradually.","is_advanced":true,"default":[429]},{"name":"drop_on","type":"int","kind":"array","description":"A list of status codes whereby the request should be considered to have failed but retries should not be attempted. This is useful for preventing wasted retries for requests that will never succeed. Note that with these status codes the _request_ is dropped, but _message_ that caused the request will not be dropped.","is_advanced":true,"default":[]},{"name":"successful_on","type":"int","kind":"array","description":"A list of status codes whereby the attempt should be considered successful, this is useful for dropping requests that return non-2XX codes indicating that the message has been dealt with, such as a 303 See Other or a 409 Conflict. All 2XX codes are considered successful unless they are present within `backoff_on` or `drop_on`, regardless of this field.","is_advanced":true,"default":[]},{"name":"proxy_url","type":"string","kind":"scalar","description":"An optional HTTP proxy URL.","is_advanced":true,"is_optional":true},{"name":"disable_http2","type":"bool","kind":"scalar","description":"Whether or not to disable disable HTTP/2","is_advanced":true,"default":false,"version":"4.44.0"},{"name":"payload","type":"string","kind":"scalar","description":"An optional payload to deliver for each request.","is_optional":true,"interpolated":true},{"name":"drop_empty_bodies","type":"bool","kind":"scalar","description":"Whether empty payloads received from the target server should be dropped.","is_advanced":true,"default":true},{"name":"stream","type":"object","kind":"scalar","description":"Allows you to set streaming mode, where requests are kept open and messages are processed line-by-line.","is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Enables streaming mode.","default":false},{"name":"reconnect","type":"bool","kind":"scalar","description":"Sets whether to re-establish the connection once it is lost.","default":true},{"name":"codec","type":"string","kind":"scalar","description":"The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.","is_deprecated":true,"is_optional":true,"examples":["lines","delim:\t","delim:foobar","gzip/csv"],"annotated_options":[["auto","EXPERIMENTAL: Attempts to derive a codec for each file based on information such as the extension. For example, a .tar.gz file would be consumed with the `gzip/tar` codec. Defaults to all-bytes."],["all-bytes","Consume the entire file as a single binary message."],["avro-ocf:marshaler=x","EXPERIMENTAL: Consume a stream of Avro OCF datum. The `marshaler` parameter is optional and has the options: `goavro` (default), `json`. Use `goavro` if OCF contains logical types."],["chunker:x","Consume the file in chunks of a given number of bytes."],["csv","Consume structured rows as comma separated values, the first row must be a header row."],["csv:x","Consume structured rows as values separated by a custom delimiter, the first row must be a header row. The custom delimiter must be a single character, e.g. the codec `\"csv:\\t\"` would consume a tab delimited file."],["csv-safe","Consume structured rows like `csv`, but sends messages with empty maps on failure to parse. Includes row number and parsing errors (if any) in the message's metadata."],["csv-safe:x","Consume structured rows like `csv:x` as values separated by a custom delimiter, but sends messages with empty maps on failure to parse. The custom delimiter must be a single character, e.g. the codec `\"csv-safe:\\t\"` would consume a tab delimited file. Includes row number and parsing errors (if any) in the message's metadata."],["delim:x","Consume the file in segments divided by a custom delimiter."],["gzip","Decompress a gzip file, this codec should precede another codec, e.g. `gzip/all-bytes`, `gzip/tar`, `gzip/csv`, etc."],["pgzip","Decompress a gzip file in parallel, this codec should precede another codec, e.g. `pgzip/all-bytes`, `pgzip/tar`, `pgzip/csv`, etc."],["lines","Consume the file in segments divided by linebreaks."],["multipart","Consumes the output of another codec and batches messages together. A batch ends when an empty message is consumed. For example, the codec `lines/multipart` could be used to consume multipart messages where an empty line indicates the end of each batch."],["regex:(?m)^\\d\\d:\\d\\d:\\d\\d","Consume the file in segments divided by regular expression."],["skipbom","Skip one or more byte order marks for each opened reader, this codec should precede another codec, e.g. `skipbom/csv`, etc."],["tar","Parse the file as a tar archive, and consume each file of the archive as a message."]]},{"name":"max_buffer","type":"int","kind":"scalar","is_deprecated":true,"default":1000000},{"name":"scanner","type":"scanner","kind":"scalar","description":"The xref:components:scanners/about.adoc[scanner] by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once.","is_optional":true,"default":{"lines":{}},"version":"4.25.0"}]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]}},{"name":"http_server","type":"input","status":"stable","plugin":true,"summary":"Receive messages POSTed over HTTP(S). HTTP 2.0 is supported when using TLS, which is enabled when key and cert files are specified.","description":"\nIf the `address` config field is left blank the xref:components:http/about.adoc[service-wide HTTP server] will be used.\n\nThe field `rate_limit` allows you to specify an optional xref:components:rate_limits/about.adoc[`rate_limit` resource], which will be applied to each HTTP request made and each websocket payload received.\n\nWhen the rate limit is breached HTTP requests will have a 429 response returned with a Retry-After header. Websocket payloads will be dropped and an optional response payload will be sent as per `ws_rate_limit_message`.\n\n== Responses\n\nIt's possible to return a response for each message received using xref:guides:sync_responses.adoc[synchronous responses]. When doing so you can customize headers with the `sync_response` field `headers`, which can also use xref:configuration:interpolation.adoc#bloblang-queries[function interpolation] in the value based on the response message contents.\n\n== Endpoints\n\nThe following fields specify endpoints that are registered for sending messages, and support path parameters of the form `/\\{foo}`, which are added to ingested messages as metadata. A path ending in `/` will match against all extensions of that path:\n\n=== `path` (defaults to `/post`)\n\nThis endpoint expects POST requests where the entire request body is consumed as a single message.\n\nIf the request contains a multipart `content-type` header as per https://www.w3.org/Protocols/rfc1341/7_2_Multipart.html[RFC1341^] then the multiple parts are consumed as a batch of messages, where each body part is a message of the batch.\n\n=== `ws_path` (defaults to `/post/ws`)\n\nCreates a websocket connection, where payloads received on the socket are passed through the pipeline as a batch of one message.\n\n\n[CAUTION]\n.Endpoint caveats\n====\nComponents within a Redpanda Connect config will register their respective endpoints in a non-deterministic order. This means that establishing precedence of endpoints that are registered via multiple `http_server` inputs or outputs (either within brokers or from cohabiting streams) is not possible in a predictable way.\n\nThis ambiguity makes it difficult to ensure that paths which are both a subset of a path registered by a separate component, and end in a slash (`/`) and will therefore match against all extensions of that path, do not prevent the more specific path from matching against requests.\n\nIt is therefore recommended that you ensure paths of separate components do not collide unless they are explicitly non-competing.\n\nFor example, if you were to deploy two separate `http_server` inputs, one with a path `/foo/` and the other with a path `/foo/bar`, it would not be possible to ensure that the path `/foo/` does not swallow requests made to `/foo/bar`.\n====\n\nYou may specify an optional `ws_welcome_message`, which is a static payload to be sent to all clients once a websocket connection is first established.\n\nIt's also possible to specify a `ws_rate_limit_message`, which is a static payload to be sent to clients that have triggered the servers rate limit.\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- http_server_user_agent\n- http_server_request_path\n- http_server_verb\n- http_server_remote_ip\n- All headers (only first values are taken)\n- All query parameters\n- All path parameters\n- All cookies\n```\n\nIf HTTPS is enabled, the following fields are added as well:\n```text\n- http_server_tls_version\n- http_server_tls_subject\n- http_server_tls_cipher_suite\n```\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].","categories":["Network"],"examples":[{"title":"Path Switching","summary":"This example shows an `http_server` input that captures all requests and processes them by switching on that path:","config":"\ninput:\n http_server:\n path: /\n allowed_verbs: [ GET, POST ]\n sync_response:\n headers:\n Content-Type: application/json\n\n processors:\n - switch:\n - check: '@http_server_request_path == \"/foo\"'\n processors:\n - mapping: |\n root.title = \"You Got Fooed!\"\n root.result = content().string().uppercase()\n\n - check: '@http_server_request_path == \"/bar\"'\n processors:\n - mapping: 'root.title = \"Bar Is Slow\"'\n - sleep: # Simulate a slow endpoint\n duration: 1s\n"},{"title":"Mock OAuth 2.0 Server","summary":"This example shows an `http_server` input that mocks an OAuth 2.0 Client Credentials flow server at the endpoint `/oauth2_test`:","config":"\ninput:\n http_server:\n path: /oauth2_test\n allowed_verbs: [ GET, POST ]\n sync_response:\n headers:\n Content-Type: application/json\n\n processors:\n - log:\n message: \"Received request\"\n level: INFO\n fields_mapping: |\n root = @\n root.body = content().string()\n\n - mapping: |\n root.access_token = \"MTQ0NjJkZmQ5OTM2NDE1ZTZjNGZmZjI3\"\n root.token_type = \"Bearer\"\n root.expires_in = 3600\n\n - sync_response: {}\n - mapping: 'root = deleted()'\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"address","type":"string","kind":"scalar","description":"An alternative address to host from. If left empty the service wide address is used.","default":""},{"name":"path","type":"string","kind":"scalar","description":"The endpoint path to listen for POST requests.","default":"/post"},{"name":"ws_path","type":"string","kind":"scalar","description":"The endpoint path to create websocket connections from.","default":"/post/ws"},{"name":"ws_welcome_message","type":"string","kind":"scalar","description":"An optional message to deliver to fresh websocket connections.","is_advanced":true,"default":""},{"name":"ws_rate_limit_message","type":"string","kind":"scalar","description":"An optional message to delivery to websocket connections that are rate limited.","is_advanced":true,"default":""},{"name":"allowed_verbs","type":"string","kind":"array","description":"An array of verbs that are allowed for the `path` endpoint.","default":["POST"],"version":"3.33.0"},{"name":"timeout","type":"string","kind":"scalar","description":"Timeout for requests. If a consumed messages takes longer than this to be delivered the connection is closed, but the message may still be delivered.","default":"5s"},{"name":"rate_limit","type":"string","kind":"scalar","description":"An optional xref:components:rate_limits/about.adoc[rate limit] to throttle requests by.","default":""},{"name":"cert_file","type":"string","kind":"scalar","description":"Enable TLS by specifying a certificate and key file. Only valid with a custom `address`.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"Enable TLS by specifying a certificate and key file. Only valid with a custom `address`.","is_advanced":true,"default":""},{"name":"cors","type":"object","kind":"scalar","description":"Adds Cross-Origin Resource Sharing headers. Only valid with a custom `address`.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to allow CORS requests.","is_advanced":true,"default":false},{"name":"allowed_origins","type":"string","kind":"array","description":"An explicit list of origins that are allowed for CORS requests.","is_advanced":true,"default":[]}],"version":"3.63.0"},{"name":"sync_response","type":"object","kind":"scalar","description":"Customize messages returned via xref:guides:sync_responses.adoc[synchronous responses].","is_advanced":true,"children":[{"name":"status","type":"string","kind":"scalar","description":"Specify the status code to return with synchronous responses. This is a string value, which allows you to customize it based on resulting payloads and their metadata.","is_advanced":true,"default":"200","interpolated":true,"examples":["${! json(\"status\") }","${! meta(\"status\") }"]},{"name":"headers","type":"string","kind":"map","description":"Specify headers to return with synchronous responses.","is_advanced":true,"default":{"Content-Type":"application/octet-stream"},"interpolated":true},{"name":"metadata_headers","type":"object","kind":"scalar","description":"Specify criteria for which metadata values are added to the response as headers.","is_advanced":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","is_advanced":true,"default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","is_advanced":true,"default":[],"examples":[[".*"],["_timestamp_unix$"]]}]}]}]}},{"name":"inproc","type":"input","status":"stable","plugin":true,"description":"\nDirectly connect to an output within a Redpanda Connect process by referencing it by a chosen ID. This allows you to hook up isolated streams whilst running Redpanda Connect in xref:guides:streams_mode/about.adoc[streams mode], it is NOT recommended that you connect the inputs of a stream with an output of the same stream, as feedback loops can lead to deadlocks in your message flow.\n\nIt is possible to connect multiple inputs to the same inproc ID, resulting in messages dispatching in a round-robin fashion to connected inputs. However, only one output can assume an inproc ID, and will replace existing outputs if a collision occurs.","categories":["Utility"],"config":{"name":"","type":"string","kind":"scalar","default":""}},{"name":"kafka","type":"input","status":"stable","plugin":true,"summary":"Connects to Kafka brokers and consumes one or more topics.","description":"\nOffsets are managed within Kafka under the specified consumer group, and partitions for each topic are automatically balanced across members of the consumer group.\n\nThe Kafka input allows parallel processing of messages from different topic partitions, and messages of the same topic partition are processed with a maximum parallelism determined by the field \u003c\u003ccheckpoint_limit,`checkpoint_limit`\u003e\u003e.\n\nIn order to enforce ordered processing of partition messages set the \u003ccheckpoint_limit,`checkpoint_limit`\u003e\u003e to `1` and this will force partitions to be processed in lock-step, where a message will only be processed once the prior message is delivered.\n\nBatching messages before processing can be enabled using the \u003c\u003cbatching,`batching`\u003e\u003e field, and this batching is performed per-partition such that messages of a batch will always originate from the same partition. This batching mechanism is capable of creating batches of greater size than the \u003c\u003ccheckpoint_limit,`checkpoint_limit`\u003e\u003e, in which case the next batch will only be created upon delivery of the current one.\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n- kafka_key\n- kafka_topic\n- kafka_partition\n- kafka_offset\n- kafka_lag\n- kafka_timestamp_ms\n- kafka_timestamp_unix\n- kafka_tombstone_message\n- All existing message headers (version 0.11+)\n\nThe field `kafka_lag` is the calculated difference between the high water mark offset of the partition at the time of ingestion and the current message offset.\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].\n\n== Ordering\n\nBy default messages of a topic partition can be processed in parallel, up to a limit determined by the field `checkpoint_limit`. However, if strict ordered processing is required then this value must be set to 1 in order to process shard messages in lock-step. When doing so it is recommended that you perform batching at this component for performance as it will not be possible to batch lock-stepped messages at the output level.\n\n== Troubleshooting\n\nIf you're seeing issues writing to or reading from Kafka with this component then it's worth trying out the newer xref:components:inputs/kafka_franz.adoc[`kafka_franz` input].\n\n- I'm seeing logs that report `Failed to connect to kafka: kafka: client has run out of available brokers to talk to (Is your cluster reachable?)`, but the brokers are definitely reachable.\n\nUnfortunately this error message will appear for a wide range of connection problems even when the broker endpoint can be reached. Double check your authentication configuration and also ensure that you have \u003c\u003ctlsenabled, enabled TLS\u003e\u003e if applicable.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"addresses","type":"string","kind":"array","description":"A list of broker addresses to connect to. If an item of the list contains commas it will be expanded into multiple addresses.","examples":[["localhost:9092"],["localhost:9041,localhost:9042"],["localhost:9041","localhost:9042"]]},{"name":"topics","type":"string","kind":"array","description":"A list of topics to consume from. Multiple comma separated topics can be listed in a single element. Partitions are automatically distributed across consumers of a topic. Alternatively, it's possible to specify explicit partitions to consume from with a colon after the topic name, e.g. `foo:0` would consume the partition 0 of the topic foo. This syntax supports ranges, e.g. `foo:0-10` would consume partitions 0 through to 10 inclusive.","examples":[["foo","bar"],["foo,bar"],["foo:0","bar:1","bar:3"],["foo:0,bar:1,bar:3"],["foo:0-5"]],"version":"3.33.0"},{"name":"target_version","type":"string","kind":"scalar","description":"The version of the Kafka protocol to use. This limits the capabilities used by the client and should ideally match the version of your brokers. Defaults to the oldest supported stable version.","is_optional":true,"examples":["2.1.0","3.1.0"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"sasl","type":"object","kind":"scalar","description":"Enables SASL authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"mechanism","type":"string","kind":"scalar","description":"The SASL authentication mechanism, if left empty SASL authentication is not used.","is_advanced":true,"default":"none","annotated_options":[["OAUTHBEARER","OAuth Bearer based authentication."],["PLAIN","Plain text authentication. NOTE: When using plain text auth it is extremely likely that you'll also need to \u003c\u003ctls-enabled, enable TLS\u003e\u003e."],["SCRAM-SHA-256","Authentication using the SCRAM-SHA-256 mechanism."],["SCRAM-SHA-512","Authentication using the SCRAM-SHA-512 mechanism."],["none","Default, no SASL authentication."]],"linter":"\nlet options = {\n \"oauthbearer\": true,\n \"plain\": true,\n \"scram-sha-256\": true,\n \"scram-sha-512\": true,\n \"none\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"user","type":"string","kind":"scalar","description":"A PLAIN username. It is recommended that you use environment variables to populate this field.","is_advanced":true,"default":"","examples":["${USER}"]},{"name":"password","type":"string","kind":"scalar","description":"A PLAIN password. It is recommended that you use environment variables to populate this field.","is_advanced":true,"is_secret":true,"default":"","examples":["${PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"access_token","type":"string","kind":"scalar","description":"A static OAUTHBEARER access token","is_advanced":true,"default":""},{"name":"token_cache","type":"string","kind":"scalar","description":"Instead of using a static `access_token` allows you to query a xref:components:caches/about.adoc[`cache`] resource to fetch OAUTHBEARER tokens from","is_advanced":true,"default":""},{"name":"token_key","type":"string","kind":"scalar","description":"Required when using a `token_cache`, the key to query the cache with for tokens.","is_advanced":true,"default":""}]},{"name":"consumer_group","type":"string","kind":"scalar","description":"An identifier for the consumer group of the connection. This field can be explicitly made empty in order to disable stored offsets for the consumed topic partitions.","default":""},{"name":"client_id","type":"string","kind":"scalar","description":"An identifier for the client connection.","is_advanced":true,"default":"benthos"},{"name":"instance_id","type":"string","kind":"scalar","description":"When using consumer groups, an identifier for this specific input so that it can be identified over restarts of this process. This should be unique per input.","is_advanced":true,"is_optional":true},{"name":"rack_id","type":"string","kind":"scalar","description":"A rack identifier for this client.","is_advanced":true,"default":""},{"name":"start_from_oldest","type":"bool","kind":"scalar","description":"Determines whether to consume from the oldest available offset, otherwise messages are consumed from the latest offset. The setting is applied when creating a new consumer group or the saved offset no longer exists.","is_advanced":true,"default":true},{"name":"checkpoint_limit","type":"int","kind":"scalar","description":"The maximum number of messages of the same topic and partition that can be processed at a given time. Increasing this limit enables parallel processing and batching at the output level to work on individual partitions. Any given offset will not be committed unless all messages under that offset are delivered in order to preserve at least once delivery guarantees.","default":1024,"version":"3.33.0"},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"commit_period","type":"string","kind":"scalar","description":"The period of time between each commit of the current partition offsets. Offsets are always committed during shutdown.","is_advanced":true,"default":"1s"},{"name":"max_processing_period","type":"string","kind":"scalar","description":"A maximum estimate for the time taken to process a message, this is used for tuning consumer group synchronization.","is_advanced":true,"default":"100ms"},{"name":"extract_tracing_map","type":"string","kind":"scalar","description":"EXPERIMENTAL: A xref:guides:bloblang/about.adoc[Bloblang mapping] that attempts to extract an object containing tracing propagation information, which will then be used as the root tracing span for the message. The specification of the extracted fields must match the format used by the service wide tracer.","is_advanced":true,"is_optional":true,"bloblang":true,"examples":["root = @","root = this.meta.span"],"version":"3.45.0"},{"name":"group","type":"object","kind":"scalar","description":"Tuning parameters for consumer group synchronization.","is_advanced":true,"children":[{"name":"session_timeout","type":"string","kind":"scalar","description":"A period after which a consumer of the group is kicked after no heartbeats.","is_advanced":true,"default":"10s"},{"name":"heartbeat_interval","type":"string","kind":"scalar","description":"A period in which heartbeats should be sent out.","is_advanced":true,"default":"3s"},{"name":"rebalance_timeout","type":"string","kind":"scalar","description":"A period after which rebalancing is abandoned if unresolved.","is_advanced":true,"default":"60s"}]},{"name":"fetch_buffer_cap","type":"int","kind":"scalar","description":"The maximum number of unprocessed messages to fetch at a given time.","is_advanced":true,"default":256},{"name":"multi_header","type":"bool","kind":"scalar","description":"Decode headers into lists to allow handling of multiple values with the same key","is_advanced":true,"default":false},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","is_advanced":true,"examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","is_advanced":true,"default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","is_advanced":true,"default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","is_advanced":true,"default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","is_advanced":true,"default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]}},{"name":"kafka_franz","type":"input","status":"beta","plugin":true,"summary":"A Kafka input using the https://github.com/twmb/franz-go[Franz Kafka client library^].","description":"\nWhen a consumer group is specified this input consumes one or more topics where partitions will automatically balance across any other connected clients with the same consumer group. When a consumer group is not specified topics can either be consumed in their entirety or with explicit partitions.\n\nThis input often out-performs the traditional `kafka` input as well as providing more useful logs and error messages.\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- kafka_key\n- kafka_topic\n- kafka_partition\n- kafka_offset\n- kafka_timestamp_ms\n- kafka_timestamp_unix\n- kafka_tombstone_message\n- All record headers\n```\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"seed_brokers","type":"string","kind":"array","description":"A list of broker addresses to connect to in order to establish connections. If an item of the list contains commas it will be expanded into multiple addresses.","examples":[["localhost:9092"],["foo:9092","bar:9092"],["foo:9092,bar:9092"]]},{"name":"client_id","type":"string","kind":"scalar","description":"An identifier for the client connection.","is_advanced":true,"default":"benthos"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"sasl","type":"object","kind":"array","description":"Specify one or more methods of SASL authentication. SASL is tried in order; if the broker supports the first mechanism, all connections will use that mechanism. If the first mechanism fails, the client will pick the first supported mechanism. If the broker does not support any client mechanisms, connections will fail.","is_advanced":true,"is_optional":true,"examples":[[{"mechanism":"SCRAM-SHA-512","password":"bar","username":"foo"}]],"children":[{"name":"mechanism","type":"string","kind":"scalar","description":"The SASL mechanism to use.","is_advanced":true,"annotated_options":[["AWS_MSK_IAM","AWS IAM based authentication as specified by the 'aws-msk-iam-auth' java library."],["OAUTHBEARER","OAuth Bearer based authentication."],["PLAIN","Plain text authentication."],["SCRAM-SHA-256","SCRAM based authentication as specified in RFC5802."],["SCRAM-SHA-512","SCRAM based authentication as specified in RFC5802."],["none","Disable sasl authentication"]],"linter":"\nlet options = {\n \"aws_msk_iam\": true,\n \"oauthbearer\": true,\n \"plain\": true,\n \"scram-sha-256\": true,\n \"scram-sha-512\": true,\n \"none\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"username","type":"string","kind":"scalar","description":"A username to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token to use for a single session's OAUTHBEARER authentication.","is_advanced":true,"default":""},{"name":"extensions","type":"string","kind":"map","description":"Key/value pairs to add to OAUTHBEARER authentication requests.","is_advanced":true,"is_optional":true},{"name":"aws","type":"object","kind":"scalar","description":"Contains AWS specific fields for when the `mechanism` is set to `AWS_MSK_IAM`.","is_advanced":true,"is_optional":true,"children":[{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]}]},{"name":"metadata_max_age","type":"string","kind":"scalar","description":"The maximum age of metadata before it is refreshed.","is_advanced":true,"default":"5m"},{"name":"request_timeout_overhead","type":"string","kind":"scalar","description":"The request time overhead. Uses the given time as overhead while deadlining requests. Roughly equivalent to request.timeout.ms, but grants additional time to requests that have timeout fields.","is_advanced":true,"default":"10s"},{"name":"conn_idle_timeout","type":"string","kind":"scalar","description":"The rough amount of time to allow connections to idle before they are closed.","is_advanced":true,"default":"20s"},{"name":"topics","type":"string","kind":"array","description":"\nA list of topics to consume from. Multiple comma separated topics can be listed in a single element. When a `consumer_group` is specified partitions are automatically distributed across consumers of a topic, otherwise all partitions are consumed.\n\nAlternatively, it's possible to specify explicit partitions to consume from with a colon after the topic name, e.g. `foo:0` would consume the partition 0 of the topic foo. This syntax supports ranges, e.g. `foo:0-10` would consume partitions 0 through to 10 inclusive.\n\nFinally, it's also possible to specify an explicit offset to consume from by adding another colon after the partition, e.g. `foo:0:10` would consume the partition 0 of the topic foo starting from the offset 10. If the offset is not present (or remains unspecified) then the field `start_from_oldest` determines which offset to start from.","examples":[["foo","bar"],["things.*"],["foo,bar"],["foo:0","bar:1","bar:3"],["foo:0,bar:1,bar:3"],["foo:0-5"]]},{"name":"regexp_topics","type":"bool","kind":"scalar","description":"Whether listed topics should be interpreted as regular expression patterns for matching multiple topics. When topics are specified with explicit partitions this field must remain set to `false`.","default":false},{"name":"rack_id","type":"string","kind":"scalar","description":"A rack specifies where the client is physically located and changes fetch requests to consume from the closest replica as opposed to the leader replica.","is_advanced":true,"default":""},{"name":"instance_id","type":"string","kind":"scalar","description":"When using a consumer group, an instance ID specifies the groups static membership, which can prevent rebalances during reconnects. When using a instance ID the client does NOT leave the group when closing. To actually leave the group one must use an external admin command to leave the group on behalf of this instance ID. This ID must be unique per consumer within the group.","is_advanced":true,"default":""},{"name":"rebalance_timeout","type":"string","kind":"scalar","description":"When using a consumer group, `rebalance_timeout` sets how long group members are allowed to take when a rebalance has begun. This timeout is how long all members are allowed to complete work and commit offsets, minus the time it took to detect the rebalance (from a heartbeat).","is_advanced":true,"default":"45s"},{"name":"session_timeout","type":"string","kind":"scalar","description":"When using a consumer group, `session_timeout` sets how long a member in hte group can go between heartbeats. If a member does not heartbeat in this timeout, the broker will remove the member from the group and initiate a rebalance.","is_advanced":true,"default":"1m"},{"name":"heartbeat_interval","type":"string","kind":"scalar","description":"When using a consumer group, `heartbeat_interval` sets how long a group member goes between heartbeats to Kafka. Kafka uses heartbeats to ensure that a group member's sesion stays active. This value should be no higher than 1/3rd of the `session_timeout`. This is equivalent to the Java heartbeat.interval.ms setting.","is_advanced":true,"default":"3s"},{"name":"start_from_oldest","type":"bool","kind":"scalar","description":"Determines whether to consume from the oldest available offset, otherwise messages are consumed from the latest offset. The setting is applied when creating a new consumer group or the saved offset no longer exists.","is_advanced":true,"is_deprecated":true,"default":true},{"name":"start_offset","type":"string","kind":"scalar","description":"Sets the offset to start consuming from, or if OffsetOutOfRange is seen while fetching, to restart consuming from.","is_advanced":true,"default":"earliest","annotated_options":[["committed","Prevents consuming a partition in a group if the partition has no prior commits. Corresponds to Kafka's `auto.offset.reset=none` option"],["earliest","Start from the earliest offset. Corresponds to Kafka's `auto.offset.reset=earliest` option."],["latest","Start from the latest offset. Corresponds to Kafka's `auto.offset.reset=latest` option."]],"linter":"\nlet options = {\n \"committed\": true,\n \"earliest\": true,\n \"latest\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"fetch_max_bytes","type":"string","kind":"scalar","description":"Sets the maximum amount of bytes a broker will try to send during a fetch. Note that brokers may not obey this limit if it has records larger than this limit. This is the equivalent to the Java fetch.max.bytes setting.","is_advanced":true,"default":"50MiB"},{"name":"fetch_max_wait","type":"string","kind":"scalar","description":"Sets the maximum amount of time a broker will wait for a fetch response to hit the minimum number of required bytes. This is the equivalent to the Java fetch.max.wait.ms setting.","is_advanced":true,"default":"5s"},{"name":"fetch_min_bytes","type":"string","kind":"scalar","description":"Sets the minimum amount of bytes a broker will try to send during a fetch. This is the equivalent to the Java fetch.min.bytes setting.","is_advanced":true,"default":"1B"},{"name":"fetch_max_partition_bytes","type":"string","kind":"scalar","description":"Sets the maximum amount of bytes that will be consumed for a single partition in a fetch request. Note that if a single batch is larger than this number, that batch will still be returned so the client can make progress. This is the equivalent to the Java fetch.max.partition.bytes setting.","is_advanced":true,"default":"1MiB"},{"name":"transaction_isolation_level","type":"string","kind":"scalar","description":"The transaction isolation level","default":"read_uncommitted","annotated_options":[["read_committed","If set, only committed transactional records are processed."],["read_uncommitted","If set, then uncommitted records are processed."]],"linter":"\nlet options = {\n \"read_committed\": true,\n \"read_uncommitted\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"consumer_group","type":"string","kind":"scalar","description":"An optional consumer group to consume as. When specified the partitions of specified topics are automatically distributed across consumers sharing a consumer group, and partition offsets are automatically committed and resumed under this name. Consumer groups are not supported when specifying explicit partitions to consume from in the `topics` field.","is_optional":true},{"name":"checkpoint_limit","type":"int","kind":"scalar","description":"Determines how many messages of the same partition can be processed in parallel before applying back pressure. When a message of a given offset is delivered to the output the offset is only allowed to be committed when all messages of prior offsets have also been delivered, this ensures at-least-once delivery guarantees. However, this mechanism also increases the likelihood of duplicates in the event of crashes or server faults, reducing the checkpoint limit will mitigate this.","is_advanced":true,"default":1024},{"name":"commit_period","type":"string","kind":"scalar","description":"The period of time between each commit of the current partition offsets. Offsets are always committed during shutdown.","is_advanced":true,"default":"5s"},{"name":"multi_header","type":"bool","kind":"scalar","description":"Decode headers into lists to allow handling of multiple values with the same key","is_advanced":true,"default":false},{"name":"batching","type":"object","kind":"","description":"Allows you to configure a xref:configuration:batching.adoc[batching policy] that applies to individual topic partitions in order to batch messages together before flushing them for processing. Batching can be beneficial for performance as well as useful for windowed processing, and doing so this way preserves the ordering of topic partitions.","is_advanced":true,"examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","is_advanced":true,"default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","is_advanced":true,"default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","is_advanced":true,"default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","is_advanced":true,"default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}],"linter":"\nlet has_topic_partitions = this.topics.any(t -\u003e t.contains(\":\"))\n\nroot = [\n if $has_topic_partitions {\n if this.consumer_group.or(\"\") != \"\" {\n \"this input does not support both a consumer group and explicit topic partitions\"\n } else if this.regexp_topics {\n \"this input does not support both regular expression topics and explicit topic partitions\"\n }\n } else {\n if this.consumer_group.or(\"\") == \"\" {\n \"a consumer group is mandatory when not using explicit topic partitions\"\n }\n },\n # We don't have any way to distinguish between start_from_oldest set explicitly to true and not set at all, so we\n # assume users will be OK if start_offset overwrites it silently\n if this.start_from_oldest == false \u0026\u0026 this.start_offset == \"earliest\" {\n \"start_from_oldest cannot be set to false when start_offset is set to earliest\"\n }\n]\n"},"version":"3.61.0"},{"name":"mongodb","type":"input","status":"experimental","plugin":true,"summary":"Executes a query and creates a message for each document received.","description":"Once the documents from the query are exhausted, this input shuts down, allowing the pipeline to gracefully terminate (or the next input in a xref:components:inputs/sequence.adoc[sequence] to execute).","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target MongoDB server.","examples":["mongodb://localhost:27017"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"database","type":"string","kind":"scalar","description":"The name of the target MongoDB database."},{"name":"username","type":"string","kind":"scalar","description":"The username to connect to the database.","default":""},{"name":"password","type":"string","kind":"scalar","description":"The password to connect to the database.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"app_name","type":"string","kind":"scalar","description":"The client application name.","is_advanced":true,"default":"benthos","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"collection","type":"string","kind":"scalar","description":"The collection to select from."},{"name":"operation","type":"string","kind":"scalar","description":"The mongodb operation to perform.","is_advanced":true,"default":"find","options":["find","aggregate"],"version":"4.2.0","linter":"\nlet options = {\n \"find\": true,\n \"aggregate\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"json_marshal_mode","type":"string","kind":"scalar","description":"The json_marshal_mode setting is optional and controls the format of the output message.","is_advanced":true,"default":"canonical","annotated_options":[["canonical","A string format that emphasizes type preservation at the expense of readability and interoperability. That is, conversion from canonical to BSON will generally preserve type information except in certain specific cases. "],["relaxed","A string format that emphasizes readability and interoperability at the expense of type preservation.That is, conversion from relaxed format to BSON can lose type information."]],"version":"4.7.0","linter":"\nlet options = {\n \"canonical\": true,\n \"relaxed\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"query","type":"string","kind":"scalar","description":"Bloblang expression describing MongoDB query.","bloblang":true,"examples":["\n root.from = {\"$lte\": timestamp_unix()}\n root.to = {\"$gte\": timestamp_unix()}\n"]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"batch_size","type":"int","kind":"scalar","description":"A explicit number of documents to batch up before flushing them for processing. Must be greater than `0`. Operations: `find`, `aggregate`","is_optional":true,"examples":[1000],"version":"4.26.0"},{"name":"sort","type":"int","kind":"map","description":"An object specifying fields to sort by, and the respective sort order (`1` ascending, `-1` descending). Note: The driver currently appears to support only one sorting key. Operations: `find`","is_optional":true,"examples":[{"name":1},{"age":-1}],"version":"4.26.0"},{"name":"limit","type":"int","kind":"scalar","description":"An explicit maximum number of documents to return. Operations: `find`","is_optional":true,"version":"4.26.0"}]},"version":"3.64.0"},{"name":"mongodb_cdc","type":"input","status":"experimental","plugin":true,"summary":"Streams changes from a MongoDB replica set.","description":"Read from a MongoDB replica set using https://www.mongodb.com/docs/manual/changeStreams/[^Change Streams]. It's only possible to watch for changes when using a sharded MongoDB or a MongoDB cluster running as a replica set.\n\nBy default MongoDB does not propagate changes in all cases. In order to capture all changes (including deletes) in a MongoDB cluster one needs to enable pre and post image saving and the collection needs to also enable saving these pre and post images. For more information see https://www.mongodb.com/docs/manual/changeStreams/#change-streams-with-document-pre--and-post-images[^MongoDB documentation].\n\n== Metadata\n\nEach message omitted by this plugin has the following metadata:\n\n- operation: either \"create\", \"replace\", \"delete\" or \"update\" for changes streamed. Documents from the initial snapshot have the operation set to \"read\".\n- collection: the collection the document was written to.\n- operation_time: the oplog time for when this operation occurred.\n ","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target MongoDB server.","examples":["mongodb://localhost:27017"]},{"name":"database","type":"string","kind":"scalar","description":"The name of the target MongoDB database."},{"name":"username","type":"string","kind":"scalar","description":"The username to connect to the database.","default":""},{"name":"password","type":"string","kind":"scalar","description":"The password to connect to the database.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"collections","type":"string","kind":"array","description":"The collections to stream changes from."},{"name":"checkpoint_key","type":"string","kind":"scalar","description":"Checkpoint cache key name.","default":"mongodb_cdc_checkpoint"},{"name":"checkpoint_cache","type":"string","kind":"scalar","description":"Checkpoint cache name."},{"name":"checkpoint_interval","type":"string","kind":"scalar","description":"The interval between writing checkpoints to the cache.","default":"5s"},{"name":"checkpoint_limit","type":"int","kind":"scalar","default":1000},{"name":"read_batch_size","type":"int","kind":"scalar","description":"The batch size of documents for MongoDB to return.","default":1000},{"name":"read_max_wait","type":"string","kind":"scalar","description":"The maximum time MongoDB waits to fulfill `read_batch_size` on the change stream before returning documents.","default":"1s"},{"name":"stream_snapshot","type":"bool","kind":"scalar","description":"If to read initial snapshot before streaming changes.","default":false},{"name":"snapshot_parallelism","type":"int","kind":"scalar","description":"Parallelism for snapshot phase.","default":1,"linter":"match {\n this \u003c 1 =\u003e [\"field snapshot_parallelism must be greater or equal to 1.\"],\n}"},{"name":"snapshot_auto_bucket_sharding","type":"bool","kind":"scalar","description":"If true, determine parallel snapshot chunks using `$bucketAuto` instead of the `splitVector` command. This allows parallel collection reading in environments where privledged access to the MongoDB cluster is not allowed such as MongoDB Atlas.","is_advanced":true,"default":false},{"name":"document_mode","type":"string","kind":"scalar","description":"The mode in which to emit documents, specifically updates and deletes.","is_advanced":true,"default":"update_lookup","annotated_options":[["partial_update","In this mode update operations only have a description of the update operation, which follows the following schema:\n {\n \"_id\": \u003cdocument_id\u003e,\n \"operations\": [\n # type == set means that the value was updated like so:\n # root.foo.\"bar.baz\" = \"world\"\n {\"path\": [\"foo\", \"bar.baz\"], \"type\": \"set\", \"value\":\"world\"},\n # type == unset means that the value was deleted like so:\n # root.qux = deleted()\n {\"path\": [\"qux\"], \"type\": \"unset\", \"value\": null},\n # type == truncatedArray means that the array at that path was truncated to value number of elements\n # root.array = this.array.slice(2)\n {\"path\": [\"array\"], \"type\": \"truncatedArray\", \"value\": 2}\n ]\n }\n "],["pre_and_post_images","Uses pre and post image collection to emit the full documents for update and delete operations. To use and configure this mode see the setup steps in the https://www.mongodb.com/docs/manual/changeStreams/#change-streams-with-document-pre--and-post-images[^MongoDB documentation]."],["update_lookup","In this mode insert, replace and update operations have the full document emitted and deletes only have the _id field populated. Documents updates lookup the full document. This corresponds to the updateLookup option, see the https://www.mongodb.com/docs/manual/changeStreams/#std-label-change-streams-updateLookup[^MongoDB documentation] for more information."]],"linter":"\nlet options = {\n \"partial_update\": true,\n \"pre_and_post_images\": true,\n \"update_lookup\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"json_marshal_mode","type":"string","kind":"scalar","description":"The json_marshal_mode setting is optional and controls the format of the output message.","is_advanced":true,"default":"canonical","annotated_options":[["canonical","A string format that emphasizes type preservation at the expense of readability and interoperability. That is, conversion from canonical to BSON will generally preserve type information except in certain specific cases. "],["relaxed","A string format that emphasizes readability and interoperability at the expense of type preservation.That is, conversion from relaxed format to BSON can lose type information."]],"linter":"\nlet options = {\n \"canonical\": true,\n \"relaxed\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"app_name","type":"string","kind":"scalar","description":"The client application name.","is_advanced":true,"default":"benthos"},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]}},{"name":"mqtt","type":"input","status":"stable","plugin":true,"summary":"Subscribe to topics on MQTT brokers.","description":"\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n- mqtt_duplicate\n- mqtt_qos\n- mqtt_retained\n- mqtt_topic\n- mqtt_message_id\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. The format should be `scheme://host:port` where `scheme` is one of `tcp`, `ssl`, or `ws`, `host` is the ip-address (or hostname) and `port` is the port on which the broker is accepting connections. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["tcp://localhost:1883"]],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"client_id","type":"string","kind":"scalar","description":"An identifier for the client connection.","default":""},{"name":"dynamic_client_id_suffix","type":"string","kind":"scalar","description":"Append a dynamically generated suffix to the specified `client_id` on each run of the pipeline. This can be useful when clustering Redpanda Connect producers.","is_advanced":true,"is_optional":true,"annotated_options":[["nanoid","append a nanoid of length 21 characters"]],"linter":"root = []"},{"name":"connect_timeout","type":"string","kind":"scalar","description":"The maximum amount of time to wait in order to establish a connection before the attempt is abandoned.","default":"30s","examples":["1s","500ms"],"version":"3.58.0"},{"name":"will","type":"object","kind":"scalar","description":"Set last will message in case of Redpanda Connect failure","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to enable last will messages.","is_advanced":true,"default":false},{"name":"qos","type":"int","kind":"scalar","description":"Set QoS for last will message. Valid values are: 0, 1, 2.","is_advanced":true,"default":0},{"name":"retained","type":"bool","kind":"scalar","description":"Set retained for last will message.","is_advanced":true,"default":false},{"name":"topic","type":"string","kind":"scalar","description":"Set topic for last will message.","is_advanced":true,"default":""},{"name":"payload","type":"string","kind":"scalar","description":"Set payload for last will message.","is_advanced":true,"default":""}]},{"name":"user","type":"string","kind":"scalar","description":"A username to connect with.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to connect with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"keepalive","type":"int","kind":"scalar","description":"Max seconds of inactivity before a keepalive message is sent.","is_advanced":true,"default":30},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"topics","type":"string","kind":"array","description":"A list of topics to consume from."},{"name":"qos","type":"int","kind":"scalar","description":"The level of delivery guarantee to enforce. Has options 0, 1, 2.","is_advanced":true,"default":1},{"name":"clean_session","type":"bool","kind":"scalar","description":"Set whether the connection is non-persistent.","is_advanced":true,"default":true},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]}},{"name":"mysql_cdc","type":"input","status":"beta","plugin":true,"summary":"Enables MySQL streaming for RedPanda Connect.","description":"\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n- operation\n- table\n- binlog_position\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"flavor","type":"string","kind":"scalar","description":"The type of MySQL database to connect to.","default":"mysql","annotated_options":[["mariadb","MariaDB flavored databases."],["mysql","MySQL flavored databases."]],"linter":"\nlet options = {\n \"mariadb\": true,\n \"mysql\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"dsn","type":"string","kind":"scalar","description":"The DSN of the MySQL database to connect to.","examples":["user:password@tcp(localhost:3306)/database"]},{"name":"tables","type":"string","kind":"array","description":"A list of tables to stream from the database.","examples":[["table1","table2"]]},{"name":"checkpoint_cache","type":"string","kind":"scalar","description":"A https://www.docs.redpanda.com/redpanda-connect/components/caches/about[cache resource^] to use for storing the current latest BinLog Position that has been successfully delivered, this allows Redpanda Connect to continue from that BinLog Position upon restart, rather than consume the entire state of the table."},{"name":"checkpoint_key","type":"string","kind":"scalar","description":"The key to use to store the snapshot position in `checkpoint_cache`. An alternative key can be provided if multiple CDC inputs share the same cache.","default":"mysql_binlog_position"},{"name":"snapshot_max_batch_size","type":"int","kind":"scalar","description":"The maximum number of rows to be streamed in a single batch when taking a snapshot.","default":1000},{"name":"stream_snapshot","type":"bool","kind":"scalar","description":"If set to true, the connector will query all the existing data as a part of snapshot process. Otherwise, it will start from the current binlog position."},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"checkpoint_limit","type":"int","kind":"scalar","description":"The maximum number of messages that can be processed at a given time. Increasing this limit enables parallel processing and batching at the output level. Any given BinLog Position will not be acknowledged unless all messages under that offset are delivered in order to preserve at least once delivery guarantees.","default":1024},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]},"version":"4.45.0"},{"name":"nanomsg","type":"input","status":"stable","plugin":true,"summary":"Consumes messages via Nanomsg sockets (scalability protocols).","description":"Currently only PULL and SUB sockets are supported.","categories":["Network"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to (or as). If an item of the list contains commas it will be expanded into multiple URLs.","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"bind","type":"bool","kind":"scalar","description":"Whether the URLs provided should be connected to, or bound as.","default":true},{"name":"socket_type","type":"string","kind":"scalar","description":"The socket type to use.","default":"PULL","options":["PULL","SUB"],"linter":"\nlet options = {\n \"pull\": true,\n \"sub\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"sub_filters","type":"string","kind":"array","description":"A list of subscription topic filters to use when consuming from a SUB socket. Specifying a single sub_filter of `''` will subscribe to everything.","default":[]},{"name":"poll_timeout","type":"string","kind":"scalar","description":"The period to wait until a poll is abandoned and reattempted.","is_advanced":true,"default":"5s"}]}},{"name":"nats","type":"input","status":"stable","plugin":true,"summary":"Subscribe to a NATS subject.","description":"\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- nats_subject\n- nats_reply_subject\n- All message headers (when supported by the connection)\n```\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].\n\n== Connection name\n\nWhen monitoring and managing a production NATS system, it is often useful to\nknow which connection a message was send/received from. This can be achieved by\nsetting the connection name option when creating a NATS connection.\n\nRedpanda Connect will automatically set the connection name based off the label of the given\nNATS component, so that monitoring tools between NATS and Redpanda Connect can stay in sync.\n\n\n== Authentication\n\nThere are several components within Redpanda Connect which uses NATS services. You will find that each of these components\nsupport optional advanced authentication parameters for https://docs.nats.io/nats-server/configuration/securing_nats/auth_intro/nkey_auth[NKeys^]\nand https://docs.nats.io/using-nats/developer/connecting/creds[User Credentials^].\n\nSee an https://docs.nats.io/running-a-nats-service/nats_admin/security/jwt[in-depth tutorial^].\n\n=== NKey file\n\nThe NATS server can use these NKeys in several ways for authentication. The simplest is for the server to be configured\nwith a list of known public keys and for the clients to respond to the challenge by signing it with its private NKey\nconfigured in the `nkey_file` or `nkey` field.\n\nhttps://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[More details^].\n\n=== User credentials\n\nNATS server supports decentralized authentication based on JSON Web Tokens (JWT). Clients need an https://docs.nats.io/nats-server/configuration/securing_nats/jwt#json-web-tokens[user JWT^]\nand a corresponding https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[NKey secret^] when connecting to a server\nwhich is configured to use this authentication scheme.\n\nThe `user_credentials_file` field should point to a file containing both the private key and the JWT and can be\ngenerated with the https://docs.nats.io/nats-tools/nsc[nsc tool^].\n\nAlternatively, the `user_jwt` field can contain a plain text JWT and the `user_nkey_seed`can contain\nthe plain text NKey Seed.\n\nhttps://docs.nats.io/using-nats/developer/connecting/creds[More details^].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["nats://127.0.0.1:4222"],["nats://username:password@127.0.0.1:4222"]]},{"name":"subject","type":"string","kind":"scalar","description":"A subject to consume from. Supports wildcards for consuming multiple subjects. Either a subject or stream must be specified.","examples":["foo.bar.baz","foo.*.baz","foo.bar.*","foo.\u003e"]},{"name":"queue","type":"string","kind":"scalar","description":"An optional queue group to consume as.","is_optional":true},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"send_ack","type":"bool","kind":"scalar","description":"Control whether ACKS are sent as a reply to each message. When enabled, these replies are sent only once the data has been delivered to all outputs.","default":true},{"name":"nak_delay","type":"string","kind":"scalar","description":"An optional delay duration on redelivering a message when negatively acknowledged.","is_advanced":true,"is_optional":true,"examples":["1m"]},{"name":"prefetch_count","type":"int","kind":"scalar","description":"The maximum number of messages to pull at a time.","is_advanced":true,"default":524288,"linter":"root = if this \u003c 0 { [\"prefetch count must be greater than or equal to zero\"] }"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auth","type":"object","kind":"scalar","description":"Optional configuration of NATS authentication parameters.","is_advanced":true,"children":[{"name":"nkey_file","type":"string","kind":"scalar","description":"An optional file containing a NKey seed.","is_advanced":true,"is_optional":true,"examples":["./seed.nk"]},{"name":"nkey","type":"string","kind":"scalar","description":"The NKey seed.","is_advanced":true,"is_optional":true,"is_secret":true,"examples":["UDXU4RCSJNZOIQHZNWXHXORDPRTGNJAHAHFRGZNEEJCPQTT2M7NLCNF4"],"version":"4.38.0","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_credentials_file","type":"string","kind":"scalar","description":"An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.","is_advanced":true,"is_optional":true,"examples":["./user.creds"]},{"name":"user_jwt","type":"string","kind":"scalar","description":"An optional plain text user JWT (given along with the corresponding user NKey Seed).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_nkey_seed","type":"string","kind":"scalar","description":"An optional plain text user NKey Seed (given along with the corresponding user JWT).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"extract_tracing_map","type":"string","kind":"scalar","description":"EXPERIMENTAL: A xref:guides:bloblang/about.adoc[Bloblang mapping] that attempts to extract an object containing tracing propagation information, which will then be used as the root tracing span for the message. The specification of the extracted fields must match the format used by the service wide tracer.","is_advanced":true,"is_optional":true,"bloblang":true,"examples":["root = @","root = this.meta.span"],"version":"4.23.0"}]}},{"name":"nats_jetstream","type":"input","status":"stable","plugin":true,"summary":"Reads messages from NATS JetStream subjects.","description":"\n== Consume mirrored streams\n\nIn the case where a stream being consumed is mirrored from a different JetStream domain the stream cannot be resolved from the subject name alone, and so the stream name as well as the subject (if applicable) must both be specified.\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- nats_subject\n- nats_sequence_stream\n- nats_sequence_consumer\n- nats_num_delivered\n- nats_num_pending\n- nats_domain\n- nats_timestamp_unix_nano\n```\n\nYou can access these metadata fields using\nxref:configuration:interpolation.adoc#bloblang-queries[function interpolation].\n\n== Connection name\n\nWhen monitoring and managing a production NATS system, it is often useful to\nknow which connection a message was send/received from. This can be achieved by\nsetting the connection name option when creating a NATS connection.\n\nRedpanda Connect will automatically set the connection name based off the label of the given\nNATS component, so that monitoring tools between NATS and Redpanda Connect can stay in sync.\n\n\n== Authentication\n\nThere are several components within Redpanda Connect which uses NATS services. You will find that each of these components\nsupport optional advanced authentication parameters for https://docs.nats.io/nats-server/configuration/securing_nats/auth_intro/nkey_auth[NKeys^]\nand https://docs.nats.io/using-nats/developer/connecting/creds[User Credentials^].\n\nSee an https://docs.nats.io/running-a-nats-service/nats_admin/security/jwt[in-depth tutorial^].\n\n=== NKey file\n\nThe NATS server can use these NKeys in several ways for authentication. The simplest is for the server to be configured\nwith a list of known public keys and for the clients to respond to the challenge by signing it with its private NKey\nconfigured in the `nkey_file` or `nkey` field.\n\nhttps://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[More details^].\n\n=== User credentials\n\nNATS server supports decentralized authentication based on JSON Web Tokens (JWT). Clients need an https://docs.nats.io/nats-server/configuration/securing_nats/jwt#json-web-tokens[user JWT^]\nand a corresponding https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[NKey secret^] when connecting to a server\nwhich is configured to use this authentication scheme.\n\nThe `user_credentials_file` field should point to a file containing both the private key and the JWT and can be\ngenerated with the https://docs.nats.io/nats-tools/nsc[nsc tool^].\n\nAlternatively, the `user_jwt` field can contain a plain text JWT and the `user_nkey_seed`can contain\nthe plain text NKey Seed.\n\nhttps://docs.nats.io/using-nats/developer/connecting/creds[More details^].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["nats://127.0.0.1:4222"],["nats://username:password@127.0.0.1:4222"]]},{"name":"queue","type":"string","kind":"scalar","description":"An optional queue group to consume as. Used to configure a push consumer.","is_optional":true},{"name":"subject","type":"string","kind":"scalar","description":"A subject to consume from. Supports wildcards for consuming multiple subjects. Either a subject or stream must be specified.","is_optional":true,"examples":["foo.bar.baz","foo.*.baz","foo.bar.*","foo.\u003e"]},{"name":"durable","type":"string","kind":"scalar","description":"Preserve the state of your consumer under a durable name. Used to configure a pull consumer.","is_optional":true},{"name":"stream","type":"string","kind":"scalar","description":"A stream to consume from. Either a subject or stream must be specified.","is_optional":true},{"name":"bind","type":"bool","kind":"scalar","description":"Indicates that the subscription should use an existing consumer.","is_optional":true},{"name":"deliver","type":"string","kind":"scalar","description":"Determines which messages to deliver when consuming without a durable subscriber.","default":"all","annotated_options":[["all","Deliver all available messages."],["last","Deliver starting with the last published messages."],["last_per_subject","Deliver starting with the last published message per subject."],["new","Deliver starting from now, not taking into account any previous messages."]],"linter":"\nlet options = {\n \"all\": true,\n \"last\": true,\n \"last_per_subject\": true,\n \"new\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"ack_wait","type":"string","kind":"scalar","description":"The maximum amount of time NATS server should wait for an ack from consumer.","is_advanced":true,"default":"30s","examples":["100ms","5m"]},{"name":"max_ack_pending","type":"int","kind":"scalar","description":"The maximum number of outstanding acks to be allowed before consuming is halted.","is_advanced":true,"default":1024},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auth","type":"object","kind":"scalar","description":"Optional configuration of NATS authentication parameters.","is_advanced":true,"children":[{"name":"nkey_file","type":"string","kind":"scalar","description":"An optional file containing a NKey seed.","is_advanced":true,"is_optional":true,"examples":["./seed.nk"]},{"name":"nkey","type":"string","kind":"scalar","description":"The NKey seed.","is_advanced":true,"is_optional":true,"is_secret":true,"examples":["UDXU4RCSJNZOIQHZNWXHXORDPRTGNJAHAHFRGZNEEJCPQTT2M7NLCNF4"],"version":"4.38.0","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_credentials_file","type":"string","kind":"scalar","description":"An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.","is_advanced":true,"is_optional":true,"examples":["./user.creds"]},{"name":"user_jwt","type":"string","kind":"scalar","description":"An optional plain text user JWT (given along with the corresponding user NKey Seed).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_nkey_seed","type":"string","kind":"scalar","description":"An optional plain text user NKey Seed (given along with the corresponding user JWT).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"extract_tracing_map","type":"string","kind":"scalar","description":"EXPERIMENTAL: A xref:guides:bloblang/about.adoc[Bloblang mapping] that attempts to extract an object containing tracing propagation information, which will then be used as the root tracing span for the message. The specification of the extracted fields must match the format used by the service wide tracer.","is_advanced":true,"is_optional":true,"bloblang":true,"examples":["root = @","root = this.meta.span"],"version":"4.23.0"}],"linter":"root = match {\n\t\t\tthis.exists(\"queue\") \u0026\u0026 this.queue != \"\" \u0026\u0026 this.exists(\"durable\") \u0026\u0026 this.durable != \"\" =\u003e [ \"both 'queue' and 'durable' can't be set simultaneously\" ],\n\t\t\t}"},"version":"3.46.0"},{"name":"nats_kv","type":"input","status":"beta","plugin":true,"summary":"Watches for updates in a NATS key-value bucket.","description":"\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n``` text\n- nats_kv_key\n- nats_kv_bucket\n- nats_kv_revision\n- nats_kv_delta\n- nats_kv_operation\n- nats_kv_created\n```\n\n== Connection name\n\nWhen monitoring and managing a production NATS system, it is often useful to\nknow which connection a message was send/received from. This can be achieved by\nsetting the connection name option when creating a NATS connection.\n\nRedpanda Connect will automatically set the connection name based off the label of the given\nNATS component, so that monitoring tools between NATS and Redpanda Connect can stay in sync.\n\n\n== Authentication\n\nThere are several components within Redpanda Connect which uses NATS services. You will find that each of these components\nsupport optional advanced authentication parameters for https://docs.nats.io/nats-server/configuration/securing_nats/auth_intro/nkey_auth[NKeys^]\nand https://docs.nats.io/using-nats/developer/connecting/creds[User Credentials^].\n\nSee an https://docs.nats.io/running-a-nats-service/nats_admin/security/jwt[in-depth tutorial^].\n\n=== NKey file\n\nThe NATS server can use these NKeys in several ways for authentication. The simplest is for the server to be configured\nwith a list of known public keys and for the clients to respond to the challenge by signing it with its private NKey\nconfigured in the `nkey_file` or `nkey` field.\n\nhttps://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[More details^].\n\n=== User credentials\n\nNATS server supports decentralized authentication based on JSON Web Tokens (JWT). Clients need an https://docs.nats.io/nats-server/configuration/securing_nats/jwt#json-web-tokens[user JWT^]\nand a corresponding https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[NKey secret^] when connecting to a server\nwhich is configured to use this authentication scheme.\n\nThe `user_credentials_file` field should point to a file containing both the private key and the JWT and can be\ngenerated with the https://docs.nats.io/nats-tools/nsc[nsc tool^].\n\nAlternatively, the `user_jwt` field can contain a plain text JWT and the `user_nkey_seed`can contain\nthe plain text NKey Seed.\n\nhttps://docs.nats.io/using-nats/developer/connecting/creds[More details^].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["nats://127.0.0.1:4222"],["nats://username:password@127.0.0.1:4222"]]},{"name":"bucket","type":"string","kind":"scalar","description":"The name of the KV bucket.","examples":["my_kv_bucket"]},{"name":"key","type":"string","kind":"scalar","description":"Key to watch for updates, can include wildcards.","default":"\u003e","examples":["foo.bar.baz","foo.*.baz","foo.bar.*","foo.\u003e"]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"ignore_deletes","type":"bool","kind":"scalar","description":"Do not send delete markers as messages.","is_advanced":true,"default":false},{"name":"include_history","type":"bool","kind":"scalar","description":"Include all the history per key, not just the last one.","is_advanced":true,"default":false},{"name":"meta_only","type":"bool","kind":"scalar","description":"Retrieve only the metadata of the entry","is_advanced":true,"default":false},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auth","type":"object","kind":"scalar","description":"Optional configuration of NATS authentication parameters.","is_advanced":true,"children":[{"name":"nkey_file","type":"string","kind":"scalar","description":"An optional file containing a NKey seed.","is_advanced":true,"is_optional":true,"examples":["./seed.nk"]},{"name":"nkey","type":"string","kind":"scalar","description":"The NKey seed.","is_advanced":true,"is_optional":true,"is_secret":true,"examples":["UDXU4RCSJNZOIQHZNWXHXORDPRTGNJAHAHFRGZNEEJCPQTT2M7NLCNF4"],"version":"4.38.0","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_credentials_file","type":"string","kind":"scalar","description":"An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.","is_advanced":true,"is_optional":true,"examples":["./user.creds"]},{"name":"user_jwt","type":"string","kind":"scalar","description":"An optional plain text user JWT (given along with the corresponding user NKey Seed).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_nkey_seed","type":"string","kind":"scalar","description":"An optional plain text user NKey Seed (given along with the corresponding user JWT).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},"version":"4.12.0"},{"name":"nats_stream","type":"input","status":"stable","plugin":true,"summary":"Subscribe to a NATS Stream subject. Joining a queue is optional and allows multiple clients of a subject to consume using queue semantics.","description":"\n[CAUTION]\n.Deprecation notice\n====\nThe NATS Streaming Server is being deprecated. Critical bug fixes and security fixes will be applied until June of 2023. NATS-enabled applications requiring persistence should use https://docs.nats.io/nats-concepts/jetstream[JetStream^].\n====\n\nTracking and persisting offsets through a durable name is also optional and works with or without a queue. If a durable name is not provided then subjects are consumed from the most recently published message.\n\nWhen a consumer closes its connection it unsubscribes, when all consumers of a durable queue do this the offsets are deleted. In order to avoid this you can stop the consumers from unsubscribing by setting the field `unsubscribe_on_close` to `false`.\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n- nats_stream_subject\n- nats_stream_sequence\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].\n\n\n\n== Authentication\n\nThere are several components within Redpanda Connect which uses NATS services. You will find that each of these components\nsupport optional advanced authentication parameters for https://docs.nats.io/nats-server/configuration/securing_nats/auth_intro/nkey_auth[NKeys^]\nand https://docs.nats.io/using-nats/developer/connecting/creds[User Credentials^].\n\nSee an https://docs.nats.io/running-a-nats-service/nats_admin/security/jwt[in-depth tutorial^].\n\n=== NKey file\n\nThe NATS server can use these NKeys in several ways for authentication. The simplest is for the server to be configured\nwith a list of known public keys and for the clients to respond to the challenge by signing it with its private NKey\nconfigured in the `nkey_file` or `nkey` field.\n\nhttps://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[More details^].\n\n=== User credentials\n\nNATS server supports decentralized authentication based on JSON Web Tokens (JWT). Clients need an https://docs.nats.io/nats-server/configuration/securing_nats/jwt#json-web-tokens[user JWT^]\nand a corresponding https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[NKey secret^] when connecting to a server\nwhich is configured to use this authentication scheme.\n\nThe `user_credentials_file` field should point to a file containing both the private key and the JWT and can be\ngenerated with the https://docs.nats.io/nats-tools/nsc[nsc tool^].\n\nAlternatively, the `user_jwt` field can contain a plain text JWT and the `user_nkey_seed`can contain\nthe plain text NKey Seed.\n\nhttps://docs.nats.io/using-nats/developer/connecting/creds[More details^].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["nats://127.0.0.1:4222"],["nats://username:password@127.0.0.1:4222"]]},{"name":"cluster_id","type":"string","kind":"scalar","description":"The ID of the cluster to consume from."},{"name":"client_id","type":"string","kind":"scalar","description":"A client ID to connect as.","default":""},{"name":"queue","type":"string","kind":"scalar","description":"The queue to consume from.","default":""},{"name":"subject","type":"string","kind":"scalar","description":"A subject to consume from.","default":""},{"name":"durable_name","type":"string","kind":"scalar","description":"Preserve the state of your consumer under a durable name.","default":""},{"name":"unsubscribe_on_close","type":"bool","kind":"scalar","description":"Whether the subscription should be destroyed when this client disconnects.","default":false},{"name":"start_from_oldest","type":"bool","kind":"scalar","description":"If a position is not found for a queue, determines whether to consume from the oldest available message, otherwise messages are consumed from the latest.","is_advanced":true,"default":true},{"name":"max_inflight","type":"int","kind":"scalar","description":"The maximum number of unprocessed messages to fetch at a given time.","is_advanced":true,"default":1024},{"name":"ack_wait","type":"string","kind":"scalar","description":"An optional duration to specify at which a message that is yet to be acked will be automatically retried.","is_advanced":true,"default":"30s"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auth","type":"object","kind":"scalar","description":"Optional configuration of NATS authentication parameters.","is_advanced":true,"children":[{"name":"nkey_file","type":"string","kind":"scalar","description":"An optional file containing a NKey seed.","is_advanced":true,"is_optional":true,"examples":["./seed.nk"]},{"name":"nkey","type":"string","kind":"scalar","description":"The NKey seed.","is_advanced":true,"is_optional":true,"is_secret":true,"examples":["UDXU4RCSJNZOIQHZNWXHXORDPRTGNJAHAHFRGZNEEJCPQTT2M7NLCNF4"],"version":"4.38.0","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_credentials_file","type":"string","kind":"scalar","description":"An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.","is_advanced":true,"is_optional":true,"examples":["./user.creds"]},{"name":"user_jwt","type":"string","kind":"scalar","description":"An optional plain text user JWT (given along with the corresponding user NKey Seed).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_nkey_seed","type":"string","kind":"scalar","description":"An optional plain text user NKey Seed (given along with the corresponding user JWT).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"extract_tracing_map","type":"string","kind":"scalar","description":"EXPERIMENTAL: A xref:guides:bloblang/about.adoc[Bloblang mapping] that attempts to extract an object containing tracing propagation information, which will then be used as the root tracing span for the message. The specification of the extracted fields must match the format used by the service wide tracer.","is_advanced":true,"is_optional":true,"bloblang":true,"examples":["root = @","root = this.meta.span"],"version":"4.23.0"}]}},{"name":"nsq","type":"input","status":"stable","plugin":true,"summary":"Subscribe to an NSQ instance topic and channel.","description":"\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n- nsq_attempts\n- nsq_id\n- nsq_nsqd_address\n- nsq_timestamp\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"nsqd_tcp_addresses","type":"string","kind":"array","description":"A list of nsqd addresses to connect to."},{"name":"lookupd_http_addresses","type":"string","kind":"array","description":"A list of nsqlookupd addresses to connect to."},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"topic","type":"string","kind":"scalar","description":"The topic to consume from."},{"name":"channel","type":"string","kind":"scalar","description":"The channel to consume from."},{"name":"user_agent","type":"string","kind":"scalar","description":"A user agent to assume when connecting.","is_optional":true},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of pending messages to consume at any given time.","default":100},{"name":"max_attempts","type":"int","kind":"scalar","description":"The maximum number of attempts to successfully consume a messages.","default":5}]}},{"name":"ockam_kafka","type":"input","status":"experimental","plugin":true,"summary":"Ockam","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"kafka","type":"object","kind":"scalar","children":[{"name":"seed_brokers","type":"string","kind":"array","description":"A list of broker addresses to connect to in order to establish connections. If an item of the list contains commas it will be expanded into multiple addresses.","is_optional":true,"examples":[["localhost:9092"],["foo:9092","bar:9092"],["foo:9092,bar:9092"]]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"topics","type":"string","kind":"array","description":"\nA list of topics to consume from. Multiple comma separated topics can be listed in a single element. When a `consumer_group` is specified partitions are automatically distributed across consumers of a topic, otherwise all partitions are consumed.\n\nAlternatively, it's possible to specify explicit partitions to consume from with a colon after the topic name, e.g. `foo:0` would consume the partition 0 of the topic foo. This syntax supports ranges, e.g. `foo:0-10` would consume partitions 0 through to 10 inclusive.\n\nFinally, it's also possible to specify an explicit offset to consume from by adding another colon after the partition, e.g. `foo:0:10` would consume the partition 0 of the topic foo starting from the offset 10. If the offset is not present (or remains unspecified) then the field `start_from_oldest` determines which offset to start from.","examples":[["foo","bar"],["things.*"],["foo,bar"],["foo:0","bar:1","bar:3"],["foo:0,bar:1,bar:3"],["foo:0-5"]]},{"name":"regexp_topics","type":"bool","kind":"scalar","description":"Whether listed topics should be interpreted as regular expression patterns for matching multiple topics. When topics are specified with explicit partitions this field must remain set to `false`.","default":false},{"name":"rack_id","type":"string","kind":"scalar","description":"A rack specifies where the client is physically located and changes fetch requests to consume from the closest replica as opposed to the leader replica.","is_advanced":true,"default":""},{"name":"instance_id","type":"string","kind":"scalar","description":"When using a consumer group, an instance ID specifies the groups static membership, which can prevent rebalances during reconnects. When using a instance ID the client does NOT leave the group when closing. To actually leave the group one must use an external admin command to leave the group on behalf of this instance ID. This ID must be unique per consumer within the group.","is_advanced":true,"default":""},{"name":"rebalance_timeout","type":"string","kind":"scalar","description":"When using a consumer group, `rebalance_timeout` sets how long group members are allowed to take when a rebalance has begun. This timeout is how long all members are allowed to complete work and commit offsets, minus the time it took to detect the rebalance (from a heartbeat).","is_advanced":true,"default":"45s"},{"name":"session_timeout","type":"string","kind":"scalar","description":"When using a consumer group, `session_timeout` sets how long a member in hte group can go between heartbeats. If a member does not heartbeat in this timeout, the broker will remove the member from the group and initiate a rebalance.","is_advanced":true,"default":"1m"},{"name":"heartbeat_interval","type":"string","kind":"scalar","description":"When using a consumer group, `heartbeat_interval` sets how long a group member goes between heartbeats to Kafka. Kafka uses heartbeats to ensure that a group member's sesion stays active. This value should be no higher than 1/3rd of the `session_timeout`. This is equivalent to the Java heartbeat.interval.ms setting.","is_advanced":true,"default":"3s"},{"name":"start_from_oldest","type":"bool","kind":"scalar","description":"Determines whether to consume from the oldest available offset, otherwise messages are consumed from the latest offset. The setting is applied when creating a new consumer group or the saved offset no longer exists.","is_advanced":true,"is_deprecated":true,"default":true},{"name":"start_offset","type":"string","kind":"scalar","description":"Sets the offset to start consuming from, or if OffsetOutOfRange is seen while fetching, to restart consuming from.","is_advanced":true,"default":"earliest","annotated_options":[["committed","Prevents consuming a partition in a group if the partition has no prior commits. Corresponds to Kafka's `auto.offset.reset=none` option"],["earliest","Start from the earliest offset. Corresponds to Kafka's `auto.offset.reset=earliest` option."],["latest","Start from the latest offset. Corresponds to Kafka's `auto.offset.reset=latest` option."]],"linter":"\nlet options = {\n \"committed\": true,\n \"earliest\": true,\n \"latest\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"fetch_max_bytes","type":"string","kind":"scalar","description":"Sets the maximum amount of bytes a broker will try to send during a fetch. Note that brokers may not obey this limit if it has records larger than this limit. This is the equivalent to the Java fetch.max.bytes setting.","is_advanced":true,"default":"50MiB"},{"name":"fetch_max_wait","type":"string","kind":"scalar","description":"Sets the maximum amount of time a broker will wait for a fetch response to hit the minimum number of required bytes. This is the equivalent to the Java fetch.max.wait.ms setting.","is_advanced":true,"default":"5s"},{"name":"fetch_min_bytes","type":"string","kind":"scalar","description":"Sets the minimum amount of bytes a broker will try to send during a fetch. This is the equivalent to the Java fetch.min.bytes setting.","is_advanced":true,"default":"1B"},{"name":"fetch_max_partition_bytes","type":"string","kind":"scalar","description":"Sets the maximum amount of bytes that will be consumed for a single partition in a fetch request. Note that if a single batch is larger than this number, that batch will still be returned so the client can make progress. This is the equivalent to the Java fetch.max.partition.bytes setting.","is_advanced":true,"default":"1MiB"},{"name":"transaction_isolation_level","type":"string","kind":"scalar","description":"The transaction isolation level","default":"read_uncommitted","annotated_options":[["read_committed","If set, only committed transactional records are processed."],["read_uncommitted","If set, then uncommitted records are processed."]],"linter":"\nlet options = {\n \"read_committed\": true,\n \"read_uncommitted\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"consumer_group","type":"string","kind":"scalar","description":"An optional consumer group to consume as. When specified the partitions of specified topics are automatically distributed across consumers sharing a consumer group, and partition offsets are automatically committed and resumed under this name. Consumer groups are not supported when specifying explicit partitions to consume from in the `topics` field.","is_optional":true},{"name":"checkpoint_limit","type":"int","kind":"scalar","description":"Determines how many messages of the same partition can be processed in parallel before applying back pressure. When a message of a given offset is delivered to the output the offset is only allowed to be committed when all messages of prior offsets have also been delivered, this ensures at-least-once delivery guarantees. However, this mechanism also increases the likelihood of duplicates in the event of crashes or server faults, reducing the checkpoint limit will mitigate this.","is_advanced":true,"default":1024},{"name":"commit_period","type":"string","kind":"scalar","description":"The period of time between each commit of the current partition offsets. Offsets are always committed during shutdown.","is_advanced":true,"default":"5s"},{"name":"multi_header","type":"bool","kind":"scalar","description":"Decode headers into lists to allow handling of multiple values with the same key","is_advanced":true,"default":false},{"name":"batching","type":"object","kind":"","description":"Allows you to configure a xref:configuration:batching.adoc[batching policy] that applies to individual topic partitions in order to batch messages together before flushing them for processing. Batching can be beneficial for performance as well as useful for windowed processing, and doing so this way preserves the ordering of topic partitions.","is_advanced":true,"examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","is_advanced":true,"default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","is_advanced":true,"default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","is_advanced":true,"default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","is_advanced":true,"default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}],"linter":"\nlet has_topic_partitions = this.topics.any(t -\u003e t.contains(\":\"))\n\nroot = [\n if $has_topic_partitions {\n if this.consumer_group.or(\"\") != \"\" {\n \"this input does not support both a consumer group and explicit topic partitions\"\n } else if this.regexp_topics {\n \"this input does not support both regular expression topics and explicit topic partitions\"\n }\n } else {\n if this.consumer_group.or(\"\") == \"\" {\n \"a consumer group is mandatory when not using explicit topic partitions\"\n }\n },\n # We don't have any way to distinguish between start_from_oldest set explicitly to true and not set at all, so we\n # assume users will be OK if start_offset overwrites it silently\n if this.start_from_oldest == false \u0026\u0026 this.start_offset == \"earliest\" {\n \"start_from_oldest cannot be set to false when start_offset is set to earliest\"\n }\n]\n"},{"name":"disable_content_encryption","type":"bool","kind":"scalar","default":false},{"name":"enrollment_ticket","type":"string","kind":"scalar","is_optional":true},{"name":"identity_name","type":"string","kind":"scalar","is_optional":true},{"name":"allow","type":"string","kind":"scalar","default":"self"},{"name":"route_to_kafka_outlet","type":"string","kind":"scalar","default":"self"},{"name":"allow_producer","type":"string","kind":"scalar","default":"self"},{"name":"relay","type":"string","kind":"scalar","is_optional":true},{"name":"node_address","type":"string","kind":"scalar","default":"127.0.0.1:6262"},{"name":"encrypted_fields","type":"string","kind":"array","description":"The fields to encrypt in the kafka messages, assuming the record is a valid JSON map. By default, the whole record is encrypted.","default":[]}]}},{"name":"parquet","type":"input","status":"experimental","plugin":true,"summary":"Reads and decodes https://parquet.apache.org/docs/[Parquet files^] into a stream of structured messages.","description":"\nThis input uses https://github.com/parquet-go/parquet-go[https://github.com/parquet-go/parquet-go^], which is itself experimental. Therefore changes could be made into how this processor functions outside of major version releases.\n\nBy default any BYTE_ARRAY or FIXED_LEN_BYTE_ARRAY value will be extracted as a byte slice (`[]byte`) unless the logical type is UTF8, in which case they are extracted as a string (`string`).\n\nWhen a value extracted as a byte slice exists within a document which is later JSON serialized by default it will be base 64 encoded into strings, which is the default for arbitrary data fields. It is possible to convert these binary values to strings (or other data types) using Bloblang transformations such as `root.foo = this.foo.string()` or `root.foo = this.foo.encode(\"hex\")`, etc.","categories":["Local"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"paths","type":"string","kind":"array","description":"A list of file paths to read from. Each file will be read sequentially until the list is exhausted, at which point the input will close. Glob patterns are supported, including super globs (double star).","examples":["/tmp/foo.parquet","/tmp/bar/*.parquet","/tmp/data/**/*.parquet"]},{"name":"batch_count","type":"int","kind":"scalar","description":"Optionally process records in batches. This can help to speed up the consumption of exceptionally large files. When the end of the file is reached the remaining records are processed as a (potentially smaller) batch.","is_advanced":true,"default":1},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]},"version":"4.8.0"},{"name":"pg_stream","type":"input","status":"deprecated","plugin":true,"summary":"Streams changes from a PostgreSQL database using logical replication.","description":"Streams changes from a PostgreSQL database for Change Data Capture (CDC).\nAdditionally, if `stream_snapshot` is set to true, then the existing data in the database is also streamed too.\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n- table (Name of the table that the message originated from)\n- operation (Type of operation that generated the message: \"read\", \"insert\", \"update\", or \"delete\". \"read\" is from messages that are read in the initial snapshot phase. This will also be \"begin\" and \"commit\" if `include_transaction_markers` is enabled)\n- lsn (the log sequence number in postgres)\n\t\t","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"dsn","type":"string","kind":"scalar","description":"The Data Source Name for the PostgreSQL database in the form of `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]`. Please note that Postgres enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.","examples":["postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable"]},{"name":"include_transaction_markers","type":"bool","kind":"scalar","description":"When set to true, empty messages with operation types BEGIN and COMMIT are generated for the beginning and end of each transaction. Messages with operation metadata set to \"begin\" or \"commit\" will have null message payloads.","default":false},{"name":"stream_snapshot","type":"bool","kind":"scalar","description":"When set to true, the plugin will first stream a snapshot of all existing data in the database before streaming changes. In order to use this the tables that are being snapshot MUST have a primary key set so that reading from the table can be parallelized.","default":false,"examples":[true]},{"name":"snapshot_memory_safety_factor","type":"float","kind":"scalar","description":"Determines the fraction of available memory that can be used for streaming the snapshot. Values between 0 and 1 represent the percentage of memory to use. Lower values make initial streaming slower but help prevent out-of-memory errors.","is_deprecated":true,"default":1,"examples":[0.2]},{"name":"snapshot_batch_size","type":"int","kind":"scalar","description":"The number of rows to fetch in each batch when querying the snapshot.","default":1000,"examples":[10000]},{"name":"schema","type":"string","kind":"scalar","description":"The PostgreSQL schema from which to replicate data.","examples":["public","\"MyCaseSensitiveSchemaNeedingQuotes\""]},{"name":"tables","type":"string","kind":"array","description":"A list of table names to include in the logical replication. Each table should be specified as a separate item.","examples":[["my_table_1","\"MyCaseSensitiveTableNeedingQuotes\""]]},{"name":"checkpoint_limit","type":"int","kind":"scalar","description":"The maximum number of messages that can be processed at a given time. Increasing this limit enables parallel processing and batching at the output level. Any given LSN will not be acknowledged unless all messages under that offset are delivered in order to preserve at least once delivery guarantees.","default":1024},{"name":"temporary_slot","type":"bool","kind":"scalar","description":"If set to true, creates a temporary replication slot that is automatically dropped when the connection is closed.","default":false},{"name":"slot_name","type":"string","kind":"scalar","description":"The name of the PostgreSQL logical replication slot to use. If not provided, a random name will be generated. You can create this slot manually before starting replication if desired.","examples":["my_test_slot"]},{"name":"pg_standby_timeout","type":"string","kind":"scalar","description":"Specify the standby timeout before refreshing an idle connection.","default":"10s","examples":["30s"]},{"name":"pg_wal_monitor_interval","type":"string","kind":"scalar","description":"How often to report changes to the replication lag.","default":"3s","examples":["6s"]},{"name":"max_parallel_snapshot_tables","type":"int","kind":"scalar","description":"Int specifies a number of tables that will be processed in parallel during the snapshot processing stage","default":1},{"name":"unchanged_toast_value","type":"unknown","kind":"scalar","description":"The value to emit when there are unchanged TOAST values in the stream. This occurs for updates and deletes where REPLICA IDENTITY is not FULL.","is_advanced":true,"default":null,"examples":["__redpanda_connect_unchanged_toast_value__"]},{"name":"heartbeat_interval","type":"string","kind":"scalar","description":"The interval at which to write heartbeat messages. Heartbeat messages are needed in scenarios when the subscribed tables are low frequency, but there are other high frequency tables writing. Due to the checkpointing mechanism for replication slots, not having new messages to acknowledge will prevent postgres from reclaiming the write ahead log, which can exhaust the local disk. Having heartbeats allows Redpanda Connect to safely acknowledge data periodically and move forward the committed point in the log so it can be reclaimed. Setting the duration to 0s will disable heartbeats entirely. Heartbeats are created by periodically writing logical messages to the write ahead log using `pg_logical_emit_message`.","is_advanced":true,"default":"1h","examples":["0s","24h"]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]},"version":"4.39.0"},{"name":"postgres_cdc","type":"input","status":"beta","plugin":true,"summary":"Streams changes from a PostgreSQL database using logical replication.","description":"Streams changes from a PostgreSQL database for Change Data Capture (CDC).\nAdditionally, if `stream_snapshot` is set to true, then the existing data in the database is also streamed too.\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n- table (Name of the table that the message originated from)\n- operation (Type of operation that generated the message: \"read\", \"insert\", \"update\", or \"delete\". \"read\" is from messages that are read in the initial snapshot phase. This will also be \"begin\" and \"commit\" if `include_transaction_markers` is enabled)\n- lsn (the log sequence number in postgres)\n\t\t","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"dsn","type":"string","kind":"scalar","description":"The Data Source Name for the PostgreSQL database in the form of `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]`. Please note that Postgres enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.","examples":["postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable"]},{"name":"include_transaction_markers","type":"bool","kind":"scalar","description":"When set to true, empty messages with operation types BEGIN and COMMIT are generated for the beginning and end of each transaction. Messages with operation metadata set to \"begin\" or \"commit\" will have null message payloads.","default":false},{"name":"stream_snapshot","type":"bool","kind":"scalar","description":"When set to true, the plugin will first stream a snapshot of all existing data in the database before streaming changes. In order to use this the tables that are being snapshot MUST have a primary key set so that reading from the table can be parallelized.","default":false,"examples":[true]},{"name":"snapshot_memory_safety_factor","type":"float","kind":"scalar","description":"Determines the fraction of available memory that can be used for streaming the snapshot. Values between 0 and 1 represent the percentage of memory to use. Lower values make initial streaming slower but help prevent out-of-memory errors.","is_deprecated":true,"default":1,"examples":[0.2]},{"name":"snapshot_batch_size","type":"int","kind":"scalar","description":"The number of rows to fetch in each batch when querying the snapshot.","default":1000,"examples":[10000]},{"name":"schema","type":"string","kind":"scalar","description":"The PostgreSQL schema from which to replicate data.","examples":["public","\"MyCaseSensitiveSchemaNeedingQuotes\""]},{"name":"tables","type":"string","kind":"array","description":"A list of table names to include in the logical replication. Each table should be specified as a separate item.","examples":[["my_table_1","\"MyCaseSensitiveTableNeedingQuotes\""]]},{"name":"checkpoint_limit","type":"int","kind":"scalar","description":"The maximum number of messages that can be processed at a given time. Increasing this limit enables parallel processing and batching at the output level. Any given LSN will not be acknowledged unless all messages under that offset are delivered in order to preserve at least once delivery guarantees.","default":1024},{"name":"temporary_slot","type":"bool","kind":"scalar","description":"If set to true, creates a temporary replication slot that is automatically dropped when the connection is closed.","default":false},{"name":"slot_name","type":"string","kind":"scalar","description":"The name of the PostgreSQL logical replication slot to use. If not provided, a random name will be generated. You can create this slot manually before starting replication if desired.","examples":["my_test_slot"]},{"name":"pg_standby_timeout","type":"string","kind":"scalar","description":"Specify the standby timeout before refreshing an idle connection.","default":"10s","examples":["30s"]},{"name":"pg_wal_monitor_interval","type":"string","kind":"scalar","description":"How often to report changes to the replication lag.","default":"3s","examples":["6s"]},{"name":"max_parallel_snapshot_tables","type":"int","kind":"scalar","description":"Int specifies a number of tables that will be processed in parallel during the snapshot processing stage","default":1},{"name":"unchanged_toast_value","type":"unknown","kind":"scalar","description":"The value to emit when there are unchanged TOAST values in the stream. This occurs for updates and deletes where REPLICA IDENTITY is not FULL.","is_advanced":true,"default":null,"examples":["__redpanda_connect_unchanged_toast_value__"]},{"name":"heartbeat_interval","type":"string","kind":"scalar","description":"The interval at which to write heartbeat messages. Heartbeat messages are needed in scenarios when the subscribed tables are low frequency, but there are other high frequency tables writing. Due to the checkpointing mechanism for replication slots, not having new messages to acknowledge will prevent postgres from reclaiming the write ahead log, which can exhaust the local disk. Having heartbeats allows Redpanda Connect to safely acknowledge data periodically and move forward the committed point in the log so it can be reclaimed. Setting the duration to 0s will disable heartbeats entirely. Heartbeats are created by periodically writing logical messages to the write ahead log using `pg_logical_emit_message`.","is_advanced":true,"default":"1h","examples":["0s","24h"]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]},"version":"4.39.0"},{"name":"pulsar","type":"input","status":"experimental","plugin":true,"summary":"Reads messages from an Apache Pulsar server.","description":"\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- pulsar_message_id\n- pulsar_key\n- pulsar_ordering_key\n- pulsar_event_time_unix\n- pulsar_publish_time_unix\n- pulsar_topic\n- pulsar_producer_name\n- pulsar_redelivery_count\n- All properties of the message\n```\n\nYou can access these metadata fields using\nxref:configuration:interpolation.adoc#bloblang-queries[function interpolation].\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"A URL to connect to.","examples":["pulsar://localhost:6650","pulsar://pulsar.us-west.example.com:6650","pulsar+ssl://pulsar.us-west.example.com:6651"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"topics","type":"string","kind":"array","description":"A list of topics to subscribe to. This or topics_pattern must be set.","is_optional":true},{"name":"topics_pattern","type":"string","kind":"scalar","description":"A regular expression matching the topics to subscribe to. This or topics must be set.","is_optional":true},{"name":"subscription_name","type":"string","kind":"scalar","description":"Specify the subscription name for this consumer."},{"name":"subscription_type","type":"string","kind":"scalar","description":"Specify the subscription type for this consumer.\n\n\u003e NOTE: Using a `key_shared` subscription type will __allow out-of-order delivery__ since nack-ing messages sets non-zero nack delivery delay - this can potentially cause consumers to stall. See https://pulsar.apache.org/docs/en/2.8.1/concepts-messaging/#negative-acknowledgement[Pulsar documentation^] and https://github.com/apache/pulsar/issues/12208[this Github issue^] for more details.","default":"shared","options":["shared","key_shared","failover","exclusive"],"linter":"\nlet options = {\n \"shared\": true,\n \"key_shared\": true,\n \"failover\": true,\n \"exclusive\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"subscription_initial_position","type":"string","kind":"scalar","description":"Specify the subscription initial position for this consumer.","default":"latest","options":["latest","earliest"],"linter":"\nlet options = {\n \"latest\": true,\n \"earliest\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"tls","type":"object","kind":"scalar","description":"Specify the path to a custom CA certificate to trust broker TLS service.","children":[{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","default":"","examples":["./root_cas.pem"]}]},{"name":"auth","type":"object","kind":"scalar","description":"Optional configuration of Pulsar authentication methods.","is_advanced":true,"is_optional":true,"children":[{"name":"oauth2","type":"object","kind":"scalar","description":"Parameters for Pulsar OAuth2 authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether OAuth2 is enabled.","is_advanced":true,"default":false},{"name":"audience","type":"string","kind":"scalar","description":"OAuth2 audience.","is_advanced":true,"default":""},{"name":"issuer_url","type":"string","kind":"scalar","description":"OAuth2 issuer URL.","is_advanced":true,"default":"","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"scope","type":"string","kind":"scalar","description":"OAuth2 scope to request.","is_advanced":true,"default":"","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"private_key_file","type":"string","kind":"scalar","description":"The path to a file containing a private key.","is_advanced":true,"default":""}]},{"name":"token","type":"object","kind":"scalar","description":"Parameters for Pulsar Token authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether Token Auth is enabled.","is_advanced":true,"default":false},{"name":"token","type":"string","kind":"scalar","description":"Actual base64 encoded token.","is_advanced":true,"default":""}]}],"version":"3.60.0"}]},"version":"3.43.0"},{"name":"read_until","type":"input","status":"stable","plugin":true,"summary":"Reads messages from a child input until a consumed message passes a xref:guides:bloblang/about.adoc[Bloblang query], at which point the input closes. It is also possible to configure a timeout after which the input is closed if no new messages arrive in that period.","description":"\nMessages are read continuously while the query check returns false, when the query returns true the message that triggered the check is sent out and the input is closed. Use this to define inputs where the stream should end once a certain message appears.\n\nIf the idle timeout is configured, the input will be closed if no new messages arrive after that period of time. Use this field if you want to empty out and close an input that doesn't have a logical end.\n\nSometimes inputs close themselves. For example, when the `file` input type reaches the end of a file it will shut down. By default this type will also shut down. If you wish for the input type to be restarted every time it shuts down until the query check is met then set `restart_input` to `true`.\n\n== Metadata\n\nA metadata key `benthos_read_until` containing the value `final` is added to the first part of the message that triggers the input to stop.","categories":["Utility"],"examples":[{"title":"Consume N Messages","summary":"A common reason to use this input is to consume only N messages from an input and then stop. This can easily be done with the xref:guides:bloblang/functions.adoc#count[`count` function]:","config":"\n# Only read 100 messages, and then exit.\ninput:\n read_until:\n check: count(\"messages\") \u003e= 100\n input:\n kafka:\n addresses: [ TODO ]\n topics: [ foo, bar ]\n consumer_group: foogroup\n"},{"title":"Read from a kafka and close when empty","summary":"A common reason to use this input is a job that consumes all messages and exits once its empty:","config":"\n# Consumes all messages and exit when the last message was consumed 5s ago.\ninput:\n read_until:\n idle_timeout: 5s\n input:\n kafka:\n addresses: [ TODO ]\n topics: [ foo, bar ]\n consumer_group: foogroup\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"input","type":"input","kind":"scalar","description":"The child input to consume from."},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether the input should now be closed.","is_optional":true,"bloblang":true,"examples":["this.type == \"foo\"","count(\"messages\") \u003e= 100"]},{"name":"idle_timeout","type":"string","kind":"scalar","description":"The maximum amount of time without receiving new messages after which the input is closed.","is_optional":true,"examples":["5s"]},{"name":"restart_input","type":"bool","kind":"scalar","description":"Whether the input should be reopened if it closes itself before the condition has resolved to true.","default":false}]}},{"name":"redis_list","type":"input","status":"stable","plugin":true,"summary":"Pops messages from the beginning of a Redis list using the BLPop command.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target Redis server. Database is optional and is supplied as the URL path.","examples":["redis://:6379","redis://localhost:6379","redis://foousername:foopassword@redisplace:6379","redis://:foopassword@redisplace:6379","redis://localhost:6379/1","redis://localhost:6379/1,redis://localhost:6380/1"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"kind","type":"string","kind":"scalar","description":"Specifies a simple, cluster-aware, or failover-aware redis client.","is_advanced":true,"default":"simple","options":["simple","cluster","failover"],"linter":"\nlet options = {\n \"simple\": true,\n \"cluster\": true,\n \"failover\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"master","type":"string","kind":"scalar","description":"Name of the redis master when `kind` is `failover`","is_advanced":true,"default":"","examples":["mymaster"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"key","type":"string","kind":"scalar","description":"The key of a list to read from."},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"max_in_flight","type":"int","kind":"scalar","description":"Optionally sets a limit on the number of messages that can be flowing through a Redpanda Connect stream pending acknowledgment from the input at any given time. Once a message has been either acknowledged or rejected (nacked) it is no longer considered pending. If the input produces logical batches then each batch is considered a single count against the maximum. **WARNING**: Batching policies at the output level will stall if this field limits the number of messages below the batching threshold. Zero (default) or lower implies no limit.","is_advanced":true,"default":0,"version":"4.9.0"},{"name":"timeout","type":"string","kind":"scalar","description":"The length of time to poll for new messages before reattempting.","is_advanced":true,"default":"5s"},{"name":"command","type":"string","kind":"scalar","description":"The command used to pop elements from the Redis list","is_advanced":true,"default":"blpop","options":["blpop","brpop"],"version":"4.22.0","linter":"\nlet options = {\n \"blpop\": true,\n \"brpop\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"}]}},{"name":"redis_pubsub","type":"input","status":"stable","plugin":true,"summary":"Consume from a Redis publish/subscribe channel using either the SUBSCRIBE or PSUBSCRIBE commands.","description":"\nIn order to subscribe to channels using the `PSUBSCRIBE` command set the field `use_patterns` to `true`, then you can include glob-style patterns in your channel names. For example:\n\n- `h?llo` subscribes to hello, hallo and hxllo\n- `h*llo` subscribes to hllo and heeeello\n- `h[ae]llo` subscribes to hello and hallo, but not hillo\n\nUse `\\` to escape special characters if you want to match them verbatim.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target Redis server. Database is optional and is supplied as the URL path.","examples":["redis://:6379","redis://localhost:6379","redis://foousername:foopassword@redisplace:6379","redis://:foopassword@redisplace:6379","redis://localhost:6379/1","redis://localhost:6379/1,redis://localhost:6380/1"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"kind","type":"string","kind":"scalar","description":"Specifies a simple, cluster-aware, or failover-aware redis client.","is_advanced":true,"default":"simple","options":["simple","cluster","failover"],"linter":"\nlet options = {\n \"simple\": true,\n \"cluster\": true,\n \"failover\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"master","type":"string","kind":"scalar","description":"Name of the redis master when `kind` is `failover`","is_advanced":true,"default":"","examples":["mymaster"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"channels","type":"string","kind":"array","description":"A list of channels to consume from."},{"name":"use_patterns","type":"bool","kind":"scalar","description":"Whether to use the PSUBSCRIBE command, allowing for glob-style patterns within target channel names.","default":false},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]}},{"name":"redis_scan","type":"input","status":"experimental","plugin":true,"summary":"Scans the set of keys in the current selected database and gets their values, using the Scan and Get commands.","description":"Optionally, iterates only elements matching a blob-style pattern. For example:\n\n- `*foo*` iterates only keys which contain `foo` in it.\n- `foo*` iterates only keys starting with `foo`.\n\nThis input generates a message for each key value pair in the following format:\n\n```json\n{\"key\":\"foo\",\"value\":\"bar\"}\n```\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target Redis server. Database is optional and is supplied as the URL path.","examples":["redis://:6379","redis://localhost:6379","redis://foousername:foopassword@redisplace:6379","redis://:foopassword@redisplace:6379","redis://localhost:6379/1","redis://localhost:6379/1,redis://localhost:6380/1"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"kind","type":"string","kind":"scalar","description":"Specifies a simple, cluster-aware, or failover-aware redis client.","is_advanced":true,"default":"simple","options":["simple","cluster","failover"],"linter":"\nlet options = {\n \"simple\": true,\n \"cluster\": true,\n \"failover\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"master","type":"string","kind":"scalar","description":"Name of the redis master when `kind` is `failover`","is_advanced":true,"default":"","examples":["mymaster"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"match","type":"string","kind":"scalar","description":"Iterates only elements matching the optional glob-style pattern. By default, it matches all elements.","default":"","examples":["*","1*","foo*","foo","*4*"]}]},"version":"4.27.0"},{"name":"redis_streams","type":"input","status":"stable","plugin":true,"summary":"Pulls messages from Redis (v5.0+) streams with the XREADGROUP command. The `client_id` should be unique for each consumer of a group.","description":"Redis stream entries are key/value pairs, as such it is necessary to specify the key that contains the body of the message. All other keys/value pairs are saved as metadata fields.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target Redis server. Database is optional and is supplied as the URL path.","examples":["redis://:6379","redis://localhost:6379","redis://foousername:foopassword@redisplace:6379","redis://:foopassword@redisplace:6379","redis://localhost:6379/1","redis://localhost:6379/1,redis://localhost:6380/1"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"kind","type":"string","kind":"scalar","description":"Specifies a simple, cluster-aware, or failover-aware redis client.","is_advanced":true,"default":"simple","options":["simple","cluster","failover"],"linter":"\nlet options = {\n \"simple\": true,\n \"cluster\": true,\n \"failover\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"master","type":"string","kind":"scalar","description":"Name of the redis master when `kind` is `failover`","is_advanced":true,"default":"","examples":["mymaster"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"body_key","type":"string","kind":"scalar","description":"The field key to extract the raw message from. All other keys will be stored in the message as metadata.","default":"body"},{"name":"streams","type":"string","kind":"array","description":"A list of streams to consume from."},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"limit","type":"int","kind":"scalar","description":"The maximum number of messages to consume from a single request.","default":10},{"name":"client_id","type":"string","kind":"scalar","description":"An identifier for the client connection.","default":""},{"name":"consumer_group","type":"string","kind":"scalar","description":"An identifier for the consumer group of the stream.","default":""},{"name":"create_streams","type":"bool","kind":"scalar","description":"Create subscribed streams if they do not exist (MKSTREAM option).","is_advanced":true,"default":true},{"name":"start_from_oldest","type":"bool","kind":"scalar","description":"If an offset is not found for a stream, determines whether to consume from the oldest available offset, otherwise messages are consumed from the latest offset.","is_advanced":true,"default":true},{"name":"commit_period","type":"string","kind":"scalar","description":"The period of time between each commit of the current offset. Offsets are always committed during shutdown.","is_advanced":true,"default":"1s"},{"name":"timeout","type":"string","kind":"scalar","description":"The length of time to poll for new messages before reattempting.","is_advanced":true,"default":"1s"}]}},{"name":"redpanda","type":"input","status":"beta","plugin":true,"summary":"A Kafka input using the https://github.com/twmb/franz-go[Franz Kafka client library^].","description":"\nWhen a consumer group is specified this input consumes one or more topics where partitions will automatically balance across any other connected clients with the same consumer group. When a consumer group is not specified topics can either be consumed in their entirety or with explicit partitions.\n\n== Delivery Guarantees\n\nWhen using consumer groups the offsets of \"delivered\" records will be committed automatically and continuously, and in the event of restarts these committed offsets will be used in order to resume from where the input left off. Redpanda Connect guarantees at least once delivery by ensuring that records are only considerd to be delivered when all configured outputs that the record is routed to have confirmed delivery.\n\n== Ordering\n\nIn order to preserve ordering of topic partitions, records consumed from each partition are processed and delivered in the order that they are received, and only one batch of records of a given partition will ever be processed at a time. This means that parallel processing can only occur when multiple topic partitions are being consumed, but ensures that data is processed in a sequential order as determined from the source partition.\n\nHowever, one way in which the order of records can be mixed is when delivery errors occur and error handling mechanisms kick in. Redpanda Connect always leans towards at least once delivery unless instructed otherwise, and this includes reattempting delivery of data when the ordering of that data can no longer be guaranteed.\n\nFor example, a batch of records may have been sent to an output broker and only a subset of records were delivered, in this case Redpanda Connect by default will reattempt to deliver the records that failed, even though these failed records may have come before records that were previously delivered successfully.\n\nIn order to avoid this scenario you must specify in your configuration an alternative way to handle delivery errors in the form of a xref:components:outputs/fallback.adoc[`fallback`] output. It is good practice to also disable the field `auto_retry_nacks` by setting it to `false` when you've added an explicit fallback output as this will improve the throughput of your pipeline. For example, the following config avoids ordering issues by specifying a fallback output into a DLQ topic, which is also retried indefinitely as a way to apply back pressure during connectivity issues:\n\n```yaml\noutput:\n fallback:\n - redpanda:\n seed_brokers: [ localhost:9092 ]\n topic: foo\n - retry:\n output:\n redpanda:\n seed_brokers: [ localhost:9092 ]\n topic: foo_dlq\n```\n\n== Batching\n\nRecords are processed and delivered from each partition in batches as received from brokers. These batch sizes are therefore dynamically sized in order to optimise throughput, but can be tuned with the config fields `fetch_max_partition_bytes` and `fetch_max_bytes`. Batches can be further broken down using the xref:components:processors/split.adoc[`split`] processor.\n\n== Metrics\n\nEmits a `redpanda_lag` metric with `topic` and `partition` labels for each consumed topic.\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- kafka_key\n- kafka_topic\n- kafka_partition\n- kafka_offset\n- kafka_lag\n- kafka_timestamp_ms\n- kafka_timestamp_unix\n- kafka_tombstone_message\n- All record headers\n```\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"seed_brokers","type":"string","kind":"array","description":"A list of broker addresses to connect to in order to establish connections. If an item of the list contains commas it will be expanded into multiple addresses.","examples":[["localhost:9092"],["foo:9092","bar:9092"],["foo:9092,bar:9092"]]},{"name":"client_id","type":"string","kind":"scalar","description":"An identifier for the client connection.","is_advanced":true,"default":"benthos"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"sasl","type":"object","kind":"array","description":"Specify one or more methods of SASL authentication. SASL is tried in order; if the broker supports the first mechanism, all connections will use that mechanism. If the first mechanism fails, the client will pick the first supported mechanism. If the broker does not support any client mechanisms, connections will fail.","is_advanced":true,"is_optional":true,"examples":[[{"mechanism":"SCRAM-SHA-512","password":"bar","username":"foo"}]],"children":[{"name":"mechanism","type":"string","kind":"scalar","description":"The SASL mechanism to use.","is_advanced":true,"annotated_options":[["AWS_MSK_IAM","AWS IAM based authentication as specified by the 'aws-msk-iam-auth' java library."],["OAUTHBEARER","OAuth Bearer based authentication."],["PLAIN","Plain text authentication."],["SCRAM-SHA-256","SCRAM based authentication as specified in RFC5802."],["SCRAM-SHA-512","SCRAM based authentication as specified in RFC5802."],["none","Disable sasl authentication"]],"linter":"\nlet options = {\n \"aws_msk_iam\": true,\n \"oauthbearer\": true,\n \"plain\": true,\n \"scram-sha-256\": true,\n \"scram-sha-512\": true,\n \"none\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"username","type":"string","kind":"scalar","description":"A username to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token to use for a single session's OAUTHBEARER authentication.","is_advanced":true,"default":""},{"name":"extensions","type":"string","kind":"map","description":"Key/value pairs to add to OAUTHBEARER authentication requests.","is_advanced":true,"is_optional":true},{"name":"aws","type":"object","kind":"scalar","description":"Contains AWS specific fields for when the `mechanism` is set to `AWS_MSK_IAM`.","is_advanced":true,"is_optional":true,"children":[{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]}]},{"name":"metadata_max_age","type":"string","kind":"scalar","description":"The maximum age of metadata before it is refreshed.","is_advanced":true,"default":"5m"},{"name":"request_timeout_overhead","type":"string","kind":"scalar","description":"The request time overhead. Uses the given time as overhead while deadlining requests. Roughly equivalent to request.timeout.ms, but grants additional time to requests that have timeout fields.","is_advanced":true,"default":"10s"},{"name":"conn_idle_timeout","type":"string","kind":"scalar","description":"The rough amount of time to allow connections to idle before they are closed.","is_advanced":true,"default":"20s"},{"name":"topics","type":"string","kind":"array","description":"\nA list of topics to consume from. Multiple comma separated topics can be listed in a single element. When a `consumer_group` is specified partitions are automatically distributed across consumers of a topic, otherwise all partitions are consumed.\n\nAlternatively, it's possible to specify explicit partitions to consume from with a colon after the topic name, e.g. `foo:0` would consume the partition 0 of the topic foo. This syntax supports ranges, e.g. `foo:0-10` would consume partitions 0 through to 10 inclusive.\n\nFinally, it's also possible to specify an explicit offset to consume from by adding another colon after the partition, e.g. `foo:0:10` would consume the partition 0 of the topic foo starting from the offset 10. If the offset is not present (or remains unspecified) then the field `start_from_oldest` determines which offset to start from.","examples":[["foo","bar"],["things.*"],["foo,bar"],["foo:0","bar:1","bar:3"],["foo:0,bar:1,bar:3"],["foo:0-5"]]},{"name":"regexp_topics","type":"bool","kind":"scalar","description":"Whether listed topics should be interpreted as regular expression patterns for matching multiple topics. When topics are specified with explicit partitions this field must remain set to `false`.","default":false},{"name":"rack_id","type":"string","kind":"scalar","description":"A rack specifies where the client is physically located and changes fetch requests to consume from the closest replica as opposed to the leader replica.","is_advanced":true,"default":""},{"name":"instance_id","type":"string","kind":"scalar","description":"When using a consumer group, an instance ID specifies the groups static membership, which can prevent rebalances during reconnects. When using a instance ID the client does NOT leave the group when closing. To actually leave the group one must use an external admin command to leave the group on behalf of this instance ID. This ID must be unique per consumer within the group.","is_advanced":true,"default":""},{"name":"rebalance_timeout","type":"string","kind":"scalar","description":"When using a consumer group, `rebalance_timeout` sets how long group members are allowed to take when a rebalance has begun. This timeout is how long all members are allowed to complete work and commit offsets, minus the time it took to detect the rebalance (from a heartbeat).","is_advanced":true,"default":"45s"},{"name":"session_timeout","type":"string","kind":"scalar","description":"When using a consumer group, `session_timeout` sets how long a member in hte group can go between heartbeats. If a member does not heartbeat in this timeout, the broker will remove the member from the group and initiate a rebalance.","is_advanced":true,"default":"1m"},{"name":"heartbeat_interval","type":"string","kind":"scalar","description":"When using a consumer group, `heartbeat_interval` sets how long a group member goes between heartbeats to Kafka. Kafka uses heartbeats to ensure that a group member's sesion stays active. This value should be no higher than 1/3rd of the `session_timeout`. This is equivalent to the Java heartbeat.interval.ms setting.","is_advanced":true,"default":"3s"},{"name":"start_from_oldest","type":"bool","kind":"scalar","description":"Determines whether to consume from the oldest available offset, otherwise messages are consumed from the latest offset. The setting is applied when creating a new consumer group or the saved offset no longer exists.","is_advanced":true,"is_deprecated":true,"default":true},{"name":"start_offset","type":"string","kind":"scalar","description":"Sets the offset to start consuming from, or if OffsetOutOfRange is seen while fetching, to restart consuming from.","is_advanced":true,"default":"earliest","annotated_options":[["committed","Prevents consuming a partition in a group if the partition has no prior commits. Corresponds to Kafka's `auto.offset.reset=none` option"],["earliest","Start from the earliest offset. Corresponds to Kafka's `auto.offset.reset=earliest` option."],["latest","Start from the latest offset. Corresponds to Kafka's `auto.offset.reset=latest` option."]],"linter":"\nlet options = {\n \"committed\": true,\n \"earliest\": true,\n \"latest\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"fetch_max_bytes","type":"string","kind":"scalar","description":"Sets the maximum amount of bytes a broker will try to send during a fetch. Note that brokers may not obey this limit if it has records larger than this limit. This is the equivalent to the Java fetch.max.bytes setting.","is_advanced":true,"default":"50MiB"},{"name":"fetch_max_wait","type":"string","kind":"scalar","description":"Sets the maximum amount of time a broker will wait for a fetch response to hit the minimum number of required bytes. This is the equivalent to the Java fetch.max.wait.ms setting.","is_advanced":true,"default":"5s"},{"name":"fetch_min_bytes","type":"string","kind":"scalar","description":"Sets the minimum amount of bytes a broker will try to send during a fetch. This is the equivalent to the Java fetch.min.bytes setting.","is_advanced":true,"default":"1B"},{"name":"fetch_max_partition_bytes","type":"string","kind":"scalar","description":"Sets the maximum amount of bytes that will be consumed for a single partition in a fetch request. Note that if a single batch is larger than this number, that batch will still be returned so the client can make progress. This is the equivalent to the Java fetch.max.partition.bytes setting.","is_advanced":true,"default":"1MiB"},{"name":"transaction_isolation_level","type":"string","kind":"scalar","description":"The transaction isolation level","default":"read_uncommitted","annotated_options":[["read_committed","If set, only committed transactional records are processed."],["read_uncommitted","If set, then uncommitted records are processed."]],"linter":"\nlet options = {\n \"read_committed\": true,\n \"read_uncommitted\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"consumer_group","type":"string","kind":"scalar","description":"An optional consumer group to consume as. When specified the partitions of specified topics are automatically distributed across consumers sharing a consumer group, and partition offsets are automatically committed and resumed under this name. Consumer groups are not supported when specifying explicit partitions to consume from in the `topics` field.","is_optional":true},{"name":"commit_period","type":"string","kind":"scalar","description":"The period of time between each commit of the current partition offsets. Offsets are always committed during shutdown.","is_advanced":true,"default":"5s"},{"name":"partition_buffer_bytes","type":"string","kind":"scalar","description":"A buffer size (in bytes) for each consumed partition, allowing records to be queued internally before flushing. Increasing this may improve throughput at the cost of higher memory utilisation. Note that each buffer can grow slightly beyond this value.","is_advanced":true,"default":"1MB"},{"name":"topic_lag_refresh_period","type":"string","kind":"scalar","description":"The period of time between each topic lag refresh cycle.","is_advanced":true,"default":"5s"},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}],"linter":"\nlet has_topic_partitions = this.topics.any(t -\u003e t.contains(\":\"))\n\nroot = [\n if $has_topic_partitions {\n if this.consumer_group.or(\"\") != \"\" {\n \"this input does not support both a consumer group and explicit topic partitions\"\n } else if this.regexp_topics {\n \"this input does not support both regular expression topics and explicit topic partitions\"\n }\n } else {\n if this.consumer_group.or(\"\") == \"\" {\n \"a consumer group is mandatory when not using explicit topic partitions\"\n }\n },\n # We don't have any way to distinguish between start_from_oldest set explicitly to true and not set at all, so we\n # assume users will be OK if start_offset overwrites it silently\n if this.start_from_oldest == false \u0026\u0026 this.start_offset == \"earliest\" {\n \"start_from_oldest cannot be set to false when start_offset is set to earliest\"\n }\n]\n"}},{"name":"redpanda_common","type":"input","status":"beta","plugin":true,"summary":"Consumes data from a Redpanda (Kafka) broker, using credentials defined in a common top-level `redpanda` config block.","description":"\nWhen a consumer group is specified this input consumes one or more topics where partitions will automatically balance across any other connected clients with the same consumer group. When a consumer group is not specified topics can either be consumed in their entirety or with explicit partitions.\n\n== Delivery Guarantees\n\nWhen using consumer groups the offsets of \"delivered\" records will be committed automatically and continuously, and in the event of restarts these committed offsets will be used in order to resume from where the input left off. Redpanda Connect guarantees at least once delivery by ensuring that records are only considerd to be delivered when all configured outputs that the record is routed to have confirmed delivery.\n\n== Ordering\n\nIn order to preserve ordering of topic partitions, records consumed from each partition are processed and delivered in the order that they are received, and only one batch of records of a given partition will ever be processed at a time. This means that parallel processing can only occur when multiple topic partitions are being consumed, but ensures that data is processed in a sequential order as determined from the source partition.\n\nHowever, one way in which the order of records can be mixed is when delivery errors occur and error handling mechanisms kick in. Redpanda Connect always leans towards at least once delivery unless instructed otherwise, and this includes reattempting delivery of data when the ordering of that data can no longer be guaranteed.\n\nFor example, a batch of records may have been sent to an output broker and only a subset of records were delivered, in this case Redpanda Connect by default will reattempt to deliver the records that failed, even though these failed records may have come before records that were previously delivered successfully.\n\nIn order to avoid this scenario you must specify in your configuration an alternative way to handle delivery errors in the form of a xref:components:outputs/fallback.adoc[`fallback`] output. It is good practice to also disable the field `auto_retry_nacks` by setting it to `false` when you've added an explicit fallback output as this will improve the throughput of your pipeline. For example, the following config avoids ordering issues by specifying a fallback output into a DLQ topic, which is also retried indefinitely as a way to apply back pressure during connectivity issues:\n\n```yaml\noutput:\n fallback:\n - redpanda_common:\n topic: foo\n - retry:\n output:\n redpanda_common:\n topic: foo_dlq\n```\n\n== Batching\n\nRecords are processed and delivered from each partition in batches as received from brokers. These batch sizes are therefore dynamically sized in order to optimise throughput, but can be tuned with the config fields `fetch_max_partition_bytes` and `fetch_max_bytes`. Batches can be further broken down using the xref:components:processors/split.adoc[`split`] processor.\n\n== Metrics\n\nEmits a `redpanda_lag` metric with `topic` and `partition` labels for each consumed topic.\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- kafka_key\n- kafka_topic\n- kafka_partition\n- kafka_offset\n- kafka_lag\n- kafka_timestamp_ms\n- kafka_timestamp_unix\n- kafka_tombstone_message\n- All record headers\n```\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"topics","type":"string","kind":"array","description":"\nA list of topics to consume from. Multiple comma separated topics can be listed in a single element. When a `consumer_group` is specified partitions are automatically distributed across consumers of a topic, otherwise all partitions are consumed.\n\nAlternatively, it's possible to specify explicit partitions to consume from with a colon after the topic name, e.g. `foo:0` would consume the partition 0 of the topic foo. This syntax supports ranges, e.g. `foo:0-10` would consume partitions 0 through to 10 inclusive.\n\nFinally, it's also possible to specify an explicit offset to consume from by adding another colon after the partition, e.g. `foo:0:10` would consume the partition 0 of the topic foo starting from the offset 10. If the offset is not present (or remains unspecified) then the field `start_from_oldest` determines which offset to start from.","examples":[["foo","bar"],["things.*"],["foo,bar"],["foo:0","bar:1","bar:3"],["foo:0,bar:1,bar:3"],["foo:0-5"]]},{"name":"regexp_topics","type":"bool","kind":"scalar","description":"Whether listed topics should be interpreted as regular expression patterns for matching multiple topics. When topics are specified with explicit partitions this field must remain set to `false`.","default":false},{"name":"rack_id","type":"string","kind":"scalar","description":"A rack specifies where the client is physically located and changes fetch requests to consume from the closest replica as opposed to the leader replica.","is_advanced":true,"default":""},{"name":"instance_id","type":"string","kind":"scalar","description":"When using a consumer group, an instance ID specifies the groups static membership, which can prevent rebalances during reconnects. When using a instance ID the client does NOT leave the group when closing. To actually leave the group one must use an external admin command to leave the group on behalf of this instance ID. This ID must be unique per consumer within the group.","is_advanced":true,"default":""},{"name":"rebalance_timeout","type":"string","kind":"scalar","description":"When using a consumer group, `rebalance_timeout` sets how long group members are allowed to take when a rebalance has begun. This timeout is how long all members are allowed to complete work and commit offsets, minus the time it took to detect the rebalance (from a heartbeat).","is_advanced":true,"default":"45s"},{"name":"session_timeout","type":"string","kind":"scalar","description":"When using a consumer group, `session_timeout` sets how long a member in hte group can go between heartbeats. If a member does not heartbeat in this timeout, the broker will remove the member from the group and initiate a rebalance.","is_advanced":true,"default":"1m"},{"name":"heartbeat_interval","type":"string","kind":"scalar","description":"When using a consumer group, `heartbeat_interval` sets how long a group member goes between heartbeats to Kafka. Kafka uses heartbeats to ensure that a group member's sesion stays active. This value should be no higher than 1/3rd of the `session_timeout`. This is equivalent to the Java heartbeat.interval.ms setting.","is_advanced":true,"default":"3s"},{"name":"start_from_oldest","type":"bool","kind":"scalar","description":"Determines whether to consume from the oldest available offset, otherwise messages are consumed from the latest offset. The setting is applied when creating a new consumer group or the saved offset no longer exists.","is_advanced":true,"is_deprecated":true,"default":true},{"name":"start_offset","type":"string","kind":"scalar","description":"Sets the offset to start consuming from, or if OffsetOutOfRange is seen while fetching, to restart consuming from.","is_advanced":true,"default":"earliest","annotated_options":[["committed","Prevents consuming a partition in a group if the partition has no prior commits. Corresponds to Kafka's `auto.offset.reset=none` option"],["earliest","Start from the earliest offset. Corresponds to Kafka's `auto.offset.reset=earliest` option."],["latest","Start from the latest offset. Corresponds to Kafka's `auto.offset.reset=latest` option."]],"linter":"\nlet options = {\n \"committed\": true,\n \"earliest\": true,\n \"latest\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"fetch_max_bytes","type":"string","kind":"scalar","description":"Sets the maximum amount of bytes a broker will try to send during a fetch. Note that brokers may not obey this limit if it has records larger than this limit. This is the equivalent to the Java fetch.max.bytes setting.","is_advanced":true,"default":"50MiB"},{"name":"fetch_max_wait","type":"string","kind":"scalar","description":"Sets the maximum amount of time a broker will wait for a fetch response to hit the minimum number of required bytes. This is the equivalent to the Java fetch.max.wait.ms setting.","is_advanced":true,"default":"5s"},{"name":"fetch_min_bytes","type":"string","kind":"scalar","description":"Sets the minimum amount of bytes a broker will try to send during a fetch. This is the equivalent to the Java fetch.min.bytes setting.","is_advanced":true,"default":"1B"},{"name":"fetch_max_partition_bytes","type":"string","kind":"scalar","description":"Sets the maximum amount of bytes that will be consumed for a single partition in a fetch request. Note that if a single batch is larger than this number, that batch will still be returned so the client can make progress. This is the equivalent to the Java fetch.max.partition.bytes setting.","is_advanced":true,"default":"1MiB"},{"name":"transaction_isolation_level","type":"string","kind":"scalar","description":"The transaction isolation level","default":"read_uncommitted","annotated_options":[["read_committed","If set, only committed transactional records are processed."],["read_uncommitted","If set, then uncommitted records are processed."]],"linter":"\nlet options = {\n \"read_committed\": true,\n \"read_uncommitted\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"consumer_group","type":"string","kind":"scalar","description":"An optional consumer group to consume as. When specified the partitions of specified topics are automatically distributed across consumers sharing a consumer group, and partition offsets are automatically committed and resumed under this name. Consumer groups are not supported when specifying explicit partitions to consume from in the `topics` field.","is_optional":true},{"name":"commit_period","type":"string","kind":"scalar","description":"The period of time between each commit of the current partition offsets. Offsets are always committed during shutdown.","is_advanced":true,"default":"5s"},{"name":"partition_buffer_bytes","type":"string","kind":"scalar","description":"A buffer size (in bytes) for each consumed partition, allowing records to be queued internally before flushing. Increasing this may improve throughput at the cost of higher memory utilisation. Note that each buffer can grow slightly beyond this value.","is_advanced":true,"default":"1MB"},{"name":"topic_lag_refresh_period","type":"string","kind":"scalar","description":"The period of time between each topic lag refresh cycle.","is_advanced":true,"default":"5s"},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}],"linter":"\nlet has_topic_partitions = this.topics.any(t -\u003e t.contains(\":\"))\n\nroot = [\n if $has_topic_partitions {\n if this.consumer_group.or(\"\") != \"\" {\n \"this input does not support both a consumer group and explicit topic partitions\"\n } else if this.regexp_topics {\n \"this input does not support both regular expression topics and explicit topic partitions\"\n }\n } else {\n if this.consumer_group.or(\"\") == \"\" {\n \"a consumer group is mandatory when not using explicit topic partitions\"\n }\n },\n # We don't have any way to distinguish between start_from_oldest set explicitly to true and not set at all, so we\n # assume users will be OK if start_offset overwrites it silently\n if this.start_from_oldest == false \u0026\u0026 this.start_offset == \"earliest\" {\n \"start_from_oldest cannot be set to false when start_offset is set to earliest\"\n }\n]\n"}},{"name":"redpanda_migrator","type":"input","status":"beta","plugin":true,"summary":"A Redpanda Migrator input using the https://github.com/twmb/franz-go[Franz Kafka client library^].","description":"\nReads a batch of messages from a Kafka broker and waits for the output to acknowledge the writes before updating the Kafka consumer group offset.\n\nThis input should be used in combination with a `redpanda_migrator` output.\n\nWhen a consumer group is specified this input consumes one or more topics where partitions will automatically balance across any other connected clients with the same consumer group. When a consumer group is not specified topics can either be consumed in their entirety or with explicit partitions.\n\nIt provides the same delivery guarantees and ordering semantics as the `redpanda` input.\n\n== Metrics\n\nEmits a `input_redpanda_migrator_lag` metric with `topic` and `partition` labels for each consumed topic.\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- kafka_key\n- kafka_topic\n- kafka_partition\n- kafka_offset\n- kafka_lag\n- kafka_timestamp_ms\n- kafka_timestamp_unix\n- kafka_tombstone_message\n- All record headers\n```\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"seed_brokers","type":"string","kind":"array","description":"A list of broker addresses to connect to in order to establish connections. If an item of the list contains commas it will be expanded into multiple addresses.","examples":[["localhost:9092"],["foo:9092","bar:9092"],["foo:9092,bar:9092"]]},{"name":"client_id","type":"string","kind":"scalar","description":"An identifier for the client connection.","is_advanced":true,"default":"benthos"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"sasl","type":"object","kind":"array","description":"Specify one or more methods of SASL authentication. SASL is tried in order; if the broker supports the first mechanism, all connections will use that mechanism. If the first mechanism fails, the client will pick the first supported mechanism. If the broker does not support any client mechanisms, connections will fail.","is_advanced":true,"is_optional":true,"examples":[[{"mechanism":"SCRAM-SHA-512","password":"bar","username":"foo"}]],"children":[{"name":"mechanism","type":"string","kind":"scalar","description":"The SASL mechanism to use.","is_advanced":true,"annotated_options":[["AWS_MSK_IAM","AWS IAM based authentication as specified by the 'aws-msk-iam-auth' java library."],["OAUTHBEARER","OAuth Bearer based authentication."],["PLAIN","Plain text authentication."],["SCRAM-SHA-256","SCRAM based authentication as specified in RFC5802."],["SCRAM-SHA-512","SCRAM based authentication as specified in RFC5802."],["none","Disable sasl authentication"]],"linter":"\nlet options = {\n \"aws_msk_iam\": true,\n \"oauthbearer\": true,\n \"plain\": true,\n \"scram-sha-256\": true,\n \"scram-sha-512\": true,\n \"none\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"username","type":"string","kind":"scalar","description":"A username to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token to use for a single session's OAUTHBEARER authentication.","is_advanced":true,"default":""},{"name":"extensions","type":"string","kind":"map","description":"Key/value pairs to add to OAUTHBEARER authentication requests.","is_advanced":true,"is_optional":true},{"name":"aws","type":"object","kind":"scalar","description":"Contains AWS specific fields for when the `mechanism` is set to `AWS_MSK_IAM`.","is_advanced":true,"is_optional":true,"children":[{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]}]},{"name":"metadata_max_age","type":"string","kind":"scalar","description":"The maximum age of metadata before it is refreshed.","is_advanced":true,"default":"5m"},{"name":"request_timeout_overhead","type":"string","kind":"scalar","description":"The request time overhead. Uses the given time as overhead while deadlining requests. Roughly equivalent to request.timeout.ms, but grants additional time to requests that have timeout fields.","is_advanced":true,"default":"10s"},{"name":"conn_idle_timeout","type":"string","kind":"scalar","description":"The rough amount of time to allow connections to idle before they are closed.","is_advanced":true,"default":"20s"},{"name":"topics","type":"string","kind":"array","description":"\nA list of topics to consume from. Multiple comma separated topics can be listed in a single element. When a `consumer_group` is specified partitions are automatically distributed across consumers of a topic, otherwise all partitions are consumed.\n\nAlternatively, it's possible to specify explicit partitions to consume from with a colon after the topic name, e.g. `foo:0` would consume the partition 0 of the topic foo. This syntax supports ranges, e.g. `foo:0-10` would consume partitions 0 through to 10 inclusive.\n\nFinally, it's also possible to specify an explicit offset to consume from by adding another colon after the partition, e.g. `foo:0:10` would consume the partition 0 of the topic foo starting from the offset 10. If the offset is not present (or remains unspecified) then the field `start_from_oldest` determines which offset to start from.","examples":[["foo","bar"],["things.*"],["foo,bar"],["foo:0","bar:1","bar:3"],["foo:0,bar:1,bar:3"],["foo:0-5"]]},{"name":"regexp_topics","type":"bool","kind":"scalar","description":"Whether listed topics should be interpreted as regular expression patterns for matching multiple topics. When topics are specified with explicit partitions this field must remain set to `false`.","default":false},{"name":"rack_id","type":"string","kind":"scalar","description":"A rack specifies where the client is physically located and changes fetch requests to consume from the closest replica as opposed to the leader replica.","is_advanced":true,"default":""},{"name":"instance_id","type":"string","kind":"scalar","description":"When using a consumer group, an instance ID specifies the groups static membership, which can prevent rebalances during reconnects. When using a instance ID the client does NOT leave the group when closing. To actually leave the group one must use an external admin command to leave the group on behalf of this instance ID. This ID must be unique per consumer within the group.","is_advanced":true,"default":""},{"name":"rebalance_timeout","type":"string","kind":"scalar","description":"When using a consumer group, `rebalance_timeout` sets how long group members are allowed to take when a rebalance has begun. This timeout is how long all members are allowed to complete work and commit offsets, minus the time it took to detect the rebalance (from a heartbeat).","is_advanced":true,"default":"45s"},{"name":"session_timeout","type":"string","kind":"scalar","description":"When using a consumer group, `session_timeout` sets how long a member in hte group can go between heartbeats. If a member does not heartbeat in this timeout, the broker will remove the member from the group and initiate a rebalance.","is_advanced":true,"default":"1m"},{"name":"heartbeat_interval","type":"string","kind":"scalar","description":"When using a consumer group, `heartbeat_interval` sets how long a group member goes between heartbeats to Kafka. Kafka uses heartbeats to ensure that a group member's sesion stays active. This value should be no higher than 1/3rd of the `session_timeout`. This is equivalent to the Java heartbeat.interval.ms setting.","is_advanced":true,"default":"3s"},{"name":"start_from_oldest","type":"bool","kind":"scalar","description":"Determines whether to consume from the oldest available offset, otherwise messages are consumed from the latest offset. The setting is applied when creating a new consumer group or the saved offset no longer exists.","is_advanced":true,"is_deprecated":true,"default":true},{"name":"start_offset","type":"string","kind":"scalar","description":"Sets the offset to start consuming from, or if OffsetOutOfRange is seen while fetching, to restart consuming from.","is_advanced":true,"default":"earliest","annotated_options":[["committed","Prevents consuming a partition in a group if the partition has no prior commits. Corresponds to Kafka's `auto.offset.reset=none` option"],["earliest","Start from the earliest offset. Corresponds to Kafka's `auto.offset.reset=earliest` option."],["latest","Start from the latest offset. Corresponds to Kafka's `auto.offset.reset=latest` option."]],"linter":"\nlet options = {\n \"committed\": true,\n \"earliest\": true,\n \"latest\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"fetch_max_bytes","type":"string","kind":"scalar","description":"Sets the maximum amount of bytes a broker will try to send during a fetch. Note that brokers may not obey this limit if it has records larger than this limit. This is the equivalent to the Java fetch.max.bytes setting.","is_advanced":true,"default":"50MiB"},{"name":"fetch_max_wait","type":"string","kind":"scalar","description":"Sets the maximum amount of time a broker will wait for a fetch response to hit the minimum number of required bytes. This is the equivalent to the Java fetch.max.wait.ms setting.","is_advanced":true,"default":"5s"},{"name":"fetch_min_bytes","type":"string","kind":"scalar","description":"Sets the minimum amount of bytes a broker will try to send during a fetch. This is the equivalent to the Java fetch.min.bytes setting.","is_advanced":true,"default":"1B"},{"name":"fetch_max_partition_bytes","type":"string","kind":"scalar","description":"Sets the maximum amount of bytes that will be consumed for a single partition in a fetch request. Note that if a single batch is larger than this number, that batch will still be returned so the client can make progress. This is the equivalent to the Java fetch.max.partition.bytes setting.","is_advanced":true,"default":"1MiB"},{"name":"transaction_isolation_level","type":"string","kind":"scalar","description":"The transaction isolation level","default":"read_uncommitted","annotated_options":[["read_committed","If set, only committed transactional records are processed."],["read_uncommitted","If set, then uncommitted records are processed."]],"linter":"\nlet options = {\n \"read_committed\": true,\n \"read_uncommitted\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"consumer_group","type":"string","kind":"scalar","description":"An optional consumer group to consume as. When specified the partitions of specified topics are automatically distributed across consumers sharing a consumer group, and partition offsets are automatically committed and resumed under this name. Consumer groups are not supported when specifying explicit partitions to consume from in the `topics` field.","is_optional":true},{"name":"commit_period","type":"string","kind":"scalar","description":"The period of time between each commit of the current partition offsets. Offsets are always committed during shutdown.","is_advanced":true,"default":"5s"},{"name":"partition_buffer_bytes","type":"string","kind":"scalar","description":"A buffer size (in bytes) for each consumed partition, allowing records to be queued internally before flushing. Increasing this may improve throughput at the cost of higher memory utilisation. Note that each buffer can grow slightly beyond this value.","is_advanced":true,"default":"1MB"},{"name":"topic_lag_refresh_period","type":"string","kind":"scalar","description":"The period of time between each topic lag refresh cycle.","is_advanced":true,"default":"5s"},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"output_resource","type":"string","kind":"scalar","description":"The label of the redpanda_migrator output in which the currently selected topics need to be created before attempting to read messages.","is_advanced":true,"is_deprecated":true,"default":"redpanda_migrator_output"},{"name":"replication_factor_override","type":"bool","kind":"scalar","description":"Use the specified replication factor when creating topics.","is_advanced":true,"is_deprecated":true,"default":true},{"name":"replication_factor","type":"int","kind":"scalar","description":"Replication factor for created topics. This is only used when `replication_factor_override` is set to `true`.","is_advanced":true,"is_deprecated":true,"default":3},{"name":"multi_header","type":"bool","kind":"scalar","description":"Decode headers into lists to allow handling of multiple values with the same key","is_advanced":true,"is_deprecated":true,"default":false},{"name":"batch_size","type":"int","kind":"scalar","description":"The maximum number of messages that should be accumulated into each batch.","is_advanced":true,"is_deprecated":true,"default":1024}],"linter":"\nlet has_topic_partitions = this.topics.any(t -\u003e t.contains(\":\"))\n\nroot = [\n if $has_topic_partitions {\n if this.consumer_group.or(\"\") != \"\" {\n \"this input does not support both a consumer group and explicit topic partitions\"\n } else if this.regexp_topics {\n \"this input does not support both regular expression topics and explicit topic partitions\"\n }\n } else {\n if this.consumer_group.or(\"\") == \"\" {\n \"a consumer group is mandatory when not using explicit topic partitions\"\n }\n },\n # We don't have any way to distinguish between start_from_oldest set explicitly to true and not set at all, so we\n # assume users will be OK if start_offset overwrites it silently\n if this.start_from_oldest == false \u0026\u0026 this.start_offset == \"earliest\" {\n \"start_from_oldest cannot be set to false when start_offset is set to earliest\"\n }\n]\n"},"version":"4.37.0"},{"name":"redpanda_migrator_bundle","type":"input","status":"experimental","plugin":true,"summary":"Redpanda Migrator bundle input","description":"All-in-one input which reads messages and schemas from a Kafka or Redpanda cluster. This input is meant to be used\ntogether with the `redpanda_migrator_bundle` output.\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"redpanda_migrator","type":"unknown","kind":"map","description":"The `redpanda_migrator` input configuration.\n"},{"name":"schema_registry","type":"unknown","kind":"map","description":"The `schema_registry` input configuration.\n"},{"name":"migrate_schemas_before_data","type":"bool","kind":"scalar","description":"Migrate all schemas first before starting to migrate data.\n","default":true}]}},{"name":"redpanda_migrator_offsets","type":"input","status":"beta","plugin":true,"summary":"Redpanda Migrator consumer group offsets input using the https://github.com/twmb/franz-go[Franz Kafka client library^].","description":"\nTODO: Description\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- kafka_key\n- kafka_topic\n- kafka_partition\n- kafka_offset\n- kafka_timestamp_unix\n- kafka_timestamp_ms\n- kafka_tombstone_message\n- kafka_offset_topic\n- kafka_offset_group\n- kafka_offset_partition\n- kafka_offset_commit_timestamp\n- kafka_offset_metadata\n- kafka_is_high_watermark\n```\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"seed_brokers","type":"string","kind":"array","description":"A list of broker addresses to connect to in order to establish connections. If an item of the list contains commas it will be expanded into multiple addresses.","examples":[["localhost:9092"],["foo:9092","bar:9092"],["foo:9092,bar:9092"]]},{"name":"client_id","type":"string","kind":"scalar","description":"An identifier for the client connection.","is_advanced":true,"default":"benthos"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"sasl","type":"object","kind":"array","description":"Specify one or more methods of SASL authentication. SASL is tried in order; if the broker supports the first mechanism, all connections will use that mechanism. If the first mechanism fails, the client will pick the first supported mechanism. If the broker does not support any client mechanisms, connections will fail.","is_advanced":true,"is_optional":true,"examples":[[{"mechanism":"SCRAM-SHA-512","password":"bar","username":"foo"}]],"children":[{"name":"mechanism","type":"string","kind":"scalar","description":"The SASL mechanism to use.","is_advanced":true,"annotated_options":[["AWS_MSK_IAM","AWS IAM based authentication as specified by the 'aws-msk-iam-auth' java library."],["OAUTHBEARER","OAuth Bearer based authentication."],["PLAIN","Plain text authentication."],["SCRAM-SHA-256","SCRAM based authentication as specified in RFC5802."],["SCRAM-SHA-512","SCRAM based authentication as specified in RFC5802."],["none","Disable sasl authentication"]],"linter":"\nlet options = {\n \"aws_msk_iam\": true,\n \"oauthbearer\": true,\n \"plain\": true,\n \"scram-sha-256\": true,\n \"scram-sha-512\": true,\n \"none\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"username","type":"string","kind":"scalar","description":"A username to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token to use for a single session's OAUTHBEARER authentication.","is_advanced":true,"default":""},{"name":"extensions","type":"string","kind":"map","description":"Key/value pairs to add to OAUTHBEARER authentication requests.","is_advanced":true,"is_optional":true},{"name":"aws","type":"object","kind":"scalar","description":"Contains AWS specific fields for when the `mechanism` is set to `AWS_MSK_IAM`.","is_advanced":true,"is_optional":true,"children":[{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]}]},{"name":"metadata_max_age","type":"string","kind":"scalar","description":"The maximum age of metadata before it is refreshed.","is_advanced":true,"default":"5m"},{"name":"request_timeout_overhead","type":"string","kind":"scalar","description":"The request time overhead. Uses the given time as overhead while deadlining requests. Roughly equivalent to request.timeout.ms, but grants additional time to requests that have timeout fields.","is_advanced":true,"default":"10s"},{"name":"conn_idle_timeout","type":"string","kind":"scalar","description":"The rough amount of time to allow connections to idle before they are closed.","is_advanced":true,"default":"20s"},{"name":"topics","type":"string","kind":"array","description":"\nA list of topics to consume from. Multiple comma separated topics can be listed in a single element. When a `consumer_group` is specified partitions are automatically distributed across consumers of a topic, otherwise all partitions are consumed.","examples":[["foo","bar"],["things.*"],["foo,bar"]],"linter":"if this.length() == 0 { [\"at least one topic must be specified\"] }"},{"name":"regexp_topics","type":"bool","kind":"scalar","description":"Whether listed topics should be interpreted as regular expression patterns for matching multiple topics.","default":false},{"name":"rack_id","type":"string","kind":"scalar","description":"A rack specifies where the client is physically located and changes fetch requests to consume from the closest replica as opposed to the leader replica.","is_advanced":true,"default":""},{"name":"consumer_group","type":"string","kind":"scalar","description":"An optional consumer group to consume as. When specified the partitions of specified topics are automatically distributed across consumers sharing a consumer group, and partition offsets are automatically committed and resumed under this name. Consumer groups are not supported when specifying explicit partitions to consume from in the `topics` field.","is_optional":true},{"name":"commit_period","type":"string","kind":"scalar","description":"The period of time between each commit of the current partition offsets. Offsets are always committed during shutdown.","is_advanced":true,"default":"5s"},{"name":"partition_buffer_bytes","type":"string","kind":"scalar","description":"A buffer size (in bytes) for each consumed partition, allowing records to be queued internally before flushing. Increasing this may improve throughput at the cost of higher memory utilisation. Note that each buffer can grow slightly beyond this value.","is_advanced":true,"default":"1MB"},{"name":"topic_lag_refresh_period","type":"string","kind":"scalar","description":"The period of time between each topic lag refresh cycle.","is_advanced":true,"default":"5s"},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]},"version":"4.45.0"},{"name":"resource","type":"input","status":"stable","plugin":true,"summary":"Resource is an input type that channels messages from a resource input, identified by its name.","description":"Resources allow you to tidy up deeply nested configs. For example, the config:\n\n```yaml\ninput:\n broker:\n inputs:\n - kafka:\n addresses: [ TODO ]\n topics: [ foo ]\n consumer_group: foogroup\n - gcp_pubsub:\n project: bar\n subscription: baz\n```\n\nCould also be expressed as:\n\n```yaml\ninput:\n broker:\n inputs:\n - resource: foo\n - resource: bar\n\ninput_resources:\n - label: foo\n kafka:\n addresses: [ TODO ]\n topics: [ foo ]\n consumer_group: foogroup\n\n - label: bar\n gcp_pubsub:\n project: bar\n subscription: baz\n```\n\nResources also allow you to reference a single input in multiple places, such as multiple streams mode configs, or multiple entries in a broker input. However, when a resource is referenced more than once the messages it produces are distributed across those references, so each message will only be directed to a single reference, not all of them.\n\nYou can find out more about resources in xref:configuration:resources.adoc[].","categories":["Utility"],"config":{"name":"","type":"string","kind":"scalar","default":""}},{"name":"schema_registry","type":"input","status":"beta","plugin":true,"summary":"Reads schemas from SchemaRegistry.","description":"\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- schema_registry_subject\n- schema_registry_version\n```\n\nYou can access these metadata fields using\nxref:configuration:interpolation.adoc#bloblang-queries[function interpolation].\n\n","categories":["Integration"],"examples":[{"title":"Read schemas","summary":"Read all schemas (including deleted) from a Schema Registry instance which are associated with subjects matching the `^foo.*` filter.","config":"\ninput:\n schema_registry:\n url: http://localhost:8081\n include_deleted: true\n subject_filter: ^foo.*\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The base URL of the schema registry service."},{"name":"include_deleted","type":"bool","kind":"scalar","description":"Include deleted entities.","is_advanced":true,"default":false},{"name":"subject_filter","type":"string","kind":"scalar","description":"Include only subjects which match the regular expression filter. All subjects are selected when not set.","is_advanced":true,"default":""},{"name":"fetch_in_order","type":"bool","kind":"scalar","description":"Fetch all schemas on connect and sort them by ID. Should be set to `true` when schema references are used.","is_advanced":true,"default":true,"version":"4.37.0"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"oauth","type":"object","kind":"scalar","description":"Allows you to specify open authentication via OAuth version 1.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use OAuth version 1 in requests.","is_advanced":true,"default":false},{"name":"consumer_key","type":"string","kind":"scalar","description":"A value used to identify the client to the service provider.","is_advanced":true,"default":""},{"name":"consumer_secret","type":"string","kind":"scalar","description":"A secret used to establish ownership of the consumer key.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"access_token","type":"string","kind":"scalar","description":"A value used to gain access to the protected resources on behalf of the user.","is_advanced":true,"default":""},{"name":"access_token_secret","type":"string","kind":"scalar","description":"A secret provided in order to establish ownership of a given access token.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"jwt","type":"object","kind":"scalar","description":"BETA: Allows you to specify JWT authentication.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use JWT authentication in requests.","is_advanced":true,"default":false},{"name":"private_key_file","type":"string","kind":"scalar","description":"A file with the PEM encoded via PKCS1 or PKCS8 as private key.","is_advanced":true,"default":""},{"name":"signing_method","type":"string","kind":"scalar","description":"A method used to sign the token such as RS256, RS384, RS512 or EdDSA.","is_advanced":true,"default":""},{"name":"claims","type":"unknown","kind":"map","description":"A value used to identify the claims that issued the JWT.","is_advanced":true,"default":{}},{"name":"headers","type":"unknown","kind":"map","description":"Add optional key/value headers to the JWT.","is_advanced":true,"default":{}}]}]},"version":"4.32.2"},{"name":"sequence","type":"input","status":"stable","plugin":true,"summary":"Reads messages from a sequence of child inputs, starting with the first and once that input gracefully terminates starts consuming from the next, and so on.","description":"This input is useful for consuming from inputs that have an explicit end but must not be consumed in parallel.","categories":["Utility"],"examples":[{"title":"End of Stream Message","summary":"A common use case for sequence might be to generate a message at the end of our main input. With the following config once the records within `./dataset.csv` are exhausted our final payload `{\"status\":\"finished\"}` will be routed through the pipeline.","config":"\ninput:\n sequence:\n inputs:\n - file:\n paths: [ ./dataset.csv ]\n scanner:\n csv: {}\n - generate:\n count: 1\n mapping: 'root = {\"status\":\"finished\"}'\n"},{"title":"Joining Data (Simple)","summary":"Redpanda Connect can be used to join unordered data from fragmented datasets in memory by specifying a common identifier field and a number of sharded iterations. For example, given two CSV files, the first called \"main.csv\", which contains rows of user data:\n\n```csv\nuuid,name,age\nAAA,Melanie,34\nBBB,Emma,28\nCCC,Geri,45\n```\n\nAnd the second called \"hobbies.csv\" that, for each user, contains zero or more rows of hobbies:\n\n```csv\nuuid,hobby\nCCC,pokemon go\nAAA,rowing\nAAA,golf\n```\n\nWe can parse and join this data into a single dataset:\n\n```json\n{\"uuid\":\"AAA\",\"name\":\"Melanie\",\"age\":34,\"hobbies\":[\"rowing\",\"golf\"]}\n{\"uuid\":\"BBB\",\"name\":\"Emma\",\"age\":28}\n{\"uuid\":\"CCC\",\"name\":\"Geri\",\"age\":45,\"hobbies\":[\"pokemon go\"]}\n```\n\nWith the following config:","config":"\ninput:\n sequence:\n sharded_join:\n type: full-outer\n id_path: uuid\n merge_strategy: array\n inputs:\n - file:\n paths:\n - ./hobbies.csv\n - ./main.csv\n scanner:\n csv: {}\n"},{"title":"Joining Data (Advanced)","summary":"In this example we are able to join unordered and fragmented data from a combination of CSV files and newline-delimited JSON documents by specifying multiple sequence inputs with their own processors for extracting the structured data.\n\nThe first file \"main.csv\" contains straight forward CSV data:\n\n```csv\nuuid,name,age\nAAA,Melanie,34\nBBB,Emma,28\nCCC,Geri,45\n```\n\nAnd the second file called \"hobbies.ndjson\" contains JSON documents, one per line, that associate an identifier with an array of hobbies. However, these data objects are in a nested format:\n\n```json\n{\"document\":{\"uuid\":\"CCC\",\"hobbies\":[{\"type\":\"pokemon go\"}]}}\n{\"document\":{\"uuid\":\"AAA\",\"hobbies\":[{\"type\":\"rowing\"},{\"type\":\"golf\"}]}}\n```\n\nAnd so we will want to map these into a flattened structure before the join, and then we will end up with a single dataset that looks like this:\n\n```json\n{\"uuid\":\"AAA\",\"name\":\"Melanie\",\"age\":34,\"hobbies\":[\"rowing\",\"golf\"]}\n{\"uuid\":\"BBB\",\"name\":\"Emma\",\"age\":28}\n{\"uuid\":\"CCC\",\"name\":\"Geri\",\"age\":45,\"hobbies\":[\"pokemon go\"]}\n```\n\nWith the following config:","config":"\ninput:\n sequence:\n sharded_join:\n type: full-outer\n id_path: uuid\n iterations: 10\n merge_strategy: array\n inputs:\n - file:\n paths: [ ./main.csv ]\n scanner:\n csv: {}\n - file:\n paths: [ ./hobbies.ndjson ]\n scanner:\n lines: {}\n processors:\n - mapping: |\n root.uuid = this.document.uuid\n root.hobbies = this.document.hobbies.map_each(this.type)\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"sharded_join","type":"object","kind":"scalar","description":"EXPERIMENTAL: Provides a way to perform outer joins of arbitrarily structured and unordered data resulting from the input sequence, even when the overall size of the data surpasses the memory available on the machine.\n\nWhen configured the sequence of inputs will be consumed one or more times according to the number of iterations, and when more than one iteration is specified each iteration will process an entirely different set of messages by sharding them by the ID field. Increasing the number of iterations reduces the memory consumption at the cost of needing to fully parse the data each time.\n\nEach message must be structured (JSON or otherwise processed into a structured form) and the fields will be aggregated with those of other messages sharing the ID. At the end of each iteration the joined messages are flushed downstream before the next iteration begins, hence keeping memory usage limited.","is_advanced":true,"children":[{"name":"type","type":"string","kind":"scalar","description":"The type of join to perform. A `full-outer` ensures that all identifiers seen in any of the input sequences are sent, and is performed by consuming all input sequences before flushing the joined results. An `outer` join consumes all input sequences but only writes data joined from the last input in the sequence, similar to a left or right outer join. With an `outer` join if an identifier appears multiple times within the final sequence input it will be flushed each time it appears. `full-outter` and `outter` have been deprecated in favour of `full-outer` and `outer`.","is_advanced":true,"default":"none","options":["none","full-outer","outer","full-outter","outter"],"linter":"\nlet options = {\n \"none\": true,\n \"full-outer\": true,\n \"outer\": true,\n \"full-outter\": true,\n \"outter\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"id_path","type":"string","kind":"scalar","description":"A xref:configuration:field_paths.adoc[dot path] that points to a common field within messages of each fragmented data set and can be used to join them. Messages that are not structured or are missing this field will be dropped. This field must be set in order to enable joins.","is_advanced":true,"default":""},{"name":"iterations","type":"int","kind":"scalar","description":"The total number of iterations (shards), increasing this number will increase the overall time taken to process the data, but reduces the memory used in the process. The real memory usage required is significantly higher than the real size of the data and therefore the number of iterations should be at least an order of magnitude higher than the available memory divided by the overall size of the dataset.","is_advanced":true,"default":1},{"name":"merge_strategy","type":"string","kind":"scalar","description":"The chosen strategy to use when a data join would otherwise result in a collision of field values. The strategy `array` means non-array colliding values are placed into an array and colliding arrays are merged. The strategy `replace` replaces old values with new values. The strategy `keep` keeps the old value.","is_advanced":true,"default":"array","options":["array","replace","keep"],"linter":"\nlet options = {\n \"array\": true,\n \"replace\": true,\n \"keep\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"}],"version":"3.40.0"},{"name":"inputs","type":"input","kind":"array","description":"An array of inputs to read from sequentially."}]}},{"name":"sftp","type":"input","status":"beta","plugin":true,"summary":"Consumes files from an SFTP server.","description":"\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n- sftp_path\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].","categories":["Network"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"address","type":"string","kind":"scalar","description":"The address of the server to connect to."},{"name":"credentials","type":"object","kind":"scalar","description":"The credentials to use to log into the target server.","children":[{"name":"username","type":"string","kind":"scalar","description":"The username to connect to the SFTP server.","default":""},{"name":"password","type":"string","kind":"scalar","description":"The password for the username to connect to the SFTP server.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"private_key_file","type":"string","kind":"scalar","description":"The private key for the username to connect to the SFTP server.","default":""},{"name":"private_key","type":"string","kind":"scalar","description":"The private key file for the username to connect to the SFTP server.","is_secret":true,"default":"","linter":"root = match { this.exists(\"private_key\") \u0026\u0026 this.exists(\"private_key_file\") =\u003e [ \"both private_key and private_key_file can't be set simultaneously\" ], }","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"private_key_pass","type":"string","kind":"scalar","description":"Optional passphrase for private key.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"paths","type":"string","kind":"array","description":"A list of paths to consume sequentially. Glob patterns are supported."},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"codec","type":"string","kind":"scalar","description":"The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.","is_deprecated":true,"is_optional":true,"examples":["lines","delim:\t","delim:foobar","gzip/csv"],"annotated_options":[["auto","EXPERIMENTAL: Attempts to derive a codec for each file based on information such as the extension. For example, a .tar.gz file would be consumed with the `gzip/tar` codec. Defaults to all-bytes."],["all-bytes","Consume the entire file as a single binary message."],["avro-ocf:marshaler=x","EXPERIMENTAL: Consume a stream of Avro OCF datum. The `marshaler` parameter is optional and has the options: `goavro` (default), `json`. Use `goavro` if OCF contains logical types."],["chunker:x","Consume the file in chunks of a given number of bytes."],["csv","Consume structured rows as comma separated values, the first row must be a header row."],["csv:x","Consume structured rows as values separated by a custom delimiter, the first row must be a header row. The custom delimiter must be a single character, e.g. the codec `\"csv:\\t\"` would consume a tab delimited file."],["csv-safe","Consume structured rows like `csv`, but sends messages with empty maps on failure to parse. Includes row number and parsing errors (if any) in the message's metadata."],["csv-safe:x","Consume structured rows like `csv:x` as values separated by a custom delimiter, but sends messages with empty maps on failure to parse. The custom delimiter must be a single character, e.g. the codec `\"csv-safe:\\t\"` would consume a tab delimited file. Includes row number and parsing errors (if any) in the message's metadata."],["delim:x","Consume the file in segments divided by a custom delimiter."],["gzip","Decompress a gzip file, this codec should precede another codec, e.g. `gzip/all-bytes`, `gzip/tar`, `gzip/csv`, etc."],["pgzip","Decompress a gzip file in parallel, this codec should precede another codec, e.g. `pgzip/all-bytes`, `pgzip/tar`, `pgzip/csv`, etc."],["lines","Consume the file in segments divided by linebreaks."],["multipart","Consumes the output of another codec and batches messages together. A batch ends when an empty message is consumed. For example, the codec `lines/multipart` could be used to consume multipart messages where an empty line indicates the end of each batch."],["regex:(?m)^\\d\\d:\\d\\d:\\d\\d","Consume the file in segments divided by regular expression."],["skipbom","Skip one or more byte order marks for each opened reader, this codec should precede another codec, e.g. `skipbom/csv`, etc."],["tar","Parse the file as a tar archive, and consume each file of the archive as a message."]]},{"name":"max_buffer","type":"int","kind":"scalar","is_deprecated":true,"default":1000000},{"name":"scanner","type":"scanner","kind":"scalar","description":"The xref:components:scanners/about.adoc[scanner] by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once.","is_optional":true,"default":{"to_the_end":{}},"version":"4.25.0"},{"name":"delete_on_finish","type":"bool","kind":"scalar","description":"Whether to delete files from the server once they are processed.","is_advanced":true,"default":false},{"name":"watcher","type":"object","kind":"scalar","description":"An experimental mode whereby the input will periodically scan the target paths for new files and consume them, when all files are consumed the input will continue polling for new files.","children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether file watching is enabled.","default":false},{"name":"minimum_age","type":"string","kind":"scalar","description":"The minimum period of time since a file was last updated before attempting to consume it. Increasing this period decreases the likelihood that a file will be consumed whilst it is still being written to.","default":"1s","examples":["10s","1m","10m"]},{"name":"poll_interval","type":"string","kind":"scalar","description":"The interval between each attempt to scan the target paths for new files.","default":"1s","examples":["100ms","1s"]},{"name":"cache","type":"string","kind":"scalar","description":"A xref:components:caches/about.adoc[cache resource] for storing the paths of files already consumed.","default":""}],"version":"3.42.0"}]},"version":"3.39.0"},{"name":"slack","type":"input","status":"experimental","plugin":true,"description":"Connects to Slack using https://api.slack.com/apis/socket-mode[^Socket Mode]. This allows for receiving events, interactions and slash commands. Each message emitted from this input has a @type metadata of the event type \"events_api\", \"interactions\" or \"slash_commands\".","categories":null,"examples":[{"title":"Echo Slackbot","summary":"A slackbot that echo messages from other users","config":"\ninput:\n slack:\n app_token: \"${APP_TOKEN:xapp-demo}\"\n bot_token: \"${BOT_TOKEN:xoxb-demo}\"\npipeline:\n processors:\n - mutation: |\n # ignore hidden or non message events\n if this.event.type != \"message\" || (this.event.hidden | false) {\n root = deleted()\n }\n # Don't respond to our own messages\n if this.authorizations.any(auth -\u003e auth.user_id == this.event.user) {\n root = deleted()\n }\noutput:\n slack_post:\n bot_token: \"${BOT_TOKEN:xoxb-demo}\"\n channel_id: \"${!this.event.channel}\"\n thread_ts: \"${!this.event.ts}\"\n text: \"ECHO: ${!this.event.text}\"\n "}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"app_token","type":"string","kind":"scalar","description":"The Slack App token to use.","linter":"\n root = if !this.has_prefix(\"xapp-\") { [ \"field must start with xapp-\" ] }\n "},{"name":"bot_token","type":"string","kind":"scalar","description":"The Slack Bot User OAuth token to use.","linter":"\n root = if !this.has_prefix(\"xoxb-\") { [ \"field must start with xoxb-\" ] }\n "},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]}},{"name":"slack_users","type":"input","status":"experimental","plugin":true,"description":"Reads all users in a slack organization (optionally filtered by a team ID).","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"bot_token","type":"string","kind":"scalar","description":"The Slack Bot User OAuth token to use.","linter":"\n root = if !this.has_prefix(\"xoxb-\") { [ \"field must start with xoxb-\" ] }\n "},{"name":"team_id","type":"string","kind":"scalar","description":"The team ID to filter by","default":""},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]}},{"name":"socket","type":"input","status":"stable","plugin":true,"summary":"Connects to a tcp or unix socket and consumes a continuous stream of messages.","categories":["Network"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"network","type":"string","kind":"scalar","description":"A network type to assume (unix|tcp).","options":["unix","tcp"],"linter":"\nlet options = {\n \"unix\": true,\n \"tcp\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"address","type":"string","kind":"scalar","description":"The address to connect to.","examples":["/tmp/benthos.sock","127.0.0.1:6000"]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"codec","type":"string","kind":"scalar","description":"The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.","is_deprecated":true,"is_optional":true,"examples":["lines","delim:\t","delim:foobar","gzip/csv"],"annotated_options":[["auto","EXPERIMENTAL: Attempts to derive a codec for each file based on information such as the extension. For example, a .tar.gz file would be consumed with the `gzip/tar` codec. Defaults to all-bytes."],["all-bytes","Consume the entire file as a single binary message."],["avro-ocf:marshaler=x","EXPERIMENTAL: Consume a stream of Avro OCF datum. The `marshaler` parameter is optional and has the options: `goavro` (default), `json`. Use `goavro` if OCF contains logical types."],["chunker:x","Consume the file in chunks of a given number of bytes."],["csv","Consume structured rows as comma separated values, the first row must be a header row."],["csv:x","Consume structured rows as values separated by a custom delimiter, the first row must be a header row. The custom delimiter must be a single character, e.g. the codec `\"csv:\\t\"` would consume a tab delimited file."],["csv-safe","Consume structured rows like `csv`, but sends messages with empty maps on failure to parse. Includes row number and parsing errors (if any) in the message's metadata."],["csv-safe:x","Consume structured rows like `csv:x` as values separated by a custom delimiter, but sends messages with empty maps on failure to parse. The custom delimiter must be a single character, e.g. the codec `\"csv-safe:\\t\"` would consume a tab delimited file. Includes row number and parsing errors (if any) in the message's metadata."],["delim:x","Consume the file in segments divided by a custom delimiter."],["gzip","Decompress a gzip file, this codec should precede another codec, e.g. `gzip/all-bytes`, `gzip/tar`, `gzip/csv`, etc."],["pgzip","Decompress a gzip file in parallel, this codec should precede another codec, e.g. `pgzip/all-bytes`, `pgzip/tar`, `pgzip/csv`, etc."],["lines","Consume the file in segments divided by linebreaks."],["multipart","Consumes the output of another codec and batches messages together. A batch ends when an empty message is consumed. For example, the codec `lines/multipart` could be used to consume multipart messages where an empty line indicates the end of each batch."],["regex:(?m)^\\d\\d:\\d\\d:\\d\\d","Consume the file in segments divided by regular expression."],["skipbom","Skip one or more byte order marks for each opened reader, this codec should precede another codec, e.g. `skipbom/csv`, etc."],["tar","Parse the file as a tar archive, and consume each file of the archive as a message."]]},{"name":"max_buffer","type":"int","kind":"scalar","is_deprecated":true,"default":1000000},{"name":"scanner","type":"scanner","kind":"scalar","description":"The xref:components:scanners/about.adoc[scanner] by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once.","is_optional":true,"default":{"lines":{}},"version":"4.25.0"}]}},{"name":"socket_server","type":"input","status":"stable","plugin":true,"summary":"Creates a server that receives a stream of messages over a TCP, UDP or Unix socket.","categories":["Network"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"network","type":"string","kind":"scalar","description":"A network type to accept.","options":["unix","tcp","udp","tls"],"linter":"\nlet options = {\n \"unix\": true,\n \"tcp\": true,\n \"udp\": true,\n \"tls\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"address","type":"string","kind":"scalar","description":"The address to listen from.","examples":["/tmp/benthos.sock","0.0.0.0:6000"]},{"name":"address_cache","type":"string","kind":"scalar","description":"An optional xref:components:caches/about.adoc[`cache`] within which this input should write it's bound address once known. The key of the cache item containing the address will be the label of the component suffixed with `_address` (e.g. `foo_address`), or `socket_server_address` when a label has not been provided. This is useful in situations where the address is dynamically allocated by the server (`127.0.0.1:0`) and you want to store the allocated address somewhere for reference by other systems and components.","is_optional":true,"version":"4.25.0"},{"name":"tls","type":"object","kind":"scalar","description":"TLS specific configuration, valid when the `network` is set to `tls`.","is_optional":true,"children":[{"name":"cert_file","type":"string","kind":"scalar","description":"PEM encoded certificate for use with TLS.","is_optional":true},{"name":"key_file","type":"string","kind":"scalar","description":"PEM encoded private key for use with TLS.","is_optional":true},{"name":"self_signed","type":"bool","kind":"scalar","description":"Whether to generate self signed certificates.","default":false}]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"codec","type":"string","kind":"scalar","description":"The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.","is_deprecated":true,"is_optional":true,"examples":["lines","delim:\t","delim:foobar","gzip/csv"],"annotated_options":[["auto","EXPERIMENTAL: Attempts to derive a codec for each file based on information such as the extension. For example, a .tar.gz file would be consumed with the `gzip/tar` codec. Defaults to all-bytes."],["all-bytes","Consume the entire file as a single binary message."],["avro-ocf:marshaler=x","EXPERIMENTAL: Consume a stream of Avro OCF datum. The `marshaler` parameter is optional and has the options: `goavro` (default), `json`. Use `goavro` if OCF contains logical types."],["chunker:x","Consume the file in chunks of a given number of bytes."],["csv","Consume structured rows as comma separated values, the first row must be a header row."],["csv:x","Consume structured rows as values separated by a custom delimiter, the first row must be a header row. The custom delimiter must be a single character, e.g. the codec `\"csv:\\t\"` would consume a tab delimited file."],["csv-safe","Consume structured rows like `csv`, but sends messages with empty maps on failure to parse. Includes row number and parsing errors (if any) in the message's metadata."],["csv-safe:x","Consume structured rows like `csv:x` as values separated by a custom delimiter, but sends messages with empty maps on failure to parse. The custom delimiter must be a single character, e.g. the codec `\"csv-safe:\\t\"` would consume a tab delimited file. Includes row number and parsing errors (if any) in the message's metadata."],["delim:x","Consume the file in segments divided by a custom delimiter."],["gzip","Decompress a gzip file, this codec should precede another codec, e.g. `gzip/all-bytes`, `gzip/tar`, `gzip/csv`, etc."],["pgzip","Decompress a gzip file in parallel, this codec should precede another codec, e.g. `pgzip/all-bytes`, `pgzip/tar`, `pgzip/csv`, etc."],["lines","Consume the file in segments divided by linebreaks."],["multipart","Consumes the output of another codec and batches messages together. A batch ends when an empty message is consumed. For example, the codec `lines/multipart` could be used to consume multipart messages where an empty line indicates the end of each batch."],["regex:(?m)^\\d\\d:\\d\\d:\\d\\d","Consume the file in segments divided by regular expression."],["skipbom","Skip one or more byte order marks for each opened reader, this codec should precede another codec, e.g. `skipbom/csv`, etc."],["tar","Parse the file as a tar archive, and consume each file of the archive as a message."]]},{"name":"max_buffer","type":"int","kind":"scalar","is_deprecated":true,"default":1000000},{"name":"scanner","type":"scanner","kind":"scalar","description":"The xref:components:scanners/about.adoc[scanner] by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once.","is_optional":true,"default":{"lines":{}},"version":"4.25.0"}]}},{"name":"spicedb_watch","type":"input","status":"stable","plugin":true,"summary":"Consume messages from the Watch API from SpiceDB.","description":"\nThe SpiceDB input allows you to consume messages from the Watch API of a SpiceDB instance.\nThis input is useful for applications that need to react to changes in the data managed by SpiceDB in real-time.\n\n== Credentials\n\nYou need to provide the endpoint of your SpiceDB instance and a Bearer token for authentication.\n\n== Cache\n\nThe zed token of the newest update consumed and acked is stored in a cache in order to start reading from it each time the input is initialised.\nIdeally this cache should be persisted across restarts.\n","categories":["Services","SpiceDB"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"endpoint","type":"string","kind":"scalar","description":"The SpiceDB endpoint.","examples":["grpc.authzed.com:443"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"bearer_token","type":"string","kind":"scalar","description":"The SpiceDB Bearer token used to authenticate against the SpiceDB instance.","is_secret":true,"default":"","examples":["t_your_token_here_1234567deadbeef"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"max_receive_message_bytes","type":"string","kind":"scalar","description":"Maximum message size in bytes the SpiceDB client can receive.","is_advanced":true,"default":"4MB","examples":["100MB","50mib"]},{"name":"cache","type":"string","kind":"scalar","description":"A cache resource to use for performing unread message backfills, the ID of the last message received will be stored in this cache and used for subsequent requests."},{"name":"cache_key","type":"string","kind":"scalar","description":"The key identifier used when storing the ID of the last message received.","is_advanced":true,"default":"authzed.com/spicedb/watch/last_zed_token"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]}]}},{"name":"splunk","type":"input","status":"beta","plugin":true,"summary":"Consumes messages from Splunk.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"Full HTTP Search API endpoint URL.","examples":["https://foobar.splunkcloud.com/services/search/v2/jobs/export"]},{"name":"user","type":"string","kind":"scalar","description":"Splunk account user."},{"name":"password","type":"string","kind":"scalar","description":"Splunk account password.","is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"query","type":"string","kind":"scalar","description":"Splunk search query."},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]},"version":"4.30.0"},{"name":"sql_raw","type":"input","status":"beta","plugin":true,"summary":"Executes a select query and creates a message for each row received.","description":"Once the rows from the query are exhausted this input shuts down, allowing the pipeline to gracefully terminate (or the next input in a xref:components:inputs/sequence.adoc[sequence] to execute).","categories":["Services"],"examples":[{"title":"Consumes an SQL table using a query as an input.","summary":"\nHere we preform an aggregate over a list of names in a table that are less than 3600 seconds old.","config":"\ninput:\n sql_raw:\n driver: postgres\n dsn: postgres://foouser:foopass@localhost:5432/testdb?sslmode=disable\n query: \"SELECT name, count(*) FROM person WHERE last_updated \u003c $1 GROUP BY name;\"\n args_mapping: |\n root = [\n now().ts_unix() - 3600\n ]\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"driver","type":"string","kind":"scalar","description":"A database \u003c\u003cdrivers, driver\u003e\u003e to use.","options":["mysql","postgres","clickhouse","mssql","sqlite","oracle","snowflake","trino","gocosmos","spanner"],"linter":"\nlet options = {\n \"mysql\": true,\n \"postgres\": true,\n \"clickhouse\": true,\n \"mssql\": true,\n \"sqlite\": true,\n \"oracle\": true,\n \"snowflake\": true,\n \"trino\": true,\n \"gocosmos\": true,\n \"spanner\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"dsn","type":"string","kind":"scalar","description":"A Data Source Name to identify the target database.\n\n==== Drivers\n\n:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community, spanner=community\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n|===\n| Driver | Data Source Name Format\n\n| `clickhouse` \n| https://github.com/ClickHouse/clickhouse-go#dsn[`clickhouse://[username[:password\\]@\\][netloc\\][:port\\]/dbname[?param1=value1\u0026...\u0026paramN=valueN\\]`^] \n\n| `mysql` \n| `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` \n\n| `postgres` \n| `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` \n\n| `mssql` \n| `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` \n\n| `sqlite` \n| `file:/path/to/filename.db[?param\u0026=value1\u0026...]` \n\n| `oracle` \n| `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` \n\n| `snowflake` \n| `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` \n\n| `trino` \n| https://github.com/trinodb/trino-go-client#dsn-data-source-name[`http[s\\]://user[:pass\\]@host[:port\\][?parameters\\]`^] \n\n| `gocosmos` \n| https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage[`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e\\][;Version=\u003ccosmosdb-api-version\u003e\\][;DefaultDb/Db=\u003cdb-name\u003e\\][;AutoId=\u003ctrue/false\u003e\\][;InsecureSkipVerify=\u003ctrue/false\u003e\\]`^] \n\n| `spanner` \n| projects/[PROJECT]/instances/[INSTANCE]/databases/[DATABASE] \n|===\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String[the docs^] for more details. For https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication[key pair authentication^], the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe https://pkg.go.dev/github.com/microsoft/gocosmos[`gocosmos`^] driver is still experimental, but it has support for https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys[hierarchical partition keys^] as well as https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query[cross-partition queries^]. Please refer to the https://github.com/microsoft/gocosmos/blob/main/SQL.md[SQL notes^] for details.","examples":["clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60","foouser:foopassword@tcp(localhost:3306)/foodb","postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable","oracle://foouser:foopass@localhost:1521/service_name"]},{"name":"query","type":"string","kind":"scalar","description":"The query to execute. The style of placeholder to use depends on the driver, some drivers require question marks (`?`) whereas others expect incrementing dollar signs (`$1`, `$2`, and so on) or colons (`:1`, `:2` and so on). The style to use is outlined in this table:\n\n| Driver | Placeholder Style |\n|---|---|\n| `clickhouse` | Dollar sign |\n| `mysql` | Question mark |\n| `postgres` | Dollar sign |\n| `mssql` | Question mark |\n| `sqlite` | Question mark |\n| `oracle` | Colon |\n| `snowflake` | Question mark |\n| `trino` | Question mark |\n| `gocosmos` | Colon |\n","examples":["SELECT * FROM footable WHERE user_id = $1;"]},{"name":"args_mapping","type":"string","kind":"scalar","description":"An optional xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `query`.","is_optional":true,"bloblang":true,"examples":["root = [ this.cat.meow, this.doc.woofs[0] ]","root = [ meta(\"user.id\") ]"]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"init_files","type":"string","kind":"array","description":"\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":[["./init/*.sql"],["./foo.sql","./bar.sql"]],"version":"4.10.0"},{"name":"init_statement","type":"string","kind":"scalar","description":"\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":["\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n"],"version":"4.10.0"},{"name":"conn_max_idle_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time.","is_advanced":true,"is_optional":true},{"name":"conn_max_life_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age.","is_advanced":true,"is_optional":true},{"name":"conn_max_idle","type":"int","kind":"scalar","description":"An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release.","is_advanced":true,"is_optional":true,"default":2},{"name":"conn_max_open","type":"int","kind":"scalar","description":"An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited).","is_advanced":true,"is_optional":true}]},"version":"4.10.0"},{"name":"sql_select","type":"input","status":"beta","plugin":true,"summary":"Executes a select query and creates a message for each row received.","description":"Once the rows from the query are exhausted this input shuts down, allowing the pipeline to gracefully terminate (or the next input in a xref:components:inputs/sequence.adoc[sequence] to execute).","categories":["Services"],"examples":[{"title":"Consume a Table (PostgreSQL)","summary":"\nHere we define a pipeline that will consume all rows from a table created within the last hour by comparing the unix timestamp stored in the row column \"created_at\":","config":"\ninput:\n sql_select:\n driver: postgres\n dsn: postgres://foouser:foopass@localhost:5432/testdb?sslmode=disable\n table: footable\n columns: [ '*' ]\n where: created_at \u003e= ?\n args_mapping: |\n root = [\n now().ts_unix() - 3600\n ]\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"driver","type":"string","kind":"scalar","description":"A database \u003c\u003cdrivers, driver\u003e\u003e to use.","options":["mysql","postgres","clickhouse","mssql","sqlite","oracle","snowflake","trino","gocosmos","spanner"],"linter":"\nlet options = {\n \"mysql\": true,\n \"postgres\": true,\n \"clickhouse\": true,\n \"mssql\": true,\n \"sqlite\": true,\n \"oracle\": true,\n \"snowflake\": true,\n \"trino\": true,\n \"gocosmos\": true,\n \"spanner\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"dsn","type":"string","kind":"scalar","description":"A Data Source Name to identify the target database.\n\n==== Drivers\n\n:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community, spanner=community\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n|===\n| Driver | Data Source Name Format\n\n| `clickhouse` \n| https://github.com/ClickHouse/clickhouse-go#dsn[`clickhouse://[username[:password\\]@\\][netloc\\][:port\\]/dbname[?param1=value1\u0026...\u0026paramN=valueN\\]`^] \n\n| `mysql` \n| `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` \n\n| `postgres` \n| `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` \n\n| `mssql` \n| `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` \n\n| `sqlite` \n| `file:/path/to/filename.db[?param\u0026=value1\u0026...]` \n\n| `oracle` \n| `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` \n\n| `snowflake` \n| `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` \n\n| `trino` \n| https://github.com/trinodb/trino-go-client#dsn-data-source-name[`http[s\\]://user[:pass\\]@host[:port\\][?parameters\\]`^] \n\n| `gocosmos` \n| https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage[`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e\\][;Version=\u003ccosmosdb-api-version\u003e\\][;DefaultDb/Db=\u003cdb-name\u003e\\][;AutoId=\u003ctrue/false\u003e\\][;InsecureSkipVerify=\u003ctrue/false\u003e\\]`^] \n\n| `spanner` \n| projects/[PROJECT]/instances/[INSTANCE]/databases/[DATABASE] \n|===\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String[the docs^] for more details. For https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication[key pair authentication^], the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe https://pkg.go.dev/github.com/microsoft/gocosmos[`gocosmos`^] driver is still experimental, but it has support for https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys[hierarchical partition keys^] as well as https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query[cross-partition queries^]. Please refer to the https://github.com/microsoft/gocosmos/blob/main/SQL.md[SQL notes^] for details.","examples":["clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60","foouser:foopassword@tcp(localhost:3306)/foodb","postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable","oracle://foouser:foopass@localhost:1521/service_name"]},{"name":"table","type":"string","kind":"scalar","description":"The table to select from.","examples":["foo"]},{"name":"columns","type":"string","kind":"array","description":"A list of columns to select.","examples":[["*"],["foo","bar","baz"]]},{"name":"where","type":"string","kind":"scalar","description":"An optional where clause to add. Placeholder arguments are populated with the `args_mapping` field. Placeholders should always be question marks, and will automatically be converted to dollar syntax when the postgres or clickhouse drivers are used.","is_optional":true,"examples":["type = ? and created_at \u003e ?","user_id = ?"]},{"name":"args_mapping","type":"string","kind":"scalar","description":"An optional xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `where`.","is_optional":true,"bloblang":true,"examples":["root = [ \"article\", now().ts_format(\"2006-01-02\") ]"]},{"name":"prefix","type":"string","kind":"scalar","description":"An optional prefix to prepend to the select query (before SELECT).","is_advanced":true,"is_optional":true},{"name":"suffix","type":"string","kind":"scalar","description":"An optional suffix to append to the select query.","is_advanced":true,"is_optional":true},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"init_files","type":"string","kind":"array","description":"\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":[["./init/*.sql"],["./foo.sql","./bar.sql"]],"version":"4.10.0"},{"name":"init_statement","type":"string","kind":"scalar","description":"\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":["\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n"],"version":"4.10.0"},{"name":"conn_max_idle_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time.","is_advanced":true,"is_optional":true},{"name":"conn_max_life_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age.","is_advanced":true,"is_optional":true},{"name":"conn_max_idle","type":"int","kind":"scalar","description":"An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release.","is_advanced":true,"is_optional":true,"default":2},{"name":"conn_max_open","type":"int","kind":"scalar","description":"An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited).","is_advanced":true,"is_optional":true}]},"version":"3.59.0"},{"name":"stdin","type":"input","status":"stable","plugin":true,"summary":"Consumes data piped to stdin, chopping it into individual messages according to the specified scanner.","categories":["Local"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"codec","type":"string","kind":"scalar","description":"The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.","is_deprecated":true,"is_optional":true,"examples":["lines","delim:\t","delim:foobar","gzip/csv"],"annotated_options":[["auto","EXPERIMENTAL: Attempts to derive a codec for each file based on information such as the extension. For example, a .tar.gz file would be consumed with the `gzip/tar` codec. Defaults to all-bytes."],["all-bytes","Consume the entire file as a single binary message."],["avro-ocf:marshaler=x","EXPERIMENTAL: Consume a stream of Avro OCF datum. The `marshaler` parameter is optional and has the options: `goavro` (default), `json`. Use `goavro` if OCF contains logical types."],["chunker:x","Consume the file in chunks of a given number of bytes."],["csv","Consume structured rows as comma separated values, the first row must be a header row."],["csv:x","Consume structured rows as values separated by a custom delimiter, the first row must be a header row. The custom delimiter must be a single character, e.g. the codec `\"csv:\\t\"` would consume a tab delimited file."],["csv-safe","Consume structured rows like `csv`, but sends messages with empty maps on failure to parse. Includes row number and parsing errors (if any) in the message's metadata."],["csv-safe:x","Consume structured rows like `csv:x` as values separated by a custom delimiter, but sends messages with empty maps on failure to parse. The custom delimiter must be a single character, e.g. the codec `\"csv-safe:\\t\"` would consume a tab delimited file. Includes row number and parsing errors (if any) in the message's metadata."],["delim:x","Consume the file in segments divided by a custom delimiter."],["gzip","Decompress a gzip file, this codec should precede another codec, e.g. `gzip/all-bytes`, `gzip/tar`, `gzip/csv`, etc."],["pgzip","Decompress a gzip file in parallel, this codec should precede another codec, e.g. `pgzip/all-bytes`, `pgzip/tar`, `pgzip/csv`, etc."],["lines","Consume the file in segments divided by linebreaks."],["multipart","Consumes the output of another codec and batches messages together. A batch ends when an empty message is consumed. For example, the codec `lines/multipart` could be used to consume multipart messages where an empty line indicates the end of each batch."],["regex:(?m)^\\d\\d:\\d\\d:\\d\\d","Consume the file in segments divided by regular expression."],["skipbom","Skip one or more byte order marks for each opened reader, this codec should precede another codec, e.g. `skipbom/csv`, etc."],["tar","Parse the file as a tar archive, and consume each file of the archive as a message."]]},{"name":"max_buffer","type":"int","kind":"scalar","is_deprecated":true,"default":1000000},{"name":"scanner","type":"scanner","kind":"scalar","description":"The xref:components:scanners/about.adoc[scanner] by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once.","is_optional":true,"default":{"lines":{}},"version":"4.25.0"},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]}},{"name":"subprocess","type":"input","status":"beta","plugin":true,"summary":"Executes a command, runs it as a subprocess, and consumes messages from it over stdout.","description":"\nMessages are consumed according to a specified codec. The command is executed once and if it terminates the input also closes down gracefully. Alternatively, the field `restart_on_close` can be set to `true` in order to have Redpanda Connect re-execute the command each time it stops.\n\nThe field `max_buffer` defines the maximum message size able to be read from the subprocess. This value should be set significantly above the real expected maximum message size.\n\nThe execution environment of the subprocess is the same as the Redpanda Connect instance, including environment variables and the current working directory.","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"name","type":"string","kind":"scalar","description":"The command to execute as a subprocess.","examples":["cat","sed","awk"]},{"name":"args","type":"string","kind":"array","description":"A list of arguments to provide the command.","default":[]},{"name":"codec","type":"string","kind":"scalar","description":"The way in which messages should be consumed from the subprocess.","default":"lines","options":["lines"],"linter":"\nlet options = {\n \"lines\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"restart_on_exit","type":"bool","kind":"scalar","description":"Whether the command should be re-executed each time the subprocess ends.","default":false},{"name":"max_buffer","type":"int","kind":"scalar","description":"The maximum expected size of an individual message.","is_advanced":true,"default":65536}]}},{"name":"timeplus","type":"input","status":"experimental","plugin":true,"summary":"Executes a query on Timeplus Enterprise and creates a message from each row received","description":"\nThis input can execute a query on Timeplus Enterprise Cloud, Timeplus Enterprise (self-hosted) or Timeplusd. A structured message will be created\nfrom each row received.\n\nIf it is a streaming query, this input will keep running until the query is terminated. If it is a table query, this input will shut down once the rows from the query are exhausted.","categories":["Services"],"examples":[{"title":"From Timeplus Enterprise Cloud via HTTP","summary":"You will need to create API Key on Timeplus Enterprise Cloud Web console first and then set the `apikey` field.","config":"\ninput:\n timeplus:\n url: https://us-west-2.timeplus.cloud\n workspace: my_workspace_id\n query: select * from iot\n apikey: \u003cYour API Key\u003e"},{"title":"From Timeplus Enterprise (self-hosted) via HTTP","summary":"For self-housted Timeplus Enterprise, you will need to specify the username and password as well as the URL of the App server","config":"\ninput:\n timeplus:\n url: http://localhost:8000\n workspace: my_workspace_id\n query: select * from iot\n username: username\n password: pw"},{"title":"From Timeplus Enterprise (self-hosted) via TCP","summary":"Make sure the the schema of url is tcp","config":"\ninput:\n timeplus:\n url: tcp://localhost:8463\n query: select * from iot\n username: timeplus\n password: timeplus"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"query","type":"string","kind":"scalar","description":"The query to run","examples":["select * from iot","select count(*) from table(iot)"]},{"name":"url","type":"string","kind":"scalar","description":"The url should always include schema and host.","default":"tcp://localhost:8463","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"workspace","type":"string","kind":"scalar","description":"ID of the workspace. Required when reads from Timeplus Enterprise.","is_optional":true},{"name":"apikey","type":"string","kind":"scalar","description":"The API key. Required when reads from Timeplus Enterprise Cloud","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"username","type":"string","kind":"scalar","description":"The username. Required when reads from Timeplus Enterprise (self-hosted) or Timeplusd","is_optional":true},{"name":"password","type":"string","kind":"scalar","description":"The password. Required when reads from Timeplus Enterprise (self-hosted) or Timeplusd","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}},{"name":"twitter_search","type":"input","status":"experimental","plugin":true,"summary":"Consumes tweets matching a given search using the Twitter recent search V2 API.","description":"Continuously polls the https://developer.twitter.com/en/docs/twitter-api/tweets/search/api-reference/get-tweets-search-recent[Twitter recent search V2 API^] for tweets that match a given search query.\n\nEach tweet received is emitted as a JSON object message, with a field `id` and `text` by default. Extra fields https://developer.twitter.com/en/docs/twitter-api/fields[can be obtained from the search API^] when listed with the `tweet_fields` field.\n\nIn order to paginate requests that are made the ID of the latest received tweet is stored in a xref:components:caches/about.adoc[cache resource], which is then used by subsequent requests to ensure only tweets after it are consumed. It is recommended that the cache you use is persistent so that Redpanda Connect can resume searches at the correct place on a restart.\n\nAuthentication is done using OAuth 2.0 credentials which can be generated within the https://developer.twitter.com[Twitter developer portal^].\n","categories":["Services","Social"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"query","type":"string","kind":"scalar","description":"A search expression to use."},{"name":"tweet_fields","type":"string","kind":"array","description":"An optional list of additional fields to obtain for each tweet, by default only the fields `id` and `text` are returned. For more info refer to the https://developer.twitter.com/en/docs/twitter-api/fields[twitter API docs^].","default":[]},{"name":"poll_period","type":"string","kind":"scalar","description":"The length of time (as a duration string) to wait between each search request. This field can be set empty, in which case requests are made at the limit set by the rate limit. This field also supports cron expressions.","default":"1m"},{"name":"backfill_period","type":"string","kind":"scalar","description":"A duration string indicating the maximum age of tweets to acquire when starting a search.","default":"5m"},{"name":"cache","type":"string","kind":"scalar","description":"A cache resource to use for request pagination."},{"name":"cache_key","type":"string","kind":"scalar","description":"The key identifier used when storing the ID of the last tweet received.","is_advanced":true,"default":"last_tweet_id"},{"name":"rate_limit","type":"string","kind":"scalar","description":"An optional rate limit resource to restrict API requests with.","is_advanced":true,"default":""},{"name":"api_key","type":"string","kind":"scalar","description":"An API key for OAuth 2.0 authentication. It is recommended that you populate this field using xref:configuration:interpolation.adoc[environment variables]."},{"name":"api_secret","type":"string","kind":"scalar","description":"An API secret for OAuth 2.0 authentication. It is recommended that you populate this field using xref:configuration:interpolation.adoc[environment variables]."}]}},{"name":"websocket","type":"input","status":"stable","plugin":true,"summary":"Connects to a websocket server and continuously receives messages.","description":"It is possible to configure an `open_message`, which when set to a non-empty string will be sent to the websocket server each time a connection is first established.","categories":["Network"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL to connect to.","examples":["ws://localhost:4195/get/ws"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"proxy_url","type":"string","kind":"scalar","description":"An optional HTTP proxy URL.","is_advanced":true,"is_optional":true,"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"open_message","type":"string","kind":"scalar","description":"An optional message to send to the server upon connection.","is_advanced":true,"is_optional":true},{"name":"open_message_type","type":"string","kind":"scalar","description":"An optional flag to indicate the data type of open_message.","is_advanced":true,"default":"binary","annotated_options":[["binary","Binary data open_message."],["text","Text data open_message. The text message payload is interpreted as UTF-8 encoded text data."]],"linter":"\nlet options = {\n \"binary\": true,\n \"text\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"connection","type":"object","kind":"scalar","description":"Customise how websocket connection attempts are made.","is_advanced":true,"is_optional":true,"children":[{"name":"max_retries","type":"int","kind":"scalar","description":"An optional limit to the number of consecutive retry attempts that will be made before abandoning the connection altogether and gracefully terminating the input. When all inputs terminate in this way the service (or stream) will shut down. If set to zero connections will never be reattempted upon a failure. If set below zero this field is ignored (effectively unset).","is_advanced":true,"is_optional":true,"examples":[-1,10]}]},{"name":"oauth","type":"object","kind":"scalar","description":"Allows you to specify open authentication via OAuth version 1.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use OAuth version 1 in requests.","is_advanced":true,"default":false},{"name":"consumer_key","type":"string","kind":"scalar","description":"A value used to identify the client to the service provider.","is_advanced":true,"default":""},{"name":"consumer_secret","type":"string","kind":"scalar","description":"A secret used to establish ownership of the consumer key.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"access_token","type":"string","kind":"scalar","description":"A value used to gain access to the protected resources on behalf of the user.","is_advanced":true,"default":""},{"name":"access_token_secret","type":"string","kind":"scalar","description":"A secret provided in order to establish ownership of a given access token.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"jwt","type":"object","kind":"scalar","description":"BETA: Allows you to specify JWT authentication.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use JWT authentication in requests.","is_advanced":true,"default":false},{"name":"private_key_file","type":"string","kind":"scalar","description":"A file with the PEM encoded via PKCS1 or PKCS8 as private key.","is_advanced":true,"default":""},{"name":"signing_method","type":"string","kind":"scalar","description":"A method used to sign the token such as RS256, RS384, RS512 or EdDSA.","is_advanced":true,"default":""},{"name":"claims","type":"unknown","kind":"map","description":"A value used to identify the claims that issued the JWT.","is_advanced":true,"default":{}},{"name":"headers","type":"unknown","kind":"map","description":"Add optional key/value headers to the JWT.","is_advanced":true,"default":{}}]}]}}],"outputs":[{"name":"amqp_0_9","type":"output","status":"stable","plugin":true,"summary":"Sends messages to an AMQP (0.91) exchange. AMQP is a messaging protocol used by various message brokers, including RabbitMQ.Connects to an AMQP (0.91) queue. AMQP is a messaging protocol used by various message brokers, including RabbitMQ.","description":"The metadata from each message are delivered as headers.\n\nIt's possible for this output type to create the target exchange by setting `exchange_declare.enabled` to `true`, if the exchange already exists then the declaration passively verifies that the settings match.\n\nTLS is automatic when connecting to an `amqps` URL, but custom settings can be enabled in the `tls` section.\n\nThe fields 'key', 'exchange' and 'type' can be dynamically set using xref:configuration:interpolation.adoc#bloblang-queries[function interpolations].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. The first URL to successfully establish a connection will be used until the connection is closed. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["amqp://guest:guest@127.0.0.1:5672/"],["amqp://127.0.0.1:5672/,amqp://127.0.0.2:5672/"],["amqp://127.0.0.1:5672/","amqp://127.0.0.2:5672/"]],"version":"3.58.0","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"exchange","type":"string","kind":"scalar","description":"An AMQP exchange to publish to.","interpolated":true},{"name":"exchange_declare","type":"object","kind":"scalar","description":"Optionally declare the target exchange (passive).","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to declare the exchange.","is_advanced":true,"default":false},{"name":"type","type":"string","kind":"scalar","description":"The type of the exchange.","is_advanced":true,"default":"direct","options":["direct","fanout","topic","headers","x-custom"],"linter":"\nlet options = {\n \"direct\": true,\n \"fanout\": true,\n \"topic\": true,\n \"headers\": true,\n \"x-custom\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"durable","type":"bool","kind":"scalar","description":"Whether the exchange should be durable.","is_advanced":true,"default":true},{"name":"arguments","type":"string","kind":"map","description":"Optional arguments specific to the server's implementation of the exchange that can be sent for exchange types which require extra parameters.","is_advanced":true,"is_optional":true,"examples":[{"alternate-exchange":"my-ae"}]}]},{"name":"key","type":"string","kind":"scalar","description":"The binding key to set for each message.","default":"","interpolated":true},{"name":"type","type":"string","kind":"scalar","description":"The type property to set for each message.","default":"","interpolated":true},{"name":"content_type","type":"string","kind":"scalar","description":"The content type attribute to set for each message.","is_advanced":true,"default":"application/octet-stream","interpolated":true},{"name":"content_encoding","type":"string","kind":"scalar","description":"The content encoding attribute to set for each message.","is_advanced":true,"default":"","interpolated":true},{"name":"correlation_id","type":"string","kind":"scalar","description":"Set the correlation ID of each message with a dynamic interpolated expression.","is_advanced":true,"default":"","interpolated":true},{"name":"reply_to","type":"string","kind":"scalar","description":"Carries response queue name - set with a dynamic interpolated expression.","is_advanced":true,"default":"","interpolated":true},{"name":"expiration","type":"string","kind":"scalar","description":"Set the per-message TTL","is_advanced":true,"default":"","interpolated":true},{"name":"message_id","type":"string","kind":"scalar","description":"Set the message ID of each message with a dynamic interpolated expression.","is_advanced":true,"default":"","interpolated":true},{"name":"user_id","type":"string","kind":"scalar","description":"Set the user ID to the name of the publisher. If this property is set by a publisher, its value must be equal to the name of the user used to open the connection.","is_advanced":true,"default":"","interpolated":true},{"name":"app_id","type":"string","kind":"scalar","description":"Set the application ID of each message with a dynamic interpolated expression.","is_advanced":true,"default":"","interpolated":true},{"name":"metadata","type":"object","kind":"scalar","description":"Specify criteria for which metadata values are attached to messages as headers.","children":[{"name":"exclude_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages.","default":[]}]},{"name":"priority","type":"string","kind":"scalar","description":"Set the priority of each message with a dynamic interpolated expression.","is_advanced":true,"default":"","interpolated":true,"examples":["0","${! meta(\"amqp_priority\") }","${! json(\"doc.priority\") }"]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"persistent","type":"bool","kind":"scalar","description":"Whether message delivery should be persistent (transient by default).","is_advanced":true,"default":false},{"name":"mandatory","type":"bool","kind":"scalar","description":"Whether to set the mandatory flag on published messages. When set if a published message is routed to zero queues it is returned.","is_advanced":true,"default":false},{"name":"immediate","type":"bool","kind":"scalar","description":"Whether to set the immediate flag on published messages. When set if there are no ready consumers of a queue then the message is dropped instead of waiting.","is_advanced":true,"default":false},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period to wait before abandoning it and reattempting. If not set, wait indefinitely.","is_advanced":true,"default":""},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]}]}},{"name":"amqp_1","type":"output","status":"stable","plugin":true,"summary":"Sends messages to an AMQP (1.0) server.","description":"\n== Metadata\n\nMessage metadata is added to each AMQP message as string annotations. In order to control which metadata keys are added use the `metadata` config field.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"A URL to connect to.","is_deprecated":true,"is_optional":true,"examples":["amqp://localhost:5672/","amqps://guest:guest@localhost:5672/"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. The first URL to successfully establish a connection will be used until the connection is closed. If an item of the list contains commas it will be expanded into multiple URLs.","is_optional":true,"examples":[["amqp://guest:guest@127.0.0.1:5672/"],["amqp://127.0.0.1:5672/,amqp://127.0.0.2:5672/"],["amqp://127.0.0.1:5672/","amqp://127.0.0.2:5672/"]],"version":"4.23.0","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"target_address","type":"string","kind":"scalar","description":"The target address to write to.","examples":["/foo","queue:/bar","topic:/baz"]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"application_properties_map","type":"string","kind":"scalar","description":"An optional Bloblang mapping that can be defined in order to set the `application-properties` on output messages.","is_advanced":true,"is_optional":true,"bloblang":true},{"name":"sasl","type":"object","kind":"scalar","description":"Enables SASL authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"mechanism","type":"string","kind":"scalar","description":"The SASL authentication mechanism to use.","is_advanced":true,"default":"none","annotated_options":[["anonymous","Anonymous SASL authentication."],["none","No SASL based authentication."],["plain","Plain text SASL authentication."]],"linter":"\nlet options = {\n \"anonymous\": true,\n \"none\": true,\n \"plain\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"user","type":"string","kind":"scalar","description":"A SASL plain text username. It is recommended that you use environment variables to populate this field.","is_advanced":true,"default":"","examples":["${USER}"]},{"name":"password","type":"string","kind":"scalar","description":"A SASL plain text password. It is recommended that you use environment variables to populate this field.","is_advanced":true,"is_secret":true,"default":"","examples":["${PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"metadata","type":"object","kind":"scalar","description":"Specify criteria for which metadata values are attached to messages as headers.","children":[{"name":"exclude_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages.","default":[]}]},{"name":"content_type","type":"string","kind":"scalar","description":"Specify the message body content type. The option `string` will transfer the message as an AMQP value of type string. Consider choosing the option `string` if your intention is to transfer UTF-8 string messages (like JSON messages) to the destination.","is_advanced":true,"default":"opaque_binary","options":["opaque_binary","string"],"linter":"\nlet options = {\n \"opaque_binary\": true,\n \"string\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"}],"linter":"\nroot = if this.url.or(\"\") == \"\" \u0026\u0026 this.urls.or([]).length() == 0 {\n \"field 'urls' must be set\"\n}\n"}},{"name":"aws_dynamodb","type":"output","status":"stable","plugin":true,"summary":"Inserts items into a DynamoDB table.","description":"\nThe field `string_columns` is a map of column names to string values, where the values are xref:configuration:interpolation.adoc#bloblang-queries[function interpolated] per message of a batch. This allows you to populate string columns of an item by extracting fields within the document payload or metadata like follows:\n\n```yml\nstring_columns:\n id: ${!json(\"id\")}\n title: ${!json(\"body.title\")}\n topic: ${!meta(\"kafka_topic\")}\n full_content: ${!content()}\n```\n\nThe field `json_map_columns` is a map of column names to json paths, where the xref:configuration:field_paths.adoc[dot path] is extracted from each document and converted into a map value. Both an empty path and the path `.` are interpreted as the root of the document. This allows you to populate map columns of an item like follows:\n\n```yml\njson_map_columns:\n user: path.to.user\n whole_document: .\n```\n\nA column name can be empty:\n\n```yml\njson_map_columns:\n \"\": .\n```\n\nIn which case the top level document fields will be written at the root of the item, potentially overwriting previously defined column values. If a path is not found within a document the column will not be populated.\n\n== Credentials\n\nBy default Redpanda Connect will use a shared credentials file when connecting to AWS services. It's also possible to set them explicitly at the component level, allowing you to transfer data across accounts. You can find out more in xref:guides:cloud/aws.adoc[].\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].\n","categories":["Services","AWS"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"table","type":"string","kind":"scalar","description":"The table to store messages in."},{"name":"string_columns","type":"string","kind":"map","description":"A map of column keys to string values to store.","default":{},"interpolated":true,"examples":[{"full_content":"${!content()}","id":"${!json(\"id\")}","title":"${!json(\"body.title\")}","topic":"${!meta(\"kafka_topic\")}"}]},{"name":"json_map_columns","type":"string","kind":"map","description":"A map of column keys to xref:configuration:field_paths.adoc[field paths] pointing to value data within messages.","default":{},"examples":[{"user":"path.to.user","whole_document":"."},{"":"."}]},{"name":"ttl","type":"string","kind":"scalar","description":"An optional TTL to set for items, calculated from the moment the message is sent.","is_advanced":true,"default":""},{"name":"ttl_key","type":"string","kind":"scalar","description":"The column key to place the TTL value within.","is_advanced":true,"default":""},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]},{"name":"max_retries","type":"int","kind":"scalar","description":"The maximum number of retries before giving up on the request. If set to zero there is no discrete limit.","is_advanced":true,"default":3},{"name":"backoff","type":"object","kind":"scalar","description":"Control time intervals between retry attempts.","is_advanced":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"1s"},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts.","is_advanced":true,"default":"5s"},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum period to wait before retry attempts are abandoned. If zero then no limit is used.","is_advanced":true,"default":"30s"}]}]},"version":"3.36.0"},{"name":"aws_kinesis","type":"output","status":"stable","plugin":true,"summary":"Sends messages to a Kinesis stream.","description":"\nBoth the `partition_key`(required) and `hash_key` (optional) fields can be dynamically set using function interpolations described xref:configuration:interpolation.adoc#bloblang-queries[here]. When sending batched messages the interpolations are performed per message part.\n\n== Credentials\n\nBy default Redpanda Connect will use a shared credentials file when connecting to AWS services. It's also possible to set them explicitly at the component level, allowing you to transfer data across accounts. You can find out more in xref:guides:cloud/aws.adoc[].\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services","AWS"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"stream","type":"string","kind":"scalar","description":"The stream to publish messages to. Streams can either be specified by their name or full ARN.","examples":["foo","arn:aws:kinesis:*:111122223333:stream/my-stream"]},{"name":"partition_key","type":"string","kind":"scalar","description":"A required key for partitioning messages.","interpolated":true},{"name":"hash_key","type":"string","kind":"scalar","description":"A optional hash key for partitioning messages.","is_advanced":true,"is_optional":true,"interpolated":true},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of parallel message batches to have in flight at any given time.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]},{"name":"max_retries","type":"int","kind":"scalar","description":"The maximum number of retries before giving up on the request. If set to zero there is no discrete limit.","is_advanced":true,"default":0},{"name":"backoff","type":"object","kind":"scalar","description":"Control time intervals between retry attempts.","is_advanced":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"1s"},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts.","is_advanced":true,"default":"5s"},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum period to wait before retry attempts are abandoned. If zero then no limit is used.","is_advanced":true,"default":"30s"}]}]},"version":"3.36.0"},{"name":"aws_kinesis_firehose","type":"output","status":"stable","plugin":true,"summary":"Sends messages to a Kinesis Firehose delivery stream.","description":"\n== Credentials\n\nBy default Redpanda Connect will use a shared credentials file when connecting to AWS services. It's also possible to set them explicitly at the component level, allowing you to transfer data across accounts. You can find out more in xref:guides:cloud/aws.adoc[].\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].\n","categories":["Services","AWS"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"stream","type":"string","kind":"scalar","description":"The stream to publish messages to."},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]},{"name":"max_retries","type":"int","kind":"scalar","description":"The maximum number of retries before giving up on the request. If set to zero there is no discrete limit.","is_advanced":true,"default":0},{"name":"backoff","type":"object","kind":"scalar","description":"Control time intervals between retry attempts.","is_advanced":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"1s"},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts.","is_advanced":true,"default":"5s"},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum period to wait before retry attempts are abandoned. If zero then no limit is used.","is_advanced":true,"default":"30s"}]}]},"version":"3.36.0"},{"name":"aws_s3","type":"output","status":"stable","plugin":true,"summary":"Sends message parts as objects to an Amazon S3 bucket. Each object is uploaded with the path specified with the `path` field.","description":"\nIn order to have a different path for each object you should use function interpolations described in xref:configuration:interpolation.adoc#bloblang-queries[Bloblang queries], which are calculated per message of a batch.\n\n== Metadata\n\nMetadata fields on messages will be sent as headers, in order to mutate these values (or remove them) check out the xref:configuration:metadata.adoc[metadata docs].\n\n== Tags\n\nThe tags field allows you to specify key/value pairs to attach to objects as tags, where the values support xref:configuration:interpolation.adoc#bloblang-queries[interpolation functions]:\n\n```yaml\noutput:\n aws_s3:\n bucket: TODO\n path: ${!counter()}-${!timestamp_unix_nano()}.tar.gz\n tags:\n Key1: Value1\n Timestamp: ${!meta(\"Timestamp\")}\n```\n\n=== Credentials\n\nBy default Redpanda Connect will use a shared credentials file when connecting to AWS services. It's also possible to set them explicitly at the component level, allowing you to transfer data across accounts. You can find out more in xref:guides:cloud/aws.adoc[].\n\n== Batching\n\nIt's common to want to upload messages to S3 as batched archives, the easiest way to do this is to batch your messages at the output level and join the batch of messages with an xref:components:processors/archive.adoc[`archive`] and/or xref:components:processors/compress.adoc[`compress`] processor.\n\nFor example, if we wished to upload messages as a .tar.gz archive of documents we could achieve that with the following config:\n\n```yaml\noutput:\n aws_s3:\n bucket: TODO\n path: ${!counter()}-${!timestamp_unix_nano()}.tar.gz\n batching:\n count: 100\n period: 10s\n processors:\n - archive:\n format: tar\n - compress:\n algorithm: gzip\n```\n\nAlternatively, if we wished to upload JSON documents as a single large document containing an array of objects we can do that with:\n\n```yaml\noutput:\n aws_s3:\n bucket: TODO\n path: ${!counter()}-${!timestamp_unix_nano()}.json\n batching:\n count: 100\n processors:\n - archive:\n format: json_array\n```\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.","categories":["Services","AWS"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"bucket","type":"string","kind":"scalar","description":"The bucket to upload messages to."},{"name":"path","type":"string","kind":"scalar","description":"The path of each message to upload.","default":"${!counter()}-${!timestamp_unix_nano()}.txt","interpolated":true,"examples":["${!counter()}-${!timestamp_unix_nano()}.txt","${!meta(\"kafka_key\")}.json","${!json(\"doc.namespace\")}/${!json(\"doc.id\")}.json"]},{"name":"tags","type":"string","kind":"map","description":"Key/value pairs to store with the object as tags.","default":{},"interpolated":true,"examples":[{"Key1":"Value1","Timestamp":"${!meta(\"Timestamp\")}"}]},{"name":"content_type","type":"string","kind":"scalar","description":"The content type to set for each object.","default":"application/octet-stream","interpolated":true},{"name":"content_encoding","type":"string","kind":"scalar","description":"An optional content encoding to set for each object.","is_advanced":true,"default":"","interpolated":true},{"name":"cache_control","type":"string","kind":"scalar","description":"The cache control to set for each object.","is_advanced":true,"default":"","interpolated":true},{"name":"content_disposition","type":"string","kind":"scalar","description":"The content disposition to set for each object.","is_advanced":true,"default":"","interpolated":true},{"name":"content_language","type":"string","kind":"scalar","description":"The content language to set for each object.","is_advanced":true,"default":"","interpolated":true},{"name":"content_md5","type":"string","kind":"scalar","description":"The content MD5 to set for each object.","is_advanced":true,"default":"","interpolated":true},{"name":"website_redirect_location","type":"string","kind":"scalar","description":"The website redirect location to set for each object.","is_advanced":true,"default":"","interpolated":true},{"name":"metadata","type":"object","kind":"scalar","description":"Specify criteria for which metadata values are attached to objects as headers.","children":[{"name":"exclude_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages.","default":[]}]},{"name":"storage_class","type":"string","kind":"scalar","description":"The storage class to set for each object.","is_advanced":true,"default":"STANDARD","interpolated":true,"options":["STANDARD","REDUCED_REDUNDANCY","GLACIER","STANDARD_IA","ONEZONE_IA","INTELLIGENT_TIERING","DEEP_ARCHIVE"],"linter":"\nlet options = {\n \"standard\": true,\n \"reduced_redundancy\": true,\n \"glacier\": true,\n \"standard_ia\": true,\n \"onezone_ia\": true,\n \"intelligent_tiering\": true,\n \"deep_archive\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"kms_key_id","type":"string","kind":"scalar","description":"An optional server side encryption key.","is_advanced":true,"default":""},{"name":"checksum_algorithm","type":"string","kind":"scalar","description":"The algorithm used to create the checksum for each object.","is_advanced":true,"default":"","options":["CRC32","CRC32C","SHA1","SHA256"],"linter":"\nlet options = {\n \"crc32\": true,\n \"crc32c\": true,\n \"sha1\": true,\n \"sha256\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"server_side_encryption","type":"string","kind":"scalar","description":"An optional server side encryption algorithm.","is_advanced":true,"default":"","version":"3.63.0"},{"name":"force_path_style_urls","type":"bool","kind":"scalar","description":"Forces the client API to use path style URLs, which helps when connecting to custom endpoints.","is_advanced":true,"default":false},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period to wait on an upload before abandoning it and reattempting.","is_advanced":true,"default":"5s"},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]},"version":"3.36.0"},{"name":"aws_sns","type":"output","status":"stable","plugin":true,"summary":"Sends messages to an AWS SNS topic.","description":"\n== Credentials\n\nBy default Redpanda Connect will use a shared credentials file when connecting to AWS services. It's also possible to set them explicitly at the component level, allowing you to transfer data across accounts. You can find out more in xref:guides:cloud/aws.adoc[].\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.","categories":["Services","AWS"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"topic_arn","type":"string","kind":"scalar","description":"The topic to publish to."},{"name":"message_group_id","type":"string","kind":"scalar","description":"An optional group ID to set for messages.","is_optional":true,"interpolated":true,"version":"3.60.0"},{"name":"message_deduplication_id","type":"string","kind":"scalar","description":"An optional deduplication ID to set for messages.","is_optional":true,"interpolated":true,"version":"3.60.0"},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"metadata","type":"object","kind":"scalar","description":"Specify criteria for which metadata values are sent as headers.","children":[{"name":"exclude_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages.","default":[]}],"version":"3.60.0"},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period to wait on an upload before abandoning it and reattempting.","is_advanced":true,"default":"5s"},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]},"version":"3.36.0"},{"name":"aws_sqs","type":"output","status":"stable","plugin":true,"summary":"Sends messages to an SQS queue.","description":"\nMetadata values are sent along with the payload as attributes with the data type String. If the number of metadata values in a message exceeds the message attribute limit (10) then the top ten keys ordered alphabetically will be selected.\n\nThe fields `message_group_id`, `message_deduplication_id` and `delay_seconds` can be set dynamically using xref:configuration:interpolation.adoc#bloblang-queries[function interpolations], which are resolved individually for each message of a batch.\n\n== Credentials\n\nBy default Redpanda Connect will use a shared credentials file when connecting to AWS services. It's also possible to set them explicitly at the component level, allowing you to transfer data across accounts. You can find out more in xref:guides:cloud/aws.adoc[].\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services","AWS"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target SQS queue.","interpolated":true},{"name":"message_group_id","type":"string","kind":"scalar","description":"An optional group ID to set for messages.","is_optional":true,"interpolated":true},{"name":"message_deduplication_id","type":"string","kind":"scalar","description":"An optional deduplication ID to set for messages.","is_optional":true,"interpolated":true},{"name":"delay_seconds","type":"string","kind":"scalar","description":"An optional delay time in seconds for message. Value between 0 and 900","is_optional":true,"interpolated":true},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of parallel message batches to have in flight at any given time.","default":64},{"name":"metadata","type":"object","kind":"scalar","description":"Specify criteria for which metadata values are sent as headers.","children":[{"name":"exclude_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages.","default":[]}]},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"max_records_per_request","type":"int","kind":"scalar","description":"Customize the maximum number of records delivered in a single SQS request. This value must be greater than 0 but no greater than 10.","is_advanced":true,"default":10,"linter":"if this \u003c= 0 || this \u003e 10 { \"this field must be \u003e0 and \u003c=10\" } "},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]},{"name":"max_retries","type":"int","kind":"scalar","description":"The maximum number of retries before giving up on the request. If set to zero there is no discrete limit.","is_advanced":true,"default":0},{"name":"backoff","type":"object","kind":"scalar","description":"Control time intervals between retry attempts.","is_advanced":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"1s"},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts.","is_advanced":true,"default":"5s"},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum period to wait before retry attempts are abandoned. If zero then no limit is used.","is_advanced":true,"default":"30s"}]}]},"version":"3.36.0"},{"name":"azure_blob_storage","type":"output","status":"beta","plugin":true,"summary":"Sends message parts as objects to an Azure Blob Storage Account container. Each object is uploaded with the filename specified with the `container` field.","description":"\nIn order to have a different path for each object you should use function\ninterpolations described xref:configuration:interpolation.adoc#bloblang-queries[here], which are\ncalculated per message of a batch.\n\nSupports multiple authentication methods but only one of the following is required:\n\n- `storage_connection_string`\n- `storage_account` and `storage_access_key`\n- `storage_account` and `storage_sas_token`\n- `storage_account` to access via https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity#DefaultAzureCredential[DefaultAzureCredential^]\n\nIf multiple are set then the `storage_connection_string` is given priority.\n\nIf the `storage_connection_string` does not contain the `AccountName` parameter, please specify it in the\n`storage_account` field.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.","categories":["Services","Azure"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"storage_account","type":"string","kind":"scalar","description":"The storage account to access. This field is ignored if `storage_connection_string` is set.","default":""},{"name":"storage_access_key","type":"string","kind":"scalar","description":"The storage account access key. This field is ignored if `storage_connection_string` is set.","default":""},{"name":"storage_connection_string","type":"string","kind":"scalar","description":"A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set.","default":""},{"name":"storage_sas_token","type":"string","kind":"scalar","description":"The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set.","default":""},{"name":"container","type":"string","kind":"scalar","description":"The container for uploading the messages to.","interpolated":true,"examples":["messages-${!timestamp(\"2006\")}"]},{"name":"path","type":"string","kind":"scalar","description":"The path of each message to upload.","default":"${!counter()}-${!timestamp_unix_nano()}.txt","interpolated":true,"examples":["${!counter()}-${!timestamp_unix_nano()}.json","${!meta(\"kafka_key\")}.json","${!json(\"doc.namespace\")}/${!json(\"doc.id\")}.json"]},{"name":"blob_type","type":"string","kind":"scalar","description":"Block and Append blobs are comprized of blocks, and each blob can support up to 50,000 blocks. The default value is `+\"`BLOCK`\"+`.`","is_advanced":true,"default":"BLOCK","interpolated":true,"options":["BLOCK","APPEND"],"linter":"\nlet options = {\n \"block\": true,\n \"append\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"public_access_level","type":"string","kind":"scalar","description":"The container's public access level. The default value is `PRIVATE`.","is_advanced":true,"default":"PRIVATE","interpolated":true,"options":["PRIVATE","BLOB","CONTAINER"],"linter":"\nlet options = {\n \"private\": true,\n \"blob\": true,\n \"container\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64}],"linter":"root = if this.storage_connection_string != \"\" \u0026\u0026 !this.storage_connection_string.contains(\"AccountName=\") \u0026\u0026 !this.storage_connection_string.contains(\"UseDevelopmentStorage=true;\") \u0026\u0026 this.storage_account == \"\" { [ \"storage_account must be set if storage_connection_string does not contain the \\\"AccountName\\\" parameter\" ] }"},"version":"3.36.0"},{"name":"azure_cosmosdb","type":"output","status":"experimental","plugin":true,"summary":"Creates or updates messages as JSON documents in https://learn.microsoft.com/en-us/azure/cosmos-db/introduction[Azure CosmosDB^].","description":"\nWhen creating documents, each message must have the `id` property (case-sensitive) set (or use `auto_id: true`). It is the unique name that identifies the document, that is, no two documents share the same `id` within a logical partition. The `id` field must not exceed 255 characters. https://learn.microsoft.com/en-us/rest/api/cosmos-db/documents[See details^].\n\nThe `partition_keys` field must resolve to the same value(s) across the entire message batch.\n\n\n== Credentials\n\nYou can use one of the following authentication mechanisms:\n\n- Set the `endpoint` field and the `account_key` field\n- Set only the `endpoint` field to use https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity#DefaultAzureCredential[DefaultAzureCredential^]\n- Set the `connection_string` field\n\n\n== Batching\n\nCosmosDB limits the maximum batch size to 100 messages and the payload must not exceed 2MB (https://learn.microsoft.com/en-us/azure/cosmos-db/concepts-limits#per-request-limits[details here^]).\n\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Azure"],"footnotes":"\n\n== CosmosDB emulator\n\nIf you wish to run the CosmosDB emulator that is referenced in the documentation https://learn.microsoft.com/en-us/azure/cosmos-db/linux-emulator[here^], the following Docker command should do the trick:\n\n```bash\n\u003e docker run --rm -it -p 8081:8081 --name=cosmosdb -e AZURE_COSMOS_EMULATOR_PARTITION_COUNT=10 -e AZURE_COSMOS_EMULATOR_ENABLE_DATA_PERSISTENCE=false mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator\n```\n\nNote: `AZURE_COSMOS_EMULATOR_PARTITION_COUNT` controls the number of partitions that will be supported by the emulator. The bigger the value, the longer it takes for the container to start up.\n\nAdditionally, instead of installing the container self-signed certificate which is exposed via `https://localhost:8081/_explorer/emulator.pem`, you can run https://mitmproxy.org/[mitmproxy^] like so:\n\n```bash\n\u003e mitmproxy -k --mode \"reverse:https://localhost:8081\"\n```\n\nThen you can access the CosmosDB UI via `http://localhost:8080/_explorer/index.html` and use `http://localhost:8080` as the CosmosDB endpoint.\n","examples":[{"title":"Create documents","summary":"Create new documents in the `blobfish` container with partition key `/habitat`.","config":"\noutput:\n azure_cosmosdb:\n endpoint: http://localhost:8080\n account_key: C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==\n database: blobbase\n container: blobfish\n partition_keys_map: root = json(\"habitat\")\n operation: Create\n"},{"title":"Patch documents","summary":"Execute the Patch operation on documents from the `blobfish` container.","config":"\noutput:\n azure_cosmosdb:\n endpoint: http://localhost:8080\n account_key: C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==\n database: testdb\n container: blobfish\n partition_keys_map: root = json(\"habitat\")\n item_id: ${! json(\"id\") }\n operation: Patch\n patch_operations:\n # Add a new /diet field\n - operation: Add\n path: /diet\n value_map: root = json(\"diet\")\n # Remove the first location from the /locations array field\n - operation: Remove\n path: /locations/0\n # Add new location at the end of the /locations array field\n - operation: Add\n path: /locations/-\n value_map: root = \"Challenger Deep\"\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"endpoint","type":"string","kind":"scalar","description":"CosmosDB endpoint.","is_optional":true,"examples":["https://localhost:8081"]},{"name":"account_key","type":"string","kind":"scalar","description":"Account key.","is_optional":true,"is_secret":true,"examples":["C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw=="],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"connection_string","type":"string","kind":"scalar","description":"Connection string.","is_optional":true,"is_secret":true,"examples":["AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==;"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"database","type":"string","kind":"scalar","description":"Database.","examples":["testdb"]},{"name":"container","type":"string","kind":"scalar","description":"Container.","examples":["testcontainer"]},{"name":"partition_keys_map","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to a single partition key value or an array of partition key values of type string, integer or boolean. Currently, hierarchical partition keys are not supported so only one value may be provided.","bloblang":true,"examples":["root = \"blobfish\"","root = 41","root = true","root = null","root = json(\"blobfish\").depth"]},{"name":"operation","type":"string","kind":"scalar","description":"Operation.","default":"Create","annotated_options":[["Create","Create operation."],["Delete","Delete operation."],["Patch","Patch operation."],["Replace","Replace operation."],["Upsert","Upsert operation."]],"linter":"\nlet options = {\n \"create\": true,\n \"delete\": true,\n \"patch\": true,\n \"replace\": true,\n \"upsert\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"patch_operations","type":"object","kind":"array","description":"Patch operations to be performed when `operation: Patch` .","is_advanced":true,"is_optional":true,"children":[{"name":"operation","type":"string","kind":"scalar","description":"Operation.","is_advanced":true,"default":"Add","annotated_options":[["Add","Add patch operation."],["Increment","Increment patch operation."],["Remove","Remove patch operation."],["Replace","Replace patch operation."],["Set","Set patch operation."]],"linter":"\nlet options = {\n \"add\": true,\n \"increment\": true,\n \"remove\": true,\n \"replace\": true,\n \"set\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"path","type":"string","kind":"scalar","description":"Path.","is_advanced":true,"examples":["/foo/bar/baz"]},{"name":"value_map","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to a value of any type that is supported by CosmosDB.","is_advanced":true,"is_optional":true,"bloblang":true,"examples":["root = \"blobfish\"","root = 41","root = true","root = json(\"blobfish\").depth","root = [1, 2, 3]"]}]},{"name":"patch_condition","type":"string","kind":"scalar","description":"Patch operation condition.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["from c where not is_defined(c.blobfish)"]},{"name":"auto_id","type":"bool","kind":"scalar","description":"Automatically set the item `id` field to a random UUID v4. If the `id` field is already set, then it will not be overwritten. Setting this to `false` can improve performance, since the messages will not have to be parsed.","is_advanced":true,"default":true},{"name":"item_id","type":"string","kind":"scalar","description":"ID of item to replace or delete. Only used by the Replace and Delete operations","is_optional":true,"interpolated":true,"examples":["${! json(\"id\") }"]},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64}],"linter":"root = []\nlet hasEndpoint = this.endpoint.or(\"\") != \"\"\nlet hasConnectionString = this.connection_string.or(\"\") != \"\"\n\nroot.\"-\" = if !$hasEndpoint \u0026\u0026 !$hasConnectionString {\n \"Either `endpoint` or `connection_string` must be set.\"\n}\n\nlet hasItemID = this.item_id.or(\"\") != \"\"\nlet hasPatchOperations = this.patch_operations.length().or(0) \u003e 0\nlet hasPatchCondition = this.patch_condition.or(\"\") != \"\"\n\nroot.\"-\" = if !$hasItemID \u0026\u0026 (this.operation == \"Replace\" || this.operation == \"Delete\" || this.operation == \"Read\" || this.operation == \"Patch\") {\n \"The `item_id` field must be set for Replace, Delete, Read and Patch operations.\"\n}\n\nroot.\"-\" = if this.operation == \"Patch\" \u0026\u0026 !$hasPatchOperations {\n \"At least one `patch_operations` must be set when `operation: Patch`.\"\n}\n\nroot.\"-\" = if $hasPatchCondition \u0026\u0026 (!$hasPatchOperations || this.operation != \"Patch\") {\n \"The `patch_condition` field only applies to `Patch` operations and it requires one or more `patch_operations`.\"\n}\n\nroot.\"-\" = if this.operation == \"Patch\" \u0026\u0026 this.patch_operations.any(o -\u003e o.operation != \"Remove\" \u0026\u0026 o.value_map.or(\"\") == \"\") {\n \"The `patch_operations` `value_map` field must be set when `operation` is not `Remove`.\"\n}\n\nroot.\"-\" = if this.operation == \"Patch\" \u0026\u0026 this.patch_operations.any(o -\u003e o.operation == \"Remove\" \u0026\u0026 o.value_map.or(\"\") != \"\") {\n \"The `patch_operations` `value_map` field must not be set when `operation` is `Remove`.\"\n}\n"},"version":"v4.25.0"},{"name":"azure_data_lake_gen2","type":"output","status":"beta","plugin":true,"summary":"Sends message parts as files to an Azure Data Lake Gen2 filesystem. Each file is uploaded with the filename specified with the `path` field.","description":"\nIn order to have a different path for each file you should use function\ninterpolations described xref:configuration:interpolation.adoc#bloblang-queries[here], which are\ncalculated per message of a batch.\n\nSupports multiple authentication methods but only one of the following is required:\n\n- `storage_connection_string`\n- `storage_account` and `storage_access_key`\n- `storage_account` and `storage_sas_token`\n- `storage_account` to access via https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity#DefaultAzureCredential[DefaultAzureCredential^]\n\nIf multiple are set then the `storage_connection_string` is given priority.\n\nIf the `storage_connection_string` does not contain the `AccountName` parameter, please specify it in the\n`storage_account` field.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.","categories":["Services","Azure"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"storage_account","type":"string","kind":"scalar","description":"The storage account to access. This field is ignored if `storage_connection_string` is set.","default":""},{"name":"storage_access_key","type":"string","kind":"scalar","description":"The storage account access key. This field is ignored if `storage_connection_string` is set.","default":""},{"name":"storage_connection_string","type":"string","kind":"scalar","description":"A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set.","default":""},{"name":"storage_sas_token","type":"string","kind":"scalar","description":"The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set.","default":""},{"name":"filesystem","type":"string","kind":"scalar","description":"The data lake storage filesystem name for uploading the messages to.","interpolated":true,"examples":["messages-${!timestamp(\"2006\")}"]},{"name":"path","type":"string","kind":"scalar","description":"The path of each message to upload within the filesystem.","default":"${!counter()}-${!timestamp_unix_nano()}.txt","interpolated":true,"examples":["${!counter()}-${!timestamp_unix_nano()}.json","${!meta(\"kafka_key\")}.json","${!json(\"doc.namespace\")}/${!json(\"doc.id\")}.json"]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64}],"linter":"root = if this.storage_connection_string != \"\" \u0026\u0026 !this.storage_connection_string.contains(\"AccountName=\") \u0026\u0026 !this.storage_connection_string.contains(\"UseDevelopmentStorage=true;\") \u0026\u0026 this.storage_account == \"\" { [ \"storage_account must be set if storage_connection_string does not contain the \\\"AccountName\\\" parameter\" ] }"},"version":"4.38.0"},{"name":"azure_queue_storage","type":"output","status":"beta","plugin":true,"summary":"Sends messages to an Azure Storage Queue.","description":"\nOnly one authentication method is required, `storage_connection_string` or `storage_account` and `storage_access_key`. If both are set then the `storage_connection_string` is given priority.\n\nIn order to set the `queue_name` you can use function interpolations described xref:configuration:interpolation.adoc#bloblang-queries[here], which are calculated per message of a batch.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services","Azure"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"storage_account","type":"string","kind":"scalar","description":"The storage account to access. This field is ignored if `storage_connection_string` is set.","default":""},{"name":"storage_access_key","type":"string","kind":"scalar","description":"The storage account access key. This field is ignored if `storage_connection_string` is set.","default":""},{"name":"storage_connection_string","type":"string","kind":"scalar","description":"A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set.","default":""},{"name":"storage_sas_token","type":"string","kind":"scalar","description":"The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set.","default":""},{"name":"queue_name","type":"string","kind":"scalar","description":"The name of the target Queue Storage queue.","interpolated":true},{"name":"ttl","type":"string","kind":"scalar","description":"The TTL of each individual message as a duration string. Defaults to 0, meaning no retention period is set","is_advanced":true,"default":"","interpolated":true,"examples":["60s","5m","36h"]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of parallel message batches to have in flight at any given time.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}],"linter":"root = if this.storage_connection_string != \"\" \u0026\u0026 !this.storage_connection_string.contains(\"AccountName=\") \u0026\u0026 !this.storage_connection_string.contains(\"UseDevelopmentStorage=true;\") \u0026\u0026 this.storage_account == \"\" { [ \"storage_account must be set if storage_connection_string does not contain the \\\"AccountName\\\" parameter\" ] }"},"version":"3.36.0"},{"name":"azure_table_storage","type":"output","status":"beta","plugin":true,"summary":"Stores messages in an Azure Table Storage table.","description":"\nOnly one authentication method is required, `storage_connection_string` or `storage_account` and `storage_access_key`. If both are set then the `storage_connection_string` is given priority.\n\nIn order to set the `table_name`, `partition_key` and `row_key` you can use function interpolations described xref:configuration:interpolation.adoc#bloblang-queries[here], which are calculated per message of a batch.\n\nIf the `properties` are not set in the config, all the `json` fields are marshalled and stored in the table, which will be created if it does not exist.\n\nThe `object` and `array` fields are marshaled as strings. e.g.:\n\nThe JSON message:\n\n```json\n{\n \"foo\": 55,\n \"bar\": {\n \"baz\": \"a\",\n \"bez\": \"b\"\n },\n \"diz\": [\"a\", \"b\"]\n}\n```\n\nWill store in the table the following properties:\n\n```yml\nfoo: '55'\nbar: '{ \"baz\": \"a\", \"bez\": \"b\" }'\ndiz: '[\"a\", \"b\"]'\n```\n\nIt's also possible to use function interpolations to get or transform the properties values, e.g.:\n\n```yml\nproperties:\n device: '${! json(\"device\") }'\n timestamp: '${! json(\"timestamp\") }'\n```\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services","Azure"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"storage_account","type":"string","kind":"scalar","description":"The storage account to access. This field is ignored if `storage_connection_string` is set.","default":""},{"name":"storage_access_key","type":"string","kind":"scalar","description":"The storage account access key. This field is ignored if `storage_connection_string` is set.","default":""},{"name":"storage_connection_string","type":"string","kind":"scalar","description":"A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set.","default":""},{"name":"storage_sas_token","type":"string","kind":"scalar","description":"The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set.","default":""},{"name":"table_name","type":"string","kind":"scalar","description":"The table to store messages into.","interpolated":true,"examples":["${! meta(\"kafka_topic\") }","${! json(\"table\") }"]},{"name":"partition_key","type":"string","kind":"scalar","description":"The partition key.","default":"","interpolated":true,"examples":["${! json(\"date\") }"]},{"name":"row_key","type":"string","kind":"scalar","description":"The row key.","default":"","interpolated":true,"examples":["${! json(\"device\")}-${!uuid_v4() }"]},{"name":"properties","type":"string","kind":"map","description":"A map of properties to store into the table.","default":{},"interpolated":true},{"name":"insert_type","type":"string","kind":"scalar","description":"Type of insert operation. Valid options are `INSERT`, `INSERT_MERGE` and `INSERT_REPLACE`","is_advanced":true,"is_deprecated":true,"default":"","interpolated":true,"examples":["${! json(\"operation\") }","${! meta(\"operation\") }","INSERT"],"options":["INSERT","INSERT_MERGE","INSERT_REPLACE"],"linter":"\nlet options = {\n \"insert\": true,\n \"insert_merge\": true,\n \"insert_replace\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"transaction_type","type":"string","kind":"scalar","description":"Type of transaction operation.","is_advanced":true,"default":"INSERT","interpolated":true,"examples":["${! json(\"operation\") }","${! meta(\"operation\") }","INSERT"],"options":["INSERT","INSERT_MERGE","INSERT_REPLACE","UPDATE_MERGE","UPDATE_REPLACE","DELETE"],"linter":"\nlet options = {\n \"insert\": true,\n \"insert_merge\": true,\n \"insert_replace\": true,\n \"update_merge\": true,\n \"update_replace\": true,\n \"delete\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of parallel message batches to have in flight at any given time.","default":64},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period to wait on an upload before abandoning it and reattempting.","is_advanced":true,"default":"5s"},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}],"linter":"root = if this.storage_connection_string != \"\" \u0026\u0026 !this.storage_connection_string.contains(\"AccountName=\") \u0026\u0026 !this.storage_connection_string.contains(\"UseDevelopmentStorage=true;\") \u0026\u0026 this.storage_account == \"\" { [ \"storage_account must be set if storage_connection_string does not contain the \\\"AccountName\\\" parameter\" ] }"},"version":"3.36.0"},{"name":"beanstalkd","type":"output","status":"experimental","plugin":true,"summary":"Write messages to a Beanstalkd queue.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"address","type":"string","kind":"scalar","description":"An address to connect to.","examples":["127.0.0.1:11300"]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase to improve throughput.","default":64}]},"version":"4.7.0"},{"name":"broker","type":"output","status":"stable","plugin":true,"summary":"Allows you to route messages to multiple child outputs using a range of brokering \u003c\u003cpatterns\u003e\u003e.","description":"\nxref:components:processors/about.adoc[Processors] can be listed to apply across individual outputs or all outputs:\n\n```yaml\noutput:\n broker:\n pattern: fan_out\n outputs:\n - resource: foo\n - resource: bar\n # Processors only applied to messages sent to bar.\n processors:\n - resource: bar_processor\n\n # Processors applied to messages sent to all brokered outputs.\n processors:\n - resource: general_processor\n```","categories":["Utility"],"footnotes":"\n== Patterns\n\nThe broker pattern determines the way in which messages are allocated and can be chosen from the following:\n\n=== `fan_out`\n\nWith the fan out pattern all outputs will be sent every message that passes through Redpanda Connect in parallel.\n\nIf an output applies back pressure it will block all subsequent messages, and if an output fails to send a message it will be retried continuously until completion or service shut down. This mechanism is in place in order to prevent one bad output from causing a larger retry loop that results in a good output from receiving unbounded message duplicates.\n\nSometimes it is useful to disable the back pressure or retries of certain fan out outputs and instead drop messages that have failed or were blocked. In this case you can wrap outputs with a xref:components:outputs/drop_on.adoc[`drop_on` output].\n\n=== `fan_out_fail_fast`\n\nThe same as the `fan_out` pattern, except that output failures will not be automatically retried. This pattern should be used with caution as busy retry loops could result in unlimited duplicates being introduced into the non-failure outputs.\n\n=== `fan_out_sequential`\n\nSimilar to the fan out pattern except outputs are written to sequentially, meaning an output is only written to once the preceding output has confirmed receipt of the same message.\n\nIf an output applies back pressure it will block all subsequent messages, and if an output fails to send a message it will be retried continuously until completion or service shut down. This mechanism is in place in order to prevent one bad output from causing a larger retry loop that results in a good output from receiving unbounded message duplicates.\n\n=== `fan_out_sequential_fail_fast`\n\nThe same as the `fan_out_sequential` pattern, except that output failures will not be automatically retried. This pattern should be used with caution as busy retry loops could result in unlimited duplicates being introduced into the non-failure outputs.\n\n=== `round_robin`\n\nWith the round robin pattern each message will be assigned a single output following their order. If an output applies back pressure it will block all subsequent messages. If an output fails to send a message then the message will be re-attempted with the next input, and so on.\n\n=== `greedy`\n\nThe greedy pattern results in higher output throughput at the cost of potentially disproportionate message allocations to those outputs. Each message is sent to a single output, which is determined by allowing outputs to claim messages as soon as they are able to process them. This results in certain faster outputs potentially processing more messages at the cost of slower outputs.","config":{"name":"","type":"object","kind":"scalar","children":[{"name":"copies","type":"int","kind":"scalar","description":"The number of copies of each configured output to spawn.","is_advanced":true,"default":1},{"name":"pattern","type":"string","kind":"scalar","description":"The brokering pattern to use.","default":"fan_out","options":["fan_out","fan_out_fail_fast","fan_out_sequential","fan_out_sequential_fail_fast","round_robin","greedy"],"linter":"\nlet options = {\n \"fan_out\": true,\n \"fan_out_fail_fast\": true,\n \"fan_out_sequential\": true,\n \"fan_out_sequential_fail_fast\": true,\n \"round_robin\": true,\n \"greedy\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"outputs","type":"output","kind":"array","description":"A list of child outputs to broker."},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]}},{"name":"cache","type":"output","status":"stable","plugin":true,"summary":"Stores each message in a xref:components:caches/about.adoc[cache].","description":"Caches are configured as xref:components:caches/about.adoc[resources], where there's a wide variety to choose from.\n\n:cache-support: aws_dynamodb=certified, aws_s3=certified, file=certified, memcached=certified, memory=certified, nats_kv=certified, redis=certified, ristretto=certified, couchbase=community, mongodb=community, sql=community, multilevel=community, ttlru=community, gcp_cloud_storage=community, lru=community, noop=community\n\nThe `target` field must reference a configured cache resource label like follows:\n\n```yaml\noutput:\n cache:\n target: foo\n key: ${!json(\"document.id\")}\n\ncache_resources:\n - label: foo\n memcached:\n addresses:\n - localhost:11211\n default_ttl: 60s\n```\n\nIn order to create a unique `key` value per item you should use function interpolations described in xref:configuration:interpolation.adoc#bloblang-queries[Bloblang queries].\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"target","type":"string","kind":"scalar","description":"The target cache to store messages in."},{"name":"key","type":"string","kind":"scalar","description":"The key to store messages by, function interpolation should be used in order to derive a unique key for each message.","default":"${!count(\"items\")}-${!timestamp_unix_nano()}","interpolated":true,"examples":["${!count(\"items\")}-${!timestamp_unix_nano()}","${!json(\"doc.id\")}","${!meta(\"kafka_key\")}"]},{"name":"ttl","type":"string","kind":"scalar","description":"The TTL of each individual item as a duration string. After this period an item will be eligible for removal during the next compaction. Not all caches support per-key TTLs, and those that do not will fall back to their generally configured TTL setting.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["60s","5m","36h"],"version":"3.33.0"},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64}]}},{"name":"cassandra","type":"output","status":"beta","plugin":true,"summary":"Runs a query against a Cassandra database for each message in order to insert data.","description":"\nQuery arguments can be set using a bloblang array for the fields using the `args_mapping` field.\n\nWhen populating timestamp columns the value must either be a string in ISO 8601 format (2006-01-02T15:04:05Z07:00), or an integer representing unix time in seconds.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":null,"examples":[{"title":"Basic Inserts","summary":"If we were to create a table with some basic columns with `CREATE TABLE foo.bar (id int primary key, content text, created_at timestamp);`, and were processing JSON documents of the form `{\"id\":\"342354354\",\"content\":\"hello world\",\"timestamp\":1605219406}` using logged batches, we could populate our table with the following config:","config":"\noutput:\n cassandra:\n addresses:\n - localhost:9042\n query: 'INSERT INTO foo.bar (id, content, created_at) VALUES (?, ?, ?)'\n args_mapping: |\n root = [\n this.id,\n this.content,\n this.timestamp\n ]\n batching:\n count: 500\n period: 1s\n"},{"title":"Insert JSON Documents","summary":"The following example inserts JSON documents into the table `footable` of the keyspace `foospace` using INSERT JSON (https://cassandra.apache.org/doc/latest/cql/json.html#insert-json).","config":"\noutput:\n cassandra:\n addresses:\n - localhost:9042\n query: 'INSERT INTO foospace.footable JSON ?'\n args_mapping: 'root = [ this ]'\n batching:\n count: 500\n period: 1s\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"addresses","type":"string","kind":"array","description":"A list of Cassandra nodes to connect to. Multiple comma separated addresses can be specified on a single line.","examples":[["localhost:9042"],["foo:9042","bar:9042"],["foo:9042,bar:9042"]]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"password_authenticator","type":"object","kind":"scalar","description":"Optional configuration of Cassandra authentication parameters.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use password authentication","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"The username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"The password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"disable_initial_host_lookup","type":"bool","kind":"scalar","description":"If enabled the driver will not attempt to get host info from the system.peers table. This can speed up queries but will mean that data_centre, rack and token information will not be available.","is_advanced":true,"default":false},{"name":"max_retries","type":"int","kind":"scalar","description":"The maximum number of retries before giving up on a request.","is_advanced":true,"default":3},{"name":"backoff","type":"object","kind":"scalar","description":"Control time intervals between retry attempts.","is_advanced":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"1s"},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts.","is_advanced":true,"default":"5s"}]},{"name":"timeout","type":"string","kind":"scalar","description":"The client connection timeout.","default":"600ms"},{"name":"query","type":"string","kind":"scalar","description":"A query to execute for each message."},{"name":"args_mapping","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] that can be used to provide arguments to Cassandra queries. The result of the query must be an array containing a matching number of elements to the query arguments.","is_optional":true,"bloblang":true,"version":"3.55.0"},{"name":"consistency","type":"string","kind":"scalar","description":"The consistency level to use.","is_advanced":true,"default":"QUORUM","options":["ANY","ONE","TWO","THREE","QUORUM","ALL","LOCAL_QUORUM","EACH_QUORUM","LOCAL_ONE"],"linter":"\nlet options = {\n \"any\": true,\n \"one\": true,\n \"two\": true,\n \"three\": true,\n \"quorum\": true,\n \"all\": true,\n \"local_quorum\": true,\n \"each_quorum\": true,\n \"local_one\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"logged_batch","type":"bool","kind":"scalar","description":"If enabled the driver will perform a logged batch. Disabling this prompts unlogged batches to be used instead, which are less efficient but necessary for alternative storages that do not support logged batches.","is_advanced":true,"default":true},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]}},{"name":"couchbase","type":"output","status":"experimental","plugin":true,"summary":"Performs operations against Couchbase for each message, allowing you to store or delete data.","description":"When inserting, replacing or upserting documents, each must have the `content` property set.\n\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Integration"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"Couchbase connection string.","examples":["couchbase://localhost:11210"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"username","type":"string","kind":"scalar","description":"Username to connect to the cluster.","is_optional":true},{"name":"password","type":"string","kind":"scalar","description":"Password to connect to the cluster.","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"bucket","type":"string","kind":"scalar","description":"Couchbase bucket."},{"name":"collection","type":"string","kind":"scalar","description":"Bucket collection.","is_advanced":true,"is_optional":true,"default":"_default"},{"name":"transcoder","type":"string","kind":"scalar","description":"Couchbase transcoder to use.","is_advanced":true,"default":"legacy","annotated_options":[["json","JSONTranscoder implements the default transcoding behavior and applies JSON transcoding to all values. This will apply the following behavior to the value: binary ([]byte) -\u003e error. default -\u003e JSON value, JSON Flags."],["legacy","LegacyTranscoder implements the behavior for a backward-compatible transcoder. This transcoder implements behavior matching that of gocb v1.This will apply the following behavior to the value: binary ([]byte) -\u003e binary bytes, Binary expectedFlags. string -\u003e string bytes, String expectedFlags. default -\u003e JSON value, JSON expectedFlags."],["raw","RawBinaryTranscoder implements passthrough behavior of raw binary data. This transcoder does not apply any serialization. This will apply the following behavior to the value: binary ([]byte) -\u003e binary bytes, binary expectedFlags. default -\u003e error."],["rawjson","RawJSONTranscoder implements passthrough behavior of JSON data. This transcoder does not apply any serialization. It will forward data across the network without incurring unnecessary parsing costs. This will apply the following behavior to the value: binary ([]byte) -\u003e JSON bytes, JSON expectedFlags. string -\u003e JSON bytes, JSON expectedFlags. default -\u003e error."],["rawstring","RawStringTranscoder implements passthrough behavior of raw string data. This transcoder does not apply any serialization. This will apply the following behavior to the value: string -\u003e string bytes, string expectedFlags. default -\u003e error."]],"linter":"\nlet options = {\n \"json\": true,\n \"legacy\": true,\n \"raw\": true,\n \"rawjson\": true,\n \"rawstring\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"timeout","type":"string","kind":"scalar","description":"Operation timeout.","is_advanced":true,"default":"15s"},{"name":"id","type":"string","kind":"scalar","description":"Document id.","interpolated":true,"examples":["${! json(\"id\") }"]},{"name":"content","type":"string","kind":"scalar","description":"Document content.","is_optional":true,"bloblang":true},{"name":"operation","type":"string","kind":"scalar","description":"Couchbase operation to perform.","default":"upsert","annotated_options":[["insert","insert a new document."],["remove","delete a document."],["replace","replace the contents of a document."],["upsert","creates a new document if it does not exist, if it does exist then it updates it."]],"linter":"\nlet options = {\n \"insert\": true,\n \"remove\": true,\n \"replace\": true,\n \"upsert\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}],"linter":"root = if ((this.operation == \"insert\" || this.operation == \"replace\" || this.operation == \"upsert\") \u0026\u0026 !this.exists(\"content\")) { [ \"content must be set for insert, replace and upsert operations.\" ] }"},"version":"4.37.0"},{"name":"cypher","type":"output","status":"experimental","plugin":true,"description":"The cypher output type writes a batch of messages to any graph database that supports the Neo4j or Bolt protocols.","categories":["Services"],"examples":[{"title":"Write to Neo4j Aura","summary":"This is an example of how to write to Neo4j Aura","config":"\noutput:\n cypher:\n uri: neo4j+s://example.databases.neo4j.io\n cypher: |\n MERGE (product:Product {id: $id})\n ON CREATE SET product.name = $product,\n product.title = $title,\n product.description = $description,\n args_mapping: |\n root = {}\n root.id = this.product.id \n root.product = this.product.summary.name\n root.title = this.product.summary.displayName\n root.description = this.product.fullDescription\n basic_auth:\n enabled: true\n username: \"${NEO4J_USER}\"\n password: \"${NEO4J_PASSWORD}\"\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"uri","type":"string","kind":"scalar","description":"The connection URI to connect to.\nSee https://neo4j.com/docs/go-manual/current/connect-advanced/[Neo4j's documentation^] for more information. ","examples":["neo4j://demo.neo4jlabs.com","neo4j+s://aura.databases.neo4j.io","neo4j+ssc://self-signed.demo.neo4jlabs.com","bolt://127.0.0.1:7687","bolt+s://core.db.server:7687","bolt+ssc://10.0.0.43"]},{"name":"cypher","type":"string","kind":"scalar","description":"The cypher expression to execute against the graph database.","examples":["MERGE (p:Person {name: $name})","MATCH (o:Organization {id: $orgId})\nMATCH (p:Person {name: $name})\nMERGE (p)-[:WORKS_FOR]-\u003e(o)"]},{"name":"database_name","type":"string","kind":"scalar","description":"Set the target database for which expressions are evaluated against.","default":""},{"name":"args_mapping","type":"string","kind":"scalar","description":"The mapping from the message to the data that is passed in as parameters to the cypher expression. Must be an object. By default the entire payload is used.","is_optional":true,"bloblang":true,"examples":["root.name = this.displayName","root = {\"orgId\": this.org.id, \"name\": this.user.name}"]},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"realm","type":"string","kind":"scalar","description":"The realm for authentication challenges.","is_advanced":true,"default":""}]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64}]},"version":"4.37.0"},{"name":"discord","type":"output","status":"experimental","plugin":true,"summary":"Writes messages to a Discord channel.","description":"\nThis output POSTs messages to the `/channels/\\{channel_id}/messages` Discord API endpoint authenticated as a bot using token based authentication.\n\nIf the format of a message is a JSON object matching the https://discord.com/developers/docs/resources/channel#message-object[Discord API message type^] then it is sent directly, otherwise an object matching the API type is created with the content of the message added as a string.\n","categories":["Services","Social"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"channel_id","type":"string","kind":"scalar","description":"A discord channel ID to write messages to."},{"name":"bot_token","type":"string","kind":"scalar","description":"A bot token used for authentication."},{"name":"rate_limit","type":"string","kind":"scalar","is_deprecated":true,"default":"An optional rate limit resource to restrict API requests with."}]}},{"name":"drop","type":"output","status":"stable","plugin":true,"summary":"Drops all messages.","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","default":{}}},{"name":"drop_on","type":"output","status":"stable","plugin":true,"summary":"Attempts to write messages to a child output and if the write fails for one of a list of configurable reasons the message is dropped (acked) instead of being reattempted (or nacked).","description":"Regular Redpanda Connect outputs will apply back pressure when downstream services aren't accessible, and Redpanda Connect retries (or nacks) all messages that fail to be delivered. However, in some circumstances, or for certain output types, we instead might want to relax these mechanisms, which is when this output becomes useful.","categories":["Utility"],"examples":[{"title":"Dropping failed HTTP requests","summary":"In this example we have a fan_out broker, where we guarantee delivery to our Kafka output, but drop messages if they fail our secondary HTTP client output.","config":"\noutput:\n broker:\n pattern: fan_out\n outputs:\n - kafka:\n addresses: [ foobar:6379 ]\n topic: foo\n - drop_on:\n error: true\n output:\n http_client:\n url: http://example.com/foo/messages\n verb: POST\n"},{"title":"Dropping from outputs that cannot connect","summary":"Most outputs that attempt to establish and long-lived connection will apply back-pressure when the connection is lost. The following example has a websocket output where if it takes longer than 10 seconds to establish a connection, or recover a lost one, pending messages are dropped.","config":"\noutput:\n drop_on:\n back_pressure: 10s\n output:\n websocket:\n url: ws://example.com/foo/messages\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"error","type":"bool","kind":"scalar","description":"Whether messages should be dropped when the child output returns an error of any type. For example, this could be when an `http_client` output gets a 4XX response code. In order to instead drop only on specific error patterns use the `error_matches` field instead.","default":false},{"name":"error_patterns","type":"string","kind":"array","description":"A list of regular expressions (re2) where if the child output returns an error that matches any part of any of these patterns the message will be dropped.","is_optional":true,"examples":[["and that was really bad$"],["roughly [0-9]+ issues occurred"]],"version":"4.27.0"},{"name":"back_pressure","type":"string","kind":"scalar","description":"An optional duration string that determines the maximum length of time to wait for a given message to be accepted by the child output before the message should be dropped instead. The most common reason for an output to block is when waiting for a lost connection to be re-established. Once a message has been dropped due to back pressure all subsequent messages are dropped immediately until the output is ready to process them again. Note that if `error` is set to `false` and this field is specified then messages dropped due to back pressure will return an error response (are nacked or reattempted).","is_optional":true,"examples":["30s","1m"]},{"name":"output","type":"output","kind":"scalar","description":"A child output to wrap with this drop mechanism."}]}},{"name":"dynamic","type":"output","status":"stable","plugin":true,"summary":"A special broker type where the outputs are identified by unique labels and can be created, changed and removed during runtime via a REST API.","description":"The broker pattern used is always `fan_out`, meaning each message will be delivered to each dynamic output.","categories":["Utility"],"footnotes":"\n== Endpoints\n\n=== GET `/outputs`\n\nReturns a JSON object detailing all dynamic outputs, providing information such as their current uptime and configuration.\n\n=== GET `/outputs/\\{id}`\n\nReturns the configuration of an output.\n\n=== POST `/outputs/\\{id}`\n\nCreates or updates an output with a configuration provided in the request body (in YAML or JSON format).\n\n=== DELETE `/outputs/\\{id}`\n\nStops and removes an output.\n\n=== GET `/outputs/\\{id}/uptime`\n\nReturns the uptime of an output as a duration string (of the form \"72h3m0.5s\").","config":{"name":"","type":"object","kind":"scalar","children":[{"name":"outputs","type":"output","kind":"map","description":"A map of outputs to statically create.","default":{}},{"name":"prefix","type":"string","kind":"scalar","description":"A path prefix for HTTP endpoints that are registered.","default":""}]}},{"name":"elasticsearch","type":"output","status":"stable","plugin":true,"summary":"Publishes messages into an Elasticsearch index. If the index does not exist then it is created with a dynamic mapping.","description":"\nBoth the `id` and `index` fields can be dynamically set using function interpolations described in xref:configuration:interpolation.adoc#bloblang-queries[Bloblang queries]. When sending batched messages these interpolations are performed per message part.\n\n== AWS\n\nIt's possible to enable AWS connectivity with this output using the `aws` fields. However, you may need to set `sniff` and `healthcheck` to false for connections to succeed.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services"],"examples":[{"title":"Elastic Cloud Serverless","summary":"This is an example of writing data to https://www.elastic.co/docs/current/serverless[Elastic Cloud serverless^].","config":"\noutput:\n elasticsearch:\n urls: [\"https://${ELASTIC_CLOUD_CLUSTER_ID}.es.us-east-1.aws.elastic.cloud:443\"]\n sniff: false\n healthcheck: false\n index: \"my-elasticsearch-index\"\n id: my-document-id-${!counter()}-${!timestamp_unix()}\n api_key: \"${ELASTIC_CLOUD_API_KEY}\"\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["http://localhost:9200"]]},{"name":"index","type":"string","kind":"scalar","description":"The index to place messages.","interpolated":true},{"name":"action","type":"string","kind":"scalar","description":"The action to take on the document. This field must resolve to one of the following action types: `create`, `index`, `update`, `upsert` or `delete`.","is_advanced":true,"default":"index","interpolated":true},{"name":"pipeline","type":"string","kind":"scalar","description":"An optional pipeline id to preprocess incoming documents.","is_advanced":true,"default":"","interpolated":true},{"name":"id","type":"string","kind":"scalar","description":"The ID for indexed messages. Interpolation should be used in order to create a unique ID for each message.","default":"${!counter()}-${!timestamp_unix()}","interpolated":true},{"name":"type","type":"string","kind":"scalar","description":"The document mapping type. This field is required for versions of elasticsearch earlier than 6.0.0, but are invalid for versions 7.0.0 or later.","default":"","interpolated":true},{"name":"routing","type":"string","kind":"scalar","description":"The routing key to use for the document.","is_advanced":true,"default":"","interpolated":true},{"name":"retry_on_conflict","type":"int","kind":"scalar","description":"When using the update or upsert action, retry_on_conflict can be used to specify how many times an update should be retried in the case of a version conflict.","is_advanced":true,"default":0},{"name":"sniff","type":"bool","kind":"scalar","description":"Prompts Redpanda Connect to sniff for brokers to connect to when establishing a connection.","is_advanced":true,"default":true},{"name":"healthcheck","type":"bool","kind":"scalar","description":"Whether to enable healthchecks.","is_advanced":true,"default":true},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum time to wait before abandoning a request (and trying again).","is_advanced":true,"default":"5s"},{"name":"api_key","type":"string","kind":"scalar","description":"The key to set in the Authorization header if using API keys for authentication.","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"max_retries","type":"int","kind":"scalar","description":"The maximum number of retries before giving up on the request. If set to zero there is no discrete limit.","is_advanced":true,"default":0},{"name":"backoff","type":"object","kind":"scalar","description":"Control time intervals between retry attempts.","is_advanced":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"1s"},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts.","is_advanced":true,"default":"5s"},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum period to wait before retry attempts are abandoned. If zero then no limit is used.","is_advanced":true,"default":"30s"}]},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"aws","type":"object","kind":"scalar","description":"Enables and customises connectivity to Amazon Elastic Service.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to connect to Amazon Elastic Service.","is_advanced":true,"default":false},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]},{"name":"gzip_compression","type":"bool","kind":"scalar","description":"Enable gzip compression on the request side.","is_advanced":true,"default":false}]}},{"name":"elasticsearch_v8","type":"output","status":"stable","plugin":true,"summary":"Publishes messages into an Elasticsearch index. If the index does not exist then it is created with a dynamic mapping.","description":"\nBoth the `id` and `index` fields can be dynamically set using function interpolations described xref:configuration:interpolation.adoc#bloblang-queries[here]. When sending batched messages these interpolations are performed per message part.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services"],"examples":[{"title":"Updating Documents","summary":"When updating documents, the request body should contain a combination of a `doc`, `upsert`, and/or `script` fields at the top level, this should be done via mapping processors. `doc` updates using a partial document, `script` performs an update using a scripting language such as the built in Painless language, and `upsert` updates an existing document or inserts a new one if it doesn’t exist. For more information on the structures and behaviors of these fields, please see the https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-update.html[Elasticsearch Update API^]","config":"\n# Partial document update\noutput:\n processors:\n - mapping: |\n meta id = this.id\n # Performs a partial update ont he document.\n root.doc = this\n elasticsearch_v8:\n urls: [localhost:9200]\n index: foo\n id: ${! @id }\n action: update\n\n# Scripted update\noutput:\n processors:\n - mapping: |\n meta id = this.id\n # Increments the field \"counter\" by 1.\n root.script.source = \"ctx._source.counter += 1\"\n elasticsearch_v8:\n urls: [localhost:9200]\n index: foo\n id: ${! @id }\n action: update\n\n# Upsert\noutput:\n processors:\n - mapping: |\n meta id = this.id\n # If the product with the ID exists, its price will be updated to 100.\n # If the product does not exist, a new document with ID 1 and a price\n # of 50 will be inserted.\n root.doc.product_price = 50\n root.upsert.product_price = 100\n elasticsearch_v8:\n urls: [localhost:9200]\n index: foo\n id: ${! @id }\n action: update\n"},{"title":"Indexing documents from Redpanda","summary":"Here we read messages from a Redpanda cluster and write them to an Elasticsearch index using a field from the message as the ID for the Elasticsearch document.","config":"\ninput:\n redpanda:\n seed_brokers: [localhost:19092]\n topics: [\"things\"]\n consumer_group: \"rpcn3\"\n processors:\n - mapping: |\n meta id = this.id\n root = this\noutput:\n elasticsearch_v8:\n urls: ['http://localhost:9200']\n index: \"things\"\n action: \"index\"\n id: ${! meta(\"id\") }\n"},{"title":"Indexing documents from S3","summary":"Here we read messages from a AWS S3 bucket and write them to an Elasticsearch index using the S3 key as the ID for the Elasticsearch document.","config":"\ninput:\n aws_s3:\n bucket: \"my-cool-bucket\"\n prefix: \"bug-facts/\"\n scanner:\n to_the_end: {}\noutput:\n elasticsearch_v8:\n urls: ['http://localhost:9200']\n index: \"cool-bug-facts\"\n action: \"index\"\n id: ${! meta(\"s3_key\") }\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["http://localhost:9200"]]},{"name":"index","type":"string","kind":"scalar","description":"The index to place messages.","interpolated":true},{"name":"action","type":"string","kind":"scalar","description":"The action to take on the document. This field must resolve to one of the following action types: `index`, `update` or `delete`. See the `Updating Documents` example for more on how the `update` action works.","interpolated":true},{"name":"id","type":"string","kind":"scalar","description":"The ID for indexed messages. Interpolation should be used in order to create a unique ID for each message.","interpolated":true,"examples":["${!counter()}-${!timestamp_unix()}"]},{"name":"pipeline","type":"string","kind":"scalar","description":"An optional pipeline id to preprocess incoming documents.","is_advanced":true,"default":"","interpolated":true},{"name":"routing","type":"string","kind":"scalar","description":"The routing key to use for the document.","is_advanced":true,"default":"","interpolated":true},{"name":"retry_on_conflict","type":"int","kind":"scalar","description":"Specify how many times should an update operation be retried when a conflict occurs","is_advanced":true,"default":0},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]}},{"name":"fallback","type":"output","status":"stable","plugin":true,"summary":"Attempts to send each message to a child output, starting from the first output on the list. If an output attempt fails then the next output in the list is attempted, and so on.","description":"\nThis pattern is useful for triggering events in the case where certain output targets have broken. For example, if you had an output type `http_client` but wished to reroute messages whenever the endpoint becomes unreachable you could use this pattern:\n\n```yaml\noutput:\n fallback:\n - http_client:\n url: http://foo:4195/post/might/become/unreachable\n retries: 3\n retry_period: 1s\n - http_client:\n url: http://bar:4196/somewhere/else\n retries: 3\n retry_period: 1s\n processors:\n - mapping: 'root = \"failed to send this message to foo: \" + content()'\n - file:\n path: /usr/local/benthos/everything_failed.jsonl\n```\n\n== Metadata\n\nWhen a given output fails the message routed to the following output will have a metadata value named `fallback_error` containing a string error message outlining the cause of the failure. The content of this string will depend on the particular output and can be used to enrich the message or provide information used to broker the data to an appropriate output using something like a `switch` output.\n\n== Batching\n\nWhen an output within a fallback sequence uses batching, like so:\n\n```yaml\noutput:\n fallback:\n - aws_dynamodb:\n table: foo\n string_columns:\n id: ${!json(\"id\")}\n content: ${!content()}\n batching:\n count: 10\n period: 1s\n - file:\n path: /usr/local/benthos/failed_stuff.jsonl\n```\n\nRedpanda Connect makes a best attempt at inferring which specific messages of the batch failed, and only propagates those individual messages to the next fallback tier.\n\nHowever, depending on the output and the error returned it is sometimes not possible to determine the individual messages that failed, in which case the whole batch is passed to the next tier in order to preserve at-least-once delivery guarantees.","categories":["Utility"],"config":{"name":"","type":"output","kind":"array","default":[]},"version":"3.58.0"},{"name":"file","type":"output","status":"stable","plugin":true,"summary":"Writes messages to files on disk based on a chosen codec.","description":"Messages can be written to different files by using xref:configuration:interpolation.adoc#bloblang-queries[interpolation functions] in the path field. However, only one file is ever open at a given time, and therefore when the path changes the previously open file is closed.","categories":["Local"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"path","type":"string","kind":"scalar","description":"The file to write to, if the file does not yet exist it will be created.","interpolated":true,"examples":["/tmp/data.txt","/tmp/${! timestamp_unix() }.txt","/tmp/${! json(\"document.id\") }.json"],"version":"3.33.0"},{"name":"codec","type":"string","kind":"scalar","description":"The way in which the bytes of messages should be written out into the output data stream. It's possible to write lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter.","default":"lines","examples":["lines","delim:\t","delim:foobar"],"annotated_options":[["all-bytes","Only applicable to file based outputs. Writes each message to a file in full, if the file already exists the old content is deleted."],["append","Append each message to the output stream without any delimiter or special encoding."],["lines","Append each message to the output stream followed by a line break."],["delim:x","Append each message to the output stream followed by a custom delimiter."]],"version":"3.33.0"}]}},{"name":"gcp_bigquery","type":"output","status":"beta","plugin":true,"summary":"Sends messages as new rows to a Google Cloud BigQuery table.","description":"\n== Credentials\n\nBy default Redpanda Connect will use a shared credentials file when connecting to GCP services. You can find out more in xref:guides:cloud/gcp.adoc[].\n\n== Format\n\nThis output currently supports only CSV, NEWLINE_DELIMITED_JSON and PARQUET, formats. Learn more about how to use GCP BigQuery with them here:\n\n- https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-json[`NEWLINE_DELIMITED_JSON`^]\n- https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-csv[`CSV`^]\n- https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-parquet[`PARQUET`^]\n\nEach message may contain multiple elements separated by newlines. For example a single message containing:\n\n```json\n{\"key\": \"1\"}\n{\"key\": \"2\"}\n```\n\nIs equivalent to two separate messages:\n\n```json\n{\"key\": \"1\"}\n```\n\nAnd:\n\n```json\n{\"key\": \"2\"}\n```\n\nThe same is true for the CSV format.\n\n=== CSV\n\nFor the CSV format when the field `csv.header` is specified a header row will be inserted as the first line of each message batch. If this field is not provided then the first message of each message batch must include a header line.\n\n=== Parquet\n\nFor parquet, the data can be encoded using the `parquet_encode` processor and each message that is sent to the output must be a full parquet message.\n\n\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["GCP","Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"project","type":"string","kind":"scalar","description":"The project ID of the dataset to insert data to. If not set, it will be inferred from the credentials or read from the GOOGLE_CLOUD_PROJECT environment variable.","default":""},{"name":"job_project","type":"string","kind":"scalar","description":"The project ID in which jobs will be exectuted. If not set, project will be used.","default":""},{"name":"dataset","type":"string","kind":"scalar","description":"The BigQuery Dataset ID."},{"name":"table","type":"string","kind":"scalar","description":"The table to insert messages to."},{"name":"format","type":"string","kind":"scalar","description":"The format of each incoming message.","default":"NEWLINE_DELIMITED_JSON","options":["NEWLINE_DELIMITED_JSON","CSV","PARQUET"],"linter":"\nlet options = {\n \"newline_delimited_json\": true,\n \"csv\": true,\n \"parquet\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of message batches to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"write_disposition","type":"string","kind":"scalar","description":"Specifies how existing data in a destination table is treated.","is_advanced":true,"default":"WRITE_APPEND","options":["WRITE_APPEND","WRITE_EMPTY","WRITE_TRUNCATE"],"linter":"\nlet options = {\n \"write_append\": true,\n \"write_empty\": true,\n \"write_truncate\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"create_disposition","type":"string","kind":"scalar","description":"Specifies the circumstances under which destination table will be created. If CREATE_IF_NEEDED is used the GCP BigQuery will create the table if it does not already exist and tables are created atomically on successful completion of a job. The CREATE_NEVER option ensures the table must already exist and will not be automatically created.","is_advanced":true,"default":"CREATE_IF_NEEDED","options":["CREATE_IF_NEEDED","CREATE_NEVER"],"linter":"\nlet options = {\n \"create_if_needed\": true,\n \"create_never\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"ignore_unknown_values","type":"bool","kind":"scalar","description":"Causes values not matching the schema to be tolerated. Unknown values are ignored. For CSV this ignores extra values at the end of a line. For JSON this ignores named values that do not match any column name. If this field is set to false (the default value), records containing unknown values are treated as bad records. The max_bad_records field can be used to customize how bad records are handled.","is_advanced":true,"default":false},{"name":"max_bad_records","type":"int","kind":"scalar","description":"The maximum number of bad records that will be ignored when reading data.","is_advanced":true,"default":0},{"name":"auto_detect","type":"bool","kind":"scalar","description":"Indicates if we should automatically infer the options and schema for CSV and JSON sources. If the table doesn't exist and this field is set to `false` the output may not be able to insert data and will throw insertion error. Be careful using this field since it delegates to the GCP BigQuery service the schema detection and values like `\"no\"` may be treated as booleans for the CSV format.","is_advanced":true,"default":false},{"name":"job_labels","type":"string","kind":"map","description":"A list of labels to add to the load job.","default":{}},{"name":"credentials_json","type":"string","kind":"scalar","description":"An optional field to set Google Service Account Credentials json.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"csv","type":"object","kind":"scalar","description":"Specify how CSV data should be interpretted.","children":[{"name":"header","type":"string","kind":"array","description":"A list of values to use as header for each batch of messages. If not specified the first line of each message will be used as header.","default":[]},{"name":"field_delimiter","type":"string","kind":"scalar","description":"The separator for fields in a CSV file, used when reading or exporting data.","default":","},{"name":"allow_jagged_rows","type":"bool","kind":"scalar","description":"Causes missing trailing optional columns to be tolerated when reading CSV data. Missing values are treated as nulls.","is_advanced":true,"default":false},{"name":"allow_quoted_newlines","type":"bool","kind":"scalar","description":"Sets whether quoted data sections containing newlines are allowed when reading CSV data.","is_advanced":true,"default":false},{"name":"encoding","type":"string","kind":"scalar","description":"Encoding is the character encoding of data to be read.","is_advanced":true,"default":"UTF-8","options":["UTF-8","ISO-8859-1"],"linter":"\nlet options = {\n \"utf-8\": true,\n \"iso-8859-1\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"skip_leading_rows","type":"int","kind":"scalar","description":"The number of rows at the top of a CSV file that BigQuery will skip when reading data. The default value is 1 since Redpanda Connect will add the specified header in the first line of each batch sent to BigQuery.","is_advanced":true,"default":1}]},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]},"version":"3.55.0"},{"name":"gcp_cloud_storage","type":"output","status":"beta","plugin":true,"summary":"Sends message parts as objects to a Google Cloud Storage bucket. Each object is uploaded with the path specified with the `path` field.","description":"\nIn order to have a different path for each object you should use function interpolations described in xref:configuration:interpolation.adoc#bloblang-queries[Bloblang queries], which are calculated per message of a batch.\n\n== Metadata\n\nMetadata fields on messages will be sent as headers, in order to mutate these values (or remove them) check out the xref:configuration:metadata.adoc[metadata docs].\n\n== Credentials\n\nBy default Redpanda Connect will use a shared credentials file when connecting to GCP services. You can find out more in xref:guides:cloud/gcp.adoc[].\n\n== Batching\n\nIt's common to want to upload messages to Google Cloud Storage as batched archives, the easiest way to do this is to batch your messages at the output level and join the batch of messages with an xref:components:processors/archive.adoc[`archive`] and/or xref:components:processors/compress.adoc[`compress`] processor.\n\nFor example, if we wished to upload messages as a .tar.gz archive of documents we could achieve that with the following config:\n\n```yaml\noutput:\n gcp_cloud_storage:\n bucket: TODO\n path: ${!counter()}-${!timestamp_unix_nano()}.tar.gz\n batching:\n count: 100\n period: 10s\n processors:\n - archive:\n format: tar\n - compress:\n algorithm: gzip\n```\n\nAlternatively, if we wished to upload JSON documents as a single large document containing an array of objects we can do that with:\n\n```yaml\noutput:\n gcp_cloud_storage:\n bucket: TODO\n path: ${!counter()}-${!timestamp_unix_nano()}.json\n batching:\n count: 100\n processors:\n - archive:\n format: json_array\n```\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services","GCP"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"bucket","type":"string","kind":"scalar","description":"The bucket to upload messages to."},{"name":"path","type":"string","kind":"scalar","description":"The path of each message to upload.","default":"${!counter()}-${!timestamp_unix_nano()}.txt","interpolated":true,"examples":["${!counter()}-${!timestamp_unix_nano()}.txt","${!meta(\"kafka_key\")}.json","${!json(\"doc.namespace\")}/${!json(\"doc.id\")}.json"]},{"name":"content_type","type":"string","kind":"scalar","description":"The content type to set for each object.","default":"application/octet-stream","interpolated":true},{"name":"content_encoding","type":"string","kind":"scalar","description":"An optional content encoding to set for each object.","is_advanced":true,"default":"","interpolated":true},{"name":"collision_mode","type":"string","kind":"scalar","description":"Determines how file path collisions should be dealt with.","default":"overwrite","annotated_options":[["append","Append the message bytes to the original file."],["error-if-exists","Return an error, this is the equivalent of a nack."],["ignore","Do not modify the original file, the new data will be dropped."],["overwrite","Replace the existing file with the new one."]],"version":"3.53.0","linter":"\nlet options = {\n \"append\": true,\n \"error-if-exists\": true,\n \"ignore\": true,\n \"overwrite\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"chunk_size","type":"int","kind":"scalar","description":"An optional chunk size which controls the maximum number of bytes of the object that the Writer will attempt to send to the server in a single request. If ChunkSize is set to zero, chunking will be disabled.","is_advanced":true,"default":16777216},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period to wait on an upload before abandoning it and reattempting.","default":"3s","examples":["1s","500ms"]},{"name":"credentials_json","type":"string","kind":"scalar","description":"An optional field to set Google Service Account Credentials json.","is_secret":true,"default":"","interpolated":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of message batches to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]},"version":"3.43.0"},{"name":"gcp_pubsub","type":"output","status":"stable","plugin":true,"summary":"Sends messages to a GCP Cloud Pub/Sub topic. xref:configuration:metadata.adoc[Metadata] from messages are sent as attributes.","description":"\nFor information on how to set up credentials, see https://cloud.google.com/docs/authentication/production[this guide^].\n\n== Troubleshooting\n\nIf you're consistently seeing `Failed to send message to gcp_pubsub: context deadline exceeded` error logs without any further information it is possible that you are encountering https://github.com/benthosdev/benthos/issues/1042, which occurs when metadata values contain characters that are not valid utf-8. This can frequently occur when consuming from Kafka as the key metadata field may be populated with an arbitrary binary value, but this issue is not exclusive to Kafka.\n\nIf you are blocked by this issue then a work around is to delete either the specific problematic keys:\n\n```yaml\npipeline:\n processors:\n - mapping: |\n meta kafka_key = deleted()\n```\n\nOr delete all keys with:\n\n```yaml\npipeline:\n processors:\n - mapping: meta = deleted()\n```","categories":["Services","GCP"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"project","type":"string","kind":"scalar","description":"The project ID of the topic to publish to."},{"name":"credentials_json","type":"string","kind":"scalar","description":"An optional field to set Google Service Account Credentials json.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"topic","type":"string","kind":"scalar","description":"The topic to publish to.","interpolated":true},{"name":"endpoint","type":"string","kind":"scalar","description":"An optional endpoint to override the default of `pubsub.googleapis.com:443`. This can be used to connect to a region specific pubsub endpoint. For a list of valid values, see https://cloud.google.com/pubsub/docs/reference/service_apis_overview#list_of_regional_endpoints[this document^].","default":"","examples":["us-central1-pubsub.googleapis.com:443","us-west3-pubsub.googleapis.com:443"]},{"name":"ordering_key","type":"string","kind":"scalar","description":"The ordering key to use for publishing messages.","is_advanced":true,"is_optional":true,"interpolated":true},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increasing this may improve throughput.","default":64},{"name":"count_threshold","type":"int","kind":"scalar","description":"Publish a pubsub buffer when it has this many messages","default":100},{"name":"delay_threshold","type":"string","kind":"scalar","description":"Publish a non-empty pubsub buffer after this delay has passed.","default":"10ms"},{"name":"byte_threshold","type":"int","kind":"scalar","description":"Publish a batch when its size in bytes reaches this value.","default":1000000},{"name":"publish_timeout","type":"string","kind":"scalar","description":"The maximum length of time to wait before abandoning a publish attempt for a message.","is_advanced":true,"default":"1m0s","examples":["10s","5m","60m"]},{"name":"metadata","type":"object","kind":"scalar","description":"Specify criteria for which metadata values are sent as attributes, all are sent by default.","is_optional":true,"children":[{"name":"exclude_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages.","default":[]}]},{"name":"flow_control","type":"object","kind":"scalar","description":"For a given topic, configures the PubSub client's internal buffer for messages to be published.","is_advanced":true,"children":[{"name":"max_outstanding_bytes","type":"int","kind":"scalar","description":"Maximum size of buffered messages to be published. If less than or equal to zero, this is disabled.","is_advanced":true,"default":-1},{"name":"max_outstanding_messages","type":"int","kind":"scalar","description":"Maximum number of buffered messages to be published. If less than or equal to zero, this is disabled.","is_advanced":true,"default":1000},{"name":"limit_exceeded_behavior","type":"string","kind":"scalar","description":"Configures the behavior when trying to publish additional messages while the flow controller is full. The available options are block (default), ignore (disable), and signal_error (publish results will return an error).","is_advanced":true,"default":"block","options":["ignore","block","signal_error"],"linter":"\nlet options = {\n \"ignore\": true,\n \"block\": true,\n \"signal_error\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"}]},{"name":"batching","type":"object","kind":"","description":"Configures a batching policy on this output. While the PubSub client maintains its own internal buffering mechanism, preparing larger batches of messages can further trade-off some latency for throughput.","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]}},{"name":"hdfs","type":"output","status":"stable","plugin":true,"summary":"Sends message parts as files to a HDFS directory.","description":"Each file is written with the path specified with the 'path' field, in order to have a different path for each object you should use function interpolations described xref:configuration:interpolation.adoc#bloblang-queries[here].\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"hosts","type":"string","kind":"array","description":"A list of target host addresses to connect to.","examples":["localhost:9000"]},{"name":"user","type":"string","kind":"scalar","description":"A user ID to connect as.","default":""},{"name":"directory","type":"string","kind":"scalar","description":"A directory to store message files within. If the directory does not exist it will be created.","interpolated":true},{"name":"path","type":"string","kind":"scalar","description":"The path to upload messages as, interpolation functions should be used in order to generate unique file paths.","default":"${!counter()}-${!timestamp_unix_nano()}.txt","interpolated":true},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]}},{"name":"http_client","type":"output","status":"stable","plugin":true,"summary":"Sends messages to an HTTP server.","description":"\nWhen the number of retries expires the output will reject the message, the behavior after this will depend on the pipeline but usually this simply means the send is attempted again until successful whilst applying back pressure.\n\nThe URL and header values of this type can be dynamically set using function interpolations described in xref:configuration:interpolation.adoc#bloblang-queries[Bloblang queries].\n\nThe body of the HTTP request is the raw contents of the message payload. If the message has multiple parts (is a batch) the request will be sent according to https://www.w3.org/Protocols/rfc1341/7_2_Multipart.html[RFC1341^]. This behavior can be disabled by setting the field \u003c\u003cbatch_as_multipart, `batch_as_multipart`\u003e\u003e to `false`.\n\n== Propagate responses\n\nIt's possible to propagate the response from each HTTP request back to the input source by setting `propagate_response` to `true`. Only inputs that support xref:guides:sync_responses.adoc[synchronous responses] are able to make use of these propagated responses.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Network"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL to connect to.","interpolated":true},{"name":"verb","type":"string","kind":"scalar","description":"A verb to connect with","default":"POST","examples":["POST","GET","DELETE"]},{"name":"headers","type":"string","kind":"map","description":"A map of headers to add to the request.","default":{},"interpolated":true,"examples":[{"Content-Type":"application/octet-stream","traceparent":"${! tracing_span().traceparent }"}]},{"name":"metadata","type":"object","kind":"scalar","description":"Specify optional matching rules to determine which metadata keys should be added to the HTTP request as headers.","is_advanced":true,"is_optional":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","is_advanced":true,"default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","is_advanced":true,"default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"dump_request_log_level","type":"string","kind":"scalar","description":"EXPERIMENTAL: Optionally set a level at which the request and response payload of each request made will be logged.","is_advanced":true,"default":"","options":["TRACE","DEBUG","INFO","WARN","ERROR","FATAL",""],"version":"4.12.0","linter":"\nlet options = {\n \"trace\": true,\n \"debug\": true,\n \"info\": true,\n \"warn\": true,\n \"error\": true,\n \"fatal\": true,\n \"\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"oauth","type":"object","kind":"scalar","description":"Allows you to specify open authentication via OAuth version 1.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use OAuth version 1 in requests.","is_advanced":true,"default":false},{"name":"consumer_key","type":"string","kind":"scalar","description":"A value used to identify the client to the service provider.","is_advanced":true,"default":""},{"name":"consumer_secret","type":"string","kind":"scalar","description":"A secret used to establish ownership of the consumer key.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"access_token","type":"string","kind":"scalar","description":"A value used to gain access to the protected resources on behalf of the user.","is_advanced":true,"default":""},{"name":"access_token_secret","type":"string","kind":"scalar","description":"A secret provided in order to establish ownership of a given access token.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"oauth2","type":"object","kind":"scalar","description":"Allows you to specify open authentication via OAuth version 2 using the client credentials token flow.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use OAuth version 2 in requests.","is_advanced":true,"default":false},{"name":"client_key","type":"string","kind":"scalar","description":"A value used to identify the client to the token provider.","is_advanced":true,"default":""},{"name":"client_secret","type":"string","kind":"scalar","description":"A secret used to establish ownership of the client key.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token_url","type":"string","kind":"scalar","description":"The URL of the token provider.","is_advanced":true,"default":"","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"scopes","type":"string","kind":"array","description":"A list of optional requested permissions.","is_advanced":true,"default":[],"version":"3.45.0"},{"name":"endpoint_params","type":"unknown","kind":"map","description":"A list of optional endpoint parameters, values should be arrays of strings.","is_advanced":true,"is_optional":true,"default":{},"examples":[{"bar":["woof"],"foo":["meow","quack"]}],"version":"4.21.0","linter":"\nroot = if this.type() == \"object\" {\n this.values().map_each(ele -\u003e if ele.type() != \"array\" {\n \"field must be an object containing arrays of strings, got %s (%v)\".format(ele.format_json(no_indent: true), ele.type())\n } else {\n ele.map_each(str -\u003e if str.type() != \"string\" {\n \"field values must be strings, got %s (%v)\".format(str.format_json(no_indent: true), str.type())\n } else { deleted() })\n }).\n flatten()\n}\n"}]},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"jwt","type":"object","kind":"scalar","description":"BETA: Allows you to specify JWT authentication.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use JWT authentication in requests.","is_advanced":true,"default":false},{"name":"private_key_file","type":"string","kind":"scalar","description":"A file with the PEM encoded via PKCS1 or PKCS8 as private key.","is_advanced":true,"default":""},{"name":"signing_method","type":"string","kind":"scalar","description":"A method used to sign the token such as RS256, RS384, RS512 or EdDSA.","is_advanced":true,"default":""},{"name":"claims","type":"unknown","kind":"map","description":"A value used to identify the claims that issued the JWT.","is_advanced":true,"default":{}},{"name":"headers","type":"unknown","kind":"map","description":"Add optional key/value headers to the JWT.","is_advanced":true,"default":{}}]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"extract_headers","type":"object","kind":"scalar","description":"Specify which response headers should be added to resulting synchronous response messages as metadata. Header keys are lowercased before matching, so ensure that your patterns target lowercased versions of the header keys that you expect. This field is not applicable unless `propagate_response` is set to `true`.","is_advanced":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","is_advanced":true,"default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","is_advanced":true,"default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"rate_limit","type":"string","kind":"scalar","description":"An optional xref:components:rate_limits/about.adoc[rate limit] to throttle requests by.","is_optional":true},{"name":"timeout","type":"string","kind":"scalar","description":"A static timeout to apply to requests.","default":"5s"},{"name":"retry_period","type":"string","kind":"scalar","description":"The base period to wait between failed requests.","is_advanced":true,"default":"1s"},{"name":"max_retry_backoff","type":"string","kind":"scalar","description":"The maximum period to wait between failed requests.","is_advanced":true,"default":"300s"},{"name":"retries","type":"int","kind":"scalar","description":"The maximum number of retry attempts to make.","is_advanced":true,"default":3},{"name":"follow_redirects","type":"bool","kind":"scalar","description":"Whether or not to transparently follow redirects, i.e. responses with 300-399 status codes. If disabled, the response message will contain the body, status, and headers from the redirect response and the processor will not make a request to the URL set in the Location header of the response.","is_advanced":true,"default":true},{"name":"backoff_on","type":"int","kind":"array","description":"A list of status codes whereby the request should be considered to have failed and retries should be attempted, but the period between them should be increased gradually.","is_advanced":true,"default":[429]},{"name":"drop_on","type":"int","kind":"array","description":"A list of status codes whereby the request should be considered to have failed but retries should not be attempted. This is useful for preventing wasted retries for requests that will never succeed. Note that with these status codes the _request_ is dropped, but _message_ that caused the request will not be dropped.","is_advanced":true,"default":[]},{"name":"successful_on","type":"int","kind":"array","description":"A list of status codes whereby the attempt should be considered successful, this is useful for dropping requests that return non-2XX codes indicating that the message has been dealt with, such as a 303 See Other or a 409 Conflict. All 2XX codes are considered successful unless they are present within `backoff_on` or `drop_on`, regardless of this field.","is_advanced":true,"default":[]},{"name":"proxy_url","type":"string","kind":"scalar","description":"An optional HTTP proxy URL.","is_advanced":true,"is_optional":true},{"name":"disable_http2","type":"bool","kind":"scalar","description":"Whether or not to disable disable HTTP/2","is_advanced":true,"default":false,"version":"4.44.0"},{"name":"batch_as_multipart","type":"bool","kind":"scalar","description":"Send message batches as a single request using https://www.w3.org/Protocols/rfc1341/7_2_Multipart.html[RFC1341^]. If disabled messages in batches will be sent as individual requests.","is_advanced":true,"default":false},{"name":"propagate_response","type":"bool","kind":"scalar","description":"Whether responses from the server should be xref:guides:sync_responses.adoc[propagated back] to the input.","is_advanced":true,"default":false},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of parallel message batches to have in flight at any given time.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"multipart","type":"object","kind":"array","description":"EXPERIMENTAL: Create explicit multipart HTTP requests by specifying an array of parts to add to the request, each part specified consists of content headers and a data field that can be populated dynamically. If this field is populated it will override the default request creation behavior.","is_advanced":true,"default":[],"children":[{"name":"content_type","type":"string","kind":"scalar","description":"The content type of the individual message part.","is_advanced":true,"default":"","interpolated":true,"examples":["application/bin"]},{"name":"content_disposition","type":"string","kind":"scalar","description":"The content disposition of the individual message part.","is_advanced":true,"default":"","interpolated":true,"examples":["form-data; name=\"bin\"; filename='${! @AttachmentName }"]},{"name":"body","type":"string","kind":"scalar","description":"The body of the individual message part.","is_advanced":true,"default":"","interpolated":true,"examples":["${! this.data.part1 }"]}],"version":"3.63.0"}]}},{"name":"http_server","type":"output","status":"stable","plugin":true,"summary":"Sets up an HTTP server that will send messages over HTTP(S) GET requests. HTTP 2.0 is supported when using TLS, which is enabled when key and cert files are specified.","description":"Sets up an HTTP server that will send messages over HTTP(S) GET requests. If the `address` config field is left blank the xref:components:http/about.adoc[service-wide HTTP server] will be used.\n\nThree endpoints will be registered at the paths specified by the fields `path`, `stream_path` and `ws_path`. Which allow you to consume a single message batch, a continuous stream of line delimited messages, or a websocket of messages for each request respectively.\n\nWhen messages are batched the `path` endpoint encodes the batch according to https://www.w3.org/Protocols/rfc1341/7_2_Multipart.html[RFC1341^]. This behavior can be overridden by xref:configuration:batching.adoc#post-batch-processing[archiving your batches].\n\nPlease note, messages are considered delivered as soon as the data is written to the client. There is no concept of at least once delivery on this output.\n\n\n[CAUTION]\n.Endpoint caveats\n====\nComponents within a Redpanda Connect config will register their respective endpoints in a non-deterministic order. This means that establishing precedence of endpoints that are registered via multiple `http_server` inputs or outputs (either within brokers or from cohabiting streams) is not possible in a predictable way.\n\nThis ambiguity makes it difficult to ensure that paths which are both a subset of a path registered by a separate component, and end in a slash (`/`) and will therefore match against all extensions of that path, do not prevent the more specific path from matching against requests.\n\nIt is therefore recommended that you ensure paths of separate components do not collide unless they are explicitly non-competing.\n\nFor example, if you were to deploy two separate `http_server` inputs, one with a path `/foo/` and the other with a path `/foo/bar`, it would not be possible to ensure that the path `/foo/` does not swallow requests made to `/foo/bar`.\n====\n","categories":["Network"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"address","type":"string","kind":"scalar","description":"An alternative address to host from. If left empty the service wide address is used.","default":""},{"name":"path","type":"string","kind":"scalar","description":"The path from which discrete messages can be consumed.","default":"/get"},{"name":"stream_path","type":"string","kind":"scalar","description":"The path from which a continuous stream of messages can be consumed.","default":"/get/stream"},{"name":"ws_path","type":"string","kind":"scalar","description":"The path from which websocket connections can be established.","default":"/get/ws"},{"name":"allowed_verbs","type":"string","kind":"array","description":"An array of verbs that are allowed for the `path` and `stream_path` HTTP endpoint.","default":["GET"]},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum time to wait before a blocking, inactive connection is dropped (only applies to the `path` endpoint).","is_advanced":true,"default":"5s"},{"name":"cert_file","type":"string","kind":"scalar","description":"Enable TLS by specifying a certificate and key file. Only valid with a custom `address`.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"Enable TLS by specifying a certificate and key file. Only valid with a custom `address`.","is_advanced":true,"default":""},{"name":"cors","type":"object","kind":"scalar","description":"Adds Cross-Origin Resource Sharing headers. Only valid with a custom `address`.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to allow CORS requests.","is_advanced":true,"default":false},{"name":"allowed_origins","type":"string","kind":"array","description":"An explicit list of origins that are allowed for CORS requests.","is_advanced":true,"default":[]}],"version":"3.63.0"}]}},{"name":"inproc","type":"output","status":"stable","plugin":true,"description":"\nSends data directly to Redpanda Connect inputs by connecting to a unique ID. This allows you to hook up isolated streams whilst running Redpanda Connect in xref:guides:streams_mode/about.adoc[streams mode], it is NOT recommended that you connect the inputs of a stream with an output of the same stream, as feedback loops can lead to deadlocks in your message flow.\n\nIt is possible to connect multiple inputs to the same inproc ID, resulting in messages dispatching in a round-robin fashion to connected inputs. However, only one output can assume an inproc ID, and will replace existing outputs if a collision occurs.","categories":["Utility"],"config":{"name":"","type":"string","kind":"scalar","default":""}},{"name":"kafka","type":"output","status":"stable","plugin":true,"summary":"The kafka output type writes a batch of messages to Kafka brokers and waits for acknowledgement before propagating it back to the input.","description":"\nThe config field `ack_replicas` determines whether we wait for acknowledgement from all replicas or just a single broker.\n\nBoth the `key` and `topic` fields can be dynamically set using function interpolations described in xref:configuration:interpolation.adoc#bloblang-queries[Bloblang queries].\n\nxref:configuration:metadata.adoc[Metadata] will be added to each message sent as headers (version 0.11+), but can be restricted using the field \u003c\u003cmetadata, `metadata`\u003e\u003e.\n\n== Strict ordering and retries\n\nWhen strict ordering is required for messages written to topic partitions it is important to ensure that both the field `max_in_flight` is set to `1` and that the field `retry_as_batch` is set to `true`.\n\nYou must also ensure that failed batches are never rerouted back to the same output. This can be done by setting the field `max_retries` to `0` and `backoff.max_elapsed_time` to empty, which will apply back pressure indefinitely until the batch is sent successfully.\n\nHowever, this also means that manual intervention will eventually be required in cases where the batch cannot be sent due to configuration problems such as an incorrect `max_msg_bytes` estimate. A less strict but automated alternative would be to route failed batches to a dead letter queue using a xref:components:outputs/fallback.adoc[`fallback` broker], but this would allow subsequent batches to be delivered in the meantime whilst those failed batches are dealt with.\n\n== Troubleshooting\n\nIf you're seeing issues writing to or reading from Kafka with this component then it's worth trying out the newer xref:components:outputs/kafka_franz.adoc[`kafka_franz` output].\n\n- I'm seeing logs that report `Failed to connect to kafka: kafka: client has run out of available brokers to talk to (Is your cluster reachable?)`, but the brokers are definitely reachable.\n\nUnfortunately this error message will appear for a wide range of connection problems even when the broker endpoint can be reached. Double check your authentication configuration and also ensure that you have \u003c\u003ctlsenabled, enabled TLS\u003e\u003e if applicable.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"addresses","type":"string","kind":"array","description":"A list of broker addresses to connect to. If an item of the list contains commas it will be expanded into multiple addresses.","examples":[["localhost:9092"],["localhost:9041,localhost:9042"],["localhost:9041","localhost:9042"]]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"sasl","type":"object","kind":"scalar","description":"Enables SASL authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"mechanism","type":"string","kind":"scalar","description":"The SASL authentication mechanism, if left empty SASL authentication is not used.","is_advanced":true,"default":"none","annotated_options":[["OAUTHBEARER","OAuth Bearer based authentication."],["PLAIN","Plain text authentication. NOTE: When using plain text auth it is extremely likely that you'll also need to \u003c\u003ctls-enabled, enable TLS\u003e\u003e."],["SCRAM-SHA-256","Authentication using the SCRAM-SHA-256 mechanism."],["SCRAM-SHA-512","Authentication using the SCRAM-SHA-512 mechanism."],["none","Default, no SASL authentication."]],"linter":"\nlet options = {\n \"oauthbearer\": true,\n \"plain\": true,\n \"scram-sha-256\": true,\n \"scram-sha-512\": true,\n \"none\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"user","type":"string","kind":"scalar","description":"A PLAIN username. It is recommended that you use environment variables to populate this field.","is_advanced":true,"default":"","examples":["${USER}"]},{"name":"password","type":"string","kind":"scalar","description":"A PLAIN password. It is recommended that you use environment variables to populate this field.","is_advanced":true,"is_secret":true,"default":"","examples":["${PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"access_token","type":"string","kind":"scalar","description":"A static OAUTHBEARER access token","is_advanced":true,"default":""},{"name":"token_cache","type":"string","kind":"scalar","description":"Instead of using a static `access_token` allows you to query a xref:components:caches/about.adoc[`cache`] resource to fetch OAUTHBEARER tokens from","is_advanced":true,"default":""},{"name":"token_key","type":"string","kind":"scalar","description":"Required when using a `token_cache`, the key to query the cache with for tokens.","is_advanced":true,"default":""}]},{"name":"topic","type":"string","kind":"scalar","description":"The topic to publish messages to.","interpolated":true},{"name":"client_id","type":"string","kind":"scalar","description":"An identifier for the client connection.","is_advanced":true,"default":"benthos"},{"name":"target_version","type":"string","kind":"scalar","description":"The version of the Kafka protocol to use. This limits the capabilities used by the client and should ideally match the version of your brokers. Defaults to the oldest supported stable version.","is_optional":true,"examples":["2.1.0","3.1.0"]},{"name":"rack_id","type":"string","kind":"scalar","description":"A rack identifier for this client.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"The key to publish messages with.","default":"","interpolated":true},{"name":"partitioner","type":"string","kind":"scalar","description":"The partitioning algorithm to use.","default":"fnv1a_hash","options":["fnv1a_hash","murmur2_hash","random","round_robin","manual"],"linter":"\nlet options = {\n \"fnv1a_hash\": true,\n \"murmur2_hash\": true,\n \"random\": true,\n \"round_robin\": true,\n \"manual\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"partition","type":"string","kind":"scalar","description":"The manually-specified partition to publish messages to, relevant only when the field `partitioner` is set to `manual`. Must be able to parse as a 32-bit integer.","is_advanced":true,"default":"","interpolated":true},{"name":"custom_topic_creation","type":"object","kind":"scalar","description":"If enabled, topics will be created with the specified number of partitions and replication factor if they do not already exist.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to enable custom topic creation.","is_advanced":true,"default":false},{"name":"partitions","type":"int","kind":"scalar","description":"The number of partitions to create for new topics. Leave at -1 to use the broker configured default. Must be \u003e= 1.","is_advanced":true,"default":-1},{"name":"replication_factor","type":"int","kind":"scalar","description":"The replication factor to use for new topics. Leave at -1 to use the broker configured default. Must be an odd number, and less then or equal to the number of brokers.","is_advanced":true,"default":-1}]},{"name":"compression","type":"string","kind":"scalar","description":"The compression algorithm to use.","default":"none","options":["none","snappy","lz4","gzip","zstd"],"linter":"\nlet options = {\n \"none\": true,\n \"snappy\": true,\n \"lz4\": true,\n \"gzip\": true,\n \"zstd\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"static_headers","type":"string","kind":"map","description":"An optional map of static headers that should be added to messages in addition to metadata.","is_optional":true,"examples":[{"first-static-header":"value-1","second-static-header":"value-2"}]},{"name":"metadata","type":"object","kind":"scalar","description":"Specify criteria for which metadata values are sent with messages as headers.","children":[{"name":"exclude_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages.","default":[]}]},{"name":"inject_tracing_map","type":"string","kind":"scalar","description":"EXPERIMENTAL: A xref:guides:bloblang/about.adoc[Bloblang mapping] used to inject an object containing tracing propagation information into outbound messages. The specification of the injected fields will match the format used by the service wide tracer.","is_advanced":true,"is_optional":true,"bloblang":true,"examples":["meta = @.merge(this)","root.meta.span = this"],"version":"3.45.0"},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"idempotent_write","type":"bool","kind":"scalar","description":"Enable the idempotent write producer option. This requires the `IDEMPOTENT_WRITE` permission on `CLUSTER` and can be disabled if this permission is not available.","is_advanced":true,"default":false},{"name":"ack_replicas","type":"bool","kind":"scalar","description":"Ensure that messages have been copied across all replicas before acknowledging receipt.","is_advanced":true,"default":false},{"name":"max_msg_bytes","type":"int","kind":"scalar","description":"The maximum size in bytes of messages sent to the target topic.","is_advanced":true,"default":1000000},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period of time to wait for message sends before abandoning the request and retrying.","is_advanced":true,"default":"5s"},{"name":"retry_as_batch","type":"bool","kind":"scalar","description":"When enabled forces an entire batch of messages to be retried if any individual message fails on a send, otherwise only the individual messages that failed are retried. Disabling this helps to reduce message duplicates during intermittent errors, but also makes it impossible to guarantee strict ordering of messages.","is_advanced":true,"default":false},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"max_retries","type":"int","kind":"scalar","description":"The maximum number of retries before giving up on the request. If set to zero there is no discrete limit.","is_advanced":true,"default":0},{"name":"backoff","type":"object","kind":"scalar","description":"Control time intervals between retry attempts.","is_advanced":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"3s","examples":["50ms","1s"]},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts","is_advanced":true,"default":"10s","examples":["5s","1m"]},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum overall period of time to spend on retry attempts before the request is aborted. Setting this value to a zeroed duration (such as `0s`) will result in unbounded retries.","is_advanced":true,"default":"30s","examples":["1m","1h"]}]},{"name":"timestamp","type":"string","kind":"scalar","description":"An optional timestamp to set for each message. When left empty, the current timestamp is used.","is_advanced":true,"is_deprecated":true,"is_optional":true,"interpolated":true,"examples":["${! timestamp_unix() }","${! metadata(\"kafka_timestamp_unix\") }"]},{"name":"timestamp_ms","type":"string","kind":"scalar","description":"An optional timestamp to set for each message expressed in milliseconds. When left empty, the current timestamp is used.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["${! timestamp_unix_milli() }","${! metadata(\"kafka_timestamp_ms\") }"]}]}},{"name":"kafka_franz","type":"output","status":"beta","plugin":true,"summary":"A Kafka output using the https://github.com/twmb/franz-go[Franz Kafka client library^].","description":"\nWrites a batch of messages to Kafka brokers and waits for acknowledgement before propagating it back to the input.\n\nThis output often out-performs the traditional `kafka` output as well as providing more useful logs and error messages.\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"seed_brokers","type":"string","kind":"array","description":"A list of broker addresses to connect to in order to establish connections. If an item of the list contains commas it will be expanded into multiple addresses.","examples":[["localhost:9092"],["foo:9092","bar:9092"],["foo:9092,bar:9092"]]},{"name":"client_id","type":"string","kind":"scalar","description":"An identifier for the client connection.","is_advanced":true,"default":"benthos"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"sasl","type":"object","kind":"array","description":"Specify one or more methods of SASL authentication. SASL is tried in order; if the broker supports the first mechanism, all connections will use that mechanism. If the first mechanism fails, the client will pick the first supported mechanism. If the broker does not support any client mechanisms, connections will fail.","is_advanced":true,"is_optional":true,"examples":[[{"mechanism":"SCRAM-SHA-512","password":"bar","username":"foo"}]],"children":[{"name":"mechanism","type":"string","kind":"scalar","description":"The SASL mechanism to use.","is_advanced":true,"annotated_options":[["AWS_MSK_IAM","AWS IAM based authentication as specified by the 'aws-msk-iam-auth' java library."],["OAUTHBEARER","OAuth Bearer based authentication."],["PLAIN","Plain text authentication."],["SCRAM-SHA-256","SCRAM based authentication as specified in RFC5802."],["SCRAM-SHA-512","SCRAM based authentication as specified in RFC5802."],["none","Disable sasl authentication"]],"linter":"\nlet options = {\n \"aws_msk_iam\": true,\n \"oauthbearer\": true,\n \"plain\": true,\n \"scram-sha-256\": true,\n \"scram-sha-512\": true,\n \"none\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"username","type":"string","kind":"scalar","description":"A username to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token to use for a single session's OAUTHBEARER authentication.","is_advanced":true,"default":""},{"name":"extensions","type":"string","kind":"map","description":"Key/value pairs to add to OAUTHBEARER authentication requests.","is_advanced":true,"is_optional":true},{"name":"aws","type":"object","kind":"scalar","description":"Contains AWS specific fields for when the `mechanism` is set to `AWS_MSK_IAM`.","is_advanced":true,"is_optional":true,"children":[{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]}]},{"name":"metadata_max_age","type":"string","kind":"scalar","description":"The maximum age of metadata before it is refreshed.","is_advanced":true,"default":"5m"},{"name":"request_timeout_overhead","type":"string","kind":"scalar","description":"The request time overhead. Uses the given time as overhead while deadlining requests. Roughly equivalent to request.timeout.ms, but grants additional time to requests that have timeout fields.","is_advanced":true,"default":"10s"},{"name":"conn_idle_timeout","type":"string","kind":"scalar","description":"The rough amount of time to allow connections to idle before they are closed.","is_advanced":true,"default":"20s"},{"name":"topic","type":"string","kind":"scalar","description":"A topic to write messages to.","interpolated":true},{"name":"key","type":"string","kind":"scalar","description":"An optional key to populate for each message.","is_optional":true,"interpolated":true},{"name":"partition","type":"string","kind":"scalar","description":"An optional explicit partition to set for each message. This field is only relevant when the `partitioner` is set to `manual`. The provided interpolation string must be a valid integer.","is_optional":true,"interpolated":true,"examples":["${! meta(\"partition\") }"]},{"name":"metadata","type":"object","kind":"scalar","description":"Determine which (if any) metadata values should be added to messages as headers.","is_optional":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"timestamp","type":"string","kind":"scalar","description":"An optional timestamp to set for each message. When left empty, the current timestamp is used.","is_advanced":true,"is_deprecated":true,"is_optional":true,"interpolated":true,"examples":["${! timestamp_unix() }","${! metadata(\"kafka_timestamp_unix\") }"]},{"name":"timestamp_ms","type":"string","kind":"scalar","description":"An optional timestamp to set for each message expressed in milliseconds. When left empty, the current timestamp is used.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["${! timestamp_unix_milli() }","${! metadata(\"kafka_timestamp_ms\") }"]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of batches to be sending in parallel at any given time.","default":10},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"rack_id","type":"string","kind":"scalar","is_deprecated":true},{"name":"partitioner","type":"string","kind":"scalar","description":"Override the default murmur2 hashing partitioner.","is_advanced":true,"is_optional":true,"annotated_options":[["least_backup","Chooses the least backed up partition (the partition with the fewest amount of buffered records). Partitions are selected per batch."],["manual","Manually select a partition for each message, requires the field `partition` to be specified."],["murmur2_hash","Kafka's default hash algorithm that uses a 32-bit murmur2 hash of the key to compute which partition the record will be on."],["round_robin","Round-robin's messages through all available partitions. This algorithm has lower throughput and causes higher CPU load on brokers, but can be useful if you want to ensure an even distribution of records to partitions."]],"linter":"\nlet options = {\n \"least_backup\": true,\n \"manual\": true,\n \"murmur2_hash\": true,\n \"round_robin\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"idempotent_write","type":"bool","kind":"scalar","description":"Enable the idempotent write producer option. This requires the `IDEMPOTENT_WRITE` permission on `CLUSTER` and can be disabled if this permission is not available.","is_advanced":true,"default":true},{"name":"compression","type":"string","kind":"scalar","description":"Optionally set an explicit compression type. The default preference is to use snappy when the broker supports it, and fall back to none if not.","is_advanced":true,"is_optional":true,"options":["lz4","snappy","gzip","none","zstd"],"linter":"\nlet options = {\n \"lz4\": true,\n \"snappy\": true,\n \"gzip\": true,\n \"none\": true,\n \"zstd\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period of time to wait for message sends before abandoning the request and retrying","is_advanced":true,"default":"10s"},{"name":"max_message_bytes","type":"string","kind":"scalar","description":"The maximum space in bytes than an individual message may take, messages larger than this value will be rejected. This field corresponds to Kafka's `max.message.bytes`.","is_advanced":true,"default":"1MiB","examples":["100MB","50mib"]},{"name":"broker_write_max_bytes","type":"string","kind":"scalar","description":"The upper bound for the number of bytes written to a broker connection in a single write. This field corresponds to Kafka's `socket.request.max.bytes`.","is_advanced":true,"default":"100MiB","examples":["128MB","50mib"]}],"linter":"root = match {\n this.partitioner == \"manual\" \u0026\u0026 this.partition.or(\"\") == \"\" =\u003e \"a partition must be specified when the partitioner is set to manual\"\n this.partitioner != \"manual\" \u0026\u0026 this.partition.or(\"\") != \"\" =\u003e \"a partition cannot be specified unless the partitioner is set to manual\"\n this.timestamp.or(\"\") != \"\" \u0026\u0026 this.timestamp_ms.or(\"\") != \"\" =\u003e \"both timestamp and timestamp_ms cannot be specified simultaneously\"\n}"},"version":"3.61.0"},{"name":"mongodb","type":"output","status":"experimental","plugin":true,"summary":"Inserts items into a MongoDB collection.","description":"\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target MongoDB server.","examples":["mongodb://localhost:27017"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"database","type":"string","kind":"scalar","description":"The name of the target MongoDB database."},{"name":"username","type":"string","kind":"scalar","description":"The username to connect to the database.","default":""},{"name":"password","type":"string","kind":"scalar","description":"The password to connect to the database.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"app_name","type":"string","kind":"scalar","description":"The client application name.","is_advanced":true,"default":"benthos","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"collection","type":"string","kind":"scalar","description":"The name of the target collection.","interpolated":true},{"name":"operation","type":"string","kind":"scalar","description":"The mongodb operation to perform.","default":"update-one","options":["insert-one","delete-one","delete-many","replace-one","update-one"],"linter":"\nlet options = {\n \"insert-one\": true,\n \"delete-one\": true,\n \"delete-many\": true,\n \"replace-one\": true,\n \"update-one\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"write_concern","type":"object","kind":"scalar","description":"The write concern settings for the mongo connection.","children":[{"name":"w","type":"string","kind":"scalar","description":"W requests acknowledgement that write operations propagate to the specified number of mongodb instances. Can be the string \"majority\" to wait for a calculated majority of nodes to acknowledge the write operation, or an integer value specifying an minimum number of nodes to acknowledge the operation, or a string specifying the name of a custom write concern configured in the cluster.","default":"majority"},{"name":"j","type":"bool","kind":"scalar","description":"J requests acknowledgement from MongoDB that write operations are written to the journal.","default":false},{"name":"w_timeout","type":"string","kind":"scalar","description":"The write concern timeout.","default":""}]},{"name":"document_map","type":"string","kind":"scalar","description":"A bloblang map representing a document to store within MongoDB, expressed as https://www.mongodb.com/docs/manual/reference/mongodb-extended-json/[extended JSON in canonical form^]. The document map is required for the operations insert-one, replace-one, update-one and aggregate.","default":"","bloblang":true,"examples":["root.a = this.foo\nroot.b = this.bar"]},{"name":"filter_map","type":"string","kind":"scalar","description":"A bloblang map representing a filter for a MongoDB command, expressed as https://www.mongodb.com/docs/manual/reference/mongodb-extended-json/[extended JSON in canonical form^]. The filter map is required for all operations except insert-one. It is used to find the document(s) for the operation. For example in a delete-one case, the filter map should have the fields required to locate the document to delete.","default":"","bloblang":true,"examples":["root.a = this.foo\nroot.b = this.bar"]},{"name":"hint_map","type":"string","kind":"scalar","description":"A bloblang map representing the hint for the MongoDB command, expressed as https://www.mongodb.com/docs/manual/reference/mongodb-extended-json/[extended JSON in canonical form^]. This map is optional and is used with all operations except insert-one. It is used to improve performance of finding the documents in the mongodb.","default":"","bloblang":true,"examples":["root.a = this.foo\nroot.b = this.bar"]},{"name":"upsert","type":"bool","kind":"scalar","description":"The upsert setting is optional and only applies for update-one and replace-one operations. If the filter specified in filter_map matches, the document is updated or replaced accordingly, otherwise it is created.","default":false,"version":"3.60.0"},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"max_retries","type":"int","kind":"scalar","description":"The maximum number of retries before giving up on the request. If set to zero there is no discrete limit.","is_advanced":true,"is_deprecated":true,"default":3},{"name":"backoff","type":"object","kind":"scalar","description":"Control time intervals between retry attempts.","is_advanced":true,"is_deprecated":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"is_deprecated":true,"default":"1s"},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts.","is_advanced":true,"is_deprecated":true,"default":"5s"},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum period to wait before retry attempts are abandoned. If zero then no limit is used.","is_advanced":true,"is_deprecated":true,"default":"30s"}]}]},"version":"3.43.0"},{"name":"mqtt","type":"output","status":"stable","plugin":true,"summary":"Pushes messages to an MQTT broker.","description":"\nThe `topic` field can be dynamically set using function interpolations described xref:configuration:interpolation.adoc#bloblang-queries[here]. When sending batched messages these interpolations are performed per message part.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. The format should be `scheme://host:port` where `scheme` is one of `tcp`, `ssl`, or `ws`, `host` is the ip-address (or hostname) and `port` is the port on which the broker is accepting connections. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["tcp://localhost:1883"]],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"client_id","type":"string","kind":"scalar","description":"An identifier for the client connection.","default":""},{"name":"dynamic_client_id_suffix","type":"string","kind":"scalar","description":"Append a dynamically generated suffix to the specified `client_id` on each run of the pipeline. This can be useful when clustering Redpanda Connect producers.","is_advanced":true,"is_optional":true,"annotated_options":[["nanoid","append a nanoid of length 21 characters"]],"linter":"root = []"},{"name":"connect_timeout","type":"string","kind":"scalar","description":"The maximum amount of time to wait in order to establish a connection before the attempt is abandoned.","default":"30s","examples":["1s","500ms"],"version":"3.58.0"},{"name":"will","type":"object","kind":"scalar","description":"Set last will message in case of Redpanda Connect failure","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to enable last will messages.","is_advanced":true,"default":false},{"name":"qos","type":"int","kind":"scalar","description":"Set QoS for last will message. Valid values are: 0, 1, 2.","is_advanced":true,"default":0},{"name":"retained","type":"bool","kind":"scalar","description":"Set retained for last will message.","is_advanced":true,"default":false},{"name":"topic","type":"string","kind":"scalar","description":"Set topic for last will message.","is_advanced":true,"default":""},{"name":"payload","type":"string","kind":"scalar","description":"Set payload for last will message.","is_advanced":true,"default":""}]},{"name":"user","type":"string","kind":"scalar","description":"A username to connect with.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to connect with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"keepalive","type":"int","kind":"scalar","description":"Max seconds of inactivity before a keepalive message is sent.","is_advanced":true,"default":30},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"topic","type":"string","kind":"scalar","description":"The topic to publish messages to.","interpolated":true},{"name":"qos","type":"int","kind":"scalar","description":"The QoS value to set for each message. Has options 0, 1, 2.","default":1},{"name":"write_timeout","type":"string","kind":"scalar","description":"The maximum amount of time to wait to write data before the attempt is abandoned.","default":"3s","examples":["1s","500ms"],"version":"3.58.0"},{"name":"retained","type":"bool","kind":"scalar","description":"Set message as retained on the topic.","default":false},{"name":"retained_interpolated","type":"string","kind":"scalar","description":"Override the value of `retained` with an interpolable value, this allows it to be dynamically set based on message contents. The value must resolve to either `true` or `false`.","is_advanced":true,"is_optional":true,"interpolated":true,"version":"3.59.0"},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64}]}},{"name":"nanomsg","type":"output","status":"stable","plugin":true,"summary":"Send messages over a Nanomsg socket.","description":"Currently only PUSH and PUB sockets are supported.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.","categories":["Network"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"bind","type":"bool","kind":"scalar","description":"Whether the URLs listed should be bind (otherwise they are connected to).","default":false},{"name":"socket_type","type":"string","kind":"scalar","description":"The socket type to send with.","default":"PUSH","options":["PUSH","PUB"],"linter":"\nlet options = {\n \"push\": true,\n \"pub\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"poll_timeout","type":"string","kind":"scalar","description":"The maximum period of time to wait for a message to send before the request is abandoned and reattempted.","default":"5s"},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64}]}},{"name":"nats","type":"output","status":"stable","plugin":true,"summary":"Publish to an NATS subject.","description":"This output will interpolate functions within the subject field, you can find a list of functions xref:configuration:interpolation.adoc#bloblang-queries[here].\n\n== Connection name\n\nWhen monitoring and managing a production NATS system, it is often useful to\nknow which connection a message was send/received from. This can be achieved by\nsetting the connection name option when creating a NATS connection.\n\nRedpanda Connect will automatically set the connection name based off the label of the given\nNATS component, so that monitoring tools between NATS and Redpanda Connect can stay in sync.\n\n\n== Authentication\n\nThere are several components within Redpanda Connect which uses NATS services. You will find that each of these components\nsupport optional advanced authentication parameters for https://docs.nats.io/nats-server/configuration/securing_nats/auth_intro/nkey_auth[NKeys^]\nand https://docs.nats.io/using-nats/developer/connecting/creds[User Credentials^].\n\nSee an https://docs.nats.io/running-a-nats-service/nats_admin/security/jwt[in-depth tutorial^].\n\n=== NKey file\n\nThe NATS server can use these NKeys in several ways for authentication. The simplest is for the server to be configured\nwith a list of known public keys and for the clients to respond to the challenge by signing it with its private NKey\nconfigured in the `nkey_file` or `nkey` field.\n\nhttps://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[More details^].\n\n=== User credentials\n\nNATS server supports decentralized authentication based on JSON Web Tokens (JWT). Clients need an https://docs.nats.io/nats-server/configuration/securing_nats/jwt#json-web-tokens[user JWT^]\nand a corresponding https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[NKey secret^] when connecting to a server\nwhich is configured to use this authentication scheme.\n\nThe `user_credentials_file` field should point to a file containing both the private key and the JWT and can be\ngenerated with the https://docs.nats.io/nats-tools/nsc[nsc tool^].\n\nAlternatively, the `user_jwt` field can contain a plain text JWT and the `user_nkey_seed`can contain\nthe plain text NKey Seed.\n\nhttps://docs.nats.io/using-nats/developer/connecting/creds[More details^].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["nats://127.0.0.1:4222"],["nats://username:password@127.0.0.1:4222"]]},{"name":"subject","type":"string","kind":"scalar","description":"The subject to publish to.","interpolated":true,"examples":["foo.bar.baz"]},{"name":"headers","type":"string","kind":"map","description":"Explicit message headers to add to messages.","default":{},"interpolated":true,"examples":[{"Content-Type":"application/json","Timestamp":"${!meta(\"Timestamp\")}"}]},{"name":"metadata","type":"object","kind":"scalar","description":"Determine which (if any) metadata values should be added to messages as headers.","is_optional":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auth","type":"object","kind":"scalar","description":"Optional configuration of NATS authentication parameters.","is_advanced":true,"children":[{"name":"nkey_file","type":"string","kind":"scalar","description":"An optional file containing a NKey seed.","is_advanced":true,"is_optional":true,"examples":["./seed.nk"]},{"name":"nkey","type":"string","kind":"scalar","description":"The NKey seed.","is_advanced":true,"is_optional":true,"is_secret":true,"examples":["UDXU4RCSJNZOIQHZNWXHXORDPRTGNJAHAHFRGZNEEJCPQTT2M7NLCNF4"],"version":"4.38.0","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_credentials_file","type":"string","kind":"scalar","description":"An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.","is_advanced":true,"is_optional":true,"examples":["./user.creds"]},{"name":"user_jwt","type":"string","kind":"scalar","description":"An optional plain text user JWT (given along with the corresponding user NKey Seed).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_nkey_seed","type":"string","kind":"scalar","description":"An optional plain text user NKey Seed (given along with the corresponding user JWT).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"inject_tracing_map","type":"string","kind":"scalar","description":"EXPERIMENTAL: A xref:guides:bloblang/about.adoc[Bloblang mapping] used to inject an object containing tracing propagation information into outbound messages. The specification of the injected fields will match the format used by the service wide tracer.","is_advanced":true,"is_optional":true,"bloblang":true,"examples":["meta = @.merge(this)","root.meta.span = this"],"version":"4.23.0"}]}},{"name":"nats_jetstream","type":"output","status":"stable","plugin":true,"summary":"Write messages to a NATS JetStream subject.","description":"== Connection name\n\nWhen monitoring and managing a production NATS system, it is often useful to\nknow which connection a message was send/received from. This can be achieved by\nsetting the connection name option when creating a NATS connection.\n\nRedpanda Connect will automatically set the connection name based off the label of the given\nNATS component, so that monitoring tools between NATS and Redpanda Connect can stay in sync.\n\n\n== Authentication\n\nThere are several components within Redpanda Connect which uses NATS services. You will find that each of these components\nsupport optional advanced authentication parameters for https://docs.nats.io/nats-server/configuration/securing_nats/auth_intro/nkey_auth[NKeys^]\nand https://docs.nats.io/using-nats/developer/connecting/creds[User Credentials^].\n\nSee an https://docs.nats.io/running-a-nats-service/nats_admin/security/jwt[in-depth tutorial^].\n\n=== NKey file\n\nThe NATS server can use these NKeys in several ways for authentication. The simplest is for the server to be configured\nwith a list of known public keys and for the clients to respond to the challenge by signing it with its private NKey\nconfigured in the `nkey_file` or `nkey` field.\n\nhttps://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[More details^].\n\n=== User credentials\n\nNATS server supports decentralized authentication based on JSON Web Tokens (JWT). Clients need an https://docs.nats.io/nats-server/configuration/securing_nats/jwt#json-web-tokens[user JWT^]\nand a corresponding https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[NKey secret^] when connecting to a server\nwhich is configured to use this authentication scheme.\n\nThe `user_credentials_file` field should point to a file containing both the private key and the JWT and can be\ngenerated with the https://docs.nats.io/nats-tools/nsc[nsc tool^].\n\nAlternatively, the `user_jwt` field can contain a plain text JWT and the `user_nkey_seed`can contain\nthe plain text NKey Seed.\n\nhttps://docs.nats.io/using-nats/developer/connecting/creds[More details^].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["nats://127.0.0.1:4222"],["nats://username:password@127.0.0.1:4222"]]},{"name":"subject","type":"string","kind":"scalar","description":"A subject to write to.","interpolated":true,"examples":["foo.bar.baz","${! meta(\"kafka_topic\") }","foo.${! json(\"meta.type\") }"]},{"name":"headers","type":"string","kind":"map","description":"Explicit message headers to add to messages.","default":{},"interpolated":true,"examples":[{"Content-Type":"application/json","Timestamp":"${!meta(\"Timestamp\")}"}],"version":"4.1.0"},{"name":"metadata","type":"object","kind":"scalar","description":"Determine which (if any) metadata values should be added to messages as headers.","is_optional":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":1024},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auth","type":"object","kind":"scalar","description":"Optional configuration of NATS authentication parameters.","is_advanced":true,"children":[{"name":"nkey_file","type":"string","kind":"scalar","description":"An optional file containing a NKey seed.","is_advanced":true,"is_optional":true,"examples":["./seed.nk"]},{"name":"nkey","type":"string","kind":"scalar","description":"The NKey seed.","is_advanced":true,"is_optional":true,"is_secret":true,"examples":["UDXU4RCSJNZOIQHZNWXHXORDPRTGNJAHAHFRGZNEEJCPQTT2M7NLCNF4"],"version":"4.38.0","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_credentials_file","type":"string","kind":"scalar","description":"An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.","is_advanced":true,"is_optional":true,"examples":["./user.creds"]},{"name":"user_jwt","type":"string","kind":"scalar","description":"An optional plain text user JWT (given along with the corresponding user NKey Seed).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_nkey_seed","type":"string","kind":"scalar","description":"An optional plain text user NKey Seed (given along with the corresponding user JWT).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"inject_tracing_map","type":"string","kind":"scalar","description":"EXPERIMENTAL: A xref:guides:bloblang/about.adoc[Bloblang mapping] used to inject an object containing tracing propagation information into outbound messages. The specification of the injected fields will match the format used by the service wide tracer.","is_advanced":true,"is_optional":true,"bloblang":true,"examples":["meta = @.merge(this)","root.meta.span = this"],"version":"4.23.0"}]},"version":"3.46.0"},{"name":"nats_kv","type":"output","status":"beta","plugin":true,"summary":"Put messages in a NATS key-value bucket.","description":"\nThe field `key` supports\nxref:configuration:interpolation.adoc#bloblang-queries[interpolation functions], allowing\nyou to create a unique key for each message.\n\n== Connection name\n\nWhen monitoring and managing a production NATS system, it is often useful to\nknow which connection a message was send/received from. This can be achieved by\nsetting the connection name option when creating a NATS connection.\n\nRedpanda Connect will automatically set the connection name based off the label of the given\nNATS component, so that monitoring tools between NATS and Redpanda Connect can stay in sync.\n\n\n== Authentication\n\nThere are several components within Redpanda Connect which uses NATS services. You will find that each of these components\nsupport optional advanced authentication parameters for https://docs.nats.io/nats-server/configuration/securing_nats/auth_intro/nkey_auth[NKeys^]\nand https://docs.nats.io/using-nats/developer/connecting/creds[User Credentials^].\n\nSee an https://docs.nats.io/running-a-nats-service/nats_admin/security/jwt[in-depth tutorial^].\n\n=== NKey file\n\nThe NATS server can use these NKeys in several ways for authentication. The simplest is for the server to be configured\nwith a list of known public keys and for the clients to respond to the challenge by signing it with its private NKey\nconfigured in the `nkey_file` or `nkey` field.\n\nhttps://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[More details^].\n\n=== User credentials\n\nNATS server supports decentralized authentication based on JSON Web Tokens (JWT). Clients need an https://docs.nats.io/nats-server/configuration/securing_nats/jwt#json-web-tokens[user JWT^]\nand a corresponding https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[NKey secret^] when connecting to a server\nwhich is configured to use this authentication scheme.\n\nThe `user_credentials_file` field should point to a file containing both the private key and the JWT and can be\ngenerated with the https://docs.nats.io/nats-tools/nsc[nsc tool^].\n\nAlternatively, the `user_jwt` field can contain a plain text JWT and the `user_nkey_seed`can contain\nthe plain text NKey Seed.\n\nhttps://docs.nats.io/using-nats/developer/connecting/creds[More details^].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["nats://127.0.0.1:4222"],["nats://username:password@127.0.0.1:4222"]]},{"name":"bucket","type":"string","kind":"scalar","description":"The name of the KV bucket.","examples":["my_kv_bucket"]},{"name":"key","type":"string","kind":"scalar","description":"The key for each message.","interpolated":true,"examples":["foo","foo.bar.baz","foo.${! json(\"meta.type\") }"]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":1024},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auth","type":"object","kind":"scalar","description":"Optional configuration of NATS authentication parameters.","is_advanced":true,"children":[{"name":"nkey_file","type":"string","kind":"scalar","description":"An optional file containing a NKey seed.","is_advanced":true,"is_optional":true,"examples":["./seed.nk"]},{"name":"nkey","type":"string","kind":"scalar","description":"The NKey seed.","is_advanced":true,"is_optional":true,"is_secret":true,"examples":["UDXU4RCSJNZOIQHZNWXHXORDPRTGNJAHAHFRGZNEEJCPQTT2M7NLCNF4"],"version":"4.38.0","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_credentials_file","type":"string","kind":"scalar","description":"An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.","is_advanced":true,"is_optional":true,"examples":["./user.creds"]},{"name":"user_jwt","type":"string","kind":"scalar","description":"An optional plain text user JWT (given along with the corresponding user NKey Seed).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_nkey_seed","type":"string","kind":"scalar","description":"An optional plain text user NKey Seed (given along with the corresponding user JWT).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},"version":"4.12.0"},{"name":"nats_stream","type":"output","status":"stable","plugin":true,"summary":"Publish to a NATS Stream subject.","description":"\n[CAUTION]\n.Deprecation notice\n====\nThe NATS Streaming Server is being deprecated. Critical bug fixes and security fixes will be applied until June of 2023. NATS-enabled applications requiring persistence should use https://docs.nats.io/nats-concepts/jetstream[JetStream^].\n====\n\n\n\n== Authentication\n\nThere are several components within Redpanda Connect which uses NATS services. You will find that each of these components\nsupport optional advanced authentication parameters for https://docs.nats.io/nats-server/configuration/securing_nats/auth_intro/nkey_auth[NKeys^]\nand https://docs.nats.io/using-nats/developer/connecting/creds[User Credentials^].\n\nSee an https://docs.nats.io/running-a-nats-service/nats_admin/security/jwt[in-depth tutorial^].\n\n=== NKey file\n\nThe NATS server can use these NKeys in several ways for authentication. The simplest is for the server to be configured\nwith a list of known public keys and for the clients to respond to the challenge by signing it with its private NKey\nconfigured in the `nkey_file` or `nkey` field.\n\nhttps://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[More details^].\n\n=== User credentials\n\nNATS server supports decentralized authentication based on JSON Web Tokens (JWT). Clients need an https://docs.nats.io/nats-server/configuration/securing_nats/jwt#json-web-tokens[user JWT^]\nand a corresponding https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[NKey secret^] when connecting to a server\nwhich is configured to use this authentication scheme.\n\nThe `user_credentials_file` field should point to a file containing both the private key and the JWT and can be\ngenerated with the https://docs.nats.io/nats-tools/nsc[nsc tool^].\n\nAlternatively, the `user_jwt` field can contain a plain text JWT and the `user_nkey_seed`can contain\nthe plain text NKey Seed.\n\nhttps://docs.nats.io/using-nats/developer/connecting/creds[More details^].\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["nats://127.0.0.1:4222"],["nats://username:password@127.0.0.1:4222"]]},{"name":"cluster_id","type":"string","kind":"scalar","description":"The cluster ID to publish to."},{"name":"subject","type":"string","kind":"scalar","description":"The subject to publish to."},{"name":"client_id","type":"string","kind":"scalar","description":"The client ID to connect with.","default":""},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auth","type":"object","kind":"scalar","description":"Optional configuration of NATS authentication parameters.","is_advanced":true,"children":[{"name":"nkey_file","type":"string","kind":"scalar","description":"An optional file containing a NKey seed.","is_advanced":true,"is_optional":true,"examples":["./seed.nk"]},{"name":"nkey","type":"string","kind":"scalar","description":"The NKey seed.","is_advanced":true,"is_optional":true,"is_secret":true,"examples":["UDXU4RCSJNZOIQHZNWXHXORDPRTGNJAHAHFRGZNEEJCPQTT2M7NLCNF4"],"version":"4.38.0","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_credentials_file","type":"string","kind":"scalar","description":"An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.","is_advanced":true,"is_optional":true,"examples":["./user.creds"]},{"name":"user_jwt","type":"string","kind":"scalar","description":"An optional plain text user JWT (given along with the corresponding user NKey Seed).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_nkey_seed","type":"string","kind":"scalar","description":"An optional plain text user NKey Seed (given along with the corresponding user JWT).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"inject_tracing_map","type":"string","kind":"scalar","description":"EXPERIMENTAL: A xref:guides:bloblang/about.adoc[Bloblang mapping] used to inject an object containing tracing propagation information into outbound messages. The specification of the injected fields will match the format used by the service wide tracer.","is_advanced":true,"is_optional":true,"bloblang":true,"examples":["meta = @.merge(this)","root.meta.span = this"],"version":"4.23.0"}]}},{"name":"nsq","type":"output","status":"stable","plugin":true,"summary":"Publish to an NSQ topic.","description":"The `topic` field can be dynamically set using function interpolations described xref:configuration:interpolation.adoc#bloblang-queries[here]. When sending batched messages these interpolations are performed per message part.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"nsqd_tcp_address","type":"string","kind":"scalar","description":"The address of the target NSQD server."},{"name":"topic","type":"string","kind":"scalar","description":"The topic to publish to.","interpolated":true},{"name":"user_agent","type":"string","kind":"scalar","description":"A user agent to assume when connecting.","is_optional":true},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64}]}},{"name":"ockam_kafka","type":"output","status":"experimental","plugin":true,"summary":"Ockam","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"kafka","type":"object","kind":"scalar","children":[{"name":"seed_brokers","type":"string","kind":"array","description":"A list of broker addresses to connect to in order to establish connections. If an item of the list contains commas it will be expanded into multiple addresses.","is_optional":true,"examples":[["localhost:9092"],["foo:9092","bar:9092"],["foo:9092,bar:9092"]]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of batches to be sending in parallel at any given time.","default":10},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"partitioner","type":"string","kind":"scalar","description":"Override the default murmur2 hashing partitioner.","is_advanced":true,"is_optional":true,"annotated_options":[["least_backup","Chooses the least backed up partition (the partition with the fewest amount of buffered records). Partitions are selected per batch."],["manual","Manually select a partition for each message, requires the field `partition` to be specified."],["murmur2_hash","Kafka's default hash algorithm that uses a 32-bit murmur2 hash of the key to compute which partition the record will be on."],["round_robin","Round-robin's messages through all available partitions. This algorithm has lower throughput and causes higher CPU load on brokers, but can be useful if you want to ensure an even distribution of records to partitions."]],"linter":"\nlet options = {\n \"least_backup\": true,\n \"manual\": true,\n \"murmur2_hash\": true,\n \"round_robin\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"idempotent_write","type":"bool","kind":"scalar","description":"Enable the idempotent write producer option. This requires the `IDEMPOTENT_WRITE` permission on `CLUSTER` and can be disabled if this permission is not available.","is_advanced":true,"default":true},{"name":"compression","type":"string","kind":"scalar","description":"Optionally set an explicit compression type. The default preference is to use snappy when the broker supports it, and fall back to none if not.","is_advanced":true,"is_optional":true,"options":["lz4","snappy","gzip","none","zstd"],"linter":"\nlet options = {\n \"lz4\": true,\n \"snappy\": true,\n \"gzip\": true,\n \"none\": true,\n \"zstd\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period of time to wait for message sends before abandoning the request and retrying","is_advanced":true,"default":"10s"},{"name":"max_message_bytes","type":"string","kind":"scalar","description":"The maximum space in bytes than an individual message may take, messages larger than this value will be rejected. This field corresponds to Kafka's `max.message.bytes`.","is_advanced":true,"default":"1MiB","examples":["100MB","50mib"]},{"name":"broker_write_max_bytes","type":"string","kind":"scalar","description":"The upper bound for the number of bytes written to a broker connection in a single write. This field corresponds to Kafka's `socket.request.max.bytes`.","is_advanced":true,"default":"100MiB","examples":["128MB","50mib"]},{"name":"topic","type":"string","kind":"scalar","description":"A topic to write messages to.","interpolated":true},{"name":"key","type":"string","kind":"scalar","description":"An optional key to populate for each message.","is_optional":true,"interpolated":true},{"name":"partition","type":"string","kind":"scalar","description":"An optional explicit partition to set for each message. This field is only relevant when the `partitioner` is set to `manual`. The provided interpolation string must be a valid integer.","is_optional":true,"interpolated":true,"examples":["${! meta(\"partition\") }"]},{"name":"metadata","type":"object","kind":"scalar","description":"Determine which (if any) metadata values should be added to messages as headers.","is_optional":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"timestamp","type":"string","kind":"scalar","description":"An optional timestamp to set for each message. When left empty, the current timestamp is used.","is_advanced":true,"is_deprecated":true,"is_optional":true,"interpolated":true,"examples":["${! timestamp_unix() }","${! metadata(\"kafka_timestamp_unix\") }"]},{"name":"timestamp_ms","type":"string","kind":"scalar","description":"An optional timestamp to set for each message expressed in milliseconds. When left empty, the current timestamp is used.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["${! timestamp_unix_milli() }","${! metadata(\"kafka_timestamp_ms\") }"]}]},{"name":"disable_content_encryption","type":"bool","kind":"scalar","default":false},{"name":"enrollment_ticket","type":"string","kind":"scalar","is_optional":true},{"name":"identity_name","type":"string","kind":"scalar","is_optional":true},{"name":"allow","type":"string","kind":"scalar","is_optional":true,"default":"self"},{"name":"route_to_kafka_outlet","type":"string","kind":"scalar","default":"self"},{"name":"allow_consumer","type":"string","kind":"scalar","default":"self"},{"name":"route_to_consumer","type":"string","kind":"scalar","default":"/ip4/127.0.0.1/tcp/6262"},{"name":"encrypted_fields","type":"string","kind":"array","description":"The fields to encrypt in the kafka messages, assuming the record is a valid JSON map. By default, the whole record is encrypted.","default":[]}]}},{"name":"opensearch","type":"output","status":"stable","plugin":true,"summary":"Publishes messages into an Elasticsearch index. If the index does not exist then it is created with a dynamic mapping.","description":"\nBoth the `id` and `index` fields can be dynamically set using function interpolations described xref:configuration:interpolation.adoc#bloblang-queries[here]. When sending batched messages these interpolations are performed per message part.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services"],"examples":[{"title":"Updating Documents","summary":"When https://opensearch.org/docs/latest/api-reference/document-apis/update-document/[updating documents^] the request body should contain a combination of a `doc`, `upsert`, and/or `script` fields at the top level, this should be done via mapping processors.","config":"\noutput:\n processors:\n - mapping: |\n meta id = this.id\n root.doc = this\n opensearch:\n urls: [ TODO ]\n index: foo\n id: ${! @id }\n action: update\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["http://localhost:9200"]]},{"name":"index","type":"string","kind":"scalar","description":"The index to place messages.","interpolated":true},{"name":"action","type":"string","kind":"scalar","description":"The action to take on the document. This field must resolve to one of the following action types: `index`, `update` or `delete`.","interpolated":true},{"name":"id","type":"string","kind":"scalar","description":"The ID for indexed messages. Interpolation should be used in order to create a unique ID for each message.","interpolated":true,"examples":["${!counter()}-${!timestamp_unix()}"]},{"name":"pipeline","type":"string","kind":"scalar","description":"An optional pipeline id to preprocess incoming documents.","is_advanced":true,"default":"","interpolated":true},{"name":"routing","type":"string","kind":"scalar","description":"The routing key to use for the document.","is_advanced":true,"default":"","interpolated":true},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"aws","type":"object","kind":"scalar","description":"Enables and customises connectivity to Amazon Elastic Service.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to connect to Amazon Elastic Service.","is_advanced":true,"default":false},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]}]}},{"name":"pinecone","type":"output","status":"experimental","plugin":true,"summary":"Inserts items into a Pinecone index.","description":"\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["AI"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"host","type":"string","kind":"scalar","description":"The host for the Pinecone index.","linter":"root = if this.has_prefix(\"https://\") { [\"host field must be a FQDN not a URL (remove the https:// prefix)\"] }"},{"name":"api_key","type":"string","kind":"scalar","description":"The Pinecone api key.","is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"operation","type":"string","kind":"scalar","description":"The operation to perform against the Pinecone index.","default":"upsert-vectors","options":["update-vector","upsert-vectors","delete-vectors"],"linter":"\nlet options = {\n \"update-vector\": true,\n \"upsert-vectors\": true,\n \"delete-vectors\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"namespace","type":"string","kind":"scalar","description":"The namespace to write to - writes to the default namespace by default.","is_advanced":true,"default":"","interpolated":true},{"name":"id","type":"string","kind":"scalar","description":"The ID for the index entry in Pinecone.","interpolated":true},{"name":"vector_mapping","type":"string","kind":"scalar","description":"The mapping to extract out the vector from the document. The result must be a floating point array. Required if not a delete operation.","is_optional":true,"bloblang":true,"examples":["root = this.embeddings_vector","root = [1.2, 0.5, 0.76]"]},{"name":"metadata_mapping","type":"string","kind":"scalar","description":"An optional mapping of message to metadata in the Pinecone index entry.","is_optional":true,"bloblang":true,"examples":["root = @","root = metadata()","root = {\"summary\": this.summary, \"foo\": this.other_field}"]}]},"version":"4.31.0"},{"name":"pulsar","type":"output","status":"experimental","plugin":true,"summary":"Write messages to an Apache Pulsar server.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"A URL to connect to.","examples":["pulsar://localhost:6650","pulsar://pulsar.us-west.example.com:6650","pulsar+ssl://pulsar.us-west.example.com:6651"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"topic","type":"string","kind":"scalar","description":"The topic to publish to."},{"name":"tls","type":"object","kind":"scalar","description":"Specify the path to a custom CA certificate to trust broker TLS service.","children":[{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","default":"","examples":["./root_cas.pem"]}]},{"name":"key","type":"string","kind":"scalar","description":"The key to publish messages with.","default":"","interpolated":true},{"name":"ordering_key","type":"string","kind":"scalar","description":"The ordering key to publish messages with.","default":"","interpolated":true},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"auth","type":"object","kind":"scalar","description":"Optional configuration of Pulsar authentication methods.","is_advanced":true,"is_optional":true,"children":[{"name":"oauth2","type":"object","kind":"scalar","description":"Parameters for Pulsar OAuth2 authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether OAuth2 is enabled.","is_advanced":true,"default":false},{"name":"audience","type":"string","kind":"scalar","description":"OAuth2 audience.","is_advanced":true,"default":""},{"name":"issuer_url","type":"string","kind":"scalar","description":"OAuth2 issuer URL.","is_advanced":true,"default":"","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"scope","type":"string","kind":"scalar","description":"OAuth2 scope to request.","is_advanced":true,"default":"","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"private_key_file","type":"string","kind":"scalar","description":"The path to a file containing a private key.","is_advanced":true,"default":""}]},{"name":"token","type":"object","kind":"scalar","description":"Parameters for Pulsar Token authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether Token Auth is enabled.","is_advanced":true,"default":false},{"name":"token","type":"string","kind":"scalar","description":"Actual base64 encoded token.","is_advanced":true,"default":""}]}],"version":"3.60.0"}]},"version":"3.43.0"},{"name":"pusher","type":"output","status":"experimental","plugin":true,"summary":"Output for publishing messages to Pusher API (https://pusher.com)","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"batching","type":"object","kind":"","description":"maximum batch size is 10 (limit of the pusher library)","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"channel","type":"string","kind":"scalar","description":"Pusher channel to publish to. Interpolation functions can also be used","interpolated":true,"examples":["my_channel","${!json(\"id\")}"]},{"name":"event","type":"string","kind":"scalar","description":"Event to publish to"},{"name":"appId","type":"string","kind":"scalar","description":"Pusher app id"},{"name":"key","type":"string","kind":"scalar","description":"Pusher key"},{"name":"secret","type":"string","kind":"scalar","description":"Pusher secret"},{"name":"cluster","type":"string","kind":"scalar","description":"Pusher cluster"},{"name":"secure","type":"bool","kind":"scalar","description":"Enable SSL encryption","default":true},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of parallel message batches to have in flight at any given time.","default":1}]},"version":"4.3.0"},{"name":"qdrant","type":"output","status":"experimental","plugin":true,"summary":"Adds items to a https://qdrant.tech/[Qdrant^] collection","description":"\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["AI"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"grpc_host","type":"string","kind":"scalar","description":"The gRPC host of the Qdrant server.","examples":["localhost:6334","xyz-example.eu-central.aws.cloud.qdrant.io:6334"]},{"name":"api_token","type":"string","kind":"scalar","description":"The Qdrant API token for authentication. Defaults to an empty string.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"tls","type":"object","kind":"scalar","description":"TLS(HTTPS) config to use when connecting","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"collection_name","type":"string","kind":"scalar","description":"The name of the collection in Qdrant.","interpolated":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of the point to insert. Can be a UUID string or positive integer.","bloblang":true,"examples":["root = \"dc88c126-679f-49f5-ab85-04b77e8c2791\"","root = 832"]},{"name":"vector_mapping","type":"string","kind":"scalar","description":"The mapping to extract the vector from the document.","bloblang":true,"examples":["root = {\"dense_vector\": [0.352,0.532,0.754],\"sparse_vector\": {\"indices\": [23,325,532],\"values\": [0.352,0.532,0.532]}, \"multi_vector\": [[0.352,0.532],[0.352,0.532]]}","root = [1.2, 0.5, 0.76]","root = this.vector","root = [[0.352,0.532,0.532,0.234],[0.352,0.532,0.532,0.234]]","root = {\"some_sparse\": {\"indices\":[23,325,532],\"values\":[0.352,0.532,0.532]}}","root = {\"some_multi\": [[0.352,0.532,0.532,0.234],[0.352,0.532,0.532,0.234]]}","root = {\"some_dense\": [0.352,0.532,0.532,0.234]}"]},{"name":"payload_mapping","type":"string","kind":"scalar","description":"An optional mapping of message to payload associated with the point.","default":"root = {}","bloblang":true,"examples":["root = {\"field\": this.value, \"field_2\": 987}","root = metadata()"]}]},"version":"4.33.0"},{"name":"questdb","type":"output","status":"experimental","plugin":true,"summary":"Pushes messages to a QuestDB table","description":"Important: We recommend that the dedupe feature is enabled on the QuestDB server. Please visit https://questdb.io/docs/ for more information about deploying, configuring, and using QuestDB.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"address","type":"string","kind":"scalar","description":"Address of the QuestDB server's HTTP port (excluding protocol)","examples":["localhost:9000"]},{"name":"username","type":"string","kind":"scalar","description":"Username for HTTP basic auth","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"password","type":"string","kind":"scalar","description":"Password for HTTP basic auth","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"Bearer token for HTTP auth (takes precedence over basic auth username \u0026 password)","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"retry_timeout","type":"string","kind":"scalar","description":"The time to continue retrying after a failed HTTP request. The interval between retries is an exponential backoff starting at 10ms and doubling after each failed attempt up to a maximum of 1 second.","is_advanced":true,"is_optional":true},{"name":"request_timeout","type":"string","kind":"scalar","description":"The time to wait for a response from the server. This is in addition to the calculation derived from the request_min_throughput parameter.","is_advanced":true,"is_optional":true},{"name":"request_min_throughput","type":"int","kind":"scalar","description":"Minimum expected throughput in bytes per second for HTTP requests. If the throughput is lower than this value, the connection will time out. This is used to calculate an additional timeout on top of request_timeout. This is useful for large requests. You can set this value to 0 to disable this logic.","is_advanced":true,"is_optional":true},{"name":"table","type":"string","kind":"scalar","description":"Destination table","examples":["trades"]},{"name":"designated_timestamp_field","type":"string","kind":"scalar","description":"Name of the designated timestamp field","is_optional":true},{"name":"designated_timestamp_unit","type":"string","kind":"scalar","description":"Designated timestamp field units","is_optional":true,"default":"auto","linter":"root = if [\"nanos\",\"micros\",\"millis\",\"seconds\",\"auto\"].contains(this) != true { [ \"valid options are \\\"nanos\\\", \\\"micros\\\", \\\"millis\\\", \\\"seconds\\\", \\\"auto\\\"\" ] }"},{"name":"timestamp_string_fields","type":"string","kind":"array","description":"String fields with textual timestamps","is_optional":true},{"name":"timestamp_string_format","type":"string","kind":"scalar","description":"Timestamp format, used when parsing timestamp string fields. Specified in golang's time.Parse layout","is_optional":true,"default":"Jan _2 15:04:05.000000Z0700"},{"name":"symbols","type":"string","kind":"array","description":"Columns that should be the SYMBOL type (string values default to STRING)","is_optional":true},{"name":"doubles","type":"string","kind":"array","description":"Columns that should be double type, (int is default)","is_optional":true},{"name":"error_on_empty_messages","type":"bool","kind":"scalar","description":"Mark a message as errored if it is empty after field validation","is_optional":true,"default":false}]}},{"name":"redis_hash","type":"output","status":"stable","plugin":true,"summary":"Sets Redis hash objects using the HMSET command.","description":"\nThe field `key` supports xref:configuration:interpolation.adoc#bloblang-queries[interpolation functions], allowing you to create a unique key for each message.\n\nThe field `fields` allows you to specify an explicit map of field names to interpolated values, also evaluated per message of a batch:\n\n```yaml\noutput:\n redis_hash:\n url: tcp://localhost:6379\n key: ${!json(\"id\")}\n fields:\n topic: ${!meta(\"kafka_topic\")}\n partition: ${!meta(\"kafka_partition\")}\n content: ${!json(\"document.text\")}\n```\n\nIf the field `walk_metadata` is set to `true` then Redpanda Connect will walk all metadata fields of messages and add them to the list of hash fields to set.\n\nIf the field `walk_json_object` is set to `true` then Redpanda Connect will walk each message as a JSON object, extracting keys and the string representation of their value and adds them to the list of hash fields to set.\n\nThe order of hash field extraction is as follows:\n\n1. Metadata (if enabled)\n2. JSON object (if enabled)\n3. Explicit fields\n\nWhere latter stages will overwrite matching field names of a former stage.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target Redis server. Database is optional and is supplied as the URL path.","examples":["redis://:6379","redis://localhost:6379","redis://foousername:foopassword@redisplace:6379","redis://:foopassword@redisplace:6379","redis://localhost:6379/1","redis://localhost:6379/1,redis://localhost:6380/1"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"kind","type":"string","kind":"scalar","description":"Specifies a simple, cluster-aware, or failover-aware redis client.","is_advanced":true,"default":"simple","options":["simple","cluster","failover"],"linter":"\nlet options = {\n \"simple\": true,\n \"cluster\": true,\n \"failover\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"master","type":"string","kind":"scalar","description":"Name of the redis master when `kind` is `failover`","is_advanced":true,"default":"","examples":["mymaster"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"key","type":"string","kind":"scalar","description":"The key for each message, function interpolations should be used to create a unique key per message.","interpolated":true,"examples":["${! @.kafka_key )}","${! this.doc.id }","${! counter() }"]},{"name":"walk_metadata","type":"bool","kind":"scalar","description":"Whether all metadata fields of messages should be walked and added to the list of hash fields to set.","default":false},{"name":"walk_json_object","type":"bool","kind":"scalar","description":"Whether to walk each message as a JSON object and add each key/value pair to the list of hash fields to set.","default":false},{"name":"fields","type":"string","kind":"map","description":"A map of key/value pairs to set as hash fields.","default":{},"interpolated":true},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64}]}},{"name":"redis_list","type":"output","status":"stable","plugin":true,"summary":"Pushes messages onto the end of a Redis list (which is created if it doesn't already exist) using the RPUSH command.","description":"The field `key` supports xref:configuration:interpolation.adoc#bloblang-queries[interpolation functions], allowing you to create a unique key for each message.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target Redis server. Database is optional and is supplied as the URL path.","examples":["redis://:6379","redis://localhost:6379","redis://foousername:foopassword@redisplace:6379","redis://:foopassword@redisplace:6379","redis://localhost:6379/1","redis://localhost:6379/1,redis://localhost:6380/1"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"kind","type":"string","kind":"scalar","description":"Specifies a simple, cluster-aware, or failover-aware redis client.","is_advanced":true,"default":"simple","options":["simple","cluster","failover"],"linter":"\nlet options = {\n \"simple\": true,\n \"cluster\": true,\n \"failover\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"master","type":"string","kind":"scalar","description":"Name of the redis master when `kind` is `failover`","is_advanced":true,"default":"","examples":["mymaster"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"key","type":"string","kind":"scalar","description":"The key for each message, function interpolations can be optionally used to create a unique key per message.","interpolated":true,"examples":["some_list","${! @.kafka_key )}","${! this.doc.id }","${! counter() }"]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"command","type":"string","kind":"scalar","description":"The command used to push elements to the Redis list","is_advanced":true,"default":"rpush","options":["rpush","lpush"],"version":"4.22.0","linter":"\nlet options = {\n \"rpush\": true,\n \"lpush\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"}]}},{"name":"redis_pubsub","type":"output","status":"stable","plugin":true,"summary":"Publishes messages through the Redis PubSub model. It is not possible to guarantee that messages have been received.","description":"\nThis output will interpolate functions within the channel field, you can find a list of functions xref:configuration:interpolation.adoc#bloblang-queries[here].\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target Redis server. Database is optional and is supplied as the URL path.","examples":["redis://:6379","redis://localhost:6379","redis://foousername:foopassword@redisplace:6379","redis://:foopassword@redisplace:6379","redis://localhost:6379/1","redis://localhost:6379/1,redis://localhost:6380/1"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"kind","type":"string","kind":"scalar","description":"Specifies a simple, cluster-aware, or failover-aware redis client.","is_advanced":true,"default":"simple","options":["simple","cluster","failover"],"linter":"\nlet options = {\n \"simple\": true,\n \"cluster\": true,\n \"failover\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"master","type":"string","kind":"scalar","description":"Name of the redis master when `kind` is `failover`","is_advanced":true,"default":"","examples":["mymaster"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"channel","type":"string","kind":"scalar","description":"The channel to publish messages to.","interpolated":true},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]}},{"name":"redis_streams","type":"output","status":"stable","plugin":true,"summary":"Pushes messages to a Redis (v5.0+) Stream (which is created if it doesn't already exist) using the XADD command.","description":"\nIt's possible to specify a maximum length of the target stream by setting it to a value greater than 0, in which case this cap is applied only when Redis is able to remove a whole macro node, for efficiency.\n\nRedis stream entries are key/value pairs, as such it is necessary to specify the key to be set to the body of the message. All metadata fields of the message will also be set as key/value pairs, if there is a key collision between a metadata item and the body then the body takes precedence.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target Redis server. Database is optional and is supplied as the URL path.","examples":["redis://:6379","redis://localhost:6379","redis://foousername:foopassword@redisplace:6379","redis://:foopassword@redisplace:6379","redis://localhost:6379/1","redis://localhost:6379/1,redis://localhost:6380/1"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"kind","type":"string","kind":"scalar","description":"Specifies a simple, cluster-aware, or failover-aware redis client.","is_advanced":true,"default":"simple","options":["simple","cluster","failover"],"linter":"\nlet options = {\n \"simple\": true,\n \"cluster\": true,\n \"failover\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"master","type":"string","kind":"scalar","description":"Name of the redis master when `kind` is `failover`","is_advanced":true,"default":"","examples":["mymaster"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"stream","type":"string","kind":"scalar","description":"The stream to add messages to.","interpolated":true},{"name":"body_key","type":"string","kind":"scalar","description":"A key to set the raw body of the message to.","default":"body"},{"name":"max_length","type":"int","kind":"scalar","description":"When greater than zero enforces a rough cap on the length of the target stream.","default":0},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"metadata","type":"object","kind":"scalar","description":"Specify criteria for which metadata values are included in the message body.","children":[{"name":"exclude_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages.","default":[]}]},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]}},{"name":"redpanda","type":"output","status":"beta","plugin":true,"summary":"A Kafka output using the https://github.com/twmb/franz-go[Franz Kafka client library^].","description":"\nWrites a batch of messages to Kafka brokers and waits for acknowledgement before propagating it back to the input.\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"seed_brokers","type":"string","kind":"array","description":"A list of broker addresses to connect to in order to establish connections. If an item of the list contains commas it will be expanded into multiple addresses.","examples":[["localhost:9092"],["foo:9092","bar:9092"],["foo:9092,bar:9092"]]},{"name":"client_id","type":"string","kind":"scalar","description":"An identifier for the client connection.","is_advanced":true,"default":"benthos"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"sasl","type":"object","kind":"array","description":"Specify one or more methods of SASL authentication. SASL is tried in order; if the broker supports the first mechanism, all connections will use that mechanism. If the first mechanism fails, the client will pick the first supported mechanism. If the broker does not support any client mechanisms, connections will fail.","is_advanced":true,"is_optional":true,"examples":[[{"mechanism":"SCRAM-SHA-512","password":"bar","username":"foo"}]],"children":[{"name":"mechanism","type":"string","kind":"scalar","description":"The SASL mechanism to use.","is_advanced":true,"annotated_options":[["AWS_MSK_IAM","AWS IAM based authentication as specified by the 'aws-msk-iam-auth' java library."],["OAUTHBEARER","OAuth Bearer based authentication."],["PLAIN","Plain text authentication."],["SCRAM-SHA-256","SCRAM based authentication as specified in RFC5802."],["SCRAM-SHA-512","SCRAM based authentication as specified in RFC5802."],["none","Disable sasl authentication"]],"linter":"\nlet options = {\n \"aws_msk_iam\": true,\n \"oauthbearer\": true,\n \"plain\": true,\n \"scram-sha-256\": true,\n \"scram-sha-512\": true,\n \"none\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"username","type":"string","kind":"scalar","description":"A username to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token to use for a single session's OAUTHBEARER authentication.","is_advanced":true,"default":""},{"name":"extensions","type":"string","kind":"map","description":"Key/value pairs to add to OAUTHBEARER authentication requests.","is_advanced":true,"is_optional":true},{"name":"aws","type":"object","kind":"scalar","description":"Contains AWS specific fields for when the `mechanism` is set to `AWS_MSK_IAM`.","is_advanced":true,"is_optional":true,"children":[{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]}]},{"name":"metadata_max_age","type":"string","kind":"scalar","description":"The maximum age of metadata before it is refreshed.","is_advanced":true,"default":"5m"},{"name":"request_timeout_overhead","type":"string","kind":"scalar","description":"The request time overhead. Uses the given time as overhead while deadlining requests. Roughly equivalent to request.timeout.ms, but grants additional time to requests that have timeout fields.","is_advanced":true,"default":"10s"},{"name":"conn_idle_timeout","type":"string","kind":"scalar","description":"The rough amount of time to allow connections to idle before they are closed.","is_advanced":true,"default":"20s"},{"name":"topic","type":"string","kind":"scalar","description":"A topic to write messages to.","interpolated":true},{"name":"key","type":"string","kind":"scalar","description":"An optional key to populate for each message.","is_optional":true,"interpolated":true},{"name":"partition","type":"string","kind":"scalar","description":"An optional explicit partition to set for each message. This field is only relevant when the `partitioner` is set to `manual`. The provided interpolation string must be a valid integer.","is_optional":true,"interpolated":true,"examples":["${! meta(\"partition\") }"]},{"name":"metadata","type":"object","kind":"scalar","description":"Determine which (if any) metadata values should be added to messages as headers.","is_optional":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"timestamp","type":"string","kind":"scalar","description":"An optional timestamp to set for each message. When left empty, the current timestamp is used.","is_advanced":true,"is_deprecated":true,"is_optional":true,"interpolated":true,"examples":["${! timestamp_unix() }","${! metadata(\"kafka_timestamp_unix\") }"]},{"name":"timestamp_ms","type":"string","kind":"scalar","description":"An optional timestamp to set for each message expressed in milliseconds. When left empty, the current timestamp is used.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["${! timestamp_unix_milli() }","${! metadata(\"kafka_timestamp_ms\") }"]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of batches to be sending in parallel at any given time.","default":256},{"name":"partitioner","type":"string","kind":"scalar","description":"Override the default murmur2 hashing partitioner.","is_advanced":true,"is_optional":true,"annotated_options":[["least_backup","Chooses the least backed up partition (the partition with the fewest amount of buffered records). Partitions are selected per batch."],["manual","Manually select a partition for each message, requires the field `partition` to be specified."],["murmur2_hash","Kafka's default hash algorithm that uses a 32-bit murmur2 hash of the key to compute which partition the record will be on."],["round_robin","Round-robin's messages through all available partitions. This algorithm has lower throughput and causes higher CPU load on brokers, but can be useful if you want to ensure an even distribution of records to partitions."]],"linter":"\nlet options = {\n \"least_backup\": true,\n \"manual\": true,\n \"murmur2_hash\": true,\n \"round_robin\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"idempotent_write","type":"bool","kind":"scalar","description":"Enable the idempotent write producer option. This requires the `IDEMPOTENT_WRITE` permission on `CLUSTER` and can be disabled if this permission is not available.","is_advanced":true,"default":true},{"name":"compression","type":"string","kind":"scalar","description":"Optionally set an explicit compression type. The default preference is to use snappy when the broker supports it, and fall back to none if not.","is_advanced":true,"is_optional":true,"options":["lz4","snappy","gzip","none","zstd"],"linter":"\nlet options = {\n \"lz4\": true,\n \"snappy\": true,\n \"gzip\": true,\n \"none\": true,\n \"zstd\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period of time to wait for message sends before abandoning the request and retrying","is_advanced":true,"default":"10s"},{"name":"max_message_bytes","type":"string","kind":"scalar","description":"The maximum space in bytes than an individual message may take, messages larger than this value will be rejected. This field corresponds to Kafka's `max.message.bytes`.","is_advanced":true,"default":"1MiB","examples":["100MB","50mib"]},{"name":"broker_write_max_bytes","type":"string","kind":"scalar","description":"The upper bound for the number of bytes written to a broker connection in a single write. This field corresponds to Kafka's `socket.request.max.bytes`.","is_advanced":true,"default":"100MiB","examples":["128MB","50mib"]}],"linter":"root = match {\n this.partitioner == \"manual\" \u0026\u0026 this.partition.or(\"\") == \"\" =\u003e \"a partition must be specified when the partitioner is set to manual\"\n this.partitioner != \"manual\" \u0026\u0026 this.partition.or(\"\") != \"\" =\u003e \"a partition cannot be specified unless the partitioner is set to manual\"\n this.timestamp.or(\"\") != \"\" \u0026\u0026 this.timestamp_ms.or(\"\") != \"\" =\u003e \"both timestamp and timestamp_ms cannot be specified simultaneously\"\n}"}},{"name":"redpanda_common","type":"output","status":"beta","plugin":true,"summary":"Sends data to a Redpanda (Kafka) broker, using credentials defined in a common top-level `redpanda` config block.","categories":["Services"],"examples":[{"title":"Simple Output","summary":"Data is generated and written to a topic bar, targetting the cluster configured within the redpanda block at the bottom. This is useful as it allows us to configure TLS and SASL only once for potentially multiple inputs and outputs.","config":"\ninput:\n generate:\n interval: 1s\n mapping: 'root.name = fake(\"name\")'\n\npipeline:\n processors:\n - mutation: |\n root.id = uuid_v4()\n root.loud_name = this.name.uppercase()\n\noutput:\n redpanda_common:\n topic: bar\n key: ${! @id }\n\nredpanda:\n seed_brokers: [ \"127.0.0.1:9092\" ]\n tls:\n enabled: true\n sasl:\n - mechanism: SCRAM-SHA-512\n password: bar\n username: foo\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"topic","type":"string","kind":"scalar","description":"A topic to write messages to.","interpolated":true},{"name":"key","type":"string","kind":"scalar","description":"An optional key to populate for each message.","is_optional":true,"interpolated":true},{"name":"partition","type":"string","kind":"scalar","description":"An optional explicit partition to set for each message. This field is only relevant when the `partitioner` is set to `manual`. The provided interpolation string must be a valid integer.","is_optional":true,"interpolated":true,"examples":["${! meta(\"partition\") }"]},{"name":"metadata","type":"object","kind":"scalar","description":"Determine which (if any) metadata values should be added to messages as headers.","is_optional":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"timestamp","type":"string","kind":"scalar","description":"An optional timestamp to set for each message. When left empty, the current timestamp is used.","is_advanced":true,"is_deprecated":true,"is_optional":true,"interpolated":true,"examples":["${! timestamp_unix() }","${! metadata(\"kafka_timestamp_unix\") }"]},{"name":"timestamp_ms","type":"string","kind":"scalar","description":"An optional timestamp to set for each message expressed in milliseconds. When left empty, the current timestamp is used.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["${! timestamp_unix_milli() }","${! metadata(\"kafka_timestamp_ms\") }"]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":10},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}],"linter":"root = match {\n this.partitioner == \"manual\" \u0026\u0026 this.partition.or(\"\") == \"\" =\u003e \"a partition must be specified when the partitioner is set to manual\"\n this.partitioner != \"manual\" \u0026\u0026 this.partition.or(\"\") != \"\" =\u003e \"a partition cannot be specified unless the partitioner is set to manual\"\n this.timestamp.or(\"\") != \"\" \u0026\u0026 this.timestamp_ms.or(\"\") != \"\" =\u003e \"both timestamp and timestamp_ms cannot be specified simultaneously\"\n}"}},{"name":"redpanda_migrator","type":"output","status":"beta","plugin":true,"summary":"A Redpanda Migrator output using the https://github.com/twmb/franz-go[Franz Kafka client library^].","description":"\nWrites a batch of messages to a Kafka broker and waits for acknowledgement before propagating it back to the input.\n\nThis output should be used in combination with a `redpanda_migrator` input identified by the label specified in\n`input_resource` which it can query for topic and ACL configurations. Once connected, the output will attempt to\ncreate all topics which the input consumes from along with their ACLs.\n\nIf the configured broker does not contain the current message topic, this output attempts to create it along with its\nACLs.\n\nACL migration adheres to the following principles:\n\n- `ALLOW WRITE` ACLs for topics are not migrated\n- `ALLOW ALL` ACLs for topics are downgraded to `ALLOW READ`\n- Only topic ACLs are migrated, group ACLs are not migrated\n","categories":["Services"],"examples":[{"title":"Transfer data","summary":"Writes messages to the configured broker and creates topics and topic ACLs if they don't exist. It also ensures that the message order is preserved.","config":"\noutput:\n redpanda_migrator:\n seed_brokers: [ \"127.0.0.1:9093\" ]\n topic: ${! metadata(\"kafka_topic\").or(throw(\"missing kafka_topic metadata\")) }\n key: ${! metadata(\"kafka_key\") }\n partitioner: manual\n partition: ${! metadata(\"kafka_partition\").or(throw(\"missing kafka_partition metadata\")) }\n timestamp_ms: ${! metadata(\"kafka_timestamp_ms\").or(timestamp_unix_milli()) }\n input_resource: redpanda_migrator_input\n max_in_flight: 1\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"seed_brokers","type":"string","kind":"array","description":"A list of broker addresses to connect to in order to establish connections. If an item of the list contains commas it will be expanded into multiple addresses.","examples":[["localhost:9092"],["foo:9092","bar:9092"],["foo:9092,bar:9092"]]},{"name":"client_id","type":"string","kind":"scalar","description":"An identifier for the client connection.","is_advanced":true,"default":"benthos"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"sasl","type":"object","kind":"array","description":"Specify one or more methods of SASL authentication. SASL is tried in order; if the broker supports the first mechanism, all connections will use that mechanism. If the first mechanism fails, the client will pick the first supported mechanism. If the broker does not support any client mechanisms, connections will fail.","is_advanced":true,"is_optional":true,"examples":[[{"mechanism":"SCRAM-SHA-512","password":"bar","username":"foo"}]],"children":[{"name":"mechanism","type":"string","kind":"scalar","description":"The SASL mechanism to use.","is_advanced":true,"annotated_options":[["AWS_MSK_IAM","AWS IAM based authentication as specified by the 'aws-msk-iam-auth' java library."],["OAUTHBEARER","OAuth Bearer based authentication."],["PLAIN","Plain text authentication."],["SCRAM-SHA-256","SCRAM based authentication as specified in RFC5802."],["SCRAM-SHA-512","SCRAM based authentication as specified in RFC5802."],["none","Disable sasl authentication"]],"linter":"\nlet options = {\n \"aws_msk_iam\": true,\n \"oauthbearer\": true,\n \"plain\": true,\n \"scram-sha-256\": true,\n \"scram-sha-512\": true,\n \"none\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"username","type":"string","kind":"scalar","description":"A username to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token to use for a single session's OAUTHBEARER authentication.","is_advanced":true,"default":""},{"name":"extensions","type":"string","kind":"map","description":"Key/value pairs to add to OAUTHBEARER authentication requests.","is_advanced":true,"is_optional":true},{"name":"aws","type":"object","kind":"scalar","description":"Contains AWS specific fields for when the `mechanism` is set to `AWS_MSK_IAM`.","is_advanced":true,"is_optional":true,"children":[{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]}]},{"name":"metadata_max_age","type":"string","kind":"scalar","description":"The maximum age of metadata before it is refreshed.","is_advanced":true,"default":"5m"},{"name":"request_timeout_overhead","type":"string","kind":"scalar","description":"The request time overhead. Uses the given time as overhead while deadlining requests. Roughly equivalent to request.timeout.ms, but grants additional time to requests that have timeout fields.","is_advanced":true,"default":"10s"},{"name":"conn_idle_timeout","type":"string","kind":"scalar","description":"The rough amount of time to allow connections to idle before they are closed.","is_advanced":true,"default":"20s"},{"name":"topic","type":"string","kind":"scalar","description":"A topic to write messages to.","interpolated":true},{"name":"key","type":"string","kind":"scalar","description":"An optional key to populate for each message.","is_optional":true,"interpolated":true},{"name":"partition","type":"string","kind":"scalar","description":"An optional explicit partition to set for each message. This field is only relevant when the `partitioner` is set to `manual`. The provided interpolation string must be a valid integer.","is_optional":true,"interpolated":true,"examples":["${! meta(\"partition\") }"]},{"name":"metadata","type":"object","kind":"scalar","description":"Determine which (if any) metadata values should be added to messages as headers.","is_optional":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"timestamp","type":"string","kind":"scalar","description":"An optional timestamp to set for each message. When left empty, the current timestamp is used.","is_advanced":true,"is_deprecated":true,"is_optional":true,"interpolated":true,"examples":["${! timestamp_unix() }","${! metadata(\"kafka_timestamp_unix\") }"]},{"name":"timestamp_ms","type":"string","kind":"scalar","description":"An optional timestamp to set for each message expressed in milliseconds. When left empty, the current timestamp is used.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["${! timestamp_unix_milli() }","${! metadata(\"kafka_timestamp_ms\") }"]},{"name":"topic_prefix","type":"string","kind":"scalar","description":"The topic prefix.","is_advanced":true,"default":"","interpolated":true},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of batches to be sending in parallel at any given time.","default":256},{"name":"input_resource","type":"string","kind":"scalar","description":"The label of the redpanda_migrator input from which to read the configurations for topics and ACLs which need to be created.","is_advanced":true,"default":"redpanda_migrator_input"},{"name":"replication_factor_override","type":"bool","kind":"scalar","description":"Use the specified replication factor when creating topics.","is_advanced":true,"default":true},{"name":"replication_factor","type":"int","kind":"scalar","description":"Replication factor for created topics. This is only used when `replication_factor_override` is set to `true`.","is_advanced":true,"default":3},{"name":"translate_schema_ids","type":"bool","kind":"scalar","description":"Translate schema IDs.","is_advanced":true,"default":false},{"name":"schema_registry_output_resource","type":"string","kind":"scalar","description":"The label of the schema_registry output to use for fetching schema IDs.","is_advanced":true,"default":"schema_registry_output"},{"name":"rack_id","type":"string","kind":"scalar","is_deprecated":true},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","is_deprecated":true,"examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","is_deprecated":true,"default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","is_deprecated":true,"default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","is_deprecated":true,"default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","is_deprecated":true,"default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_deprecated":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"partitioner","type":"string","kind":"scalar","description":"Override the default murmur2 hashing partitioner.","is_advanced":true,"is_optional":true,"annotated_options":[["least_backup","Chooses the least backed up partition (the partition with the fewest amount of buffered records). Partitions are selected per batch."],["manual","Manually select a partition for each message, requires the field `partition` to be specified."],["murmur2_hash","Kafka's default hash algorithm that uses a 32-bit murmur2 hash of the key to compute which partition the record will be on."],["round_robin","Round-robin's messages through all available partitions. This algorithm has lower throughput and causes higher CPU load on brokers, but can be useful if you want to ensure an even distribution of records to partitions."]],"linter":"\nlet options = {\n \"least_backup\": true,\n \"manual\": true,\n \"murmur2_hash\": true,\n \"round_robin\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"idempotent_write","type":"bool","kind":"scalar","description":"Enable the idempotent write producer option. This requires the `IDEMPOTENT_WRITE` permission on `CLUSTER` and can be disabled if this permission is not available.","is_advanced":true,"default":true},{"name":"compression","type":"string","kind":"scalar","description":"Optionally set an explicit compression type. The default preference is to use snappy when the broker supports it, and fall back to none if not.","is_advanced":true,"is_optional":true,"options":["lz4","snappy","gzip","none","zstd"],"linter":"\nlet options = {\n \"lz4\": true,\n \"snappy\": true,\n \"gzip\": true,\n \"none\": true,\n \"zstd\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period of time to wait for message sends before abandoning the request and retrying","is_advanced":true,"default":"10s"},{"name":"max_message_bytes","type":"string","kind":"scalar","description":"The maximum space in bytes than an individual message may take, messages larger than this value will be rejected. This field corresponds to Kafka's `max.message.bytes`.","is_advanced":true,"default":"1MiB","examples":["100MB","50mib"]},{"name":"broker_write_max_bytes","type":"string","kind":"scalar","description":"The upper bound for the number of bytes written to a broker connection in a single write. This field corresponds to Kafka's `socket.request.max.bytes`.","is_advanced":true,"default":"100MiB","examples":["128MB","50mib"]}],"linter":"root = match {\n this.partitioner == \"manual\" \u0026\u0026 this.partition.or(\"\") == \"\" =\u003e \"a partition must be specified when the partitioner is set to manual\"\n this.partitioner != \"manual\" \u0026\u0026 this.partition.or(\"\") != \"\" =\u003e \"a partition cannot be specified unless the partitioner is set to manual\"\n this.timestamp.or(\"\") != \"\" \u0026\u0026 this.timestamp_ms.or(\"\") != \"\" =\u003e \"both timestamp and timestamp_ms cannot be specified simultaneously\"\n}"},"version":"4.37.0"},{"name":"redpanda_migrator_bundle","type":"output","status":"experimental","plugin":true,"summary":"Redpanda Migrator bundle output","description":"All-in-one output which writes messages and schemas to a Kafka or Redpanda cluster. This output is meant to be used\ntogether with the `redpanda_migrator_bundle` input.\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"redpanda_migrator","type":"unknown","kind":"map","description":"The `redpanda_migrator` output configuration.\n"},{"name":"schema_registry","type":"unknown","kind":"map","description":"The `schema_registry` output configuration. The `subject` field must be left empty.\n"},{"name":"translate_schema_ids","type":"bool","kind":"scalar","description":"Allow the target Schema Registry instance to allocate different schema IDs for migrated schemas. This is useful\nwhen it already contains some schemas which differ from the ones being migrated.\n","default":false}]}},{"name":"redpanda_migrator_offsets","type":"output","status":"beta","plugin":true,"summary":"Redpanda Migrator consumer group offsets output using the https://github.com/twmb/franz-go[Franz Kafka client library^].","description":"This output can be used in combination with the `kafka_franz` input that is configured to read the `__consumer_offsets` topic.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"seed_brokers","type":"string","kind":"array","description":"A list of broker addresses to connect to in order to establish connections. If an item of the list contains commas it will be expanded into multiple addresses.","examples":[["localhost:9092"],["foo:9092","bar:9092"],["foo:9092,bar:9092"]]},{"name":"client_id","type":"string","kind":"scalar","description":"An identifier for the client connection.","is_advanced":true,"default":"benthos"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"sasl","type":"object","kind":"array","description":"Specify one or more methods of SASL authentication. SASL is tried in order; if the broker supports the first mechanism, all connections will use that mechanism. If the first mechanism fails, the client will pick the first supported mechanism. If the broker does not support any client mechanisms, connections will fail.","is_advanced":true,"is_optional":true,"examples":[[{"mechanism":"SCRAM-SHA-512","password":"bar","username":"foo"}]],"children":[{"name":"mechanism","type":"string","kind":"scalar","description":"The SASL mechanism to use.","is_advanced":true,"annotated_options":[["AWS_MSK_IAM","AWS IAM based authentication as specified by the 'aws-msk-iam-auth' java library."],["OAUTHBEARER","OAuth Bearer based authentication."],["PLAIN","Plain text authentication."],["SCRAM-SHA-256","SCRAM based authentication as specified in RFC5802."],["SCRAM-SHA-512","SCRAM based authentication as specified in RFC5802."],["none","Disable sasl authentication"]],"linter":"\nlet options = {\n \"aws_msk_iam\": true,\n \"oauthbearer\": true,\n \"plain\": true,\n \"scram-sha-256\": true,\n \"scram-sha-512\": true,\n \"none\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"username","type":"string","kind":"scalar","description":"A username to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token to use for a single session's OAUTHBEARER authentication.","is_advanced":true,"default":""},{"name":"extensions","type":"string","kind":"map","description":"Key/value pairs to add to OAUTHBEARER authentication requests.","is_advanced":true,"is_optional":true},{"name":"aws","type":"object","kind":"scalar","description":"Contains AWS specific fields for when the `mechanism` is set to `AWS_MSK_IAM`.","is_advanced":true,"is_optional":true,"children":[{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]}]},{"name":"metadata_max_age","type":"string","kind":"scalar","description":"The maximum age of metadata before it is refreshed.","is_advanced":true,"default":"5m"},{"name":"request_timeout_overhead","type":"string","kind":"scalar","description":"The request time overhead. Uses the given time as overhead while deadlining requests. Roughly equivalent to request.timeout.ms, but grants additional time to requests that have timeout fields.","is_advanced":true,"default":"10s"},{"name":"conn_idle_timeout","type":"string","kind":"scalar","description":"The rough amount of time to allow connections to idle before they are closed.","is_advanced":true,"default":"20s"},{"name":"offset_topic","type":"string","kind":"scalar","description":"Kafka offset topic.","default":"${! @kafka_offset_topic }","interpolated":true},{"name":"offset_topic_prefix","type":"string","kind":"scalar","description":"Kafka offset topic prefix.","is_advanced":true,"default":"","interpolated":true},{"name":"offset_group","type":"string","kind":"scalar","description":"Kafka offset group.","default":"${! @kafka_offset_group }","interpolated":true},{"name":"offset_partition","type":"string","kind":"scalar","description":"Kafka offset partition.","default":"${! @kafka_offset_partition }","interpolated":true},{"name":"offset_commit_timestamp","type":"string","kind":"scalar","description":"Kafka offset commit timestamp.","default":"${! @kafka_offset_commit_timestamp }","interpolated":true},{"name":"offset_metadata","type":"string","kind":"scalar","description":"Kafka offset metadata value.","default":"${! @kafka_offset_metadata }","interpolated":true},{"name":"is_high_watermark","type":"string","kind":"scalar","description":"Indicates if the update represents the high watermark of the Kafka topic partition.","default":"${! @kafka_is_high_watermark }","interpolated":true},{"name":"kafka_key","type":"string","kind":"scalar","description":"Kafka key.","is_deprecated":true,"default":"${! @kafka_key }","interpolated":true},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of batches to be sending in parallel at any given time.","is_deprecated":true,"default":1},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period of time to wait for message sends before abandoning the request and retrying","is_advanced":true,"default":"10s"},{"name":"max_message_bytes","type":"string","kind":"scalar","description":"The maximum space in bytes than an individual message may take, messages larger than this value will be rejected. This field corresponds to Kafka's `max.message.bytes`.","is_advanced":true,"default":"1MiB","examples":["100MB","50mib"]},{"name":"broker_write_max_bytes","type":"string","kind":"scalar","description":"The upper bound for the number of bytes written to a broker connection in a single write. This field corresponds to Kafka's `socket.request.max.bytes`.","is_advanced":true,"default":"100MiB","examples":["128MB","50mib"]},{"name":"max_retries","type":"int","kind":"scalar","description":"The maximum number of retries before giving up on the request. If set to zero there is no discrete limit.","is_advanced":true,"default":0},{"name":"backoff","type":"object","kind":"scalar","description":"Control time intervals between retry attempts.","is_advanced":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"1s"},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts.","is_advanced":true,"default":"5s"},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum period to wait before retry attempts are abandoned. If zero then no limit is used.","is_advanced":true,"default":"30s"}]}]},"version":"4.37.0"},{"name":"reject","type":"output","status":"stable","plugin":true,"summary":"Rejects all messages, treating them as though the output destination failed to publish them.","description":"\nThe routing of messages after this output depends on the type of input it came from. For inputs that support propagating nacks upstream such as AMQP or NATS the message will be nacked. However, for inputs that are sequential such as files or Kafka the messages will simply be reprocessed from scratch.\n\nTo learn when this output could be useful, see [the \u003c\u003cexamples\u003e\u003e.","categories":["Utility"],"examples":[{"title":"Rejecting Failed Messages","summary":"\nThis input is particularly useful for routing messages that have failed during processing, where instead of routing them to some sort of dead letter queue we wish to push the error upstream. We can do this with a switch broker:","config":"\noutput:\n switch:\n retry_until_success: false\n cases:\n - check: '!errored()'\n output:\n amqp_1:\n urls: [ amqps://guest:guest@localhost:5672/ ]\n target_address: queue:/the_foos\n\n - output:\n reject: \"processing failed due to: ${! error() }\"\n"}],"config":{"name":"","type":"string","kind":"scalar","default":""}},{"name":"reject_errored","type":"output","status":"stable","plugin":true,"summary":"Rejects messages that have failed their processing steps, resulting in nack behavior at the input level, otherwise sends them to a child output.","description":"\nThe routing of messages rejected by this output depends on the type of input it came from. For inputs that support propagating nacks upstream such as AMQP or NATS the message will be nacked. However, for inputs that are sequential such as files or Kafka the messages will simply be reprocessed from scratch.","categories":["Utility"],"examples":[{"title":"Rejecting Failed Messages","summary":"\nThe most straight forward use case for this output type is to nack messages that have failed their processing steps. In this example our mapping might fail, in which case the messages that failed are rejected and will be nacked by our input:","config":"\ninput:\n nats_jetstream:\n urls: [ nats://127.0.0.1:4222 ]\n subject: foos.pending\n\npipeline:\n processors:\n - mutation: 'root.age = this.fuzzy.age.int64()'\n\noutput:\n reject_errored:\n nats_jetstream:\n urls: [ nats://127.0.0.1:4222 ]\n subject: foos.processed\n"},{"title":"DLQing Failed Messages","summary":"\nAnother use case for this output is to send failed messages straight into a dead-letter queue. You use it within a xref:components:outputs/fallback.adoc[fallback output] that allows you to specify where these failed messages should go to next.","config":"\npipeline:\n processors:\n - mutation: 'root.age = this.fuzzy.age.int64()'\n\noutput:\n fallback:\n - reject_errored:\n http_client:\n url: http://foo:4195/post/might/become/unreachable\n retries: 3\n retry_period: 1s\n - http_client:\n url: http://bar:4196/somewhere/else\n retries: 3\n retry_period: 1s\n"}],"config":{"name":"","type":"output","kind":"scalar"}},{"name":"resource","type":"output","status":"stable","plugin":true,"summary":"Resource is an output type that channels messages to a resource output, identified by its name.","description":"Resources allow you to tidy up deeply nested configs. For example, the config:\n\n```yaml\noutput:\n broker:\n pattern: fan_out\n outputs:\n - kafka:\n addresses: [ TODO ]\n topic: foo\n - gcp_pubsub:\n project: bar\n topic: baz\n```\n\nCould also be expressed as:\n\n```yaml\noutput:\n broker:\n pattern: fan_out\n outputs:\n - resource: foo\n - resource: bar\n\noutput_resources:\n - label: foo\n kafka:\n addresses: [ TODO ]\n topic: foo\n\n - label: bar\n gcp_pubsub:\n project: bar\n topic: baz\n```\n\nYou can find out more about resources in xref:configuration:resources.adoc[]","categories":["Utility"],"config":{"name":"","type":"string","kind":"scalar","default":""}},{"name":"retry","type":"output","status":"stable","plugin":true,"summary":"Attempts to write messages to a child output and if the write fails for any reason the message is retried either until success or, if the retries or max elapsed time fields are non-zero, either is reached.","description":"\nAll messages in Redpanda Connect are always retried on an output error, but this would usually involve propagating the error back to the source of the message, whereby it would be reprocessed before reaching the output layer once again.\n\nThis output type is useful whenever we wish to avoid reprocessing a message on the event of a failed send. We might, for example, have a deduplication processor that we want to avoid reapplying to the same message more than once in the pipeline.\n\nRather than retrying the same output you may wish to retry the send using a different output target (a dead letter queue). In which case you should instead use the xref:components:outputs/fallback.adoc[`fallback`] output type.","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"max_retries","type":"int","kind":"scalar","description":"The maximum number of retries before giving up on the request. If set to zero there is no discrete limit.","is_advanced":true,"default":0},{"name":"backoff","type":"object","kind":"scalar","description":"Control time intervals between retry attempts.","is_advanced":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"500ms"},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts.","is_advanced":true,"default":"3s"},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum period to wait before retry attempts are abandoned. If zero then no limit is used.","is_advanced":true,"default":"0s"}]},{"name":"output","type":"output","kind":"scalar","description":"A child output."}]}},{"name":"schema_registry","type":"output","status":"beta","plugin":true,"summary":"Publishes schemas to SchemaRegistry.","description":"\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.","categories":["Integration"],"examples":[{"title":"Write schemas","summary":"Write schemas to a Schema Registry instance and log errors for schemas which already exist.","config":"\noutput:\n fallback:\n - schema_registry:\n url: http://localhost:8082\n subject: ${! @schema_registry_subject }\n - switch:\n cases:\n - check: '@fallback_error == \"request returned status: 422\"'\n output:\n drop: {}\n processors:\n - log:\n message: |\n Subject '${! @schema_registry_subject }' version ${! @schema_registry_version } already has schema: ${! content() }\n - output:\n reject: ${! @fallback_error }\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The base URL of the schema registry service."},{"name":"subject","type":"string","kind":"scalar","description":"Subject.","interpolated":true},{"name":"backfill_dependencies","type":"bool","kind":"scalar","description":"Backfill schema references and previous versions.","is_advanced":true,"default":true},{"name":"translate_ids","type":"bool","kind":"scalar","description":"Translate schema IDs.","is_advanced":true,"default":false},{"name":"input_resource","type":"string","kind":"scalar","description":"The label of the schema_registry input from which to read source schemas.","is_advanced":true,"default":"schema_registry_input"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"oauth","type":"object","kind":"scalar","description":"Allows you to specify open authentication via OAuth version 1.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use OAuth version 1 in requests.","is_advanced":true,"default":false},{"name":"consumer_key","type":"string","kind":"scalar","description":"A value used to identify the client to the service provider.","is_advanced":true,"default":""},{"name":"consumer_secret","type":"string","kind":"scalar","description":"A secret used to establish ownership of the consumer key.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"access_token","type":"string","kind":"scalar","description":"A value used to gain access to the protected resources on behalf of the user.","is_advanced":true,"default":""},{"name":"access_token_secret","type":"string","kind":"scalar","description":"A secret provided in order to establish ownership of a given access token.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"jwt","type":"object","kind":"scalar","description":"BETA: Allows you to specify JWT authentication.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use JWT authentication in requests.","is_advanced":true,"default":false},{"name":"private_key_file","type":"string","kind":"scalar","description":"A file with the PEM encoded via PKCS1 or PKCS8 as private key.","is_advanced":true,"default":""},{"name":"signing_method","type":"string","kind":"scalar","description":"A method used to sign the token such as RS256, RS384, RS512 or EdDSA.","is_advanced":true,"default":""},{"name":"claims","type":"unknown","kind":"map","description":"A value used to identify the claims that issued the JWT.","is_advanced":true,"default":{}},{"name":"headers","type":"unknown","kind":"map","description":"Add optional key/value headers to the JWT.","is_advanced":true,"default":{}}]}]},"version":"4.32.2"},{"name":"sftp","type":"output","status":"beta","plugin":true,"summary":"Writes files to an SFTP server.","description":"In order to have a different path for each object you should use function interpolations described xref:configuration:interpolation.adoc#bloblang-queries[here].\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.","categories":["Network"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"address","type":"string","kind":"scalar","description":"The address of the server to connect to."},{"name":"path","type":"string","kind":"scalar","description":"The file to save the messages to on the server.","interpolated":true},{"name":"codec","type":"string","kind":"scalar","description":"The way in which the bytes of messages should be written out into the output data stream. It's possible to write lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter.","default":"all-bytes","examples":["lines","delim:\t","delim:foobar"],"annotated_options":[["all-bytes","Only applicable to file based outputs. Writes each message to a file in full, if the file already exists the old content is deleted."],["append","Append each message to the output stream without any delimiter or special encoding."],["delim:x","Append each message to the output stream followed by a custom delimiter."],["lines","Append each message to the output stream followed by a line break."]]},{"name":"credentials","type":"object","kind":"scalar","description":"The credentials to use to log into the target server.","children":[{"name":"username","type":"string","kind":"scalar","description":"The username to connect to the SFTP server.","default":""},{"name":"password","type":"string","kind":"scalar","description":"The password for the username to connect to the SFTP server.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"private_key_file","type":"string","kind":"scalar","description":"The private key for the username to connect to the SFTP server.","default":""},{"name":"private_key","type":"string","kind":"scalar","description":"The private key file for the username to connect to the SFTP server.","is_secret":true,"default":"","linter":"root = match { this.exists(\"private_key\") \u0026\u0026 this.exists(\"private_key_file\") =\u003e [ \"both private_key and private_key_file can't be set simultaneously\" ], }","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"private_key_pass","type":"string","kind":"scalar","description":"Optional passphrase for private key.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64}]},"version":"3.39.0"},{"name":"slack_post","type":"output","status":"experimental","plugin":true,"description":"Post a new message to a Slack channel using https://api.slack.com/methods/chat.postMessage[^chat.postMessage]","categories":null,"examples":[{"title":"Echo Slackbot","summary":"A slackbot that echo messages from other users","config":"\ninput:\n slack:\n app_token: \"${APP_TOKEN:xapp-demo}\"\n bot_token: \"${BOT_TOKEN:xoxb-demo}\"\npipeline:\n processors:\n - mutation: |\n # ignore hidden or non message events\n if this.event.type != \"message\" || (this.event.hidden | false) {\n root = deleted()\n }\n # Don't respond to our own messages\n if this.authorizations.any(auth -\u003e auth.user_id == this.event.user) {\n root = deleted()\n }\noutput:\n slack_post:\n bot_token: \"${BOT_TOKEN:xoxb-demo}\"\n channel_id: \"${!this.event.channel}\"\n thread_ts: \"${!this.event.ts}\"\n text: \"ECHO: ${!this.event.text}\"\n "}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"bot_token","type":"string","kind":"scalar","description":"The Slack Bot User OAuth token to use.","linter":"\n root = if !this.has_prefix(\"xoxb-\") { [ \"field must start with xoxb-\" ] }\n "},{"name":"channel_id","type":"string","kind":"scalar","description":"The channel ID to post messages to.","interpolated":true},{"name":"thread_ts","type":"string","kind":"scalar","description":"Optional thread timestamp to post messages to.","default":"","interpolated":true},{"name":"text","type":"string","kind":"scalar","description":"The text content of the message. Mutually exclusive with `blocks`.","default":"","interpolated":true},{"name":"blocks","type":"string","kind":"scalar","description":"A Bloblang query that should return a JSON array of Slack blocks (see https://api.slack.com/reference/block-kit/blocks[Blocks in Slack documentation]). Mutually exclusive with `text`.","is_optional":true,"bloblang":true},{"name":"markdown","type":"bool","kind":"scalar","description":"Enable markdown formatting in the message.","default":true},{"name":"unfurl_links","type":"bool","kind":"scalar","description":"Enable link unfurling in the message.","default":false},{"name":"unfurl_media","type":"bool","kind":"scalar","description":"Enable media unfurling in the message.","default":true},{"name":"link_names","type":"bool","kind":"scalar","description":"Enable link names in the message.","default":false}]}},{"name":"snowflake_put","type":"output","status":"beta","plugin":true,"summary":"Sends messages to Snowflake stages and, optionally, calls Snowpipe to load this data into one or more tables.","description":"\nIn order to use a different stage and / or Snowpipe for each message, you can use function interpolations as described in\nxref:configuration:interpolation.adoc#bloblang-queries[Bloblang queries]. When using batching, messages are grouped by the calculated\nstage and Snowpipe and are streamed to individual files in their corresponding stage and, optionally, a Snowpipe\n`insertFiles` REST API call will be made for each individual file.\n\n== Credentials\n\nTwo authentication mechanisms are supported:\n\n- User/password\n- Key Pair Authentication\n\n=== User/password\n\nThis is a basic authentication mechanism which allows you to PUT data into a stage. However, it is not compatible with\nSnowpipe.\n\n=== Key pair authentication\n\nThis authentication mechanism allows Snowpipe functionality, but it does require configuring an SSH Private Key\nbeforehand. Please consult the https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication[documentation^]\nfor details on how to set it up and assign the Public Key to your user.\n\nNote that the Snowflake documentation https://twitter.com/felipehoffa/status/1560811785606684672[used to suggest^]\nusing this command:\n\n```bash\nopenssl genrsa 2048 | openssl pkcs8 -topk8 -inform PEM -out rsa_key.p8\n```\n\nto generate an encrypted SSH private key. However, in this case, it uses an encryption algorithm called\n`pbeWithMD5AndDES-CBC`, which is part of the PKCS#5 v1.5 and is considered insecure. Due to this, Redpanda Connect does not\nsupport it and, if you wish to use password-protected keys directly, you must use PKCS#5 v2.0 to encrypt them by using\nthe following command (as the current Snowflake docs suggest):\n\n```bash\nopenssl genrsa 2048 | openssl pkcs8 -topk8 -v2 des3 -inform PEM -out rsa_key.p8\n```\n\nIf you have an existing key encrypted with PKCS#5 v1.5, you can re-encrypt it with PKCS#5 v2.0 using this command:\n\n```bash\nopenssl pkcs8 -in rsa_key_original.p8 -topk8 -v2 des3 -out rsa_key.p8\n```\n\nPlease consult the https://linux.die.net/man/1/pkcs8[pkcs8 command documentation^] for details on PKCS#5 algorithms.\n\n== Batching\n\nIt's common to want to upload messages to Snowflake as batched archives. The easiest way to do this is to batch your\nmessages at the output level and join the batch of messages with an\nxref:components:processors/archive.adoc[`archive`] and/or xref:components:processors/compress.adoc[`compress`]\nprocessor.\n\nFor the optimal batch size, please consult the Snowflake https://docs.snowflake.com/en/user-guide/data-load-considerations-prepare.html[documentation^].\n\n== Snowpipe\n\nGiven a table called `BENTHOS_TBL` with one column of type `variant`:\n\n```sql\nCREATE OR REPLACE TABLE BENTHOS_DB.PUBLIC.BENTHOS_TBL(RECORD variant)\n```\n\nand the following `BENTHOS_PIPE` Snowpipe:\n\n```sql\nCREATE OR REPLACE PIPE BENTHOS_DB.PUBLIC.BENTHOS_PIPE AUTO_INGEST = FALSE AS COPY INTO BENTHOS_DB.PUBLIC.BENTHOS_TBL FROM (SELECT * FROM @%BENTHOS_TBL) FILE_FORMAT = (TYPE = JSON COMPRESSION = AUTO)\n```\n\nyou can configure Redpanda Connect to use the implicit table stage `@%BENTHOS_TBL` as the `stage` and\n`BENTHOS_PIPE` as the `snowpipe`. In this case, you must set `compression` to `AUTO` and, if\nusing message batching, you'll need to configure an xref:components:processors/archive.adoc[`archive`] processor\nwith the `concatenate` format. Since the `compression` is set to `AUTO`, the\nhttps://github.com/snowflakedb/gosnowflake[gosnowflake^] client library will compress the messages automatically so you\ndon't need to add a xref:components:processors/compress.adoc[`compress`] processor for message batches.\n\nIf you add `STRIP_OUTER_ARRAY = TRUE` in your Snowpipe `FILE_FORMAT`\ndefinition, then you must use `json_array` instead of `concatenate` as the archive processor format.\n\nNOTE: Only Snowpipes with `FILE_FORMAT` `TYPE` `JSON` are currently supported.\n\n== Snowpipe troubleshooting\n\nSnowpipe https://docs.snowflake.com/en/user-guide/data-load-snowpipe-rest-apis.html[provides^] the `insertReport`\nand `loadHistoryScan` REST API endpoints which can be used to get information about recent Snowpipe calls. In\norder to query them, you'll first need to generate a valid JWT token for your Snowflake account. There are two methods\nfor doing so:\n\n- Using the `snowsql` https://docs.snowflake.com/en/user-guide/snowsql.html[utility^]:\n\n```bash\nsnowsql --private-key-path rsa_key.p8 --generate-jwt -a \u003caccount\u003e -u \u003cuser\u003e\n```\n\n- Using the Python `sql-api-generate-jwt` https://docs.snowflake.com/en/developer-guide/sql-api/authenticating.html#generating-a-jwt-in-python[utility^]:\n\n```bash\npython3 sql-api-generate-jwt.py --private_key_file_path=rsa_key.p8 --account=\u003caccount\u003e --user=\u003cuser\u003e\n```\n\nOnce you successfully generate a JWT token and store it into the `JWT_TOKEN` environment variable, then you can,\nfor example, query the `insertReport` endpoint using `curl`:\n\n```bash\ncurl -H \"Authorization: Bearer ${JWT_TOKEN}\" \"https://\u003caccount\u003e.snowflakecomputing.com/v1/data/pipes/\u003cdatabase\u003e.\u003cschema\u003e.\u003csnowpipe\u003e/insertReport\"\n```\n\nIf you need to pass in a valid `requestId` to any of these Snowpipe REST API endpoints, you can set a\nxref:guides:bloblang/functions.adoc#uuid_v4[uuid_v4()] string in a metadata field called\n`request_id`, log it via the xref:components:processors/log.adoc[`log`] processor and\nthen configure `request_id: ${ @request_id }` ). Alternatively, you can xref:components:logger/about.adoc[enable debug logging]\n and Redpanda Connect will print the Request IDs that it sends to Snowpipe.\n\n== General troubleshooting\n\nThe underlying https://github.com/snowflakedb/gosnowflake[`gosnowflake` driver^] requires write access to\nthe default directory to use for temporary files. Please consult the https://pkg.go.dev/os#TempDir[`os.TempDir`^]\ndocs for details on how to change this directory via environment variables.\n\nA silent failure can occur due to https://github.com/snowflakedb/gosnowflake/issues/701[this issue^], where the\nunderlying https://github.com/snowflakedb/gosnowflake[`gosnowflake` driver^] doesn't return an error and doesn't\nlog a failure if it can't figure out the current username. One way to trigger this behavior is by running Redpanda Connect in a\nDocker container with a non-existent user ID (such as `--user 1000:1000`).\n\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services"],"examples":[{"title":"Kafka / realtime brokers","summary":"Upload message batches from realtime brokers such as Kafka persisting the batch partition and offsets in the stage path and filename similarly to the https://docs.snowflake.com/en/user-guide/kafka-connector-ts.html#step-1-view-the-copy-history-for-the-table[Kafka Connector scheme^] and call Snowpipe to load them into a table. When batching is configured at the input level, it is done per-partition.","config":"\ninput:\n kafka:\n addresses:\n - localhost:9092\n topics:\n - foo\n consumer_group: benthos\n batching:\n count: 10\n period: 3s\n processors:\n - mapping: |\n meta kafka_start_offset = meta(\"kafka_offset\").from(0)\n meta kafka_end_offset = meta(\"kafka_offset\").from(-1)\n meta batch_timestamp = if batch_index() == 0 { now() }\n - mapping: |\n meta batch_timestamp = if batch_index() != 0 { meta(\"batch_timestamp\").from(0) }\n\noutput:\n snowflake_put:\n account: benthos\n user: test@benthos.dev\n private_key_file: path_to_ssh_key.pem\n role: ACCOUNTADMIN\n database: BENTHOS_DB\n warehouse: COMPUTE_WH\n schema: PUBLIC\n stage: \"@%BENTHOS_TBL\"\n path: benthos/BENTHOS_TBL/${! @kafka_partition }\n file_name: ${! @kafka_start_offset }_${! @kafka_end_offset }_${! meta(\"batch_timestamp\") }\n upload_parallel_threads: 4\n compression: NONE\n snowpipe: BENTHOS_PIPE\n"},{"title":"No compression","summary":"Upload concatenated messages into a `.json` file to a table stage without calling Snowpipe.","config":"\noutput:\n snowflake_put:\n account: benthos\n user: test@benthos.dev\n private_key_file: path_to_ssh_key.pem\n role: ACCOUNTADMIN\n database: BENTHOS_DB\n warehouse: COMPUTE_WH\n schema: PUBLIC\n stage: \"@%BENTHOS_TBL\"\n path: benthos\n upload_parallel_threads: 4\n compression: NONE\n batching:\n count: 10\n period: 3s\n processors:\n - archive:\n format: concatenate\n"},{"title":"Parquet format with snappy compression","summary":"Upload concatenated messages into a `.parquet` file to a table stage without calling Snowpipe.","config":"\noutput:\n snowflake_put:\n account: benthos\n user: test@benthos.dev\n private_key_file: path_to_ssh_key.pem\n role: ACCOUNTADMIN\n database: BENTHOS_DB\n warehouse: COMPUTE_WH\n schema: PUBLIC\n stage: \"@%BENTHOS_TBL\"\n path: benthos\n file_extension: parquet\n upload_parallel_threads: 4\n compression: NONE\n batching:\n count: 10\n period: 3s\n processors:\n - parquet_encode:\n schema:\n - name: ID\n type: INT64\n - name: CONTENT\n type: BYTE_ARRAY\n default_compression: snappy\n"},{"title":"Automatic compression","summary":"Upload concatenated messages compressed automatically into a `.gz` archive file to a table stage without calling Snowpipe.","config":"\noutput:\n snowflake_put:\n account: benthos\n user: test@benthos.dev\n private_key_file: path_to_ssh_key.pem\n role: ACCOUNTADMIN\n database: BENTHOS_DB\n warehouse: COMPUTE_WH\n schema: PUBLIC\n stage: \"@%BENTHOS_TBL\"\n path: benthos\n upload_parallel_threads: 4\n compression: AUTO\n batching:\n count: 10\n period: 3s\n processors:\n - archive:\n format: concatenate\n"},{"title":"DEFLATE compression","summary":"Upload concatenated messages compressed into a `.deflate` archive file to a table stage and call Snowpipe to load them into a table.","config":"\noutput:\n snowflake_put:\n account: benthos\n user: test@benthos.dev\n private_key_file: path_to_ssh_key.pem\n role: ACCOUNTADMIN\n database: BENTHOS_DB\n warehouse: COMPUTE_WH\n schema: PUBLIC\n stage: \"@%BENTHOS_TBL\"\n path: benthos\n upload_parallel_threads: 4\n compression: DEFLATE\n snowpipe: BENTHOS_PIPE\n batching:\n count: 10\n period: 3s\n processors:\n - archive:\n format: concatenate\n - mapping: |\n root = content().compress(\"zlib\")\n"},{"title":"RAW_DEFLATE compression","summary":"Upload concatenated messages compressed into a `.raw_deflate` archive file to a table stage and call Snowpipe to load them into a table.","config":"\noutput:\n snowflake_put:\n account: benthos\n user: test@benthos.dev\n private_key_file: path_to_ssh_key.pem\n role: ACCOUNTADMIN\n database: BENTHOS_DB\n warehouse: COMPUTE_WH\n schema: PUBLIC\n stage: \"@%BENTHOS_TBL\"\n path: benthos\n upload_parallel_threads: 4\n compression: RAW_DEFLATE\n snowpipe: BENTHOS_PIPE\n batching:\n count: 10\n period: 3s\n processors:\n - archive:\n format: concatenate\n - mapping: |\n root = content().compress(\"flate\")\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"account","type":"string","kind":"scalar","description":"Account name, which is the same as the https://docs.snowflake.com/en/user-guide/admin-account-identifier.html#where-are-account-identifiers-used[Account Identifier^].\nHowever, when using an https://docs.snowflake.com/en/user-guide/admin-account-identifier.html#using-an-account-locator-as-an-identifier[Account Locator^],\nthe Account Identifier is formatted as `\u003caccount_locator\u003e.\u003cregion_id\u003e.\u003ccloud\u003e` and this field needs to be\npopulated using the `\u003caccount_locator\u003e` part.\n"},{"name":"region","type":"string","kind":"scalar","description":"Optional region field which needs to be populated when using\nan https://docs.snowflake.com/en/user-guide/admin-account-identifier.html#using-an-account-locator-as-an-identifier[Account Locator^]\nand it must be set to the `\u003cregion_id\u003e` part of the Account Identifier\n(`\u003caccount_locator\u003e.\u003cregion_id\u003e.\u003ccloud\u003e`).\n","is_optional":true,"examples":["us-west-2"]},{"name":"cloud","type":"string","kind":"scalar","description":"Optional cloud platform field which needs to be populated\nwhen using an https://docs.snowflake.com/en/user-guide/admin-account-identifier.html#using-an-account-locator-as-an-identifier[Account Locator^]\nand it must be set to the `\u003ccloud\u003e` part of the Account Identifier\n(`\u003caccount_locator\u003e.\u003cregion_id\u003e.\u003ccloud\u003e`).\n","is_optional":true,"examples":["aws","gcp","azure"]},{"name":"user","type":"string","kind":"scalar","description":"Username."},{"name":"password","type":"string","kind":"scalar","description":"An optional password.","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"private_key","type":"string","kind":"scalar","description":"The private SSH key. `private_key_pass` is required when using encrypted keys.","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"private_key_file","type":"string","kind":"scalar","description":"The path to a file containing the private SSH key. `private_key_pass` is required when using encrypted keys.","is_optional":true},{"name":"private_key_pass","type":"string","kind":"scalar","description":"An optional private SSH key passphrase.","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"role","type":"string","kind":"scalar","description":"Role."},{"name":"database","type":"string","kind":"scalar","description":"Database."},{"name":"warehouse","type":"string","kind":"scalar","description":"Warehouse."},{"name":"schema","type":"string","kind":"scalar","description":"Schema."},{"name":"stage","type":"string","kind":"scalar","description":"Stage name. Use either one of the\n\t\thttps://docs.snowflake.com/en/user-guide/data-load-local-file-system-create-stage.html[supported^] stage types.","interpolated":true},{"name":"path","type":"string","kind":"scalar","description":"Stage path.","default":"","interpolated":true},{"name":"file_name","type":"string","kind":"scalar","description":"Stage file name. Will be equal to the Request ID if not set or empty.","is_optional":true,"default":"","interpolated":true,"version":"v4.12.0"},{"name":"file_extension","type":"string","kind":"scalar","description":"Stage file extension. Will be derived from the configured `compression` if not set or empty.","is_optional":true,"default":"","interpolated":true,"examples":["csv","parquet"],"version":"v4.12.0"},{"name":"upload_parallel_threads","type":"int","kind":"scalar","description":"Specifies the number of threads to use for uploading files.","is_advanced":true,"default":4,"linter":"root = if this \u003c 1 || this \u003e 99 { [ \"upload_parallel_threads must be between 1 and 99\" ] }"},{"name":"compression","type":"string","kind":"scalar","description":"Compression type.","default":"AUTO","annotated_options":[["AUTO","Compression (gzip) is applied automatically by the output and messages must contain plain-text JSON. Default `file_extension`: `gz`."],["DEFLATE","Messages must be pre-compressed using the zlib algorithm (with zlib header, RFC1950). Default `file_extension`: `deflate`."],["GZIP","Messages must be pre-compressed using the gzip algorithm. Default `file_extension`: `gz`."],["NONE","No compression is applied and messages must contain plain-text JSON. Default `file_extension`: `json`."],["RAW_DEFLATE","Messages must be pre-compressed using the flate algorithm (without header, RFC1951). Default `file_extension`: `raw_deflate`."],["ZSTD","Messages must be pre-compressed using the Zstandard algorithm. Default `file_extension`: `zst`."]],"linter":"\nlet options = {\n \"auto\": true,\n \"deflate\": true,\n \"gzip\": true,\n \"none\": true,\n \"raw_deflate\": true,\n \"zstd\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"request_id","type":"string","kind":"scalar","description":"Request ID. Will be assigned a random UUID (v4) string if not set or empty.","is_optional":true,"default":"","interpolated":true,"version":"v4.12.0"},{"name":"snowpipe","type":"string","kind":"scalar","description":"An optional Snowpipe name. Use the `\u003csnowpipe\u003e` part from `\u003cdatabase\u003e.\u003cschema\u003e.\u003csnowpipe\u003e`. `private_key` or `private_key_file` must be set when using this feature.","is_optional":true,"interpolated":true},{"name":"client_session_keep_alive","type":"bool","kind":"scalar","description":"Enable Snowflake keepalive mechanism to prevent the client session from expiring after 4 hours (error 390114).","is_advanced":true,"default":false},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of parallel message batches to have in flight at any given time.","default":1}],"linter":"root = match {\n (!this.exists(\"password\") || this.password == \"\") \u0026\u0026 (!this.exists(\"private_key\") || this.private_key == \"\") \u0026\u0026 (!this.exists(\"private_key_file\") || this.private_key_file == \"\") =\u003e [ \"either `password` or `private_key` or `private_key_file` must be set\" ],\n this.exists(\"password\") \u0026\u0026 this.password != \"\" \u0026\u0026 (this.exists(\"private_key\") \u0026\u0026 this.private_key != \"\" || this.exists(\"private_key_file\") \u0026\u0026 this.private_key_file != \"\") =\u003e [ \"only one of `password`, `private_key` and `private_key_file` can be set\" ],\n this.exists(\"snowpipe\") \u0026\u0026 this.snowpipe != \"\" \u0026\u0026 !((this.exists(\"private_key\") \u0026\u0026 this.private_key != \"\") || (this.exists(\"private_key_file\") \u0026\u0026 this.private_key_file != \"\")) =\u003e [ \"either `private_key` or `private_key_file` must be set when using `snowpipe`\" ],\n}"},"version":"4.0.0"},{"name":"snowflake_streaming","type":"output","status":"experimental","plugin":true,"summary":"Ingest data into Snowflake using Snowpipe Streaming.","description":"\nIngest data into Snowflake using Snowpipe Streaming.\n\n[%header,format=dsv]\n|===\nSnowflake column type:Allowed format in Redpanda Connect\nCHAR, VARCHAR:string\nBINARY:[]byte\nNUMBER:any numeric type, string\nFLOAT:any numeric type\nBOOLEAN:bool,any numeric type,string parsable according to `strconv.ParseBool`\nTIME,DATE,TIMESTAMP:unix or RFC 3339 with nanoseconds timestamps\nVARIANT,ARRAY,OBJECT:any data type is converted into JSON\nGEOGRAPHY,GEOMETRY: Not supported\n|===\n\nFor TIMESTAMP, TIME and DATE columns, you can parse different string formats using a bloblang `mapping`.\n\nAuthentication can be configured using a https://docs.snowflake.com/en/user-guide/key-pair-auth[RSA Key Pair^].\n\nThere are https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview#limitations[limitations^] of what data types can be loaded into Snowflake using this method.\n\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].\n\nIt is recommended that each batches results in at least 16MiB of compressed output being written to Snowflake.\nYou can monitor the output batch size using the `snowflake_compressed_output_size_bytes` metric.\n","categories":["Services"],"examples":[{"title":"Exactly once CDC into Snowflake","summary":"How to send data from a PostgreSQL table into Snowflake exactly once using Postgres Logical Replication.\n\nNOTE: If attempting to do exactly-once it's important that rows are delivered in order to the output. Be sure to read the documentation for offset_token first.\nRemoving the offset_token is a safer option that will instruct Redpanda Connect to use its default at-least-once delivery model instead.","config":"\ninput:\n postgres_cdc:\n dsn: postgres://foouser:foopass@localhost:5432/foodb\n schema: \"public\"\n slot_name: \"my_repl_slot\"\n tables: [\"my_pg_table\"]\n # We want very large batches - each batch will be sent to Snowflake individually\n # so to optimize query performance we want as big of files as we have memory for\n batching:\n count: 50000\n period: 45s\n # Prevent multiple batches from being in flight at once, so that we never send\n # a batch while another batch is being retried, this is important to ensure that\n # the Snowflake Snowpipe Streaming channel does not see older data - as it will\n # assume that the older data is already committed.\n checkpoint_limit: 1\noutput:\n snowflake_streaming:\n # We use the log sequence number in the WAL from Postgres to ensure we\n # only upload data exactly once, these are already lexicographically\n # ordered.\n offset_token: \"${!@lsn}\"\n # Since we're sending a single ordered log, we can only send one thing\n # at a time to ensure that we're properly incrementing our offset_token\n # and only using a single channel at a time.\n max_in_flight: 1\n account: \"MYSNOW-ACCOUNT\"\n user: MYUSER\n role: ACCOUNTADMIN\n database: \"MYDATABASE\"\n schema: \"PUBLIC\"\n table: \"MY_PG_TABLE\"\n private_key_file: \"my/private/key.p8\"\n"},{"title":"Ingesting data exactly once from Redpanda","summary":"How to ingest data from Redpanda with consumer groups, decode the schema using the schema registry, then write the corresponding data into Snowflake exactly once.\n\nNOTE: If attempting to do exactly-once its important that records are delivered in order to the output and correctly partitioned. Be sure to read the documentation for \nchannel_name and offset_token first. Removing the offset_token is a safer option that will instruct Redpanda Connect to use its default at-least-once delivery model instead.","config":"\ninput:\n redpanda_common:\n topics: [\"my_topic_going_to_snow\"]\n consumer_group: \"redpanda_connect_to_snowflake\"\n # We want very large batches - each batch will be sent to Snowflake individually\n # so to optimize query performance we want as big of files as we have memory for\n fetch_max_bytes: 100MiB\n fetch_min_bytes: 50MiB\n partition_buffer_bytes: 100MiB\npipeline:\n processors:\n - schema_registry_decode:\n url: \"redpanda.example.com:8081\"\n basic_auth:\n enabled: true\n username: MY_USER_NAME\n password: \"${TODO}\"\noutput:\n fallback:\n - snowflake_streaming:\n # To ensure that we write an ordered stream each partition in kafka gets its own\n # channel.\n channel_name: \"partition-${!@kafka_partition}\"\n # Ensure that our offsets are lexicographically sorted in string form by padding with\n # leading zeros\n offset_token: offset-${!\"%016X\".format(@kafka_offset)}\n account: \"MYSNOW-ACCOUNT\"\n user: MYUSER\n role: ACCOUNTADMIN\n database: \"MYDATABASE\"\n schema: \"PUBLIC\"\n table: \"MYTABLE\"\n private_key_file: \"my/private/key.p8\"\n schema_evolution:\n enabled: true\n # In order to prevent delivery orders from messing with the order of delivered records\n # it's important that failures are immediately sent to a dead letter queue and not retried\n # to Snowflake. See the ordering documentation for the \"redpanda\" input for more details.\n - retry:\n output:\n redpanda_common:\n topic: \"dead_letter_queue\"\n"},{"title":"HTTP Server to push data to Snowflake","summary":"This example demonstrates how to create an HTTP server input that can recieve HTTP PUT requests\nwith JSON payloads, that are buffered locally then written to Snowflake in batches.\n\nNOTE: This example uses a buffer to respond to the HTTP request immediately, so it's possible that failures to deliver data could result in data loss.\nSee the documentation about xref:components:buffers/memory.adoc[buffers] for more information, or remove the buffer entirely to respond to the HTTP request only once the data is written to Snowflake.","config":"\ninput:\n http_server:\n path: /snowflake\nbuffer:\n memory:\n # Max inflight data before applying backpressure\n limit: 524288000 # 50MiB\n # Batching policy, influences how large the generated files sent to Snowflake are\n batch_policy:\n enabled: true\n byte_size: 33554432 # 32MiB\n period: \"10s\"\noutput:\n snowflake_streaming:\n account: \"MYSNOW-ACCOUNT\"\n user: MYUSER\n role: ACCOUNTADMIN\n database: \"MYDATABASE\"\n schema: \"PUBLIC\"\n table: \"MYTABLE\"\n private_key_file: \"my/private/key.p8\"\n # By default there is only a single channel per output table allowed\n # if we want to have multiple Redpanda Connect streams writing data\n # then we need a unique channel prefix per stream. We'll use the host\n # name to get unique prefixes in this example.\n channel_prefix: \"snowflake-channel-for-${HOST}\"\n schema_evolution:\n enabled: true\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"account","type":"string","kind":"scalar","description":"The Snowflake https://docs.snowflake.com/en/user-guide/admin-account-identifier.html#using-an-account-locator-as-an-identifier[Account name^]. Which should be formatted as `\u003corgname\u003e-\u003caccount_name\u003e` where `\u003corgname\u003e` is the name of your Snowflake organization and `\u003caccount_name\u003e` is the unique name of your account within your organization.\n","examples":["ORG-ACCOUNT"]},{"name":"url","type":"string","kind":"scalar","description":"Override the default URL used to connect to Snowflake which is https://ORG-ACCOUNT.snowflakecomputing.com","is_advanced":true,"is_optional":true,"examples":["https://org-account.privatelink.snowflakecomputing.com"]},{"name":"user","type":"string","kind":"scalar","description":"The user to run the Snowpipe Stream as. See https://docs.snowflake.com/en/user-guide/admin-user-management[Snowflake Documentation^] on how to create a user."},{"name":"role","type":"string","kind":"scalar","description":"The role for the `user` field. The role must have the https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview#required-access-privileges[required privileges^] to call the Snowpipe Streaming APIs. See https://docs.snowflake.com/en/user-guide/admin-user-management#user-roles[Snowflake Documentation^] for more information about roles.","examples":["ACCOUNTADMIN"]},{"name":"database","type":"string","kind":"scalar","description":"The Snowflake database to ingest data into.","examples":["MY_DATABASE"]},{"name":"schema","type":"string","kind":"scalar","description":"The Snowflake schema to ingest data into.","examples":["PUBLIC"]},{"name":"table","type":"string","kind":"scalar","description":"The Snowflake table to ingest data into.","interpolated":true,"examples":["MY_TABLE"]},{"name":"private_key","type":"string","kind":"scalar","description":"The PEM encoded private RSA key to use for authenticating with Snowflake. Either this or `private_key_file` must be specified.","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"private_key_file","type":"string","kind":"scalar","description":"The file to load the private RSA key from. This should be a `.p8` PEM encoded file. Either this or `private_key` must be specified.","is_optional":true},{"name":"private_key_pass","type":"string","kind":"scalar","description":"The RSA key passphrase if the RSA key is encrypted.","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"mapping","type":"string","kind":"scalar","description":"A bloblang mapping to execute on each message.","is_optional":true,"bloblang":true},{"name":"init_statement","type":"string","kind":"scalar","description":"\nOptional SQL statements to execute immediately upon the first connection. This is a useful way to initialize tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n","is_optional":true,"examples":["\nCREATE TABLE IF NOT EXISTS mytable (amount NUMBER);\n","\nALTER TABLE t1 ALTER COLUMN c1 DROP NOT NULL;\nALTER TABLE t1 ADD COLUMN a2 NUMBER;\n"]},{"name":"schema_evolution","type":"object","kind":"scalar","description":"Options to control schema evolution within the pipeline as new columns are added to the pipeline.","is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether schema evolution is enabled."},{"name":"ignore_nulls","type":"bool","kind":"scalar","description":"If `true`, then new columns that are `null` are ignored and schema evolution is not triggered. If `false` then null columns trigger schema migrations in Snowflake. NOTE: unless you already know what type this column will be in advance, it's highly encouraged to ignore null values.","is_advanced":true,"default":true},{"name":"new_column_type_mapping","type":"string","kind":"scalar","description":"\nThe mapping function from Redpanda Connect type to column type in Snowflake. Overriding this can allow for customization of the datatype if there is specific information that you know about the data types in use. This mapping should result in the `root` variable being assigned a string with the data type for the new column in Snowflake.\n\n The input to this mapping is either the output of `processors` if specified, otherwise it is an object with the value and the name of the new column, the original message and table being written too. The metadata is unchanged from the original message that caused the schema to change. For example: `{\"value\": 42.3, \"name\":\"new_data_field\", \"message\": {\"existing_data_field\": 42, \"new_data_field\": \"foo\"}, \"db\": MY_DATABASE\", \"schema\": \"MY_SCHEMA\", \"table\": \"MY_TABLE\"}","is_deprecated":true,"is_optional":true,"bloblang":true},{"name":"processors","type":"processor","kind":"array","description":"\nA series of processors to execute when new columns are added to the table. Specifying this can support running side effects when the schema evolves or enriching the message with additional data to guide the schema changes. For example, one could read the schema the message was produced with from the schema registry and use that to decide which type the new column in Snowflake should be.\n\n The input to these processors is an object with the value and the name of the new column, the original message and table being written too. The metadata is unchanged from the original message that caused the schema to change. For example: `{\"value\": 42.3, \"name\":\"new_data_field\", \"message\": {\"existing_data_field\": 42, \"new_data_field\": \"foo\"}, \"db\": MY_DATABASE\", \"schema\": \"MY_SCHEMA\", \"table\": \"MY_TABLE\"}`. The output of these series of processors should be a single message, where the contents of the message is a string indicating the column data type to use (FLOAT, VARIANT, NUMBER(38, 0), etc. An ALTER TABLE statement will then be executed on the table in Snowflake to add the column with the corresponding data type.","is_advanced":true,"is_optional":true,"examples":[[{"mapping":"root = match this.value.type() {\n this == \"string\" =\u003e \"STRING\"\n this == \"bytes\" =\u003e \"BINARY\"\n this == \"number\" =\u003e \"DOUBLE\"\n this == \"bool\" =\u003e \"BOOLEAN\"\n this == \"timestamp\" =\u003e \"TIMESTAMP\"\n _ =\u003e \"VARIANT\"\n}"}]]}]},{"name":"parallelism","type":"int","kind":"scalar","description":"The maximum amount of parallelism to use when building the output for Snowflake. The metric to watch to see if you need to change this is `snowflake_build_output_latency_ns`.","is_advanced":true,"is_deprecated":true,"is_optional":true},{"name":"build_options","type":"object","kind":"scalar","description":"Options to optimize the time to build output data that is sent to Snowflake. The metric to watch to see if you need to change this is `snowflake_build_output_latency_ns`.","is_advanced":true,"children":[{"name":"parallelism","type":"int","kind":"scalar","description":"The maximum amount of parallelism to use.","is_advanced":true,"default":1,"linter":"root = if this \u003c 1 { [\"parallelism must be positive\"] }"},{"name":"chunk_size","type":"int","kind":"scalar","description":"The number of rows to chunk for parallelization.","is_advanced":true,"default":50000,"linter":"root = if this \u003c 1 { [\"chunk_size must be positive\"] }"}]},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":4},{"name":"channel_prefix","type":"string","kind":"scalar","description":"The prefix to use when creating a channel name.\nDuplicate channel names will result in errors and prevent multiple instances of Redpanda Connect from writing at the same time.\nBy default if neither `channel_prefix` or `channel_name is specified then the output will create a channel name that is based on the table FQN so there will only be a single stream per table.\n\nAt most `max_in_flight` channels will be opened.\n\nThis option is mutually exclusive with `channel_name`.\n\nNOTE: There is a limit of 10,000 streams per table - if using more than 10k streams please reach out to Snowflake support.","is_advanced":true,"is_optional":true,"examples":["channel-${HOST}"]},{"name":"channel_name","type":"string","kind":"scalar","description":"The channel name to use.\nDuplicate channel names will result in errors and prevent multiple instances of Redpanda Connect from writing at the same time.\nNote that batches are assumed to all contain messages for the same channel, so this interpolation is only executed on the first\nmessage in each batch. It's recommended to batch at the input level to ensure that batches contain messages for the same channel\nif using an input that is partitioned (such as an Apache Kafka topic).\n\nThis option is mutually exclusive with `channel_prefix`.\n\nNOTE: There is a limit of 10,000 streams per table - if using more than 10k streams please reach out to Snowflake support.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["partition-${!@kafka_partition}"]},{"name":"offset_token","type":"string","kind":"scalar","description":"The offset token to use for exactly once delivery of data in the pipeline. When data is sent on a channel, each message in a batch's offset token\nis compared to the latest token for a channel. If the offset token is lexicographically less than the latest in the channel, it's assumed the message is a duplicate and\nis dropped. This means it is *very important* to have ordered delivery to the output, any out of order messages to the output will be seen as duplicates and dropped.\nSpecifically this means that retried messages could be seen as duplicates if later messages have succeeded in the meantime, so in most circumstances a dead letter queue\noutput should be employed for failed messages.\n\nNOTE: It's assumed that messages within a batch are in increasing order by offset token, additionally if you're using a numeric value as an offset token, make sure to pad\n the value so that it's lexicographically ordered in its string representation, since offset tokens are compared in string form.\n\nFor more information about offset tokens, see https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview#offset-tokens[^Snowflake Documentation]","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["offset-${!\"%016X\".format(@kafka_offset)}","postgres-${!@lsn}"]},{"name":"commit_timeout","type":"string","kind":"scalar","description":"The max duration to wait until the data has been asynchronously committed to Snowflake.","is_advanced":true,"default":"60s","examples":["10s","10m"]}],"linter":"root = match {\n this.exists(\"channel_prefix\") \u0026\u0026 this.exists(\"channel_name\") =\u003e [ \"both `channel_prefix` and `channel_name` can't be set simultaneously\" ],\n}"},"version":"4.39.0"},{"name":"socket","type":"output","status":"stable","plugin":true,"summary":"Connects to a (tcp/udp/unix) server and sends a continuous stream of data, dividing messages according to the specified codec.","categories":["Network"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"network","type":"string","kind":"scalar","description":"A network type to connect as.","options":["unix","tcp","udp"],"linter":"\nlet options = {\n \"unix\": true,\n \"tcp\": true,\n \"udp\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"address","type":"string","kind":"scalar","description":"The address to connect to.","examples":["/tmp/benthos.sock","127.0.0.1:6000"]},{"name":"codec","type":"string","kind":"scalar","description":"The way in which the bytes of messages should be written out into the output data stream. It's possible to write lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter.","default":"lines","examples":["lines","delim:\t","delim:foobar"],"annotated_options":[["all-bytes","Only applicable to file based outputs. Writes each message to a file in full, if the file already exists the old content is deleted."],["append","Append each message to the output stream without any delimiter or special encoding."],["lines","Append each message to the output stream followed by a line break."],["delim:x","Append each message to the output stream followed by a custom delimiter."]]}]}},{"name":"splunk_hec","type":"output","status":"beta","plugin":true,"summary":"Publishes messages to a Splunk HTTP Endpoint Collector (HEC).","description":"\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"Full HTTP Endpoint Collector (HEC) URL.","examples":["https://foobar.splunkcloud.com/services/collector/event"]},{"name":"token","type":"string","kind":"scalar","description":"A bot token used for authentication.","is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"gzip","type":"bool","kind":"scalar","description":"Enable gzip compression","default":false},{"name":"event_host","type":"string","kind":"scalar","description":"Set the host value to assign to the event data. Overrides existing host field if present.","is_optional":true},{"name":"event_source","type":"string","kind":"scalar","description":"Set the source value to assign to the event data. Overrides existing source field if present.","is_optional":true},{"name":"event_sourcetype","type":"string","kind":"scalar","description":"Set the sourcetype value to assign to the event data. Overrides existing sourcetype field if present.","is_optional":true},{"name":"event_index","type":"string","kind":"scalar","description":"Set the index value to assign to the event data. Overrides existing index field if present.","is_optional":true},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"skip_cert_verify","type":"bool","kind":"scalar","is_deprecated":true,"is_optional":true},{"name":"batching_count","type":"int","kind":"scalar","is_deprecated":true,"is_optional":true},{"name":"batching_period","type":"string","kind":"scalar","is_deprecated":true,"is_optional":true},{"name":"batching_byte_size","type":"int","kind":"scalar","is_deprecated":true,"is_optional":true},{"name":"rate_limit","type":"string","kind":"scalar","is_deprecated":true,"is_optional":true}]},"version":"4.30.0"},{"name":"sql","type":"output","status":"deprecated","plugin":true,"summary":"Executes an arbitrary SQL query for each message.","description":"\n== Alternatives\n\nFor basic inserts use the xref:components:outputs/sql.adoc[`sql_insert`] output. For more complex queries use the xref:components:outputs/sql_raw.adoc[`sql_raw`] output.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"driver","type":"string","kind":"scalar","description":"A database \u003c\u003cdrivers, driver\u003e\u003e to use.","options":["mysql","postgres","clickhouse","mssql","sqlite","oracle","snowflake","trino","gocosmos","spanner"],"linter":"\nlet options = {\n \"mysql\": true,\n \"postgres\": true,\n \"clickhouse\": true,\n \"mssql\": true,\n \"sqlite\": true,\n \"oracle\": true,\n \"snowflake\": true,\n \"trino\": true,\n \"gocosmos\": true,\n \"spanner\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"data_source_name","type":"string","kind":"scalar","description":"Data source name."},{"name":"query","type":"string","kind":"scalar","description":"The query to execute. The style of placeholder to use depends on the driver, some drivers require question marks (`?`) whereas others expect incrementing dollar signs (`$1`, `$2`, and so on) or colons (`:1`, `:2` and so on). The style to use is outlined in this table:\n\n| Driver | Placeholder Style |\n|---|---|\n| `clickhouse` | Dollar sign |\n| `mysql` | Question mark |\n| `postgres` | Dollar sign |\n| `mssql` | Question mark |\n| `sqlite` | Question mark |\n| `oracle` | Colon |\n| `snowflake` | Question mark |\n| `trino` | Question mark |\n| `gocosmos` | Colon |\n","examples":["INSERT INTO footable (foo, bar, baz) VALUES (?, ?, ?);"]},{"name":"args_mapping","type":"string","kind":"scalar","description":"An optional xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `query`.","is_optional":true,"bloblang":true,"examples":["root = [ this.cat.meow, this.doc.woofs[0] ]","root = [ meta(\"user.id\") ]"]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of inserts to run in parallel.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]},"version":"3.65.0"},{"name":"sql_insert","type":"output","status":"stable","plugin":true,"summary":"Inserts a row into an SQL database for each message.","categories":["Services"],"examples":[{"title":"Table Insert (MySQL)","summary":"\nHere we insert rows into a database by populating the columns id, name and topic with values extracted from messages and metadata:","config":"\noutput:\n sql_insert:\n driver: mysql\n dsn: foouser:foopassword@tcp(localhost:3306)/foodb\n table: footable\n columns: [ id, name, topic ]\n args_mapping: |\n root = [\n this.user.id,\n this.user.name,\n meta(\"kafka_topic\"),\n ]\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"driver","type":"string","kind":"scalar","description":"A database \u003c\u003cdrivers, driver\u003e\u003e to use.","options":["mysql","postgres","clickhouse","mssql","sqlite","oracle","snowflake","trino","gocosmos","spanner"],"linter":"\nlet options = {\n \"mysql\": true,\n \"postgres\": true,\n \"clickhouse\": true,\n \"mssql\": true,\n \"sqlite\": true,\n \"oracle\": true,\n \"snowflake\": true,\n \"trino\": true,\n \"gocosmos\": true,\n \"spanner\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"dsn","type":"string","kind":"scalar","description":"A Data Source Name to identify the target database.\n\n==== Drivers\n\n:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community, spanner=community\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n|===\n| Driver | Data Source Name Format\n\n| `clickhouse` \n| https://github.com/ClickHouse/clickhouse-go#dsn[`clickhouse://[username[:password\\]@\\][netloc\\][:port\\]/dbname[?param1=value1\u0026...\u0026paramN=valueN\\]`^] \n\n| `mysql` \n| `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` \n\n| `postgres` \n| `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` \n\n| `mssql` \n| `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` \n\n| `sqlite` \n| `file:/path/to/filename.db[?param\u0026=value1\u0026...]` \n\n| `oracle` \n| `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` \n\n| `snowflake` \n| `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` \n\n| `trino` \n| https://github.com/trinodb/trino-go-client#dsn-data-source-name[`http[s\\]://user[:pass\\]@host[:port\\][?parameters\\]`^] \n\n| `gocosmos` \n| https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage[`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e\\][;Version=\u003ccosmosdb-api-version\u003e\\][;DefaultDb/Db=\u003cdb-name\u003e\\][;AutoId=\u003ctrue/false\u003e\\][;InsecureSkipVerify=\u003ctrue/false\u003e\\]`^] \n\n| `spanner` \n| projects/[PROJECT]/instances/[INSTANCE]/databases/[DATABASE] \n|===\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String[the docs^] for more details. For https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication[key pair authentication^], the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe https://pkg.go.dev/github.com/microsoft/gocosmos[`gocosmos`^] driver is still experimental, but it has support for https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys[hierarchical partition keys^] as well as https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query[cross-partition queries^]. Please refer to the https://github.com/microsoft/gocosmos/blob/main/SQL.md[SQL notes^] for details.","examples":["clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60","foouser:foopassword@tcp(localhost:3306)/foodb","postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable","oracle://foouser:foopass@localhost:1521/service_name"]},{"name":"table","type":"string","kind":"scalar","description":"The table to insert to.","examples":["foo"]},{"name":"columns","type":"string","kind":"array","description":"A list of columns to insert.","examples":[["foo","bar","baz"]]},{"name":"args_mapping","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of columns specified.","bloblang":true,"examples":["root = [ this.cat.meow, this.doc.woofs[0] ]","root = [ meta(\"user.id\") ]"]},{"name":"prefix","type":"string","kind":"scalar","description":"An optional prefix to prepend to the insert query (before INSERT).","is_advanced":true,"is_optional":true},{"name":"suffix","type":"string","kind":"scalar","description":"An optional suffix to append to the insert query.","is_advanced":true,"is_optional":true,"examples":["ON CONFLICT (name) DO NOTHING"]},{"name":"options","type":"string","kind":"array","description":"A list of keyword options to add before the INTO clause of the query.","is_advanced":true,"is_optional":true,"examples":[["DELAYED","IGNORE"]]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of inserts to run in parallel.","default":64},{"name":"init_files","type":"string","kind":"array","description":"\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":[["./init/*.sql"],["./foo.sql","./bar.sql"]],"version":"4.10.0"},{"name":"init_statement","type":"string","kind":"scalar","description":"\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":["\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n"],"version":"4.10.0"},{"name":"conn_max_idle_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time.","is_advanced":true,"is_optional":true},{"name":"conn_max_life_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age.","is_advanced":true,"is_optional":true},{"name":"conn_max_idle","type":"int","kind":"scalar","description":"An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release.","is_advanced":true,"is_optional":true,"default":2},{"name":"conn_max_open","type":"int","kind":"scalar","description":"An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited).","is_advanced":true,"is_optional":true},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]},"version":"3.59.0"},{"name":"sql_raw","type":"output","status":"stable","plugin":true,"summary":"Executes an arbitrary SQL query for each message.","categories":["Services"],"examples":[{"title":"Table Insert (MySQL)","summary":"\nHere we insert rows into a database by populating the columns id, name and topic with values extracted from messages and metadata:","config":"\noutput:\n sql_raw:\n driver: mysql\n dsn: foouser:foopassword@tcp(localhost:3306)/foodb\n query: \"INSERT INTO footable (id, name, topic) VALUES (?, ?, ?);\"\n args_mapping: |\n root = [\n this.user.id,\n this.user.name,\n meta(\"kafka_topic\"),\n ]\n"},{"title":"Dynamically Creating Tables (PostgreSQL)","summary":"Here we dynamically create output tables transactionally with inserting a record into the newly created table.","config":"\noutput:\n processors:\n - mapping: |\n root = this\n # Prevent SQL injection when using unsafe_dynamic_query\n meta table_name = \"\\\"\" + metadata(\"table_name\").replace_all(\"\\\"\", \"\\\"\\\"\") + \"\\\"\"\n sql_raw:\n driver: postgres\n dsn: postgres://localhost/postgres\n unsafe_dynamic_query: true\n queries:\n - query: |\n CREATE TABLE IF NOT EXISTS ${!metadata(\"table_name\")} (id varchar primary key, document jsonb);\n - query: |\n INSERT INTO ${!metadata(\"table_name\")} (id, document) VALUES ($1, $2)\n ON CONFLICT (id) DO UPDATE SET document = EXCLUDED.document;\n args_mapping: |\n root = [ this.id, this.document.string() ]\n\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"driver","type":"string","kind":"scalar","description":"A database \u003c\u003cdrivers, driver\u003e\u003e to use.","options":["mysql","postgres","clickhouse","mssql","sqlite","oracle","snowflake","trino","gocosmos","spanner"],"linter":"\nlet options = {\n \"mysql\": true,\n \"postgres\": true,\n \"clickhouse\": true,\n \"mssql\": true,\n \"sqlite\": true,\n \"oracle\": true,\n \"snowflake\": true,\n \"trino\": true,\n \"gocosmos\": true,\n \"spanner\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"dsn","type":"string","kind":"scalar","description":"A Data Source Name to identify the target database.\n\n==== Drivers\n\n:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community, spanner=community\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n|===\n| Driver | Data Source Name Format\n\n| `clickhouse` \n| https://github.com/ClickHouse/clickhouse-go#dsn[`clickhouse://[username[:password\\]@\\][netloc\\][:port\\]/dbname[?param1=value1\u0026...\u0026paramN=valueN\\]`^] \n\n| `mysql` \n| `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` \n\n| `postgres` \n| `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` \n\n| `mssql` \n| `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` \n\n| `sqlite` \n| `file:/path/to/filename.db[?param\u0026=value1\u0026...]` \n\n| `oracle` \n| `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` \n\n| `snowflake` \n| `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` \n\n| `trino` \n| https://github.com/trinodb/trino-go-client#dsn-data-source-name[`http[s\\]://user[:pass\\]@host[:port\\][?parameters\\]`^] \n\n| `gocosmos` \n| https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage[`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e\\][;Version=\u003ccosmosdb-api-version\u003e\\][;DefaultDb/Db=\u003cdb-name\u003e\\][;AutoId=\u003ctrue/false\u003e\\][;InsecureSkipVerify=\u003ctrue/false\u003e\\]`^] \n\n| `spanner` \n| projects/[PROJECT]/instances/[INSTANCE]/databases/[DATABASE] \n|===\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String[the docs^] for more details. For https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication[key pair authentication^], the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe https://pkg.go.dev/github.com/microsoft/gocosmos[`gocosmos`^] driver is still experimental, but it has support for https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys[hierarchical partition keys^] as well as https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query[cross-partition queries^]. Please refer to the https://github.com/microsoft/gocosmos/blob/main/SQL.md[SQL notes^] for details.","examples":["clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60","foouser:foopassword@tcp(localhost:3306)/foodb","postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable","oracle://foouser:foopass@localhost:1521/service_name"]},{"name":"query","type":"string","kind":"scalar","description":"The query to execute. The style of placeholder to use depends on the driver, some drivers require question marks (`?`) whereas others expect incrementing dollar signs (`$1`, `$2`, and so on) or colons (`:1`, `:2` and so on). The style to use is outlined in this table:\n\n| Driver | Placeholder Style |\n|---|---|\n| `clickhouse` | Dollar sign |\n| `mysql` | Question mark |\n| `postgres` | Dollar sign |\n| `mssql` | Question mark |\n| `sqlite` | Question mark |\n| `oracle` | Colon |\n| `snowflake` | Question mark |\n| `trino` | Question mark |\n| `gocosmos` | Colon |\n","is_optional":true,"examples":["INSERT INTO footable (foo, bar, baz) VALUES (?, ?, ?);"]},{"name":"unsafe_dynamic_query","type":"bool","kind":"scalar","description":"Whether to enable xref:configuration:interpolation.adoc#bloblang-queries[interpolation functions] in the query. Great care should be made to ensure your queries are defended against injection attacks.","is_advanced":true,"default":false},{"name":"args_mapping","type":"string","kind":"scalar","description":"An optional xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `query`.","is_optional":true,"bloblang":true,"examples":["root = [ this.cat.meow, this.doc.woofs[0] ]","root = [ meta(\"user.id\") ]"]},{"name":"queries","type":"object","kind":"array","description":"A list of statements to run in addition to `query`. When specifying multiple statements, they are all executed within a transaction.","is_optional":true,"children":[{"name":"query","type":"string","kind":"scalar","description":"The query to execute. The style of placeholder to use depends on the driver, some drivers require question marks (`?`) whereas others expect incrementing dollar signs (`$1`, `$2`, and so on) or colons (`:1`, `:2` and so on). The style to use is outlined in this table:\n\n| Driver | Placeholder Style |\n|---|---|\n| `clickhouse` | Dollar sign |\n| `mysql` | Question mark |\n| `postgres` | Dollar sign |\n| `mssql` | Question mark |\n| `sqlite` | Question mark |\n| `oracle` | Colon |\n| `snowflake` | Question mark |\n| `trino` | Question mark |\n| `gocosmos` | Colon |\n"},{"name":"args_mapping","type":"string","kind":"scalar","description":"An optional xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `query`.","is_optional":true,"bloblang":true,"examples":["root = [ this.cat.meow, this.doc.woofs[0] ]","root = [ meta(\"user.id\") ]"]}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of statements to execute in parallel.","default":64},{"name":"init_files","type":"string","kind":"array","description":"\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":[["./init/*.sql"],["./foo.sql","./bar.sql"]],"version":"4.10.0"},{"name":"init_statement","type":"string","kind":"scalar","description":"\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":["\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n"],"version":"4.10.0"},{"name":"conn_max_idle_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time.","is_advanced":true,"is_optional":true},{"name":"conn_max_life_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age.","is_advanced":true,"is_optional":true},{"name":"conn_max_idle","type":"int","kind":"scalar","description":"An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release.","is_advanced":true,"is_optional":true,"default":2},{"name":"conn_max_open","type":"int","kind":"scalar","description":"An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited).","is_advanced":true,"is_optional":true},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}],"linter":"root = match {\n !this.exists(\"queries\") \u0026\u0026 !this.exists(\"query\") =\u003e [ \"either `query` or `queries` is required\" ],\n }"},"version":"3.65.0"},{"name":"stdout","type":"output","status":"stable","plugin":true,"summary":"Prints messages to stdout as a continuous stream of data.","categories":["Local"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"codec","type":"string","kind":"scalar","description":"The way in which the bytes of messages should be written out into the output data stream. It's possible to write lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter.","default":"lines","examples":["lines","delim:\t","delim:foobar"],"annotated_options":[["all-bytes","Only applicable to file based outputs. Writes each message to a file in full, if the file already exists the old content is deleted."],["append","Append each message to the output stream without any delimiter or special encoding."],["lines","Append each message to the output stream followed by a line break."],["delim:x","Append each message to the output stream followed by a custom delimiter."]],"version":"3.46.0"}]}},{"name":"subprocess","type":"output","status":"beta","plugin":true,"summary":"Executes a command, runs it as a subprocess, and writes messages to it over stdin.","description":"\nMessages are written according to a specified codec. The process is expected to terminate gracefully when stdin is closed.\n\nIf the subprocess exits unexpectedly then Redpanda Connect will log anything printed to stderr and will log the exit code, and will attempt to execute the command again until success.\n\nThe execution environment of the subprocess is the same as the Redpanda Connect instance, including environment variables and the current working directory.","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"name","type":"string","kind":"scalar","description":"The command to execute as a subprocess."},{"name":"args","type":"string","kind":"array","description":"A list of arguments to provide the command.","default":[]},{"name":"codec","type":"string","kind":"scalar","description":"The way in which messages should be written to the subprocess.","default":"lines","options":["lines"],"linter":"\nlet options = {\n \"lines\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"}]}},{"name":"switch","type":"output","status":"stable","plugin":true,"summary":"The switch output type allows you to route messages to different outputs based on their contents.","description":"Messages that do not pass the check of a single output case are effectively dropped. In order to prevent this outcome set the field \u003c\u003cstrict_mode, `strict_mode`\u003e\u003e to `true`, in which case messages that do not pass at least one case are considered failed and will be nacked and/or reprocessed depending on your input.","categories":["Utility"],"examples":[{"title":"Basic Multiplexing","summary":"\nThe most common use for a switch output is to multiplex messages across a range of output destinations. The following config checks the contents of the field `type` of messages and sends `foo` type messages to an `amqp_1` output, `bar` type messages to a `gcp_pubsub` output, and everything else to a `redis_streams` output.\n\nOutputs can have their own processors associated with them, and in this example the `redis_streams` output has a processor that enforces the presence of a type field before sending it.","config":"\noutput:\n switch:\n cases:\n - check: this.type == \"foo\"\n output:\n amqp_1:\n urls: [ amqps://guest:guest@localhost:5672/ ]\n target_address: queue:/the_foos\n\n - check: this.type == \"bar\"\n output:\n gcp_pubsub:\n project: dealing_with_mike\n topic: mikes_bars\n\n - output:\n redis_streams:\n url: tcp://localhost:6379\n stream: everything_else\n processors:\n - mapping: |\n root = this\n root.type = this.type | \"unknown\"\n"},{"title":"Control Flow","summary":"\nThe `continue` field allows messages that have passed a case to be tested against the next one also. This can be useful when combining non-mutually-exclusive case checks.\n\nIn the following example a message that passes both the check of the first case as well as the second will be routed to both.","config":"\noutput:\n switch:\n cases:\n - check: 'this.user.interests.contains(\"walks\").catch(false)'\n output:\n amqp_1:\n urls: [ amqps://guest:guest@localhost:5672/ ]\n target_address: queue:/people_what_think_good\n continue: true\n\n - check: 'this.user.dislikes.contains(\"videogames\").catch(false)'\n output:\n gcp_pubsub:\n project: people\n topic: that_i_dont_want_to_hang_with\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"retry_until_success","type":"bool","kind":"scalar","description":"\nIf a selected output fails to send a message this field determines whether it is reattempted indefinitely. If set to false the error is instead propagated back to the input level.\n\nIf a message can be routed to \u003e1 outputs it is usually best to set this to true in order to avoid duplicate messages being routed to an output.","default":false},{"name":"strict_mode","type":"bool","kind":"scalar","description":"This field determines whether an error should be reported if no condition is met. If set to true, an error is propagated back to the input level. The default behavior is false, which will drop the message.","is_advanced":true,"default":false},{"name":"cases","type":"object","kind":"array","description":"A list of switch cases, outlining outputs that can be routed to.","examples":[[{"check":"this.urls.contains(\"http://benthos.dev\")","continue":true,"output":{"cache":{"key":"${!json(\"id\")}","target":"foo"}}},{"output":{"s3":{"bucket":"bar","path":"${!json(\"id\")}"}}}]],"children":[{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should be routed to the case output. If left empty the case always passes.","default":"","bloblang":true,"examples":["this.type == \"foo\"","this.contents.urls.contains(\"https://benthos.dev/\")"]},{"name":"output","type":"output","kind":"scalar","description":"An xref:components:outputs/about.adoc[output] for messages that pass the check to be routed to."},{"name":"continue","type":"bool","kind":"scalar","description":"Indicates whether, if this case passes for a message, the next case should also be tested.","is_advanced":true,"default":false}]}],"linter":"if this.exists(\"retry_until_success\") \u0026\u0026 this.retry_until_success {\n if this.cases.or([]).any(oconf -\u003e oconf.output.type.or(\"\") == \"reject\" || oconf.output.reject.type() == \"string\" ) {\n \"a 'switch' output with a 'reject' case output must have the field 'switch.retry_until_success' set to 'false', otherwise the 'reject' child output will result in infinite retries\"\n }\n}"}},{"name":"sync_response","type":"output","status":"stable","plugin":true,"summary":"Returns the final message payload back to the input origin of the message, where it is dealt with according to that specific input type.","description":"\nFor most inputs this mechanism is ignored entirely, in which case the sync response is dropped without penalty. It is therefore safe to use this output even when combining input types that might not have support for sync responses. An example of an input able to utilize this is the `http_server`.\n\nIt is safe to combine this output with others using broker types. For example, with the `http_server` input we could send the payload to a Kafka topic and also send a modified payload back with:\n\n```yaml\ninput:\n http_server:\n path: /post\noutput:\n broker:\n pattern: fan_out\n outputs:\n - kafka:\n addresses: [ TODO:9092 ]\n topic: foo_topic\n - sync_response: {}\n processors:\n - mapping: 'root = content().uppercase()'\n```\n\nUsing the above example and posting the message 'hello world' to the endpoint `/post` Redpanda Connect would send it unchanged to the topic `foo_topic` and also respond with 'HELLO WORLD'.\n\nFor more information please read xref:guides:sync_responses.adoc[synchronous responses].","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","default":{}}},{"name":"timeplus","type":"output","status":"experimental","plugin":true,"summary":"Sends message to a Timeplus Enterprise stream via ingest endpoint","description":"\nThis output can send message to Timeplus Enterprise Cloud, Timeplus Enterprise (self-hosted) or directly to timeplusd.\n\nThis output accepts structured message only. It also expects all message contains the same keys and matches the schema of the destination stream. If the upstream source or pipeline returns\nunstructured message such as string, please refer to the \"Unstructured message\" example.","categories":["Services"],"examples":[{"title":"To Timeplus Enterprise Cloud","summary":"You will need to create API Key on Timeplus Enterprise Cloud Web console first and then set the `apikey` field.","config":"\noutput:\n timeplus:\n workspace: my_workspace_id\n stream: mystream\n apikey: \u003cYour API Key\u003e"},{"title":"To Timeplus Enterprise (self-hosted)","summary":"For self-housted Timeplus Enterprise, you will need to specify the username and password as well as the URL of the App server","config":"\noutput:\n timeplus:\n url: http://localhost:8000\n workspace: my_workspace_id\n stream: mystream\n username: username\n password: pw"},{"title":"To Timeplusd","summary":"This output writes to Timeplusd via HTTP so make sure you specify the HTTP port of the Timeplusd.","config":"\noutput:\n timeplus:\n url: http://localhost:3218\n stream: mystream\n username: username\n password: pw"},{"title":"Unstructured message","summary":"If the upstream source or pipeline returns unstructured message such as string, you can leverage the output processors to wrap it into a stucture message and then pass it to the output. This example create a strcutre mesasge with `raw` field and store the original string content into this field. You can modify the name of this `raw` field to whatever you want. Please make sure the destiation stream contains such field","config":"\noutput:\n timeplus:\n workspace: my_workspace_id\n stream: mystream\n apikey: \u003cApi key genereated on web console\u003e\n\n processors:\n - mapping: |\n root = {}\n root.raw = content().string()"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"target","type":"string","kind":"scalar","description":"The destination type, either Timeplus Enterprise or timeplusd","default":"timeplus","options":["timeplus","timeplusd"],"linter":"\nlet options = {\n \"timeplus\": true,\n \"timeplusd\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"url","type":"string","kind":"scalar","description":"The url should always include schema and host.","default":"https://us-west-2.timeplus.cloud","examples":["http://localhost:8000","http://127.0.0.1:3218"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"workspace","type":"string","kind":"scalar","description":"ID of the workspace. Required if target is `timeplus`.","is_optional":true},{"name":"stream","type":"string","kind":"scalar","description":"The name of the stream. Make sure the schema of the stream matches the input"},{"name":"apikey","type":"string","kind":"scalar","description":"The API key. Required if you are sending message to Timeplus Enterprise Cloud","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"username","type":"string","kind":"scalar","description":"The username. Required if you are sending message to Timeplus Enterprise (self-hosted) or timeplusd","is_optional":true},{"name":"password","type":"string","kind":"scalar","description":"The password. Required if you are sending message to Timeplus Enterprise (self-hosted) or timeplusd","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]}},{"name":"websocket","type":"output","status":"stable","plugin":true,"summary":"Sends messages to an HTTP server via a websocket connection.","categories":["Network"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL to connect to.","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"proxy_url","type":"string","kind":"scalar","description":"An optional HTTP proxy URL.","is_advanced":true,"is_optional":true,"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"oauth","type":"object","kind":"scalar","description":"Allows you to specify open authentication via OAuth version 1.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use OAuth version 1 in requests.","is_advanced":true,"default":false},{"name":"consumer_key","type":"string","kind":"scalar","description":"A value used to identify the client to the service provider.","is_advanced":true,"default":""},{"name":"consumer_secret","type":"string","kind":"scalar","description":"A secret used to establish ownership of the consumer key.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"access_token","type":"string","kind":"scalar","description":"A value used to gain access to the protected resources on behalf of the user.","is_advanced":true,"default":""},{"name":"access_token_secret","type":"string","kind":"scalar","description":"A secret provided in order to establish ownership of a given access token.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"jwt","type":"object","kind":"scalar","description":"BETA: Allows you to specify JWT authentication.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use JWT authentication in requests.","is_advanced":true,"default":false},{"name":"private_key_file","type":"string","kind":"scalar","description":"A file with the PEM encoded via PKCS1 or PKCS8 as private key.","is_advanced":true,"default":""},{"name":"signing_method","type":"string","kind":"scalar","description":"A method used to sign the token such as RS256, RS384, RS512 or EdDSA.","is_advanced":true,"default":""},{"name":"claims","type":"unknown","kind":"map","description":"A value used to identify the claims that issued the JWT.","is_advanced":true,"default":{}},{"name":"headers","type":"unknown","kind":"map","description":"Add optional key/value headers to the JWT.","is_advanced":true,"default":{}}]}]}}],"processors":[{"name":"archive","type":"processor","status":"stable","plugin":true,"summary":"Archives all the messages of a batch into a single message according to the selected archive format.","description":"\nSome archive formats (such as tar, zip) treat each archive item (message part) as a file with a path. Since message parts only contain raw data a unique path must be generated for each part. This can be done by using function interpolations on the 'path' field as described in xref:configuration:interpolation.adoc#bloblang-queries[Bloblang queries]. For types that aren't file based (such as binary) the file field is ignored.\n\nThe resulting archived message adopts the metadata of the _first_ message part of the batch.\n\nThe functionality of this processor depends on being applied across messages that are batched. You can find out more about batching xref:configuration:batching.adoc[in this doc].","categories":["Parsing","Utility"],"examples":[{"title":"Tar Archive","summary":"\nIf we had JSON messages in a batch each of the form:\n\n```json\n{\"doc\":{\"id\":\"foo\",\"body\":\"hello world 1\"}}\n```\n\nAnd we wished to tar archive them, setting their filenames to their respective unique IDs (with the extension `.json`), our config might look like\nthis:","config":"\npipeline:\n processors:\n - archive:\n format: tar\n path: ${!json(\"doc.id\")}.json\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"format","type":"string","kind":"scalar","description":"The archiving format to apply.","annotated_options":[["binary","Archive messages to a https://github.com/redpanda-data/benthos/blob/main/internal/message/message.go#L96[binary blob format^]."],["concatenate","Join the raw contents of each message into a single binary message."],["json_array","Attempt to parse each message as a JSON document and append the result to an array, which becomes the contents of the resulting message."],["lines","Join the raw contents of each message and insert a line break between each one."],["tar","Archive messages to a unix standard tape archive."],["zip","Archive messages to a zip file."]],"linter":"\nlet options = {\n \"binary\": true,\n \"concatenate\": true,\n \"json_array\": true,\n \"lines\": true,\n \"tar\": true,\n \"zip\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"path","type":"string","kind":"scalar","description":"The path to set for each message in the archive (when applicable).","default":"","interpolated":true,"examples":["${!count(\"files\")}-${!timestamp_unix_nano()}.txt","${!meta(\"kafka_key\")}-${!json(\"id\")}.json"]}]}},{"name":"avro","type":"processor","status":"beta","plugin":true,"summary":"Performs Avro based operations on messages based on a schema.","description":"\nWARNING: If you are consuming or generating messages using a schema registry service then it is likely this processor will fail as those services require messages to be prefixed with the identifier of the schema version being used. Instead, try the xref:components:processors/schema_registry_encode.adoc[`schema_registry_encode`] and xref:components:processors/schema_registry_decode.adoc[`schema_registry_decode`] processors.\n\n== Operators\n\n=== `to_json`\n\nConverts Avro documents into a JSON structure. This makes it easier to\nmanipulate the contents of the document within Benthos. The encoding field\nspecifies how the source documents are encoded.\n\n=== `from_json`\n\nAttempts to convert JSON documents into Avro documents according to the\nspecified encoding.","categories":["Parsing"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"operator","type":"string","kind":"scalar","description":"The \u003c\u003coperators, operator\u003e\u003e to execute","options":["to_json","from_json"],"linter":"\nlet options = {\n \"to_json\": true,\n \"from_json\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"encoding","type":"string","kind":"scalar","description":"An Avro encoding format to use for conversions to and from a schema.","default":"textual","options":["textual","binary","single"],"linter":"\nlet options = {\n \"textual\": true,\n \"binary\": true,\n \"single\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"schema","type":"string","kind":"scalar","description":"A full Avro schema to use.","default":""},{"name":"schema_path","type":"string","kind":"scalar","description":"The path of a schema document to apply. Use either this or the `schema` field.","default":"","examples":["file://path/to/spec.avsc","http://localhost:8081/path/to/spec/versions/1"]}]}},{"name":"awk","type":"processor","status":"stable","plugin":true,"summary":"Executes an AWK program on messages. This processor is very powerful as it offers a range of \u003c\u003cawk-functions,custom functions\u003e\u003e for querying and mutating message contents and metadata.","description":"\nWorks by feeding message contents as the program input based on a chosen \u003c\u003ccodecs,codec\u003e\u003e and replaces the contents of each message with the result. If the result is empty (nothing is printed by the program) then the original message contents remain unchanged.\n\nComes with a wide range of \u003c\u003cawk-functions,custom functions\u003e\u003e for accessing message metadata, json fields, printing logs, etc. These functions can be overridden by functions within the program.\n\nCheck out the \u003c\u003cexamples,examples section\u003e\u003e in order to see how this processor can be used.\n\nThis processor uses https://github.com/benhoyt/goawk[GoAWK^], in order to understand the differences in how the program works you can read more about it in https://github.com/benhoyt/goawk#differences-from-awk[goawk.differences^].","categories":["Mapping"],"footnotes":"\n== Codecs\n\nThe chosen codec determines how the contents of the message are fed into the\nprogram. Codecs only impact the input string and variables initialized for your\nprogram, they do not change the range of custom functions available.\n\n=== `none`\n\nAn empty string is fed into the program. Functions can still be used in order to\nextract and mutate metadata and message contents.\n\nThis is useful for when your program only uses functions and doesn't need the\nfull text of the message to be parsed by the program, as it is significantly\nfaster.\n\n=== `text`\n\nThe full contents of the message are fed into the program as a string, allowing\nyou to reference tokenized segments of the message with variables ($0, $1, etc).\nCustom functions can still be used with this codec.\n\nThis is the default codec as it behaves most similar to typical usage of the awk\ncommand line tool.\n\n=== `json`\n\nAn empty string is fed into the program, and variables are automatically\ninitialized before execution of your program by walking the flattened JSON\nstructure. Each value is converted into a variable by taking its full path,\ne.g. the object:\n\n```json\n{\n\t\"foo\": {\n\t\t\"bar\": {\n\t\t\t\"value\": 10\n\t\t},\n\t\t\"created_at\": \"2018-12-18T11:57:32\"\n\t}\n}\n```\n\nWould result in the following variable declarations:\n\n```\nfoo_bar_value = 10\nfoo_created_at = \"2018-12-18T11:57:32\"\n```\n\nCustom functions can also still be used with this codec.\n\n== AWK functions\n\n=== `json_get`\n\nSignature: `json_get(path)`\n\nAttempts to find a JSON value in the input message payload by a\nxref:configuration:field_paths.adoc[dot separated path] and returns it as a string.\n\n=== `json_set`\n\nSignature: `json_set(path, value)`\n\nAttempts to set a JSON value in the input message payload identified by a\nxref:configuration:field_paths.adoc[dot separated path], the value argument will be interpreted\nas a string.\n\nIn order to set non-string values use one of the following typed varieties:\n\n- `json_set_int(path, value)`\n- `json_set_float(path, value)`\n- `json_set_bool(path, value)`\n\n=== `json_append`\n\nSignature: `json_append(path, value)`\n\nAttempts to append a value to an array identified by a\nxref:configuration:field_paths.adoc[dot separated path]. If the target does not\nexist it will be created. If the target exists but is not already an array then\nit will be converted into one, with its original contents set to the first\nelement of the array.\n\nThe value argument will be interpreted as a string. In order to append\nnon-string values use one of the following typed varieties:\n\n- `json_append_int(path, value)`\n- `json_append_float(path, value)`\n- `json_append_bool(path, value)`\n\n=== `json_delete`\n\nSignature: `json_delete(path)`\n\nAttempts to delete a JSON field from the input message payload identified by a\nxref:configuration:field_paths.adoc[dot separated path].\n\n=== `json_length`\n\nSignature: `json_length(path)`\n\nReturns the size of the string or array value of JSON field from the input\nmessage payload identified by a xref:configuration:field_paths.adoc[dot separated path].\n\nIf the target field does not exist, or is not a string or array type, then zero\nis returned. In order to explicitly check the type of a field use `json_type`.\n\n=== `json_type`\n\nSignature: `json_type(path)`\n\nReturns the type of a JSON field from the input message payload identified by a\nxref:configuration:field_paths.adoc[dot separated path].\n\nPossible values are: \"string\", \"int\", \"float\", \"bool\", \"undefined\", \"null\",\n\"array\", \"object\".\n\n=== `create_json_object`\n\nSignature: `create_json_object(key1, val1, key2, val2, ...)`\n\nGenerates a valid JSON object of key value pair arguments. The arguments are\nvariadic, meaning any number of pairs can be listed. The value will always\nresolve to a string regardless of the value type. E.g. the following call:\n\n`create_json_object(\"a\", \"1\", \"b\", 2, \"c\", \"3\")`\n\nWould result in this string:\n\n`\\{\"a\":\"1\",\"b\":\"2\",\"c\":\"3\"}`\n\n=== `create_json_array`\n\nSignature: `create_json_array(val1, val2, ...)`\n\nGenerates a valid JSON array of value arguments. The arguments are variadic,\nmeaning any number of values can be listed. The value will always resolve to a\nstring regardless of the value type. E.g. the following call:\n\n`create_json_array(\"1\", 2, \"3\")`\n\nWould result in this string:\n\n`[\"1\",\"2\",\"3\"]`\n\n=== `metadata_set`\n\nSignature: `metadata_set(key, value)`\n\nSet a metadata key for the message to a value. The value will always resolve to\na string regardless of the value type.\n\n=== `metadata_get`\n\nSignature: `metadata_get(key) string`\n\nGet the value of a metadata key from the message.\n\n=== `timestamp_unix`\n\nSignature: `timestamp_unix() int`\n\nReturns the current unix timestamp (the number of seconds since 01-01-1970).\n\n=== `timestamp_unix`\n\nSignature: `timestamp_unix(date) int`\n\nAttempts to parse a date string by detecting its format and returns the\nequivalent unix timestamp (the number of seconds since 01-01-1970).\n\n=== `timestamp_unix`\n\nSignature: `timestamp_unix(date, format) int`\n\nAttempts to parse a date string according to a format and returns the equivalent\nunix timestamp (the number of seconds since 01-01-1970).\n\nThe format is defined by showing how the reference time, defined to be\n`Mon Jan 2 15:04:05 -0700 MST 2006` would be displayed if it were the value.\n\n=== `timestamp_unix_nano`\n\nSignature: `timestamp_unix_nano() int`\n\nReturns the current unix timestamp in nanoseconds (the number of nanoseconds\nsince 01-01-1970).\n\n=== `timestamp_unix_nano`\n\nSignature: `timestamp_unix_nano(date) int`\n\nAttempts to parse a date string by detecting its format and returns the\nequivalent unix timestamp in nanoseconds (the number of nanoseconds since\n01-01-1970).\n\n=== `timestamp_unix_nano`\n\nSignature: `timestamp_unix_nano(date, format) int`\n\nAttempts to parse a date string according to a format and returns the equivalent\nunix timestamp in nanoseconds (the number of nanoseconds since 01-01-1970).\n\nThe format is defined by showing how the reference time, defined to be\n`Mon Jan 2 15:04:05 -0700 MST 2006` would be displayed if it were the value.\n\n=== `timestamp_format`\n\nSignature: `timestamp_format(unix, format) string`\n\nFormats a unix timestamp. The format is defined by showing how the reference\ntime, defined to be `Mon Jan 2 15:04:05 -0700 MST 2006` would be displayed if it\nwere the value.\n\nThe format is optional, and if omitted RFC3339 (`2006-01-02T15:04:05Z07:00`)\nwill be used.\n\n=== `timestamp_format_nano`\n\nSignature: `timestamp_format_nano(unixNano, format) string`\n\nFormats a unix timestamp in nanoseconds. The format is defined by showing how\nthe reference time, defined to be `Mon Jan 2 15:04:05 -0700 MST 2006` would be\ndisplayed if it were the value.\n\nThe format is optional, and if omitted RFC3339 (`2006-01-02T15:04:05Z07:00`)\nwill be used.\n\n=== `print_log`\n\nSignature: `print_log(message, level)`\n\nPrints a Redpanda Connect log message at a particular log level. The log level is\noptional, and if omitted the level `INFO` will be used.\n\n=== `base64_encode`\n\nSignature: `base64_encode(data)`\n\nEncodes the input data to a base64 string.\n\n=== `base64_decode`\n\nSignature: `base64_decode(data)`\n\nAttempts to base64-decode the input data and returns the decoded string if\nsuccessful. It will emit an error otherwise.\n\n","examples":[{"title":"JSON Mapping and Arithmetic","summary":"\nBecause AWK is a full programming language it's much easier to map documents and perform arithmetic with it than with other Redpanda Connect processors. For example, if we were expecting documents of the form:\n\n```json\n{\"doc\":{\"val1\":5,\"val2\":10},\"id\":\"1\",\"type\":\"add\"}\n{\"doc\":{\"val1\":5,\"val2\":10},\"id\":\"2\",\"type\":\"multiply\"}\n```\n\nAnd we wished to perform the arithmetic specified in the `type` field,\non the values `val1` and `val2` and, finally, map the result into the\ndocument, giving us the following resulting documents:\n\n```json\n{\"doc\":{\"result\":15,\"val1\":5,\"val2\":10},\"id\":\"1\",\"type\":\"add\"}\n{\"doc\":{\"result\":50,\"val1\":5,\"val2\":10},\"id\":\"2\",\"type\":\"multiply\"}\n```\n\nWe can do that with the following:","config":"\npipeline:\n processors:\n - awk:\n codec: none\n program: |\n function map_add_vals() {\n json_set_int(\"doc.result\", json_get(\"doc.val1\") + json_get(\"doc.val2\"));\n }\n function map_multiply_vals() {\n json_set_int(\"doc.result\", json_get(\"doc.val1\") * json_get(\"doc.val2\"));\n }\n function map_unknown(type) {\n json_set(\"error\",\"unknown document type\");\n print_log(\"Document type not recognised: \" type, \"ERROR\");\n }\n {\n type = json_get(\"type\");\n if (type == \"add\")\n map_add_vals();\n else if (type == \"multiply\")\n map_multiply_vals();\n else\n map_unknown(type);\n }\n"},{"title":"Stuff With Arrays","summary":"\nIt's possible to iterate JSON arrays by appending an index value to the path, this can be used to do things like removing duplicates from arrays. For example, given the following input document:\n\n```json\n{\"path\":{\"to\":{\"foos\":[\"one\",\"two\",\"three\",\"two\",\"four\"]}}}\n```\n\nWe could create a new array `foos_unique` from `foos` giving us the result:\n\n```json\n{\"path\":{\"to\":{\"foos\":[\"one\",\"two\",\"three\",\"two\",\"four\"],\"foos_unique\":[\"one\",\"two\",\"three\",\"four\"]}}}\n```\n\nWith the following config:","config":"\npipeline:\n processors:\n - awk:\n codec: none\n program: |\n {\n array_path = \"path.to.foos\"\n array_len = json_length(array_path)\n\n for (i = 0; i \u003c array_len; i++) {\n ele = json_get(array_path \".\" i)\n if ( ! ( ele in seen ) ) {\n json_append(array_path \"_unique\", ele)\n seen[ele] = 1\n }\n }\n }\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"codec","type":"string","kind":"scalar","description":"A \u003c\u003ccodecs,codec\u003e\u003e defines how messages should be inserted into the AWK program as variables. The codec does not change which \u003c\u003cawk-functions,custom Redpanda Connect functions\u003e\u003e are available. The `text` codec is the closest to a typical AWK use case.","options":["none","text","json"],"linter":"\nlet options = {\n \"none\": true,\n \"text\": true,\n \"json\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"program","type":"string","kind":"scalar","description":"An AWK program to execute"}]}},{"name":"aws_bedrock_chat","type":"processor","status":"experimental","plugin":true,"summary":"Generates responses to messages in a chat conversation, using the AWS Bedrock API.","description":"This processor sends prompts to your chosen large language model (LLM) and generates text from the responses, using the AWS Bedrock API.\nFor more information, see the https://docs.aws.amazon.com/bedrock/latest/userguide[AWS Bedrock documentation^].","categories":["AI"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]},{"name":"model","type":"string","kind":"scalar","description":"The model ID to use. For a full list see the https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids.html[AWS Bedrock documentation^].","examples":["amazon.titan-text-express-v1","anthropic.claude-3-5-sonnet-20240620-v1:0","cohere.command-text-v14","meta.llama3-1-70b-instruct-v1:0","mistral.mistral-large-2402-v1:0"]},{"name":"prompt","type":"string","kind":"scalar","description":"The prompt you want to generate a response for. By default, the processor submits the entire payload as a string.","is_optional":true},{"name":"system_prompt","type":"string","kind":"scalar","description":"The system prompt to submit to the AWS Bedrock LLM.","is_optional":true},{"name":"max_tokens","type":"int","kind":"scalar","description":"The maximum number of tokens to allow in the generated response.","is_optional":true},{"name":"temperature","type":"float","kind":"scalar","description":"The likelihood of the model selecting higher-probability options while generating a response. A lower value makes the model omre likely to choose higher-probability options, while a higher value makes the model more likely to choose lower-probability options.","is_optional":true,"linter":"root = if this \u003c 0 || this \u003e 1 { [\"field must be between 0.0-1.0\"] }"},{"name":"stop","type":"string","kind":"array","description":"A list of stop sequences. A stop sequence is a sequence of characters that causes the model to stop generating the response.","is_advanced":true,"is_optional":true},{"name":"top_p","type":"float","kind":"scalar","description":"The percentage of most-likely candidates that the model considers for the next token. For example, if you choose a value of 0.8, the model selects from the top 80% of the probability distribution of tokens that could be next in the sequence. ","is_advanced":true,"is_optional":true,"linter":"root = if this \u003c 0 || this \u003e 1 { [\"field must be between 0.0-1.0\"] }"}]},"version":"4.34.0"},{"name":"aws_bedrock_embeddings","type":"processor","status":"experimental","plugin":true,"summary":"Computes vector embeddings on text, using the AWS Bedrock API.","description":"This processor sends text to your chosen large language model (LLM) and computes vector embeddings, using the AWS Bedrock API.\nFor more information, see the https://docs.aws.amazon.com/bedrock/latest/userguide[AWS Bedrock documentation^].","categories":["AI"],"examples":[{"title":"Store embedding vectors in Clickhouse","summary":"Compute embeddings for some generated data and store it within https://clickhouse.com/[Clickhouse^]","config":"input:\n generate:\n interval: 1s\n mapping: |\n root = {\"text\": fake(\"paragraph\")}\npipeline:\n processors:\n - branch:\n request_map: |\n root = this.text\n processors:\n - aws_bedrock_embeddings:\n model: amazon.titan-embed-text-v1\n result_map: |\n root.embeddings = this\noutput:\n sql_insert:\n driver: clickhouse\n dsn: \"clickhouse://localhost:9000\"\n table: searchable_text\n columns: [\"id\", \"text\", \"vector\"]\n args_mapping: \"root = [uuid_v4(), this.text, this.embeddings]\"\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]},{"name":"model","type":"string","kind":"scalar","description":"The model ID to use. For a full list see the https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids.html[AWS Bedrock documentation^].","examples":["amazon.titan-embed-text-v1","amazon.titan-embed-text-v2:0","cohere.embed-english-v3","cohere.embed-multilingual-v3"]},{"name":"text","type":"string","kind":"scalar","description":"The prompt you want to generate a response for. By default, the processor submits the entire payload as a string.","is_optional":true}]},"version":"4.37.0"},{"name":"aws_dynamodb_partiql","type":"processor","status":"experimental","plugin":true,"summary":"Executes a PartiQL expression against a DynamoDB table for each message.","description":"Both writes or reads are supported, when the query is a read the contents of the message will be replaced with the result. This processor is more efficient when messages are pre-batched as the whole batch will be executed in a single call.","categories":["Integration"],"examples":[{"title":"Insert","summary":"The following example inserts rows into the table footable with the columns foo, bar and baz populated with values extracted from messages:","config":"\npipeline:\n processors:\n - aws_dynamodb_partiql:\n query: \"INSERT INTO footable VALUE {'foo':'?','bar':'?','baz':'?'}\"\n args_mapping: |\n root = [\n { \"S\": this.foo },\n { \"S\": meta(\"kafka_topic\") },\n { \"S\": this.document.content },\n ]\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"query","type":"string","kind":"scalar","description":"A PartiQL query to execute for each message."},{"name":"unsafe_dynamic_query","type":"bool","kind":"scalar","description":"Whether to enable dynamic queries that support interpolation functions.","is_advanced":true,"default":false},{"name":"args_mapping","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] that, for each message, creates a list of arguments to use with the query.","default":"","bloblang":true},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]},"version":"3.48.0"},{"name":"aws_lambda","type":"processor","status":"stable","plugin":true,"summary":"Invokes an AWS lambda for each message. The contents of the message is the payload of the request, and the result of the invocation will become the new contents of the message.","description":"The `rate_limit` field can be used to specify a rate limit xref:components:rate_limits/about.adoc[resource] to cap the rate of requests across parallel components service wide.\n\nIn order to map or encode the payload to a specific request body, and map the response back into the original payload instead of replacing it entirely, you can use the xref:components:processors/branch.adoc[`branch` processor].\n\n== Error handling\n\nWhen Redpanda Connect is unable to connect to the AWS endpoint or is otherwise unable to invoke the target lambda function it will retry the request according to the configured number of retries. Once these attempts have been exhausted the failed message will continue through the pipeline with it's contents unchanged, but flagged as having failed, allowing you to use xref:configuration:error_handling.adoc[standard processor error handling patterns].\n\nHowever, if the invocation of the function is successful but the function itself throws an error, then the message will have it's contents updated with a JSON payload describing the reason for the failure, and a metadata field `lambda_function_error` will be added to the message allowing you to detect and handle function errors with a xref:components:processors/branch.adoc[`branch`]:\n\n```yaml\npipeline:\n processors:\n - branch:\n processors:\n - aws_lambda:\n function: foo\n result_map: |\n root = if meta().exists(\"lambda_function_error\") {\n throw(\"Invocation failed due to %v: %v\".format(this.errorType, this.errorMessage))\n } else {\n this\n }\noutput:\n switch:\n retry_until_success: false\n cases:\n - check: errored()\n output:\n reject: ${! error() }\n - output:\n resource: somewhere_else\n```\n\n== Credentials\n\nBy default Redpanda Connect will use a shared credentials file when connecting to AWS services. It's also possible to set them explicitly at the component level, allowing you to transfer data across accounts. You can find out more in xref:guides:cloud/aws.adoc[].","categories":["Integration"],"examples":[{"title":"Branched Invoke","summary":"\nThis example uses a xref:components:processors/branch.adoc[`branch` processor] to map a new payload for triggering a lambda function with an ID and username from the original message, and the result of the lambda is discarded, meaning the original message is unchanged.","config":"\npipeline:\n processors:\n - branch:\n request_map: '{\"id\":this.doc.id,\"username\":this.user.name}'\n processors:\n - aws_lambda:\n function: trigger_user_update\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"parallel","type":"bool","kind":"scalar","description":"Whether messages of a batch should be dispatched in parallel.","default":false},{"name":"function","type":"string","kind":"scalar","description":"The function to invoke."},{"name":"rate_limit","type":"string","kind":"scalar","description":"An optional xref:components:rate_limits/about.adoc[`rate_limit`] to throttle invocations by.","is_advanced":true,"default":""},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period of time to wait before abandoning an invocation.","is_advanced":true,"default":"5s"},{"name":"retries","type":"int","kind":"scalar","description":"The maximum number of retry attempts for each message.","is_advanced":true,"default":3}]},"version":"3.36.0"},{"name":"azure_cosmosdb","type":"processor","status":"experimental","plugin":true,"summary":"Creates or updates messages as JSON documents in https://learn.microsoft.com/en-us/azure/cosmos-db/introduction[Azure CosmosDB^].","description":"\nWhen creating documents, each message must have the `id` property (case-sensitive) set (or use `auto_id: true`). It is the unique name that identifies the document, that is, no two documents share the same `id` within a logical partition. The `id` field must not exceed 255 characters. https://learn.microsoft.com/en-us/rest/api/cosmos-db/documents[See details^].\n\nThe `partition_keys` field must resolve to the same value(s) across the entire message batch.\n\n\n== Credentials\n\nYou can use one of the following authentication mechanisms:\n\n- Set the `endpoint` field and the `account_key` field\n- Set only the `endpoint` field to use https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity#DefaultAzureCredential[DefaultAzureCredential^]\n- Set the `connection_string` field\n\n\n== Metadata\n\nThis component adds the following metadata fields to each message:\n```\n- activity_id\n- request_charge\n```\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].\n\n\n== Batching\n\nCosmosDB limits the maximum batch size to 100 messages and the payload must not exceed 2MB (https://learn.microsoft.com/en-us/azure/cosmos-db/concepts-limits#per-request-limits[details here^]).\n","categories":["Azure"],"footnotes":"\n\n== CosmosDB emulator\n\nIf you wish to run the CosmosDB emulator that is referenced in the documentation https://learn.microsoft.com/en-us/azure/cosmos-db/linux-emulator[here^], the following Docker command should do the trick:\n\n```bash\n\u003e docker run --rm -it -p 8081:8081 --name=cosmosdb -e AZURE_COSMOS_EMULATOR_PARTITION_COUNT=10 -e AZURE_COSMOS_EMULATOR_ENABLE_DATA_PERSISTENCE=false mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator\n```\n\nNote: `AZURE_COSMOS_EMULATOR_PARTITION_COUNT` controls the number of partitions that will be supported by the emulator. The bigger the value, the longer it takes for the container to start up.\n\nAdditionally, instead of installing the container self-signed certificate which is exposed via `https://localhost:8081/_explorer/emulator.pem`, you can run https://mitmproxy.org/[mitmproxy^] like so:\n\n```bash\n\u003e mitmproxy -k --mode \"reverse:https://localhost:8081\"\n```\n\nThen you can access the CosmosDB UI via `http://localhost:8080/_explorer/index.html` and use `http://localhost:8080` as the CosmosDB endpoint.\n","examples":[{"title":"Patch documents","summary":"Query documents from a container and patch them.","config":"\ninput:\n azure_cosmosdb:\n endpoint: http://localhost:8080\n account_key: C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==\n database: blobbase\n container: blobfish\n partition_keys_map: root = \"AbyssalPlain\"\n query: SELECT * FROM blobfish\n\n processors:\n - mapping: |\n root = \"\"\n meta habitat = json(\"habitat\")\n meta id = this.id\n - azure_cosmosdb:\n endpoint: http://localhost:8080\n account_key: C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==\n database: testdb\n container: blobfish\n partition_keys_map: root = json(\"habitat\")\n item_id: ${! meta(\"id\") }\n operation: Patch\n patch_operations:\n # Add a new /diet field\n - operation: Add\n path: /diet\n value_map: root = json(\"diet\")\n # Remove the first location from the /locations array field\n - operation: Remove\n path: /locations/0\n # Add new location at the end of the /locations array field\n - operation: Add\n path: /locations/-\n value_map: root = \"Challenger Deep\"\n # Return the updated document\n enable_content_response_on_write: true\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"endpoint","type":"string","kind":"scalar","description":"CosmosDB endpoint.","is_optional":true,"examples":["https://localhost:8081"]},{"name":"account_key","type":"string","kind":"scalar","description":"Account key.","is_optional":true,"is_secret":true,"examples":["C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw=="],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"connection_string","type":"string","kind":"scalar","description":"Connection string.","is_optional":true,"is_secret":true,"examples":["AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==;"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"database","type":"string","kind":"scalar","description":"Database.","examples":["testdb"]},{"name":"container","type":"string","kind":"scalar","description":"Container.","examples":["testcontainer"]},{"name":"partition_keys_map","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to a single partition key value or an array of partition key values of type string, integer or boolean. Currently, hierarchical partition keys are not supported so only one value may be provided.","bloblang":true,"examples":["root = \"blobfish\"","root = 41","root = true","root = null","root = json(\"blobfish\").depth"]},{"name":"operation","type":"string","kind":"scalar","description":"Operation.","default":"Create","annotated_options":[["Create","Create operation."],["Delete","Delete operation."],["Patch","Patch operation."],["Read","Read operation."],["Replace","Replace operation."],["Upsert","Upsert operation."]],"linter":"\nlet options = {\n \"create\": true,\n \"delete\": true,\n \"patch\": true,\n \"read\": true,\n \"replace\": true,\n \"upsert\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"patch_operations","type":"object","kind":"array","description":"Patch operations to be performed when `operation: Patch` .","is_advanced":true,"is_optional":true,"children":[{"name":"operation","type":"string","kind":"scalar","description":"Operation.","is_advanced":true,"default":"Add","annotated_options":[["Add","Add patch operation."],["Increment","Increment patch operation."],["Remove","Remove patch operation."],["Replace","Replace patch operation."],["Set","Set patch operation."]],"linter":"\nlet options = {\n \"add\": true,\n \"increment\": true,\n \"remove\": true,\n \"replace\": true,\n \"set\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"path","type":"string","kind":"scalar","description":"Path.","is_advanced":true,"examples":["/foo/bar/baz"]},{"name":"value_map","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to a value of any type that is supported by CosmosDB.","is_advanced":true,"is_optional":true,"bloblang":true,"examples":["root = \"blobfish\"","root = 41","root = true","root = json(\"blobfish\").depth","root = [1, 2, 3]"]}]},{"name":"patch_condition","type":"string","kind":"scalar","description":"Patch operation condition.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["from c where not is_defined(c.blobfish)"]},{"name":"auto_id","type":"bool","kind":"scalar","description":"Automatically set the item `id` field to a random UUID v4. If the `id` field is already set, then it will not be overwritten. Setting this to `false` can improve performance, since the messages will not have to be parsed.","is_advanced":true,"default":true},{"name":"item_id","type":"string","kind":"scalar","description":"ID of item to replace or delete. Only used by the Replace and Delete operations","is_optional":true,"interpolated":true,"examples":["${! json(\"id\") }"]},{"name":"enable_content_response_on_write","type":"bool","kind":"scalar","description":"Enable content response on write operations. To save some bandwidth, set this to false if you don't need to receive the updated message(s) from the server, in which case the processor will not modify the content of the messages which are fed into it. Applies to every operation except Read.","is_advanced":true,"default":true}],"linter":"root = []\nlet hasEndpoint = this.endpoint.or(\"\") != \"\"\nlet hasConnectionString = this.connection_string.or(\"\") != \"\"\n\nroot.\"-\" = if !$hasEndpoint \u0026\u0026 !$hasConnectionString {\n \"Either `endpoint` or `connection_string` must be set.\"\n}\n\nlet hasItemID = this.item_id.or(\"\") != \"\"\nlet hasPatchOperations = this.patch_operations.length().or(0) \u003e 0\nlet hasPatchCondition = this.patch_condition.or(\"\") != \"\"\n\nroot.\"-\" = if !$hasItemID \u0026\u0026 (this.operation == \"Replace\" || this.operation == \"Delete\" || this.operation == \"Read\" || this.operation == \"Patch\") {\n \"The `item_id` field must be set for Replace, Delete, Read and Patch operations.\"\n}\n\nroot.\"-\" = if this.operation == \"Patch\" \u0026\u0026 !$hasPatchOperations {\n \"At least one `patch_operations` must be set when `operation: Patch`.\"\n}\n\nroot.\"-\" = if $hasPatchCondition \u0026\u0026 (!$hasPatchOperations || this.operation != \"Patch\") {\n \"The `patch_condition` field only applies to `Patch` operations and it requires one or more `patch_operations`.\"\n}\n\nroot.\"-\" = if this.operation == \"Patch\" \u0026\u0026 this.patch_operations.any(o -\u003e o.operation != \"Remove\" \u0026\u0026 o.value_map.or(\"\") == \"\") {\n \"The `patch_operations` `value_map` field must be set when `operation` is not `Remove`.\"\n}\n\nroot.\"-\" = if this.operation == \"Patch\" \u0026\u0026 this.patch_operations.any(o -\u003e o.operation == \"Remove\" \u0026\u0026 o.value_map.or(\"\") != \"\") {\n \"The `patch_operations` `value_map` field must not be set when `operation` is `Remove`.\"\n}\n"},"version":"v4.25.0"},{"name":"benchmark","type":"processor","status":"experimental","plugin":true,"summary":"Logs basic throughput statistics of messages that pass through this processor.","description":"Logs messages per second and bytes per second of messages that are processed at a regular interval. A summary of the amount of messages processed over the entire lifetime of the processor will also be printed when the processor shuts down.","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"interval","type":"string","kind":"scalar","description":"How often to emit rolling statistics. If set to 0, only a summary will be logged when the processor shuts down.","default":"5s"},{"name":"count_bytes","type":"bool","kind":"scalar","description":"Whether or not to measure the number of bytes per second of throughput. Counting the number of bytes requires serializing structured data, which can cause an unnecessary performance hit if serialization is not required elsewhere in the pipeline.","default":true}]}},{"name":"bloblang","type":"processor","status":"stable","plugin":true,"summary":"Executes a xref:guides:bloblang/about.adoc[Bloblang] mapping on messages.","description":"\nBloblang is a powerful language that enables a wide range of mapping, transformation and filtering tasks. For more information see xref:guides:bloblang/about.adoc[].\n\nIf your mapping is large and you'd prefer for it to live in a separate file then you can execute a mapping directly from a file with the expression `from \"\u003cpath\u003e\"`, where the path must be absolute, or relative from the location that Redpanda Connect is executed from.\n\n== Component rename\n\nThis processor was recently renamed to the xref:components:processors/mapping.adoc[`mapping` processor] in order to make the purpose of the processor more prominent. It is still valid to use the existing `bloblang` name but eventually it will be deprecated and replaced by the new name in example configs.","categories":["Mapping","Parsing"],"footnotes":"\n== Error handling\n\nBloblang mappings can fail, in which case the message remains unchanged, errors are logged, and the message is flagged as having failed, allowing you to use\nxref:configuration:error_handling.adoc[standard processor error handling patterns].\n\nHowever, Bloblang itself also provides powerful ways of ensuring your mappings do not fail by specifying desired fallback behavior, which you can read about in xref:guides:bloblang/about#error-handling.adoc[Error handling].","examples":[{"title":"Mapping","summary":"\nGiven JSON documents containing an array of fans:\n\n```json\n{\n \"id\":\"foo\",\n \"description\":\"a show about foo\",\n \"fans\":[\n {\"name\":\"bev\",\"obsession\":0.57},\n {\"name\":\"grace\",\"obsession\":0.21},\n {\"name\":\"ali\",\"obsession\":0.89},\n {\"name\":\"vic\",\"obsession\":0.43}\n ]\n}\n```\n\nWe can reduce the fans to only those with an obsession score above 0.5, giving us:\n\n```json\n{\n \"id\":\"foo\",\n \"description\":\"a show about foo\",\n \"fans\":[\n {\"name\":\"bev\",\"obsession\":0.57},\n {\"name\":\"ali\",\"obsession\":0.89}\n ]\n}\n```\n\nWith the following config:","config":"\npipeline:\n processors:\n - bloblang: |\n root = this\n root.fans = this.fans.filter(fan -\u003e fan.obsession \u003e 0.5)\n"},{"title":"More Mapping","summary":"\nWhen receiving JSON documents of the form:\n\n```json\n{\n \"locations\": [\n {\"name\": \"Seattle\", \"state\": \"WA\"},\n {\"name\": \"New York\", \"state\": \"NY\"},\n {\"name\": \"Bellevue\", \"state\": \"WA\"},\n {\"name\": \"Olympia\", \"state\": \"WA\"}\n ]\n}\n```\n\nWe could collapse the location names from the state of Washington into a field `Cities`:\n\n```json\n{\"Cities\": \"Bellevue, Olympia, Seattle\"}\n```\n\nWith the following config:","config":"\npipeline:\n processors:\n - bloblang: |\n root.Cities = this.locations.\n filter(loc -\u003e loc.state == \"WA\").\n map_each(loc -\u003e loc.name).\n sort().join(\", \")\n"}],"config":{"name":"","type":"string","kind":"scalar","default":"","bloblang":true}},{"name":"bounds_check","type":"processor","status":"stable","plugin":true,"summary":"Removes messages (and batches) that do not fit within certain size boundaries.","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"max_part_size","type":"int","kind":"scalar","description":"The maximum size of a message to allow (in bytes)","default":1073741824},{"name":"min_part_size","type":"int","kind":"scalar","description":"The minimum size of a message to allow (in bytes)","default":1},{"name":"max_parts","type":"int","kind":"scalar","description":"The maximum size of message batches to allow (in message count)","is_advanced":true,"default":100},{"name":"min_parts","type":"int","kind":"scalar","description":"The minimum size of message batches to allow (in message count)","is_advanced":true,"default":1}]}},{"name":"branch","type":"processor","status":"stable","plugin":true,"summary":"The `branch` processor allows you to create a new request message via a xref:guides:bloblang/about.adoc[Bloblang mapping], execute a list of processors on the request messages, and, finally, map the result back into the source message using another mapping.","description":"\nThis is useful for preserving the original message contents when using processors that would otherwise replace the entire contents.\n\n== Metadata\n\nMetadata fields that are added to messages during branch processing will not be automatically copied into the resulting message. In order to do this you should explicitly declare in your `result_map` either a wholesale copy with `meta = metadata()`, or selective copies with `meta foo = metadata(\"bar\")` and so on. It is also possible to reference the metadata of the origin message in the `result_map` using the xref:guides:bloblang/about.adoc#metadata[`@` operator].\n\n== Error handling\n\nIf the `request_map` fails the child processors will not be executed. If the child processors themselves result in an (uncaught) error then the `result_map` will not be executed. If the `result_map` fails the message will remain unchanged. Under any of these conditions standard xref:configuration:error_handling.adoc[error handling methods] can be used in order to filter, DLQ or recover the failed messages.\n\n== Conditional branching\n\nIf the root of your request map is set to `deleted()` then the branch processors are skipped for the given message, this allows you to conditionally branch messages.","categories":["Composition"],"examples":[{"title":"HTTP Request","summary":"\nThis example strips the request message into an empty body, grabs an HTTP payload, and places the result back into the original message at the path `image.pull_count`:","config":"\npipeline:\n processors:\n - branch:\n request_map: 'root = \"\"'\n processors:\n - http:\n url: https://hub.docker.com/v2/repositories/jeffail/benthos\n verb: GET\n headers:\n Content-Type: application/json\n result_map: root.image.pull_count = this.pull_count\n\n# Example input: {\"id\":\"foo\",\"some\":\"pre-existing data\"}\n# Example output: {\"id\":\"foo\",\"some\":\"pre-existing data\",\"image\":{\"pull_count\":1234}}\n"},{"title":"Non Structured Results","summary":"\nWhen the result of your branch processors is unstructured and you wish to simply set a resulting field to the raw output use the content function to obtain the raw bytes of the resulting message and then coerce it into your value type of choice:","config":"\npipeline:\n processors:\n - branch:\n request_map: 'root = this.document.id'\n processors:\n - cache:\n resource: descriptions_cache\n key: ${! content() }\n operator: get\n result_map: root.document.description = content().string()\n\n# Example input: {\"document\":{\"id\":\"foo\",\"content\":\"hello world\"}}\n# Example output: {\"document\":{\"id\":\"foo\",\"content\":\"hello world\",\"description\":\"this is a cool doc\"}}\n"},{"title":"Lambda Function","summary":"\nThis example maps a new payload for triggering a lambda function with an ID and username from the original message, and the result of the lambda is discarded, meaning the original message is unchanged.","config":"\npipeline:\n processors:\n - branch:\n request_map: '{\"id\":this.doc.id,\"username\":this.user.name}'\n processors:\n - aws_lambda:\n function: trigger_user_update\n\n# Example input: {\"doc\":{\"id\":\"foo\",\"body\":\"hello world\"},\"user\":{\"name\":\"fooey\"}}\n# Output matches the input, which is unchanged\n"},{"title":"Conditional Caching","summary":"\nThis example caches a document by a message ID only when the type of the document is a foo:","config":"\npipeline:\n processors:\n - branch:\n request_map: |\n meta id = this.id\n root = if this.type == \"foo\" {\n this.document\n } else {\n deleted()\n }\n processors:\n - cache:\n resource: TODO\n operator: set\n key: ${! @id }\n value: ${! content() }\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"request_map","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] that describes how to create a request payload suitable for the child processors of this branch. If left empty then the branch will begin with an exact copy of the origin message (including metadata).","default":"","bloblang":true,"examples":["root = {\n\t\"id\": this.doc.id,\n\t\"content\": this.doc.body.text\n}","root = if this.type == \"foo\" {\n\tthis.foo.request\n} else {\n\tdeleted()\n}"]},{"name":"processors","type":"processor","kind":"array","description":"A list of processors to apply to mapped requests. When processing message batches the resulting batch must match the size and ordering of the input batch, therefore filtering, grouping should not be performed within these processors."},{"name":"result_map","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] that describes how the resulting messages from branched processing should be mapped back into the original payload. If left empty the origin message will remain unchanged (including metadata).","default":"","bloblang":true,"examples":["meta foo_code = metadata(\"code\")\nroot.foo_result = this","meta = metadata()\nroot.bar.body = this.body\nroot.bar.id = this.user.id","root.raw_result = content().string()","root.enrichments.foo = if metadata(\"request_failed\") != null {\n throw(metadata(\"request_failed\"))\n} else {\n this\n}","# Retain only the updated metadata fields which were present in the origin message\nmeta = metadata().filter(v -\u003e @.get(v.key) != null)"]}]}},{"name":"cache","type":"processor","status":"stable","plugin":true,"summary":"Performs operations against a xref:components:caches/about.adoc[cache resource] for each message, allowing you to store or retrieve data within message payloads.","description":"\nFor use cases where you wish to cache the result of processors consider using the xref:components:processors/cached.adoc[`cached` processor] instead.\n\nThis processor will interpolate functions within the `key` and `value` fields individually for each message. This allows you to specify dynamic keys and values based on the contents of the message payloads and metadata. You can find a list of functions in xref:configuration:interpolation.adoc#bloblang-queries[Bloblang queries].","categories":["Integration"],"footnotes":"\n== Operators\n\n=== `set`\n\nSet a key in the cache to a value. If the key already exists the contents are\noverridden.\n\n=== `add`\n\nSet a key in the cache to a value. If the key already exists the action fails\nwith a 'key already exists' error, which can be detected with\nxref:configuration:error_handling.adoc[processor error handling].\n\n=== `get`\n\nRetrieve the contents of a cached key and replace the original message payload\nwith the result. If the key does not exist the action fails with an error, which\ncan be detected with xref:configuration:error_handling.adoc[processor error handling].\n\n=== `delete`\n\nDelete a key and its contents from the cache. If the key does not exist the\naction is a no-op and will not fail with an error.\n\n=== `exists`\n\nCheck if a given key exists in the cache and replace the original message payload\nwith `true` or `false`.","examples":[{"title":"Deduplication","summary":"\nDeduplication can be done using the add operator with a key extracted from the message payload, since it fails when a key already exists we can remove the duplicates using a xref:components:processors/mapping.adoc[`mapping` processor]:","config":"\npipeline:\n processors:\n - cache:\n resource: foocache\n operator: add\n key: '${! json(\"message.id\") }'\n value: \"storeme\"\n - mapping: root = if errored() { deleted() }\n\ncache_resources:\n - label: foocache\n redis:\n url: tcp://TODO:6379\n"},{"title":"Deduplication Batch-Wide","summary":"\nSometimes it's necessary to deduplicate a batch of messages (also known as a window) by a single identifying value. This can be done by introducing a xref:components:processors/branch.adoc[`branch` processor], which executes the cache only once on behalf of the batch, in this case with a value make from a field extracted from the first and last messages of the batch:","config":"\npipeline:\n processors:\n # Try and add one message to a cache that identifies the whole batch\n - branch:\n request_map: |\n root = if batch_index() == 0 {\n json(\"id\").from(0) + json(\"meta.tail_id\").from(-1)\n } else { deleted() }\n processors:\n - cache:\n resource: foocache\n operator: add\n key: ${! content() }\n value: t\n # Delete all messages if we failed\n - mapping: |\n root = if errored().from(0) {\n deleted()\n }\n"},{"title":"Hydration","summary":"\nIt's possible to enrich payloads with content previously stored in a cache by using the xref:components:processors/branch.adoc[`branch`] processor:","config":"\npipeline:\n processors:\n - branch:\n processors:\n - cache:\n resource: foocache\n operator: get\n key: '${! json(\"message.document_id\") }'\n result_map: 'root.message.document = this'\n\n # NOTE: If the data stored in the cache is not valid JSON then use\n # something like this instead:\n # result_map: 'root.message.document = content().string()'\n\ncache_resources:\n - label: foocache\n memcached:\n addresses: [ \"TODO:11211\" ]\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"resource","type":"string","kind":"scalar","description":"The xref:components:caches/about.adoc[`cache` resource] to target with this processor."},{"name":"operator","type":"string","kind":"scalar","description":"The \u003c\u003coperators, operation\u003e\u003e to perform with the cache.","options":["set","add","get","delete","exists"],"linter":"\nlet options = {\n \"set\": true,\n \"add\": true,\n \"get\": true,\n \"delete\": true,\n \"exists\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"key","type":"string","kind":"scalar","description":"A key to use with the cache.","interpolated":true},{"name":"value","type":"string","kind":"scalar","description":"A value to use with the cache (when applicable).","is_optional":true,"interpolated":true},{"name":"ttl","type":"string","kind":"scalar","description":"The TTL of each individual item as a duration string. After this period an item will be eligible for removal during the next compaction. Not all caches support per-key TTLs, those that do will have a configuration field `default_ttl`, and those that do not will fall back to their generally configured TTL setting.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["60s","5m","36h"],"version":"3.33.0"}]}},{"name":"cached","type":"processor","status":"experimental","plugin":true,"summary":"Cache the result of applying one or more processors to messages identified by a key. If the key already exists within the cache the contents of the message will be replaced with the cached result instead of applying the processors. This component is therefore useful in situations where an expensive set of processors need only be executed periodically.","description":"The format of the data when stored within the cache is a custom and versioned schema chosen to balance performance and storage space. It is therefore not possible to point this processor to a cache that is pre-populated with data that this processor has not created itself.","categories":["Utility"],"examples":[{"title":"Cached Enrichment","summary":"In the following example we want to we enrich messages consumed from Kafka with data specific to the origin topic partition, we do this by placing an `http` processor within a `branch`, where the HTTP URL contains interpolation functions with the topic and partition in the path.\n\nHowever, it would be inefficient to make this HTTP request for every single message as the result is consistent for all data of a given topic partition. We can solve this by placing our enrichment call within a `cached` processor where the key contains the topic and partition, resulting in messages that originate from the same topic/partition combination using the cached result of the prior.","config":"\npipeline:\n processors:\n - branch:\n processors:\n - cached:\n key: '${! meta(\"kafka_topic\") }-${! meta(\"kafka_partition\") }'\n cache: foo_cache\n processors:\n - mapping: 'root = \"\"'\n - http:\n url: http://example.com/enrichment/${! meta(\"kafka_topic\") }/${! meta(\"kafka_partition\") }\n verb: GET\n result_map: 'root.enrichment = this'\n\ncache_resources:\n - label: foo_cache\n memory:\n # Disable compaction so that cached items never expire\n compaction_interval: \"\"\n"},{"title":"Periodic Global Enrichment","summary":"In the following example we enrich all messages with the same data obtained from a static URL with an `http` processor within a `branch`. However, we expect the data from this URL to change roughly every 10 minutes, so we configure a `cached` processor with a static key (since this request is consistent for all messages) and a TTL of `10m`.","config":"\npipeline:\n processors:\n - branch:\n request_map: 'root = \"\"'\n processors:\n - cached:\n key: static_foo\n cache: foo_cache\n ttl: 10m\n processors:\n - http:\n url: http://example.com/get/foo.json\n verb: GET\n result_map: 'root.foo = this'\n\ncache_resources:\n - label: foo_cache\n memory: {}\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"cache","type":"string","kind":"scalar","description":"The cache resource to read and write processor results from."},{"name":"skip_on","type":"string","kind":"scalar","description":"A condition that can be used to skip caching the results from the processors.","is_optional":true,"bloblang":true,"examples":["errored()"]},{"name":"key","type":"string","kind":"scalar","description":"A key to be resolved for each message, if the key already exists in the cache then the cached result is used, otherwise the processors are applied and the result is cached under this key. The key could be static and therefore apply generally to all messages or it could be an interpolated expression that is potentially unique for each message.","interpolated":true,"examples":["my_foo_result","${! this.document.id }","${! meta(\"kafka_key\") }","${! meta(\"kafka_topic\") }"]},{"name":"ttl","type":"string","kind":"scalar","description":"An optional expiry period to set for each cache entry. Some caches only have a general TTL and will therefore ignore this setting.","is_optional":true,"interpolated":true},{"name":"processors","type":"processor","kind":"array","description":"The list of processors whose result will be cached."}]},"version":"4.3.0"},{"name":"catch","type":"processor","status":"stable","plugin":true,"summary":"Applies a list of child processors _only_ when a previous processing step has failed.","description":"\nBehaves similarly to the xref:components:processors/for_each.adoc[`for_each`] processor, where a list of child processors are applied to individual messages of a batch. However, processors are only applied to messages that failed a processing step prior to the catch.\n\nFor example, with the following config:\n\n```yaml\npipeline:\n processors:\n - resource: foo\n - catch:\n - resource: bar\n - resource: baz\n```\n\nIf the processor `foo` fails for a particular message, that message will be fed into the processors `bar` and `baz`. Messages that do not fail for the processor `foo` will skip these processors.\n\nWhen messages leave the catch block their fail flags are cleared. This processor is useful for when it's possible to recover failed messages, or when special actions (such as logging/metrics) are required before dropping them.\n\nMore information about error handling can be found in xref:configuration:error_handling.adoc[].","categories":["Composition"],"config":{"name":"","type":"processor","kind":"array","default":[]}},{"name":"cohere_chat","type":"processor","status":"experimental","plugin":true,"summary":"Generates responses to messages in a chat conversation, using the Cohere API.","description":"\nThis processor sends the contents of user prompts to the Cohere API, which generates responses. By default, the processor submits the entire payload of each message as a string, unless you use the `prompt` configuration field to customize it.\n\nTo learn more about chat completion, see the https://docs.cohere.com/docs/chat-api[Cohere API documentation^].","categories":["AI"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"base_url","type":"string","kind":"scalar","description":"The base URL to use for API requests.","default":"https://api.cohere.com"},{"name":"api_key","type":"string","kind":"scalar","description":"The API key for the Cohere API.","is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"model","type":"string","kind":"scalar","description":"The name of the Cohere model to use.","examples":["command-r-plus","command-r","command","command-light"]},{"name":"prompt","type":"string","kind":"scalar","description":"The user prompt you want to generate a response for. By default, the processor submits the entire payload as a string.","is_optional":true,"interpolated":true},{"name":"system_prompt","type":"string","kind":"scalar","description":"The system prompt to submit along with the user prompt.","is_optional":true,"interpolated":true},{"name":"max_tokens","type":"int","kind":"scalar","description":"The maximum number of tokens that can be generated in the chat completion.","is_optional":true},{"name":"temperature","type":"float","kind":"scalar","description":"What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic.\n\nWe generally recommend altering this or top_p but not both.","is_optional":true,"linter":"root = if this \u003e 2 || this \u003c 0 { [ \"field must be between 0 and 2\" ] }"},{"name":"response_format","type":"string","kind":"scalar","description":"Specify the model's output format. If `json_schema` is specified, then additionally a `json_schema` or `schema_registry` must be configured.","default":"text","options":["text","json","json_schema"],"linter":"\nlet options = {\n \"text\": true,\n \"json\": true,\n \"json_schema\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"json_schema","type":"string","kind":"scalar","description":"The JSON schema to use when responding in `json_schema` format. To learn more about what JSON schema is supported see the https://docs.cohere.com/docs/structured-outputs-json[Cohere documentation^].","is_optional":true},{"name":"schema_registry","type":"object","kind":"scalar","description":"The schema registry to dynamically load schemas from when responding in `json_schema` format. Schemas themselves must be in JSON format. To learn more about what JSON schema is supported see the https://docs.cohere.com/docs/structured-outputs-json[Cohere documentation^].","is_advanced":true,"is_optional":true,"children":[{"name":"url","type":"string","kind":"scalar","description":"The base URL of the schema registry service.","is_advanced":true,"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"subject","type":"string","kind":"scalar","description":"The subject name to fetch the schema for.","is_advanced":true},{"name":"refresh_interval","type":"string","kind":"scalar","description":"The refresh rate for getting the latest schema. If not specified the schema does not refresh.","is_advanced":true,"is_optional":true},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"oauth","type":"object","kind":"scalar","description":"Allows you to specify open authentication via OAuth version 1.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use OAuth version 1 in requests.","is_advanced":true,"default":false},{"name":"consumer_key","type":"string","kind":"scalar","description":"A value used to identify the client to the service provider.","is_advanced":true,"default":""},{"name":"consumer_secret","type":"string","kind":"scalar","description":"A secret used to establish ownership of the consumer key.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"access_token","type":"string","kind":"scalar","description":"A value used to gain access to the protected resources on behalf of the user.","is_advanced":true,"default":""},{"name":"access_token_secret","type":"string","kind":"scalar","description":"A secret provided in order to establish ownership of a given access token.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"jwt","type":"object","kind":"scalar","description":"BETA: Allows you to specify JWT authentication.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use JWT authentication in requests.","is_advanced":true,"default":false},{"name":"private_key_file","type":"string","kind":"scalar","description":"A file with the PEM encoded via PKCS1 or PKCS8 as private key.","is_advanced":true,"default":""},{"name":"signing_method","type":"string","kind":"scalar","description":"A method used to sign the token such as RS256, RS384, RS512 or EdDSA.","is_advanced":true,"default":""},{"name":"claims","type":"unknown","kind":"map","description":"A value used to identify the claims that issued the JWT.","is_advanced":true,"default":{}},{"name":"headers","type":"unknown","kind":"map","description":"Add optional key/value headers to the JWT.","is_advanced":true,"default":{}}]}]},{"name":"top_p","type":"float","kind":"scalar","description":"An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.\n\nWe generally recommend altering this or temperature but not both.","is_advanced":true,"is_optional":true,"linter":"root = if this \u003e 1 || this \u003c 0 { [ \"field must be between 0 and 1\" ] }"},{"name":"frequency_penalty","type":"float","kind":"scalar","description":"Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim.","is_advanced":true,"is_optional":true,"linter":"root = if this \u003e 2 || this \u003c -2 { [ \"field must be less than 2 and greater than -2\" ] }"},{"name":"presence_penalty","type":"float","kind":"scalar","description":"Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics.","is_advanced":true,"is_optional":true,"linter":"root = if this \u003e 2 || this \u003c -2 { [ \"field must be less than 2 and greater than -2\" ] }"},{"name":"seed","type":"int","kind":"scalar","description":"If specified, our system will make a best effort to sample deterministically, such that repeated requests with the same seed and parameters should return the same result. Determinism is not guaranteed.","is_advanced":true,"is_optional":true},{"name":"stop","type":"string","kind":"array","description":"Up to 4 sequences where the API will stop generating further tokens.","is_advanced":true,"is_optional":true},{"name":"max_tool_calls","type":"int","kind":"scalar","description":"Maximum number of tool calls the model can do.","default":10},{"name":"tools","type":"object","kind":"array","description":"The tools to allow the LLM to invoke. This allows building subpipelines that the LLM can choose to invoke to execute agentic-like actions.","default":[],"children":[{"name":"name","type":"string","kind":"scalar","description":"The name of this tool."},{"name":"description","type":"string","kind":"scalar","description":"A description of this tool, the LLM uses this to decide if the tool should be used."},{"name":"parameters","type":"object","kind":"scalar","description":"The parameters the LLM needs to provide to invoke this tool.","children":[{"name":"required","type":"string","kind":"array","description":"The required parameters for this pipeline.","default":[]},{"name":"properties","type":"object","kind":"map","description":"The properties for the processor's input data","children":[{"name":"type","type":"string","kind":"scalar","description":"The type of this parameter."},{"name":"description","type":"string","kind":"scalar","description":"A description of this parameter."},{"name":"enum","type":"string","kind":"array","description":"Specifies that this parameter is an enum and only these specific values should be used.","default":[]}]}]},{"name":"processors","type":"processor","kind":"array","description":"The pipeline to execute when the LLM uses this tool.","is_optional":true}]}],"linter":"\n root = match {\n this.exists(\"json_schema\") \u0026\u0026 this.exists(\"schema_registry\") =\u003e [\"cannot set both `json_schema` and `schema_registry`\"]\n this.response_format == \"json_schema\" \u0026\u0026 !this.exists(\"json_schema\") \u0026\u0026 !this.exists(\"schema_registry\") =\u003e [\"schema must be specified using either `json_schema` or `schema_registry`\"]\n }\n "},"version":"4.37.0"},{"name":"cohere_embeddings","type":"processor","status":"experimental","plugin":true,"summary":"Generates vector embeddings to represent input text, using the Cohere API.","description":"\nThis processor sends text strings to the Cohere API, which generates vector embeddings. By default, the processor submits the entire payload of each message as a string, unless you use the `text_mapping` configuration field to customize it.\n\nTo learn more about vector embeddings, see the https://docs.cohere.com/docs/embeddings[Cohere API documentation^].","categories":["AI"],"examples":[{"title":"Store embedding vectors in Qdrant","summary":"Compute embeddings for some generated data and store it within xrefs:component:outputs/qdrant.adoc[Qdrant]","config":"input:\n generate:\n interval: 1s\n mapping: |\n root = {\"text\": fake(\"paragraph\")}\npipeline:\n processors:\n - cohere_embeddings:\n model: embed-english-v3\n api_key: \"${COHERE_API_KEY}\"\n text_mapping: \"root = this.text\"\noutput:\n qdrant:\n grpc_host: localhost:6334\n collection_name: \"example_collection\"\n id: \"root = uuid_v4()\"\n vector_mapping: \"root = this\""}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"base_url","type":"string","kind":"scalar","description":"The base URL to use for API requests.","default":"https://api.cohere.com"},{"name":"api_key","type":"string","kind":"scalar","description":"The API key for the Cohere API.","is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"model","type":"string","kind":"scalar","description":"The name of the Cohere model to use.","examples":["embed-english-v3.0","embed-english-light-v3.0","embed-multilingual-v3.0","embed-multilingual-light-v3.0"]},{"name":"text_mapping","type":"string","kind":"scalar","description":"The text you want to generate a vector embedding for. By default, the processor submits the entire payload as a string.","is_optional":true,"bloblang":true},{"name":"input_type","type":"string","kind":"scalar","description":"Specifies the type of input passed to the model.","default":"search_document","annotated_options":[["classification","Used for embeddings passed through a text classifier."],["clustering","Used for the embeddings run through a clustering algorithm."],["search_document","Used for embeddings stored in a vector database for search use-cases."],["search_query","Used for embeddings of search queries run against a vector DB to find relevant documents."]],"linter":"\nlet options = {\n \"classification\": true,\n \"clustering\": true,\n \"search_document\": true,\n \"search_query\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"dimensions","type":"int","kind":"scalar","description":"The number of dimensions of the output embedding. This is only available for embed-v4 and newer models. Possible values are 256, 512, 1024, and 1536.","is_optional":true}]},"version":"4.37.0"},{"name":"cohere_rerank","type":"processor","status":"experimental","plugin":true,"summary":"Generates vector embeddings to represent input text, using the Cohere API.","description":"\nThis processor sends document strings to the Cohere API, which reranks them based on the relevance to the query.\n\nTo learn more about reranking, see the https://docs.cohere.com/docs/rerank-2[Cohere API documentation^].\n\nThe output of this processor is an array of strings that are ordered by their relevance.\n\n== Metadata\n\nrelevance_scores: an array of scores for each document, indicating how relevant it is to the query. The scores are in the same order as the documents in the input. The higher the score, the more relevant the document is to the query.\n\n\t\t","categories":["AI"],"examples":[{"title":"Rerank some documents based on a query","summary":"Rerank some documents based on a query","config":"input:\n generate:\n interval: 1s\n mapping: |\n root = {\n \"query\": fake(\"sentence\"),\n \"docs\": [fake(\"paragraph\"), fake(\"paragraph\"), fake(\"paragraph\")],\n }\npipeline:\n processors:\n - cohere_rerank:\n model: rerank-v3.5\n api_key: \"${COHERE_API_KEY}\"\n query: \"${!this.query}\"\n documents: \"root = this.docs\"\noutput:\n stdout: {}"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"base_url","type":"string","kind":"scalar","description":"The base URL to use for API requests.","default":"https://api.cohere.com"},{"name":"api_key","type":"string","kind":"scalar","description":"The API key for the Cohere API.","is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"model","type":"string","kind":"scalar","description":"The name of the Cohere model to use.","examples":["rerank-v3.5"]},{"name":"query","type":"string","kind":"scalar","description":"The search query","interpolated":true},{"name":"documents","type":"string","kind":"scalar","description":"A list of texts that will be compared to the query. For optimal performance Cohere recommends against sending more than 1000 documents in a single request. NOTE: structured data should be formatted as YAML for best performance.","bloblang":true},{"name":"top_n","type":"int","kind":"scalar","description":"The number of documents to return, if 0 all documents are returned.","default":0},{"name":"max_tokens_per_doc","type":"int","kind":"scalar","description":"Long documents will be automatically truncated to the specified number of tokens.","default":4096}]},"version":"4.37.0"},{"name":"command","type":"processor","status":"experimental","plugin":true,"summary":"Executes a command for each message.","description":"\nThe specified command is executed for each message processed, with the raw bytes of the message being fed into the stdin of the command process, and the resulting message having its contents replaced with the stdout of it.\n\n== Performance\n\nSince this processor executes a new process for each message performance will likely be an issue for high throughput streams. If this is the case then consider using the xref:components:processors/subprocess.adoc[`subprocess` processor] instead as it keeps the underlying process alive long term and uses codecs to insert and extract inputs and outputs to it via stdin/stdout.\n\n== Error handling\n\nIf a non-zero error code is returned by the command then an error containing the entirety of stderr (or a generic message if nothing is written) is set on the message. These failed messages will continue through the pipeline unchanged, but can be dropped or placed in a dead letter queue according to your config, you can read about xref:configuration:error_handling.adoc[these patterns].\n\nIf the command is successful but stderr is written to then a metadata field `command_stderr` is populated with its contents.\n","categories":["Integration"],"examples":[{"title":"Cron Scheduled Command","summary":"This example uses a xref:components:inputs/generate.adoc[`generate` input] to trigger a command on a cron schedule:","config":"\ninput:\n generate:\n interval: '0,30 */2 * * * *'\n mapping: 'root = \"\"' # Empty string as we do not need to pipe anything to stdin\n processors:\n - command:\n name: df\n args_mapping: '[ \"-h\" ]'\n"},{"title":"Dynamic Command Execution","summary":"This example config takes structured messages of the form `{\"command\":\"echo\",\"args\":[\"foo\"]}` and uses their contents to execute the contained command and arguments dynamically, replacing its contents with the command result printed to stdout:","config":"\npipeline:\n processors:\n - command:\n name: ${! this.command }\n args_mapping: 'this.args'\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"name","type":"string","kind":"scalar","description":"The name of the command to execute.","interpolated":true,"examples":["bash","go","${! @command }"]},{"name":"args_mapping","type":"string","kind":"scalar","description":"An optional xref:guides:bloblang/about.adoc[Bloblang mapping] that, when specified, should resolve into an array of arguments to pass to the command. Command arguments are expressed this way in order to support dynamic behavior.","is_optional":true,"bloblang":true,"examples":["[ \"-c\", this.script_path ]"]}]},"version":"4.21.0"},{"name":"compress","type":"processor","status":"stable","plugin":true,"summary":"Compresses messages according to the selected algorithm. Supported compression algorithms are: [flate gzip lz4 pgzip snappy zlib]","description":"The 'level' field might not apply to all algorithms.","categories":["Parsing"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"algorithm","type":"string","kind":"scalar","description":"The compression algorithm to use.","options":["flate","gzip","lz4","pgzip","snappy","zlib"]},{"name":"level","type":"int","kind":"scalar","description":"The level of compression to use. May not be applicable to all algorithms.","default":-1}]}},{"name":"couchbase","type":"processor","status":"experimental","plugin":true,"summary":"Performs operations against Couchbase for each message, allowing you to store or retrieve data within message payloads.","description":"When inserting, replacing or upserting documents, each must have the `content` property set.","categories":["Integration"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"Couchbase connection string.","examples":["couchbase://localhost:11210"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"username","type":"string","kind":"scalar","description":"Username to connect to the cluster.","is_optional":true},{"name":"password","type":"string","kind":"scalar","description":"Password to connect to the cluster.","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"bucket","type":"string","kind":"scalar","description":"Couchbase bucket."},{"name":"collection","type":"string","kind":"scalar","description":"Bucket collection.","is_advanced":true,"is_optional":true,"default":"_default"},{"name":"transcoder","type":"string","kind":"scalar","description":"Couchbase transcoder to use.","is_advanced":true,"default":"legacy","annotated_options":[["json","JSONTranscoder implements the default transcoding behavior and applies JSON transcoding to all values. This will apply the following behavior to the value: binary ([]byte) -\u003e error. default -\u003e JSON value, JSON Flags."],["legacy","LegacyTranscoder implements the behavior for a backward-compatible transcoder. This transcoder implements behavior matching that of gocb v1.This will apply the following behavior to the value: binary ([]byte) -\u003e binary bytes, Binary expectedFlags. string -\u003e string bytes, String expectedFlags. default -\u003e JSON value, JSON expectedFlags."],["raw","RawBinaryTranscoder implements passthrough behavior of raw binary data. This transcoder does not apply any serialization. This will apply the following behavior to the value: binary ([]byte) -\u003e binary bytes, binary expectedFlags. default -\u003e error."],["rawjson","RawJSONTranscoder implements passthrough behavior of JSON data. This transcoder does not apply any serialization. It will forward data across the network without incurring unnecessary parsing costs. This will apply the following behavior to the value: binary ([]byte) -\u003e JSON bytes, JSON expectedFlags. string -\u003e JSON bytes, JSON expectedFlags. default -\u003e error."],["rawstring","RawStringTranscoder implements passthrough behavior of raw string data. This transcoder does not apply any serialization. This will apply the following behavior to the value: string -\u003e string bytes, string expectedFlags. default -\u003e error."]],"linter":"\nlet options = {\n \"json\": true,\n \"legacy\": true,\n \"raw\": true,\n \"rawjson\": true,\n \"rawstring\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"timeout","type":"string","kind":"scalar","description":"Operation timeout.","is_advanced":true,"default":"15s"},{"name":"id","type":"string","kind":"scalar","description":"Document id.","interpolated":true,"examples":["${! json(\"id\") }"]},{"name":"content","type":"string","kind":"scalar","description":"Document content.","is_optional":true,"bloblang":true},{"name":"operation","type":"string","kind":"scalar","description":"Couchbase operation to perform.","default":"get","annotated_options":[["get","fetch a document."],["insert","insert a new document."],["remove","delete a document."],["replace","replace the contents of a document."],["upsert","creates a new document if it does not exist, if it does exist then it updates it."]],"linter":"\nlet options = {\n \"get\": true,\n \"insert\": true,\n \"remove\": true,\n \"replace\": true,\n \"upsert\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"}],"linter":"root = if ((this.operation == \"insert\" || this.operation == \"replace\" || this.operation == \"upsert\") \u0026\u0026 !this.exists(\"content\")) { [ \"content must be set for insert, replace and upsert operations.\" ] }"},"version":"4.11.0"},{"name":"crash","type":"processor","status":"beta","plugin":true,"summary":"Crashes the process using a fatal log message. The log message can be set using function interpolations described in xref:configuration:interpolation.adoc#bloblang-queries[Bloblang queries] which allows you to log the contents and metadata of messages.","categories":["Utility"],"config":{"name":"","type":"string","kind":"scalar","interpolated":true}},{"name":"decompress","type":"processor","status":"stable","plugin":true,"summary":"Decompresses messages according to the selected algorithm. Supported decompression algorithms are: [bzip2 flate gzip lz4 pgzip snappy zlib]","categories":["Parsing"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"algorithm","type":"string","kind":"scalar","description":"The decompression algorithm to use.","options":["bzip2","flate","gzip","lz4","pgzip","snappy","zlib"]}]}},{"name":"dedupe","type":"processor","status":"stable","plugin":true,"summary":"Deduplicates messages by storing a key value in a cache using the `add` operator. If the key already exists within the cache it is dropped.","description":"\nCaches must be configured as resources, for more information check out the xref:components:caches/about.adoc[cache documentation].\n\nWhen using this processor with an output target that might fail you should always wrap the output within an indefinite xref:components:outputs/retry.adoc[`retry`] block. This ensures that during outages your messages aren't reprocessed after failures, which would result in messages being dropped.\n\n== Batch deduplication\n\nThis processor enacts on individual messages only, in order to perform a deduplication on behalf of a batch (or window) of messages instead use the xref:components:processors/cache.adoc#examples[`cache` processor].\n\n== Delivery guarantees\n\nPerforming deduplication on a stream using a distributed cache voids any at-least-once guarantees that it previously had. This is because the cache will preserve message signatures even if the message fails to leave the Redpanda Connect pipeline, which would cause message loss in the event of an outage at the output sink followed by a restart of the Redpanda Connect instance (or a server crash, etc).\n\nThis problem can be mitigated by using an in-memory cache and distributing messages to horizontally scaled Redpanda Connect pipelines partitioned by the deduplication key. However, in situations where at-least-once delivery guarantees are important it is worth avoiding deduplication in favour of implement idempotent behavior at the edge of your stream pipelines.","categories":["Utility"],"examples":[{"title":"Deduplicate based on Kafka key","summary":"The following configuration demonstrates a pipeline that deduplicates messages based on the Kafka key.","config":"\npipeline:\n processors:\n - dedupe:\n cache: keycache\n key: ${! meta(\"kafka_key\") }\n\ncache_resources:\n - label: keycache\n memory:\n default_ttl: 60s\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"cache","type":"string","kind":"scalar","description":"The xref:components:caches/about.adoc[`cache` resource] to target with this processor."},{"name":"key","type":"string","kind":"scalar","description":"An interpolated string yielding the key to deduplicate by for each message.","interpolated":true,"examples":["${! meta(\"kafka_key\") }","${! content().hash(\"xxhash64\") }"]},{"name":"drop_on_err","type":"bool","kind":"scalar","description":"Whether messages should be dropped when the cache returns a general error such as a network issue.","default":true}]}},{"name":"for_each","type":"processor","status":"stable","plugin":true,"summary":"A processor that applies a list of child processors to messages of a batch as though they were each a batch of one message.","description":"\nThis is useful for forcing batch wide processors such as xref:components:processors/dedupe.adoc[`dedupe`] or interpolations such as the `value` field of the `metadata` processor to execute on individual message parts of a batch instead.\n\nPlease note that most processors already process per message of a batch, and this processor is not needed in those cases.","categories":["Composition"],"config":{"name":"","type":"processor","kind":"array","default":[]}},{"name":"gcp_bigquery_select","type":"processor","status":"experimental","plugin":true,"summary":"Executes a `SELECT` query against BigQuery and replaces messages with the rows returned.","categories":["Integration"],"examples":[{"title":"Word count","summary":"\nGiven a stream of English terms, enrich the messages with the word count from Shakespeare's public works:","config":"\npipeline:\n processors:\n - branch:\n processors:\n - gcp_bigquery_select:\n project: test-project\n table: bigquery-public-data.samples.shakespeare\n columns:\n - word\n - sum(word_count) as total_count\n where: word = ?\n suffix: |\n GROUP BY word\n ORDER BY total_count DESC\n LIMIT 10\n args_mapping: root = [ this.term ]\n result_map: |\n root.count = this.get(\"0.total_count\")\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"project","type":"string","kind":"scalar","description":"GCP project where the query job will execute."},{"name":"credentials_json","type":"string","kind":"scalar","description":"An optional field to set Google Service Account Credentials json.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"table","type":"string","kind":"scalar","description":"Fully-qualified BigQuery table name to query.","examples":["bigquery-public-data.samples.shakespeare"]},{"name":"columns","type":"string","kind":"array","description":"A list of columns to query."},{"name":"where","type":"string","kind":"scalar","description":"An optional where clause to add. Placeholder arguments are populated with the `args_mapping` field. Placeholders should always be question marks (`?`).","is_optional":true,"examples":["type = ? and created_at \u003e ?","user_id = ?"]},{"name":"job_labels","type":"string","kind":"map","description":"A list of labels to add to the query job.","default":{}},{"name":"args_mapping","type":"string","kind":"scalar","description":"An optional xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `where`.","is_optional":true,"bloblang":true,"examples":["root = [ \"article\", now().ts_format(\"2006-01-02\") ]"]},{"name":"prefix","type":"string","kind":"scalar","description":"An optional prefix to prepend to the select query (before SELECT).","is_optional":true},{"name":"suffix","type":"string","kind":"scalar","description":"An optional suffix to append to the select query.","is_optional":true}]},"version":"3.64.0"},{"name":"gcp_vertex_ai_chat","type":"processor","status":"experimental","plugin":true,"summary":"Generates responses to messages in a chat conversation, using the Vertex AI API.","description":"This processor sends prompts to your chosen large language model (LLM) and generates text from the responses, using the Vertex AI API.\n\nFor more information, see the https://cloud.google.com/vertex-ai/docs[Vertex AI documentation^].","categories":["AI"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"project","type":"string","kind":"scalar","description":"GCP project ID to use"},{"name":"credentials_json","type":"string","kind":"scalar","description":"An optional field to set google Service Account Credentials json.","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"location","type":"string","kind":"scalar","description":"The location of the model if using a fined tune model. For base models this can be omitted","is_optional":true,"examples":["us-central1"]},{"name":"model","type":"string","kind":"scalar","description":"The name of the LLM to use. For a full list of models, see the https://console.cloud.google.com/vertex-ai/model-garden[Vertex AI Model Garden].","examples":["gemini-1.5-pro-001","gemini-1.5-flash-001"]},{"name":"prompt","type":"string","kind":"scalar","description":"The prompt you want to generate a response for. By default, the processor submits the entire payload as a string.","is_optional":true,"interpolated":true},{"name":"system_prompt","type":"string","kind":"scalar","description":"The system prompt to submit to the Vertex AI LLM.","is_advanced":true,"is_optional":true,"interpolated":true},{"name":"attachment","type":"string","kind":"scalar","description":"Additional data like an image to send with the prompt to the model. The result of the mapping must be a byte array, and the content type is automatically detected.","is_optional":true,"bloblang":true,"examples":["root = this.image.decode(\"base64\") # decode base64 encoded image"],"version":"4.38.0"},{"name":"temperature","type":"float","kind":"scalar","description":"Controls the randomness of predications.","is_optional":true,"linter":"root = if this \u003c 0 || this \u003e 2 { [\"field must be between 0.0-2.0\"] }"},{"name":"max_tokens","type":"int","kind":"scalar","description":"The maximum number of output tokens to generate per message.","is_optional":true},{"name":"response_format","type":"string","kind":"scalar","description":"The response format of generated type, the model must also be prompted to output the appropriate response type.","default":"text","options":["text","json"],"linter":"\nlet options = {\n \"text\": true,\n \"json\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"top_p","type":"float","kind":"scalar","description":"If specified, nucleus sampling will be used.","is_advanced":true,"is_optional":true,"linter":"root = if this \u003c 0 || this \u003e 1 { [\"field must be between 0.0-1.0\"] }"},{"name":"top_k","type":"int","kind":"scalar","description":"If specified top-k sampling will be used.","is_advanced":true,"is_optional":true,"linter":"root = if this \u003c 1 || this \u003e 40 { [\"field must be between 1-40\"] }"},{"name":"stop","type":"string","kind":"array","description":"Stop sequences to when the model will stop generating further tokens.","is_advanced":true,"is_optional":true},{"name":"presence_penalty","type":"float","kind":"scalar","description":"Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics.","is_advanced":true,"is_optional":true,"linter":"root = if this \u003c -2 || this \u003e 2 { [\"field must be greater than -2.0 and less than 2.0\"] }"},{"name":"frequency_penalty","type":"float","kind":"scalar","description":"Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim.","is_advanced":true,"is_optional":true,"linter":"root = if this \u003c -2 || this \u003e 2 { [\"field must be greater than -2.0 and less than 2.0\"] }"}]},"version":"4.34.0"},{"name":"gcp_vertex_ai_embeddings","type":"processor","status":"experimental","plugin":true,"summary":"Generates vector embeddings to represent input text, using the Vertex AI API.","description":"This processor sends text strings to the Vertex AI API, which generates vector embeddings. By default, the processor submits the entire payload of each message as a string, unless you use the `text` configuration field to customize it.\n\nFor more information, see the https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings[Vertex AI documentation^].","categories":["AI"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"project","type":"string","kind":"scalar","description":"GCP project ID to use"},{"name":"credentials_json","type":"string","kind":"scalar","description":"An optional field to set google Service Account Credentials json.","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"location","type":"string","kind":"scalar","description":"The location of the model.","default":"us-central1"},{"name":"model","type":"string","kind":"scalar","description":"The name of the LLM to use. For a full list of models, see the https://console.cloud.google.com/vertex-ai/model-garden[Vertex AI Model Garden].","examples":["text-embedding-004","text-multilingual-embedding-002"]},{"name":"task_type","type":"string","kind":"scalar","description":"The way to optimize embeddings that the model generates for specific use cases.","default":"RETRIEVAL_DOCUMENT","annotated_options":[["CLASSIFICATION","optimize for being able classify texts according to preset labels"],["CLUSTERING","optimize for clustering texts based on their similarities"],["FACT_VERIFICATION","optimize for queries that are proving or disproving a fact such as \"apples grow underground\""],["QUESTION_ANSWERING","optimize for search proper questions such as \"Why is the sky blue?\""],["RETRIEVAL_DOCUMENT","optimize for documents that will be searched (also known as a corpus)"],["RETRIEVAL_QUERY","optimize for queries such as \"What is the best fish recipe?\" or \"best restaurant in Chicago\""],["SEMANTIC_SIMILARITY","optimize for text similarity"]],"linter":"\nlet options = {\n \"classification\": true,\n \"clustering\": true,\n \"fact_verification\": true,\n \"question_answering\": true,\n \"retrieval_document\": true,\n \"retrieval_query\": true,\n \"semantic_similarity\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"text","type":"string","kind":"scalar","description":"The text you want to compute vector embeddings for. By default, the processor submits the entire payload as a string.","is_optional":true,"interpolated":true},{"name":"output_dimensions","type":"int","kind":"scalar","description":"The maximum length for the output embedding size. If set, the output embeddings will be truncated to this size.","is_optional":true}]},"version":"4.37.0"},{"name":"google_drive_download","type":"processor","status":"experimental","plugin":true,"summary":"Downloads files from Google Drive","description":"\nCan download a file from Google Drive based on a file ID.\n== Authentication\nBy default, this connector will use Google Application Default Credentials (ADC) to authenticate with Google APIs.\n\nTo use this mechanism locally, the following gcloud commands can be used:\n\n\t# Login for the application default credentials and add scopes for readonly drive access\n\tgcloud auth application-default login --scopes='openid,https://www.googleapis.com/auth/userinfo.email,https://www.googleapis.com/auth/cloud-platform,https://www.googleapis.com/auth/drive.readonly'\n\t# When logging in with a user account, you may need to set the quota project for the application default credentials\n\tgcloud auth application-default set-quota-project \u003cproject-id\u003e\n\nOtherwise if using a service account, you can create a JSON key for the service account and set it in the `credentials_json` field.\nIn order for a service account to access files in Google Drive either files need to be explicitly shared with the service account email, otherwise https://support.google.com/a/answer/162106[^domain wide delegation] can be used to share all files within a Google Workspace.\n","categories":["Unstructured"],"examples":[{"title":"Download files from Google Drive","summary":"This examples downloads all the files from Google Drive","config":"\npipeline:\n processors:\n - google_drive_search:\n query: \"name = 'Test Doc'\"\n - google_drive_download:\n file_id: \"${!this.id}\"\n mime_type: \"${!this.mimeType}\"\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"credentials_json","type":"string","kind":"scalar","description":"A service account credentials JSON file. If left unset then the application default credentials are used.","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"file_id","type":"string","kind":"scalar","description":"The file ID of the file to download.","interpolated":true},{"name":"mime_type","type":"string","kind":"scalar","description":"The mime type of the file in drive.","interpolated":true},{"name":"export_mime_types","type":"string","kind":"map","description":"A map of Google Drive MIME types to their export formats. The key is the MIME type, and the value is the export format. See https://developers.google.com/workspace/drive/api/guides/ref-export-formats[^Google Drive API Documentation] for a list of supported export types","is_advanced":true,"default":{"application/vnd.google-apps.document":"text/markdown","application/vnd.google-apps.drawing":"image/png","application/vnd.google-apps.presentation":"application/pdf","application/vnd.google-apps.script":"application/vnd.google-apps.script+json","application/vnd.google-apps.spreadsheet":"text/csv"},"examples":[{"application/vnd.google-apps.document":"application/pdf","application/vnd.google-apps.drawing":"application/pdf","application/vnd.google-apps.presentation":"application/pdf","application/vnd.google-apps.spreadsheet":"application/pdf"},{"application/vnd.google-apps.document":"application/vnd.openxmlformats-officedocument.wordprocessingml.document","application/vnd.google-apps.drawing":"image/svg+xml","application/vnd.google-apps.presentation":"application/vnd.openxmlformats-officedocument.presentationml.presentation","application/vnd.google-apps.spreadsheet":"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}]}]}},{"name":"google_drive_list_labels","type":"processor","status":"experimental","plugin":true,"summary":"Lists labels for a file in Google Drive","description":"\nCan list all labels from Google Drive.\n\t\t== Authentication\nBy default, this connector will use Google Application Default Credentials (ADC) to authenticate with Google APIs.\n\nTo use this mechanism locally, the following gcloud commands can be used:\n\n\t# Login for the application default credentials and add scopes for readonly drive access\n\tgcloud auth application-default login --scopes='openid,https://www.googleapis.com/auth/userinfo.email,https://www.googleapis.com/auth/cloud-platform,https://www.googleapis.com/auth/drive.labels.readonly'\n\t# When logging in with a user account, you may need to set the quota project for the application default credentials\n\tgcloud auth application-default set-quota-project \u003cproject-id\u003e\n\nOtherwise if using a service account, you can create a JSON key for the service account and set it in the `credentials_json` field.\nIn order for a service account to access files in Google Drive either files need to be explicitly shared with the service account email, otherwise https://support.google.com/a/answer/162106[^domain wide delegation] can be used to share all files within a Google Workspace.\n","categories":["Unstructured"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"credentials_json","type":"string","kind":"scalar","description":"A service account credentials JSON file. If left unset then the application default credentials are used.","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}},{"name":"google_drive_search","type":"processor","status":"experimental","plugin":true,"summary":"Searches Google Drive for files matching the provided query.","description":"\nThis processor searches for files in Google Drive using the provided query.\n\nSearch results are emitted as message batch, where each message is a https://developers.google.com/workspace/drive/api/reference/rest/v3/files#File[^Google Drive File]\n\n== Authentication\nBy default, this connector will use Google Application Default Credentials (ADC) to authenticate with Google APIs.\n\nTo use this mechanism locally, the following gcloud commands can be used:\n\n\t# Login for the application default credentials and add scopes for readonly drive access\n\tgcloud auth application-default login --scopes='openid,https://www.googleapis.com/auth/userinfo.email,https://www.googleapis.com/auth/cloud-platform,https://www.googleapis.com/auth/drive.readonly'\n\t# When logging in with a user account, you may need to set the quota project for the application default credentials\n\tgcloud auth application-default set-quota-project \u003cproject-id\u003e\n\nOtherwise if using a service account, you can create a JSON key for the service account and set it in the `credentials_json` field.\nIn order for a service account to access files in Google Drive either files need to be explicitly shared with the service account email, otherwise https://support.google.com/a/answer/162106[^domain wide delegation] can be used to share all files within a Google Workspace.\n","categories":["Unstructured"],"examples":[{"title":"Search \u0026 download files from Google Drive","summary":"This examples downloads all the files from Google Drive that are returned in the query","config":"\ninput:\n stdin: {}\npipeline:\n processors:\n - google_drive_search:\n query: \"${!content().string()}\"\n - mutation: 'meta path = this.name'\n - google_drive_download:\n file_id: \"${!this.id}\"\n mime_type: \"${!this.mimeType}\"\noutput:\n file:\n path: \"${!@path}\"\n codec: all-bytes\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"credentials_json","type":"string","kind":"scalar","description":"A service account credentials JSON file. If left unset then the application default credentials are used.","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"query","type":"string","kind":"scalar","description":"The search query to use for finding files in Google Drive. Supports the same query format as the Google Drive UI.","interpolated":true},{"name":"projection","type":"string","kind":"array","description":"The partial fields to include in the result.","default":["id","name","mimeType","size","labelInfo"]},{"name":"include_label_ids","type":"string","kind":"scalar","description":"A comma delimited list of label IDs to include in the result","default":"","interpolated":true},{"name":"max_results","type":"int","kind":"scalar","description":"The maximum number of results to return.","default":64}]}},{"name":"grok","type":"processor","status":"stable","plugin":true,"summary":"Parses messages into a structured format by attempting to apply a list of Grok expressions, the first expression to result in at least one value replaces the original message with a JSON object containing the values.","description":"\nType hints within patterns are respected, therefore with the pattern `%\\{WORD:first},%{INT:second:int}` and a payload of `foo,1` the resulting payload would be `\\{\"first\":\"foo\",\"second\":1}`.\n\n== Performance\n\nThis processor currently uses the https://golang.org/s/re2syntax[Go RE2^] regular expression engine, which is guaranteed to run in time linear to the size of the input. However, this property often makes it less performant than PCRE based implementations of grok. For more information, see https://swtch.com/~rsc/regexp/regexp1.html.","categories":["Parsing"],"footnotes":"\n== Default patterns\n\nFor summary of the default patterns on offer, see https://github.com/Jeffail/grok/blob/master/patterns.go#L5.","examples":[{"title":"VPC Flow Logs","summary":"\nGrok can be used to parse unstructured logs such as VPC flow logs that look like this:\n\n```text\n2 123456789010 eni-1235b8ca123456789 172.31.16.139 172.31.16.21 20641 22 6 20 4249 1418530010 1418530070 ACCEPT OK\n```\n\nInto structured objects that look like this:\n\n```json\n{\"accountid\":\"123456789010\",\"action\":\"ACCEPT\",\"bytes\":4249,\"dstaddr\":\"172.31.16.21\",\"dstport\":22,\"end\":1418530070,\"interfaceid\":\"eni-1235b8ca123456789\",\"logstatus\":\"OK\",\"packets\":20,\"protocol\":6,\"srcaddr\":\"172.31.16.139\",\"srcport\":20641,\"start\":1418530010,\"version\":2}\n```\n\nWith the following config:","config":"\npipeline:\n processors:\n - grok:\n expressions:\n - '%{VPCFLOWLOG}'\n pattern_definitions:\n VPCFLOWLOG: '%{NUMBER:version:int} %{NUMBER:accountid} %{NOTSPACE:interfaceid} %{NOTSPACE:srcaddr} %{NOTSPACE:dstaddr} %{NOTSPACE:srcport:int} %{NOTSPACE:dstport:int} %{NOTSPACE:protocol:int} %{NOTSPACE:packets:int} %{NOTSPACE:bytes:int} %{NUMBER:start:int} %{NUMBER:end:int} %{NOTSPACE:action} %{NOTSPACE:logstatus}'\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"expressions","type":"string","kind":"array","description":"One or more Grok expressions to attempt against incoming messages. The first expression to match at least one value will be used to form a result."},{"name":"pattern_definitions","type":"string","kind":"map","description":"A map of pattern definitions that can be referenced within `patterns`.","default":{}},{"name":"pattern_paths","type":"string","kind":"array","description":"A list of paths to load Grok patterns from. This field supports wildcards, including super globs (double star).","default":[]},{"name":"named_captures_only","type":"bool","kind":"scalar","description":"Whether to only capture values from named patterns.","is_advanced":true,"default":true},{"name":"use_default_patterns","type":"bool","kind":"scalar","description":"Whether to use a \u003c\u003cdefault-patterns, default set of patterns\u003e\u003e.","is_advanced":true,"default":true},{"name":"remove_empty_values","type":"bool","kind":"scalar","description":"Whether to remove values that are empty from the resulting structure.","is_advanced":true,"default":true}]}},{"name":"group_by","type":"processor","status":"stable","plugin":true,"summary":"Splits a xref:configuration:batching.adoc[batch of messages] into N batches, where each resulting batch contains a group of messages determined by a xref:guides:bloblang/about.adoc[Bloblang query].","description":"\nOnce the groups are established a list of processors are applied to their respective grouped batch, which can be used to label the batch as per their grouping. Messages that do not pass the check of any specified group are placed in their own group.\n\nThe functionality of this processor depends on being applied across messages that are batched. You can find out more about batching xref:configuration:batching.adoc[in this doc].","categories":["Composition"],"examples":[{"title":"Grouped Processing","summary":"Imagine we have a batch of messages that we wish to split into a group of foos and everything else, which should be sent to different output destinations based on those groupings. We also need to send the foos as a tar gzip archive. For this purpose we can use the `group_by` processor with a xref:components:outputs/switch.adoc[`switch`] output:","config":"\npipeline:\n processors:\n - group_by:\n - check: content().contains(\"this is a foo\")\n processors:\n - archive:\n format: tar\n - compress:\n algorithm: gzip\n - mapping: 'meta grouping = \"foo\"'\n\noutput:\n switch:\n cases:\n - check: meta(\"grouping\") == \"foo\"\n output:\n gcp_pubsub:\n project: foo_prod\n topic: only_the_foos\n - output:\n gcp_pubsub:\n project: somewhere_else\n topic: no_foos_here\n"}],"config":{"name":"","type":"object","kind":"array","children":[{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message belongs to a given group.","bloblang":true,"examples":["this.type == \"foo\"","this.contents.urls.contains(\"https://benthos.dev/\")","true"]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to execute on the newly formed group.","default":[]}]}},{"name":"group_by_value","type":"processor","status":"stable","plugin":true,"summary":"Splits a batch of messages into N batches, where each resulting batch contains a group of messages determined by a xref:configuration:interpolation.adoc#bloblang-queries[function interpolated string] evaluated per message.","description":"\nThis allows you to group messages using arbitrary fields within their content or metadata, process them individually, and send them to unique locations as per their group.\n\nThe functionality of this processor depends on being applied across messages that are batched. You can find out more about batching xref:configuration:batching.adoc[in this doc].","categories":["Composition"],"footnotes":"\n== Examples\n\nIf we were consuming Kafka messages and needed to group them by their key, archive the groups, and send them to S3 with the key as part of the path we could achieve that with the following:\n\n```yaml\npipeline:\n processors:\n - group_by_value:\n value: ${! meta(\"kafka_key\") }\n - archive:\n format: tar\n - compress:\n algorithm: gzip\noutput:\n aws_s3:\n bucket: TODO\n path: docs/${! meta(\"kafka_key\") }/${! count(\"files\") }-${! timestamp_unix_nano() }.tar.gz\n```","config":{"name":"","type":"object","kind":"scalar","children":[{"name":"value","type":"string","kind":"scalar","description":"The interpolated string to group based on.","interpolated":true,"examples":["${! meta(\"kafka_key\") }","${! json(\"foo.bar\") }-${! meta(\"baz\") }"]}]}},{"name":"http","type":"processor","status":"stable","plugin":true,"summary":"Performs an HTTP request using a message batch as the request body, and replaces the original message parts with the body of the response.","description":"\nThe `rate_limit` field can be used to specify a rate limit xref:components:rate_limits/about.adoc[resource] to cap the rate of requests across all parallel components service wide.\n\nThe URL and header values of this type can be dynamically set using function interpolations described xref:configuration:interpolation.adoc#bloblang-queries[here].\n\nIn order to map or encode the payload to a specific request body, and map the response back into the original payload instead of replacing it entirely, you can use the xref:components:processors/branch.adoc[`branch` processor].\n\n== Response codes\n\nRedpanda Connect considers any response code between 200 and 299 inclusive to indicate a successful response, you can add more success status codes with the field `successful_on`.\n\nWhen a request returns a response code within the `backoff_on` field it will be retried after increasing intervals.\n\nWhen a request returns a response code within the `drop_on` field it will not be reattempted and is immediately considered a failed request.\n\n== Add metadata\n\nIf the request returns an error response code this processor sets a metadata field `http_status_code` on the resulting message.\n\nUse the field `extract_headers` to specify rules for which other headers should be copied into the resulting message from the response.\n\n== Error handling\n\nWhen all retry attempts for a message are exhausted the processor cancels the attempt. These failed messages will continue through the pipeline unchanged, but can be dropped or placed in a dead letter queue according to your config, you can read about xref:configuration:error_handling.adoc[these patterns].","categories":["Integration"],"examples":[{"title":"Branched Request","summary":"This example uses a xref:components:processors/branch.adoc[`branch` processor] to strip the request message into an empty body, grab an HTTP payload, and place the result back into the original message at the path `repo.status`:","config":"\npipeline:\n processors:\n - branch:\n request_map: 'root = \"\"'\n processors:\n - http:\n url: https://hub.docker.com/v2/repositories/jeffail/benthos\n verb: GET\n headers:\n Content-Type: application/json\n result_map: 'root.repo.status = this'\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL to connect to.","interpolated":true},{"name":"verb","type":"string","kind":"scalar","description":"A verb to connect with","default":"POST","examples":["POST","GET","DELETE"]},{"name":"headers","type":"string","kind":"map","description":"A map of headers to add to the request.","default":{},"interpolated":true,"examples":[{"Content-Type":"application/octet-stream","traceparent":"${! tracing_span().traceparent }"}]},{"name":"metadata","type":"object","kind":"scalar","description":"Specify optional matching rules to determine which metadata keys should be added to the HTTP request as headers.","is_advanced":true,"is_optional":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","is_advanced":true,"default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","is_advanced":true,"default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"dump_request_log_level","type":"string","kind":"scalar","description":"EXPERIMENTAL: Optionally set a level at which the request and response payload of each request made will be logged.","is_advanced":true,"default":"","options":["TRACE","DEBUG","INFO","WARN","ERROR","FATAL",""],"version":"4.12.0","linter":"\nlet options = {\n \"trace\": true,\n \"debug\": true,\n \"info\": true,\n \"warn\": true,\n \"error\": true,\n \"fatal\": true,\n \"\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"oauth","type":"object","kind":"scalar","description":"Allows you to specify open authentication via OAuth version 1.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use OAuth version 1 in requests.","is_advanced":true,"default":false},{"name":"consumer_key","type":"string","kind":"scalar","description":"A value used to identify the client to the service provider.","is_advanced":true,"default":""},{"name":"consumer_secret","type":"string","kind":"scalar","description":"A secret used to establish ownership of the consumer key.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"access_token","type":"string","kind":"scalar","description":"A value used to gain access to the protected resources on behalf of the user.","is_advanced":true,"default":""},{"name":"access_token_secret","type":"string","kind":"scalar","description":"A secret provided in order to establish ownership of a given access token.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"oauth2","type":"object","kind":"scalar","description":"Allows you to specify open authentication via OAuth version 2 using the client credentials token flow.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use OAuth version 2 in requests.","is_advanced":true,"default":false},{"name":"client_key","type":"string","kind":"scalar","description":"A value used to identify the client to the token provider.","is_advanced":true,"default":""},{"name":"client_secret","type":"string","kind":"scalar","description":"A secret used to establish ownership of the client key.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token_url","type":"string","kind":"scalar","description":"The URL of the token provider.","is_advanced":true,"default":"","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"scopes","type":"string","kind":"array","description":"A list of optional requested permissions.","is_advanced":true,"default":[],"version":"3.45.0"},{"name":"endpoint_params","type":"unknown","kind":"map","description":"A list of optional endpoint parameters, values should be arrays of strings.","is_advanced":true,"is_optional":true,"default":{},"examples":[{"bar":["woof"],"foo":["meow","quack"]}],"version":"4.21.0","linter":"\nroot = if this.type() == \"object\" {\n this.values().map_each(ele -\u003e if ele.type() != \"array\" {\n \"field must be an object containing arrays of strings, got %s (%v)\".format(ele.format_json(no_indent: true), ele.type())\n } else {\n ele.map_each(str -\u003e if str.type() != \"string\" {\n \"field values must be strings, got %s (%v)\".format(str.format_json(no_indent: true), str.type())\n } else { deleted() })\n }).\n flatten()\n}\n"}]},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"jwt","type":"object","kind":"scalar","description":"BETA: Allows you to specify JWT authentication.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use JWT authentication in requests.","is_advanced":true,"default":false},{"name":"private_key_file","type":"string","kind":"scalar","description":"A file with the PEM encoded via PKCS1 or PKCS8 as private key.","is_advanced":true,"default":""},{"name":"signing_method","type":"string","kind":"scalar","description":"A method used to sign the token such as RS256, RS384, RS512 or EdDSA.","is_advanced":true,"default":""},{"name":"claims","type":"unknown","kind":"map","description":"A value used to identify the claims that issued the JWT.","is_advanced":true,"default":{}},{"name":"headers","type":"unknown","kind":"map","description":"Add optional key/value headers to the JWT.","is_advanced":true,"default":{}}]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"extract_headers","type":"object","kind":"scalar","description":"Specify which response headers should be added to resulting messages as metadata. Header keys are lowercased before matching, so ensure that your patterns target lowercased versions of the header keys that you expect.","is_advanced":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","is_advanced":true,"default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","is_advanced":true,"default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"rate_limit","type":"string","kind":"scalar","description":"An optional xref:components:rate_limits/about.adoc[rate limit] to throttle requests by.","is_optional":true},{"name":"timeout","type":"string","kind":"scalar","description":"A static timeout to apply to requests.","default":"5s"},{"name":"retry_period","type":"string","kind":"scalar","description":"The base period to wait between failed requests.","is_advanced":true,"default":"1s"},{"name":"max_retry_backoff","type":"string","kind":"scalar","description":"The maximum period to wait between failed requests.","is_advanced":true,"default":"300s"},{"name":"retries","type":"int","kind":"scalar","description":"The maximum number of retry attempts to make.","is_advanced":true,"default":3},{"name":"follow_redirects","type":"bool","kind":"scalar","description":"Whether or not to transparently follow redirects, i.e. responses with 300-399 status codes. If disabled, the response message will contain the body, status, and headers from the redirect response and the processor will not make a request to the URL set in the Location header of the response.","is_advanced":true,"default":true},{"name":"backoff_on","type":"int","kind":"array","description":"A list of status codes whereby the request should be considered to have failed and retries should be attempted, but the period between them should be increased gradually.","is_advanced":true,"default":[429]},{"name":"drop_on","type":"int","kind":"array","description":"A list of status codes whereby the request should be considered to have failed but retries should not be attempted. This is useful for preventing wasted retries for requests that will never succeed. Note that with these status codes the _request_ is dropped, but _message_ that caused the request will not be dropped.","is_advanced":true,"default":[]},{"name":"successful_on","type":"int","kind":"array","description":"A list of status codes whereby the attempt should be considered successful, this is useful for dropping requests that return non-2XX codes indicating that the message has been dealt with, such as a 303 See Other or a 409 Conflict. All 2XX codes are considered successful unless they are present within `backoff_on` or `drop_on`, regardless of this field.","is_advanced":true,"default":[]},{"name":"proxy_url","type":"string","kind":"scalar","description":"An optional HTTP proxy URL.","is_advanced":true,"is_optional":true},{"name":"disable_http2","type":"bool","kind":"scalar","description":"Whether or not to disable disable HTTP/2","is_advanced":true,"default":false,"version":"4.44.0"},{"name":"batch_as_multipart","type":"bool","kind":"scalar","description":"Send message batches as a single request using https://www.w3.org/Protocols/rfc1341/7_2_Multipart.html[RFC1341^].","is_advanced":true,"default":false},{"name":"parallel","type":"bool","kind":"scalar","description":"When processing batched messages, whether to send messages of the batch in parallel, otherwise they are sent serially.","default":false}]}},{"name":"insert_part","type":"processor","status":"stable","plugin":true,"summary":"Insert a new message into a batch at an index. If the specified index is greater than the length of the existing batch it will be appended to the end.","description":"\nThe index can be negative, and if so the message will be inserted from the end counting backwards starting from -1. E.g. if index = -1 then the new message will become the last of the batch, if index = -2 then the new message will be inserted before the last message, and so on. If the negative index is greater than the length of the existing batch it will be inserted at the beginning.\n\nThe new message will have metadata copied from the first pre-existing message of the batch.\n\nThis processor will interpolate functions within the 'content' field, you can find a list of functions xref:configuration:interpolation.adoc#bloblang-queries[here].","categories":["Composition"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"index","type":"int","kind":"scalar","description":"The index within the batch to insert the message at.","default":-1},{"name":"content","type":"string","kind":"scalar","description":"The content of the message being inserted.","default":"","interpolated":true}]}},{"name":"javascript","type":"processor","status":"experimental","plugin":true,"summary":"Executes a provided JavaScript code block or file for each message.","description":"\nThe https://github.com/dop251/goja[execution engine^] behind this processor provides full ECMAScript 5.1 support (including regex and strict mode). Most of the ECMAScript 6 spec is implemented but this is a work in progress.\n\nImports via `require` should work similarly to NodeJS, and access to the console is supported which will print via the Redpanda Connect logger. More caveats can be found on https://github.com/dop251/goja#known-incompatibilities-and-caveats[GitHub^].\n\nThis processor is implemented using the https://github.com/dop251/goja[github.com/dop251/goja^] library.","categories":["Mapping"],"footnotes":"\n== Runtime\n\nIn order to optimize code execution JS runtimes are created on demand (in order to support parallel execution) and are reused across invocations. Therefore, it is important to understand that global state created by your programs will outlive individual invocations. In order for your programs to avoid failing after the first invocation ensure that you do not define variables at the global scope.\n\nAlthough technically possible, it is recommended that you do not rely on the global state for maintaining state across invocations as the pooling nature of the runtimes will prevent deterministic behavior. We aim to support deterministic strategies for mutating global state in the future.\n\n== Functions\n\n### `benthos.v0_fetch`\n\nExecutes an HTTP request synchronously and returns the result as an object of the form `{\"status\":200,\"body\":\"foo\"}`.\n\n#### Parameters\n\n**`url`** \u0026lt;string\u0026gt; The URL to fetch \n**`headers`** \u0026lt;object(string,string)\u0026gt; An object of string/string key/value pairs to add the request as headers. \n**`method`** \u0026lt;string\u0026gt; The method of the request. \n**`body`** \u0026lt;(optional) string\u0026gt; A body to send. \n\n#### Examples\n\n```javascript\nlet result = benthos.v0_fetch(\"http://example.com\", {}, \"GET\", \"\")\nbenthos.v0_msg_set_structured(result);\n```\n\n### `benthos.v0_msg_as_string`\n\nObtain the raw contents of the processed message as a string.\n\n#### Examples\n\n```javascript\nlet contents = benthos.v0_msg_as_string();\n```\n\n### `benthos.v0_msg_as_structured`\n\nObtain the root of the processed message as a structured value. If the message is not valid JSON or has not already been expanded into a structured form this function will throw an error.\n\n#### Examples\n\n```javascript\nlet foo = benthos.v0_msg_as_structured().foo;\n```\n\n### `benthos.v0_msg_exists_meta`\n\nCheck that a metadata key exists.\n\n#### Parameters\n\n**`name`** \u0026lt;string\u0026gt; The metadata key to search for. \n\n#### Examples\n\n```javascript\nif (benthos.v0_msg_exists_meta(\"kafka_key\")) {}\n```\n\n### `benthos.v0_msg_get_meta`\n\nGet the value of a metadata key from the processed message.\n\n#### Parameters\n\n**`name`** \u0026lt;string\u0026gt; The metadata key to search for. \n\n#### Examples\n\n```javascript\nlet key = benthos.v0_msg_get_meta(\"kafka_key\");\n```\n\n### `benthos.v0_msg_set_meta`\n\nSet a metadata key on the processed message to a value.\n\n#### Parameters\n\n**`name`** \u0026lt;string\u0026gt; The metadata key to set. \n**`value`** \u0026lt;anything\u0026gt; The value to set it to. \n\n#### Examples\n\n```javascript\nbenthos.v0_msg_set_meta(\"thing\", \"hello world\");\n```\n\n### `benthos.v0_msg_set_string`\n\nSet the contents of the processed message to a given string.\n\n#### Parameters\n\n**`value`** \u0026lt;string\u0026gt; The value to set it to. \n\n#### Examples\n\n```javascript\nbenthos.v0_msg_set_string(\"hello world\");\n```\n\n### `benthos.v0_msg_set_structured`\n\nSet the root of the processed message to a given value of any type.\n\n#### Parameters\n\n**`value`** \u0026lt;anything\u0026gt; The value to set it to. \n\n#### Examples\n\n```javascript\nbenthos.v0_msg_set_structured({\n \"foo\": \"a thing\",\n \"bar\": \"something else\",\n \"baz\": 1234\n});\n```\n\n","examples":[{"title":"Simple mutation","summary":"In this example we define a simple function that performs a basic mutation against messages, treating their contents as raw strings.","config":"\npipeline:\n processors:\n - javascript:\n code: 'benthos.v0_msg_set_string(benthos.v0_msg_as_string() + \"hello world\");'\n"},{"title":"Structured mutation","summary":"In this example we define a function that performs basic mutations against a structured message. Note that we encapsulate the logic within an anonymous function that is called for each invocation, this is required in order to avoid duplicate variable declarations in the global state.","config":"\npipeline:\n processors:\n - javascript:\n code: |\n (() =\u003e {\n let thing = benthos.v0_msg_as_structured();\n thing.num_keys = Object.keys(thing).length;\n delete thing[\"b\"];\n benthos.v0_msg_set_structured(thing);\n })();\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"code","type":"string","kind":"scalar","description":"An inline JavaScript program to run. One of `code` or `file` must be defined.","is_optional":true},{"name":"file","type":"string","kind":"scalar","description":"A file containing a JavaScript program to run. One of `code` or `file` must be defined.","is_optional":true},{"name":"global_folders","type":"string","kind":"array","description":"List of folders that will be used to load modules from if the requested JS module is not found elsewhere.","default":[]}],"linter":"\nlet codeLen = (this.code | \"\").length()\nlet fileLen = (this.file | \"\").length()\nroot = if $codeLen == 0 \u0026\u0026 $fileLen == 0 {\n \"either the code or file field must be specified\"\n} else if $codeLen \u003e 0 \u0026\u0026 $fileLen \u003e 0 {\n \"cannot specify both the code and file fields\"\n}"},"version":"4.14.0"},{"name":"jmespath","type":"processor","status":"stable","plugin":true,"summary":"Executes a http://jmespath.org/[JMESPath query] on JSON documents and replaces the message with the resulting document.","description":"\n[TIP]\n.Try out Bloblang\n====\nFor better performance and improved capabilities try native Redpanda Connect mapping with the xref:components:processors/mapping.adoc[`mapping` processor].\n====\n","categories":["Mapping"],"examples":[{"title":"Mapping","summary":"\nWhen receiving JSON documents of the form:\n\n```json\n{\n \"locations\": [\n {\"name\": \"Seattle\", \"state\": \"WA\"},\n {\"name\": \"New York\", \"state\": \"NY\"},\n {\"name\": \"Bellevue\", \"state\": \"WA\"},\n {\"name\": \"Olympia\", \"state\": \"WA\"}\n ]\n}\n```\n\nWe could collapse the location names from the state of Washington into a field `Cities`:\n\n```json\n{\"Cities\": \"Bellevue, Olympia, Seattle\"}\n```\n\nWith the following config:","config":"\npipeline:\n processors:\n - jmespath:\n query: \"locations[?state == 'WA'].name | sort(@) | {Cities: join(', ', @)}\"\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"query","type":"string","kind":"scalar","description":"The JMESPath query to apply to messages."}]}},{"name":"jq","type":"processor","status":"stable","plugin":true,"summary":"Transforms and filters messages using jq queries.","description":"\n[TIP]\n.Try out Bloblang\n====\nFor better performance and improved capabilities try out native Redpanda Connect mapping with the xref:components:processors/mapping.adoc[`mapping` processor].\n====\n\nThe provided query is executed on each message, targeting either the contents as a structured JSON value or as a raw string using the field `raw`, and the message is replaced with the query result.\n\nMessage metadata is also accessible within the query from the variable `$metadata`.\n\nThis processor uses the https://github.com/itchyny/gojq[gojq library^], and therefore does not require jq to be installed as a dependency. However, this also means there are some https://github.com/itchyny/gojq#difference-to-jq[differences in how these queries are executed^] versus the jq cli.\n\nIf the query does not emit any value then the message is filtered, if the query returns multiple values then the resulting message will be an array containing all values.\n\nThe full query syntax is described in https://stedolan.github.io/jq/manual/[jq's documentation^].\n\n== Error handling\n\nQueries can fail, in which case the message remains unchanged, errors are logged, and the message is flagged as having failed, allowing you to use xref:configuration:error_handling.adoc[standard processor error handling patterns].","categories":["Mapping"],"examples":[{"title":"Mapping","summary":"\nWhen receiving JSON documents of the form:\n\n```json\n{\n \"locations\": [\n {\"name\": \"Seattle\", \"state\": \"WA\"},\n {\"name\": \"New York\", \"state\": \"NY\"},\n {\"name\": \"Bellevue\", \"state\": \"WA\"},\n {\"name\": \"Olympia\", \"state\": \"WA\"}\n ]\n}\n```\n\nWe could collapse the location names from the state of Washington into a field `Cities`:\n\n```json\n{\"Cities\": \"Bellevue, Olympia, Seattle\"}\n```\n\nWith the following config:","config":"\npipeline:\n processors:\n - jq:\n query: '{Cities: .locations | map(select(.state == \"WA\").name) | sort | join(\", \") }'\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"query","type":"string","kind":"scalar","description":"The jq query to filter and transform messages with."},{"name":"raw","type":"bool","kind":"scalar","description":"Whether to process the input as a raw string instead of as JSON.","is_advanced":true,"default":false},{"name":"output_raw","type":"bool","kind":"scalar","description":"Whether to output raw text (unquoted) instead of JSON strings when the emitted values are string types.","is_advanced":true,"default":false}]}},{"name":"json_schema","type":"processor","status":"stable","plugin":true,"summary":"Checks messages against a provided JSONSchema definition but does not change the payload under any circumstances. If a message does not match the schema it can be caught using xref:configuration:error_handling.adoc[error handling methods].","description":"Please refer to the https://json-schema.org/[JSON Schema website^] for information and tutorials regarding the syntax of the schema.","categories":["Mapping"],"footnotes":"\n== Examples\n\nWith the following JSONSchema document:\n\n```json\n{\n\t\"$id\": \"https://example.com/person.schema.json\",\n\t\"$schema\": \"http://json-schema.org/draft-07/schema#\",\n\t\"title\": \"Person\",\n\t\"type\": \"object\",\n\t\"properties\": {\n\t \"firstName\": {\n\t\t\"type\": \"string\",\n\t\t\"description\": \"The person's first name.\"\n\t },\n\t \"lastName\": {\n\t\t\"type\": \"string\",\n\t\t\"description\": \"The person's last name.\"\n\t },\n\t \"age\": {\n\t\t\"description\": \"Age in years which must be equal to or greater than zero.\",\n\t\t\"type\": \"integer\",\n\t\t\"minimum\": 0\n\t }\n\t}\n}\n```\n\nAnd the following Redpanda Connect configuration:\n\n```yaml\npipeline:\n processors:\n - json_schema:\n schema_path: \"file://path_to_schema.json\"\n - catch:\n - log:\n level: ERROR\n message: \"Schema validation failed due to: ${!error()}\"\n - mapping: 'root = deleted()' # Drop messages that fail\n```\n\nIf a payload being processed looked like:\n\n```json\n{\"firstName\":\"John\",\"lastName\":\"Doe\",\"age\":-21}\n```\n\nThen a log message would appear explaining the fault and the payload would be\ndropped.","config":{"name":"","type":"object","kind":"scalar","children":[{"name":"schema","type":"string","kind":"scalar","description":"A schema to apply. Use either this or the `schema_path` field.","is_optional":true},{"name":"schema_path","type":"string","kind":"scalar","description":"The path of a schema document to apply. Use either this or the `schema` field.","is_optional":true}]}},{"name":"log","type":"processor","status":"stable","plugin":true,"summary":"Prints a log event for each message. Messages always remain unchanged. The log message can be set using function interpolations described in xref:configuration:interpolation.adoc#bloblang-queries[Bloblang queries] which allows you to log the contents and metadata of messages.","description":"\nThe `level` field determines the log level of the printed events and can be any of the following values: TRACE, DEBUG, INFO, WARN, ERROR.\n\n== Structured fields\n\nIt's also possible add custom fields to logs when the format is set to a structured form such as `json` or `logfmt` with the config field \u003c\u003cfields_mapping, `fields_mapping`\u003e\u003e:\n\n```yaml\npipeline:\n processors:\n - log:\n level: DEBUG\n message: hello world\n fields_mapping: |\n root.reason = \"cus I wana\"\n root.id = this.id\n root.age = this.user.age\n root.kafka_topic = meta(\"kafka_topic\")\n```\n","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"level","type":"string","kind":"scalar","description":"The log level to use.","default":"INFO","options":["ERROR","WARN","INFO","DEBUG","TRACE"]},{"name":"fields_mapping","type":"string","kind":"scalar","description":"An optional xref:guides:bloblang/about.adoc[Bloblang mapping] that can be used to specify extra fields to add to the log. If log fields are also added with the `fields` field then those values will override matching keys from this mapping.","is_optional":true,"bloblang":true,"examples":["root.reason = \"cus I wana\"\nroot.id = this.id\nroot.age = this.user.age.number()\nroot.kafka_topic = meta(\"kafka_topic\")"]},{"name":"message","type":"string","kind":"scalar","description":"The message to print.","default":"","interpolated":true},{"name":"fields","type":"string","kind":"map","description":"A map of fields to print along with the log message.","is_deprecated":true,"is_optional":true,"interpolated":true}]}},{"name":"mapping","type":"processor","status":"stable","plugin":true,"summary":"Executes a xref:guides:bloblang/about.adoc[Bloblang] mapping on messages, creating a new document that replaces (or filters) the original message.","description":"\nBloblang is a powerful language that enables a wide range of mapping, transformation and filtering tasks. For more information, see xref:guides:bloblang/about.adoc[].\n\nIf your mapping is large and you'd prefer for it to live in a separate file then you can execute a mapping directly from a file with the expression `from \"\u003cpath\u003e\"`, where the path must be absolute, or relative from the location that Redpanda Connect is executed from.\n\nNote: This processor is equivalent to the xref:components:processors/bloblang.adoc#component-rename[Bloblang] one. The latter will be deprecated in a future release.\n\n== Input document immutability\n\nMapping operates by creating an entirely new object during assignments, this has the advantage of treating the original referenced document as immutable and therefore queryable at any stage of your mapping. For example, with the following mapping:\n\n```coffeescript\nroot.id = this.id\nroot.invitees = this.invitees.filter(i -\u003e i.mood \u003e= 0.5)\nroot.rejected = this.invitees.filter(i -\u003e i.mood \u003c 0.5)\n```\n\nNotice that we mutate the value of `invitees` in the resulting document by filtering out objects with a lower mood. However, even after doing so we're still able to reference the unchanged original contents of this value from the input document in order to populate a second field. Within this mapping we also have the flexibility to reference the mutable mapped document by using the keyword `root` (i.e. `root.invitees`) on the right-hand side instead.\n\nMapping documents is advantageous in situations where the result is a document with a dramatically different shape to the input document, since we are effectively rebuilding the document in its entirety and might as well keep a reference to the unchanged input document throughout. However, in situations where we are only performing minor alterations to the input document, the rest of which is unchanged, it might be more efficient to use the xref:components:processors/mutation.adoc[`mutation` processor] instead.\n\n== Error handling\n\nBloblang mappings can fail, in which case the message remains unchanged, errors are logged, and the message is flagged as having failed, allowing you to use xref:configuration:error_handling.adoc[standard processor error handling patterns].\n\nHowever, Bloblang itself also provides powerful ways of ensuring your mappings do not fail by specifying desired xref:guides:bloblang/about.adoc#error-handling[fallback behavior].\n\t\t\t","categories":["Mapping","Parsing"],"examples":[{"title":"Mapping","summary":"\nGiven JSON documents containing an array of fans:\n\n```json\n{\n \"id\":\"foo\",\n \"description\":\"a show about foo\",\n \"fans\":[\n {\"name\":\"bev\",\"obsession\":0.57},\n {\"name\":\"grace\",\"obsession\":0.21},\n {\"name\":\"ali\",\"obsession\":0.89},\n {\"name\":\"vic\",\"obsession\":0.43}\n ]\n}\n```\n\nWe can reduce the documents down to just the ID and only those fans with an obsession score above 0.5, giving us:\n\n```json\n{\n \"id\":\"foo\",\n \"fans\":[\n {\"name\":\"bev\",\"obsession\":0.57},\n {\"name\":\"ali\",\"obsession\":0.89}\n ]\n}\n```\n\nWith the following config:","config":"\npipeline:\n processors:\n - mapping: |\n root.id = this.id\n root.fans = this.fans.filter(fan -\u003e fan.obsession \u003e 0.5)\n"},{"title":"More Mapping","summary":"\nWhen receiving JSON documents of the form:\n\n```json\n{\n \"locations\": [\n {\"name\": \"Seattle\", \"state\": \"WA\"},\n {\"name\": \"New York\", \"state\": \"NY\"},\n {\"name\": \"Bellevue\", \"state\": \"WA\"},\n {\"name\": \"Olympia\", \"state\": \"WA\"}\n ]\n}\n```\n\nWe could collapse the location names from the state of Washington into a field `Cities`:\n\n```json\n{\"Cities\": \"Bellevue, Olympia, Seattle\"}\n```\n\nWith the following config:","config":"\npipeline:\n processors:\n - mapping: |\n root.Cities = this.locations.\n filter(loc -\u003e loc.state == \"WA\").\n map_each(loc -\u003e loc.name).\n sort().join(\", \")\n"}],"config":{"name":"","type":"string","kind":"scalar","bloblang":true},"version":"4.5.0"},{"name":"metric","type":"processor","status":"stable","plugin":true,"summary":"Emit custom metrics by extracting values from messages.","description":"\nThis processor works by evaluating an xref:configuration:interpolation.adoc#bloblang-queries[interpolated field `value`] for each message and updating a emitted metric according to the \u003c\u003ctypes, type\u003e\u003e.\n\nCustom metrics such as these are emitted along with Redpanda Connect internal metrics, where you can customize where metrics are sent, which metric names are emitted and rename them as/when appropriate. For more information see the xref:components:metrics/about.adoc[metrics docs].","categories":["Utility"],"footnotes":"\n== Types\n\n=== `counter`\n\nIncrements a counter by exactly 1, the contents of `value` are ignored\nby this type.\n\n=== `counter_by`\n\nIf the contents of `value` can be parsed as a positive integer value\nthen the counter is incremented by this value.\n\nFor example, the following configuration will increment the value of the\n`count.custom.field` metric by the contents of `field.some.value`:\n\n```yaml\npipeline:\n processors:\n - metric:\n type: counter_by\n name: CountCustomField\n value: ${!json(\"field.some.value\")}\n```\n\n=== `gauge`\n\nIf the contents of `value` can be parsed as a positive integer value\nthen the gauge is set to this value.\n\nFor example, the following configuration will set the value of the\n`gauge.custom.field` metric to the contents of `field.some.value`:\n\n```yaml\npipeline:\n processors:\n - metric:\n type: gauge\n name: GaugeCustomField\n value: ${!json(\"field.some.value\")}\n```\n\n=== `timing`\n\nEquivalent to `gauge` where instead the metric is a timing. It is recommended that timing values are recorded in nanoseconds in order to be consistent with standard Redpanda Connect timing metrics, as in some cases these values are automatically converted into other units such as when exporting timings as histograms with Prometheus metrics.","examples":[{"title":"Counter","summary":"In this example we emit a counter metric called `Foos`, which increments for every message processed, and we label the metric with some metadata about where the message came from and a field from the document that states what type it is. We also configure our metrics to emit to CloudWatch, and explicitly only allow our custom metric and some internal Redpanda Connect metrics to emit.","config":"\npipeline:\n processors:\n - metric:\n name: Foos\n type: counter\n labels:\n topic: ${! meta(\"kafka_topic\") }\n partition: ${! meta(\"kafka_partition\") }\n type: ${! json(\"document.type\").or(\"unknown\") }\n\nmetrics:\n mapping: |\n root = if ![\n \"Foos\",\n \"input_received\",\n \"output_sent\"\n ].contains(this) { deleted() }\n aws_cloudwatch:\n namespace: ProdConsumer\n"},{"title":"Gauge","summary":"In this example we emit a gauge metric called `FooSize`, which is given a value extracted from JSON messages at the path `foo.size`. We then also configure our Prometheus metric exporter to only emit this custom metric and nothing else. We also label the metric with some metadata.","config":"\npipeline:\n processors:\n - metric:\n name: FooSize\n type: gauge\n labels:\n topic: ${! meta(\"kafka_topic\") }\n value: ${! json(\"foo.size\") }\n\nmetrics:\n mapping: 'if this != \"FooSize\" { deleted() }'\n prometheus: {}\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"type","type":"string","kind":"scalar","description":"The metric \u003c\u003ctypes, type\u003e\u003e to create.","options":["counter","counter_by","gauge","timing"],"linter":"\nlet options = {\n \"counter\": true,\n \"counter_by\": true,\n \"gauge\": true,\n \"timing\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"name","type":"string","kind":"scalar","description":"The name of the metric to create, this must be unique across all Redpanda Connect components otherwise it will overwrite those other metrics."},{"name":"labels","type":"string","kind":"map","description":"A map of label names and values that can be used to enrich metrics. Labels are not supported by some metric destinations, in which case the metrics series are combined.","is_optional":true,"interpolated":true,"examples":[{"topic":"${! meta(\"kafka_topic\") }","type":"${! json(\"doc.type\") }"}]},{"name":"value","type":"string","kind":"scalar","description":"For some metric types specifies a value to set, increment. Certain metrics exporters such as Prometheus support floating point values, but those that do not will cast a floating point value into an integer.","default":"","interpolated":true}]}},{"name":"mongodb","type":"processor","status":"experimental","plugin":true,"summary":"Performs operations against MongoDB for each message, allowing you to store or retrieve data within message payloads.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target MongoDB server.","examples":["mongodb://localhost:27017"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"database","type":"string","kind":"scalar","description":"The name of the target MongoDB database."},{"name":"username","type":"string","kind":"scalar","description":"The username to connect to the database.","default":""},{"name":"password","type":"string","kind":"scalar","description":"The password to connect to the database.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"app_name","type":"string","kind":"scalar","description":"The client application name.","is_advanced":true,"default":"benthos","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"collection","type":"string","kind":"scalar","description":"The name of the target collection."},{"name":"operation","type":"string","kind":"scalar","description":"The mongodb operation to perform.","default":"insert-one","options":["insert-one","delete-one","delete-many","replace-one","update-one","find-one","aggregate"],"linter":"\nlet options = {\n \"insert-one\": true,\n \"delete-one\": true,\n \"delete-many\": true,\n \"replace-one\": true,\n \"update-one\": true,\n \"find-one\": true,\n \"aggregate\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"write_concern","type":"object","kind":"scalar","description":"The write concern settings for the mongo connection.","children":[{"name":"w","type":"string","kind":"scalar","description":"W requests acknowledgement that write operations propagate to the specified number of mongodb instances. Can be the string \"majority\" to wait for a calculated majority of nodes to acknowledge the write operation, or an integer value specifying an minimum number of nodes to acknowledge the operation, or a string specifying the name of a custom write concern configured in the cluster.","default":"majority"},{"name":"j","type":"bool","kind":"scalar","description":"J requests acknowledgement from MongoDB that write operations are written to the journal.","default":false},{"name":"w_timeout","type":"string","kind":"scalar","description":"The write concern timeout.","default":""}]},{"name":"document_map","type":"string","kind":"scalar","description":"A bloblang map representing a document to store within MongoDB, expressed as https://www.mongodb.com/docs/manual/reference/mongodb-extended-json/[extended JSON in canonical form^]. The document map is required for the operations insert-one, replace-one, update-one and aggregate.","default":"","bloblang":true,"examples":["root.a = this.foo\nroot.b = this.bar"]},{"name":"filter_map","type":"string","kind":"scalar","description":"A bloblang map representing a filter for a MongoDB command, expressed as https://www.mongodb.com/docs/manual/reference/mongodb-extended-json/[extended JSON in canonical form^]. The filter map is required for all operations except insert-one. It is used to find the document(s) for the operation. For example in a delete-one case, the filter map should have the fields required to locate the document to delete.","default":"","bloblang":true,"examples":["root.a = this.foo\nroot.b = this.bar"]},{"name":"hint_map","type":"string","kind":"scalar","description":"A bloblang map representing the hint for the MongoDB command, expressed as https://www.mongodb.com/docs/manual/reference/mongodb-extended-json/[extended JSON in canonical form^]. This map is optional and is used with all operations except insert-one. It is used to improve performance of finding the documents in the mongodb.","default":"","bloblang":true,"examples":["root.a = this.foo\nroot.b = this.bar"]},{"name":"upsert","type":"bool","kind":"scalar","description":"The upsert setting is optional and only applies for update-one and replace-one operations. If the filter specified in filter_map matches, the document is updated or replaced accordingly, otherwise it is created.","default":false,"version":"3.60.0"},{"name":"json_marshal_mode","type":"string","kind":"scalar","description":"The json_marshal_mode setting is optional and controls the format of the output message.","is_advanced":true,"default":"canonical","annotated_options":[["canonical","A string format that emphasizes type preservation at the expense of readability and interoperability. That is, conversion from canonical to BSON will generally preserve type information except in certain specific cases. "],["relaxed","A string format that emphasizes readability and interoperability at the expense of type preservation. That is, conversion from relaxed format to BSON can lose type information."]],"version":"3.60.0","linter":"\nlet options = {\n \"canonical\": true,\n \"relaxed\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"max_retries","type":"int","kind":"scalar","description":"The maximum number of retries before giving up on the request. If set to zero there is no discrete limit.","is_advanced":true,"is_deprecated":true,"default":3},{"name":"backoff","type":"object","kind":"scalar","description":"Control time intervals between retry attempts.","is_advanced":true,"is_deprecated":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"is_deprecated":true,"default":"1s"},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts.","is_advanced":true,"is_deprecated":true,"default":"5s"},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum period to wait before retry attempts are abandoned. If zero then no limit is used.","is_advanced":true,"is_deprecated":true,"default":"30s"}]}]},"version":"3.43.0"},{"name":"msgpack","type":"processor","status":"beta","plugin":true,"summary":"Converts messages to or from the https://msgpack.org/[MessagePack^] format.","categories":["Parsing"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"operator","type":"string","kind":"scalar","description":"The operation to perform on messages.","annotated_options":[["from_json","Convert JSON messages to MessagePack format"],["to_json","Convert MessagePack messages to JSON format"]],"linter":"\nlet options = {\n \"from_json\": true,\n \"to_json\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"}]},"version":"3.59.0"},{"name":"mutation","type":"processor","status":"stable","plugin":true,"summary":"Executes a xref:guides:bloblang/about.adoc[Bloblang] mapping and directly transforms the contents of messages, mutating (or deleting) them.","description":"\nBloblang is a powerful language that enables a wide range of mapping, transformation and filtering tasks. For more information, see xref:guides:bloblang/about.adoc[].\n\nIf your mapping is large and you'd prefer for it to live in a separate file then you can execute a mapping directly from a file with the expression `from \"\u003cpath\u003e\"`, where the path must be absolute, or relative from the location that Redpanda Connect is executed from.\n\n== Input document mutability\n\nA mutation is a mapping that transforms input documents directly, this has the advantage of reducing the need to copy the data fed into the mapping. However, this also means that the referenced document is mutable and therefore changes throughout the mapping. For example, with the following Bloblang:\n\n```coffeescript\nroot.rejected = this.invitees.filter(i -\u003e i.mood \u003c 0.5)\nroot.invitees = this.invitees.filter(i -\u003e i.mood \u003e= 0.5)\n```\n\nNotice that we create a field `rejected` by copying the array field `invitees` and filtering out objects with a high mood. We then overwrite the field `invitees` by filtering out objects with a low mood, resulting in two array fields that are each a subset of the original. If we were to reverse the ordering of these assignments like so:\n\n```coffeescript\nroot.invitees = this.invitees.filter(i -\u003e i.mood \u003e= 0.5)\nroot.rejected = this.invitees.filter(i -\u003e i.mood \u003c 0.5)\n```\n\nThen the new field `rejected` would be empty as we have already mutated `invitees` to exclude the objects that it would be populated by. We can solve this problem either by carefully ordering our assignments or by capturing the original array using a variable (`let invitees = this.invitees`).\n\nMutations are advantageous over a standard mapping in situations where the result is a document with mostly the same shape as the input document, since we can avoid unnecessarily copying data from the referenced input document. However, in situations where we are creating an entirely new document shape it can be more convenient to use the traditional xref:components:processors/mapping.adoc[`mapping` processor] instead.\n\n== Error handling\n\nBloblang mappings can fail, in which case the error is logged and the message is flagged as having failed, allowing you to use xref:configuration:error_handling.adoc[standard processor error handling patterns].\n\nHowever, Bloblang itself also provides powerful ways of ensuring your mappings do not fail by specifying desired xref:guides:bloblang/about.adoc#error-handling[fallback behavior].\n\t\t\t","categories":["Mapping","Parsing"],"examples":[{"title":"Mapping","summary":"\nGiven JSON documents containing an array of fans:\n\n```json\n{\n \"id\":\"foo\",\n \"description\":\"a show about foo\",\n \"fans\":[\n {\"name\":\"bev\",\"obsession\":0.57},\n {\"name\":\"grace\",\"obsession\":0.21},\n {\"name\":\"ali\",\"obsession\":0.89},\n {\"name\":\"vic\",\"obsession\":0.43}\n ]\n}\n```\n\nWe can reduce the documents down to just the ID and only those fans with an obsession score above 0.5, giving us:\n\n```json\n{\n \"id\":\"foo\",\n \"fans\":[\n {\"name\":\"bev\",\"obsession\":0.57},\n {\"name\":\"ali\",\"obsession\":0.89}\n ]\n}\n```\n\nWith the following config:","config":"\npipeline:\n processors:\n - mutation: |\n root.description = deleted()\n root.fans = this.fans.filter(fan -\u003e fan.obsession \u003e 0.5)\n"},{"title":"More Mapping","summary":"\nWhen receiving JSON documents of the form:\n\n```json\n{\n \"locations\": [\n {\"name\": \"Seattle\", \"state\": \"WA\"},\n {\"name\": \"New York\", \"state\": \"NY\"},\n {\"name\": \"Bellevue\", \"state\": \"WA\"},\n {\"name\": \"Olympia\", \"state\": \"WA\"}\n ]\n}\n```\n\nWe could collapse the location names from the state of Washington into a field `Cities`:\n\n```json\n{\"Cities\": \"Bellevue, Olympia, Seattle\"}\n```\n\nWith the following config:","config":"\npipeline:\n processors:\n - mutation: |\n root.Cities = this.locations.\n filter(loc -\u003e loc.state == \"WA\").\n map_each(loc -\u003e loc.name).\n sort().join(\", \")\n"}],"config":{"name":"","type":"string","kind":"scalar","bloblang":true},"version":"4.5.0"},{"name":"nats_kv","type":"processor","status":"beta","plugin":true,"summary":"Perform operations on a NATS key-value bucket.","description":"\n== KV operations\n\nThe NATS KV processor supports a multitude of KV operations via the \u003c\u003coperation\u003e\u003e field. Along with `get`, `put`, and `delete`, this processor supports atomic operations like `update` and `create`, as well as utility operations like `purge`, `history`, and `keys`.\n\n== Metadata\n\nThis processor adds the following metadata fields to each message, depending on the chosen `operation`:\n\n=== get, get_revision\n``` text\n- nats_kv_key\n- nats_kv_bucket\n- nats_kv_revision\n- nats_kv_delta\n- nats_kv_operation\n- nats_kv_created\n```\n\n=== create, update, delete, purge\n``` text\n- nats_kv_key\n- nats_kv_bucket\n- nats_kv_revision\n- nats_kv_operation\n```\n\n=== keys\n``` text\n- nats_kv_bucket\n```\n\n== Connection name\n\nWhen monitoring and managing a production NATS system, it is often useful to\nknow which connection a message was send/received from. This can be achieved by\nsetting the connection name option when creating a NATS connection.\n\nRedpanda Connect will automatically set the connection name based off the label of the given\nNATS component, so that monitoring tools between NATS and Redpanda Connect can stay in sync.\n\n\n== Authentication\n\nThere are several components within Redpanda Connect which uses NATS services. You will find that each of these components\nsupport optional advanced authentication parameters for https://docs.nats.io/nats-server/configuration/securing_nats/auth_intro/nkey_auth[NKeys^]\nand https://docs.nats.io/using-nats/developer/connecting/creds[User Credentials^].\n\nSee an https://docs.nats.io/running-a-nats-service/nats_admin/security/jwt[in-depth tutorial^].\n\n=== NKey file\n\nThe NATS server can use these NKeys in several ways for authentication. The simplest is for the server to be configured\nwith a list of known public keys and for the clients to respond to the challenge by signing it with its private NKey\nconfigured in the `nkey_file` or `nkey` field.\n\nhttps://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[More details^].\n\n=== User credentials\n\nNATS server supports decentralized authentication based on JSON Web Tokens (JWT). Clients need an https://docs.nats.io/nats-server/configuration/securing_nats/jwt#json-web-tokens[user JWT^]\nand a corresponding https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[NKey secret^] when connecting to a server\nwhich is configured to use this authentication scheme.\n\nThe `user_credentials_file` field should point to a file containing both the private key and the JWT and can be\ngenerated with the https://docs.nats.io/nats-tools/nsc[nsc tool^].\n\nAlternatively, the `user_jwt` field can contain a plain text JWT and the `user_nkey_seed`can contain\nthe plain text NKey Seed.\n\nhttps://docs.nats.io/using-nats/developer/connecting/creds[More details^].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["nats://127.0.0.1:4222"],["nats://username:password@127.0.0.1:4222"]]},{"name":"bucket","type":"string","kind":"scalar","description":"The name of the KV bucket.","examples":["my_kv_bucket"]},{"name":"operation","type":"string","kind":"scalar","description":"The operation to perform on the KV bucket.","annotated_options":[["create","Adds the key/value pair if it does not exist. Returns an error if it already exists."],["delete","Deletes the key/value pair, but keeps historical values."],["get","Returns the latest value for `key`."],["get_revision","Returns the value of `key` for the specified `revision`."],["history","Returns historical values of `key` as an array of objects containing the following fields: `key`, `value`, `bucket`, `revision`, `delta`, `operation`, `created`."],["keys","Returns the keys in the `bucket` which match the `keys_filter` as an array of strings."],["purge","Deletes the key/value pair and all historical values."],["put","Places a new value for the key into the store."],["update","Updates the value for `key` only if the `revision` matches the latest revision."]],"linter":"\nlet options = {\n \"create\": true,\n \"delete\": true,\n \"get\": true,\n \"get_revision\": true,\n \"history\": true,\n \"keys\": true,\n \"purge\": true,\n \"put\": true,\n \"update\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"key","type":"string","kind":"scalar","description":"The key for each message. Supports https://docs.nats.io/nats-concepts/subjects#wildcards[wildcards^] for the `history` and `keys` operations.","interpolated":true,"examples":["foo","foo.bar.baz","foo.*","foo.\u003e","foo.${! json(\"meta.type\") }"],"linter":"if this == \"\" {[ \"'key' must be set to a non-empty string\" ]}"},{"name":"revision","type":"string","kind":"scalar","description":"The revision of the key to operate on. Used for `get_revision` and `update` operations.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["42","${! @nats_kv_revision }"]},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period to wait on an operation before aborting and returning an error.","is_advanced":true,"default":"5s"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auth","type":"object","kind":"scalar","description":"Optional configuration of NATS authentication parameters.","is_advanced":true,"children":[{"name":"nkey_file","type":"string","kind":"scalar","description":"An optional file containing a NKey seed.","is_advanced":true,"is_optional":true,"examples":["./seed.nk"]},{"name":"nkey","type":"string","kind":"scalar","description":"The NKey seed.","is_advanced":true,"is_optional":true,"is_secret":true,"examples":["UDXU4RCSJNZOIQHZNWXHXORDPRTGNJAHAHFRGZNEEJCPQTT2M7NLCNF4"],"version":"4.38.0","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_credentials_file","type":"string","kind":"scalar","description":"An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.","is_advanced":true,"is_optional":true,"examples":["./user.creds"]},{"name":"user_jwt","type":"string","kind":"scalar","description":"An optional plain text user JWT (given along with the corresponding user NKey Seed).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_nkey_seed","type":"string","kind":"scalar","description":"An optional plain text user NKey Seed (given along with the corresponding user JWT).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}],"linter":"root = match {\n [\"get_revision\", \"update\"].contains(this.operation) \u0026\u0026 !this.exists(\"revision\") =\u003e [ \"'revision' must be set when operation is '\" + this.operation + \"'\" ],\n ![\"get_revision\", \"update\"].contains(this.operation) \u0026\u0026 this.exists(\"revision\") =\u003e [ \"'revision' cannot be set when operation is '\" + this.operation + \"'\" ],\n }"},"version":"4.12.0"},{"name":"nats_request_reply","type":"processor","status":"experimental","plugin":true,"summary":"Sends a message to a NATS subject and expects a reply, from a NATS subscriber acting as a responder, back.","description":"\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- nats_subject\n- nats_sequence_stream\n- nats_sequence_consumer\n- nats_num_delivered\n- nats_num_pending\n- nats_domain\n- nats_timestamp_unix_nano\n```\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].\n\n== Connection name\n\nWhen monitoring and managing a production NATS system, it is often useful to\nknow which connection a message was send/received from. This can be achieved by\nsetting the connection name option when creating a NATS connection.\n\nRedpanda Connect will automatically set the connection name based off the label of the given\nNATS component, so that monitoring tools between NATS and Redpanda Connect can stay in sync.\n\n\n== Authentication\n\nThere are several components within Redpanda Connect which uses NATS services. You will find that each of these components\nsupport optional advanced authentication parameters for https://docs.nats.io/nats-server/configuration/securing_nats/auth_intro/nkey_auth[NKeys^]\nand https://docs.nats.io/using-nats/developer/connecting/creds[User Credentials^].\n\nSee an https://docs.nats.io/running-a-nats-service/nats_admin/security/jwt[in-depth tutorial^].\n\n=== NKey file\n\nThe NATS server can use these NKeys in several ways for authentication. The simplest is for the server to be configured\nwith a list of known public keys and for the clients to respond to the challenge by signing it with its private NKey\nconfigured in the `nkey_file` or `nkey` field.\n\nhttps://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[More details^].\n\n=== User credentials\n\nNATS server supports decentralized authentication based on JSON Web Tokens (JWT). Clients need an https://docs.nats.io/nats-server/configuration/securing_nats/jwt#json-web-tokens[user JWT^]\nand a corresponding https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[NKey secret^] when connecting to a server\nwhich is configured to use this authentication scheme.\n\nThe `user_credentials_file` field should point to a file containing both the private key and the JWT and can be\ngenerated with the https://docs.nats.io/nats-tools/nsc[nsc tool^].\n\nAlternatively, the `user_jwt` field can contain a plain text JWT and the `user_nkey_seed`can contain\nthe plain text NKey Seed.\n\nhttps://docs.nats.io/using-nats/developer/connecting/creds[More details^].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["nats://127.0.0.1:4222"],["nats://username:password@127.0.0.1:4222"]]},{"name":"subject","type":"string","kind":"scalar","description":"A subject to write to.","interpolated":true,"examples":["foo.bar.baz","${! meta(\"kafka_topic\") }","foo.${! json(\"meta.type\") }"]},{"name":"inbox_prefix","type":"string","kind":"scalar","description":"Set an explicit inbox prefix for the response subject","is_advanced":true,"is_optional":true,"examples":["_INBOX_joe"]},{"name":"headers","type":"string","kind":"map","description":"Explicit message headers to add to messages.","default":{},"interpolated":true,"examples":[{"Content-Type":"application/json","Timestamp":"${!meta(\"Timestamp\")}"}]},{"name":"metadata","type":"object","kind":"scalar","description":"Determine which (if any) metadata values should be added to messages as headers.","is_optional":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"timeout","type":"string","kind":"scalar","description":"A duration string is a possibly signed sequence of decimal numbers, each with optional fraction and a unit suffix, such as 300ms, -1.5h or 2h45m. Valid time units are ns, us (or Β΅s), ms, s, m, h.","is_optional":true,"default":"3s"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auth","type":"object","kind":"scalar","description":"Optional configuration of NATS authentication parameters.","is_advanced":true,"children":[{"name":"nkey_file","type":"string","kind":"scalar","description":"An optional file containing a NKey seed.","is_advanced":true,"is_optional":true,"examples":["./seed.nk"]},{"name":"nkey","type":"string","kind":"scalar","description":"The NKey seed.","is_advanced":true,"is_optional":true,"is_secret":true,"examples":["UDXU4RCSJNZOIQHZNWXHXORDPRTGNJAHAHFRGZNEEJCPQTT2M7NLCNF4"],"version":"4.38.0","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_credentials_file","type":"string","kind":"scalar","description":"An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.","is_advanced":true,"is_optional":true,"examples":["./user.creds"]},{"name":"user_jwt","type":"string","kind":"scalar","description":"An optional plain text user JWT (given along with the corresponding user NKey Seed).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_nkey_seed","type":"string","kind":"scalar","description":"An optional plain text user NKey Seed (given along with the corresponding user JWT).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},"version":"4.27.0"},{"name":"noop","type":"processor","status":"stable","plugin":true,"summary":"Noop is a processor that does nothing, the message passes through unchanged. Why? Sometimes doing nothing is the braver option.","categories":null,"config":{"name":"","type":"object","kind":"scalar","default":{}}},{"name":"ollama_chat","type":"processor","status":"experimental","plugin":true,"summary":"Generates responses to messages in a chat conversation, using the Ollama API.","description":"This processor sends prompts to your chosen Ollama large language model (LLM) and generates text from the responses, using the Ollama API.\n\nBy default, the processor starts and runs a locally installed Ollama server. Alternatively, to use an already running Ollama server, add your server details to the `server_address` field. You can https://ollama.com/download[download and install Ollama from the Ollama website^].\n\nFor more information, see the https://github.com/ollama/ollama/tree/main/docs[Ollama documentation^].","categories":["AI"],"examples":[{"title":"Use Llava to analyze an image","summary":"This example fetches image URLs from stdin and has a multimodal LLM describe the image.","config":"\ninput:\n stdin:\n scanner:\n lines: {}\npipeline:\n processors:\n - http:\n verb: GET\n url: \"${!content().string()}\"\n - ollama_chat:\n model: llava\n prompt: \"Describe the following image\"\n image: \"root = content()\"\noutput:\n stdout:\n codec: lines\n"},{"title":"Use subpipelines as tool calls","summary":"This example allows llama3.2 to execute a subpipeline as a tool call to get more data.","config":"\ninput:\n generate:\n count: 1\n mapping: |\n root = \"What is the weather like in Chicago?\"\npipeline:\n processors:\n - ollama_chat:\n model: llama3.2\n prompt: \"${!content().string()}\"\n tools:\n - name: GetWeather\n description: \"Retrieve the weather for a specific city\"\n parameters:\n required: [\"city\"]\n properties:\n city:\n type: string\n description: the city to lookup the weather for\n processors:\n - http:\n verb: GET\n url: 'https://wttr.in/${!this.city}?T'\n headers:\n # Spoof curl user-ageent to get a plaintext text\n User-Agent: curl/8.11.1\noutput:\n stdout: {}\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"model","type":"string","kind":"scalar","description":"The name of the Ollama LLM to use. For a full list of models, see the https://ollama.com/models[Ollama website].","examples":["llama3.1","gemma2","qwen2","phi3"]},{"name":"prompt","type":"string","kind":"scalar","description":"The prompt you want to generate a response for. By default, the processor submits the entire payload as a string.","is_optional":true,"interpolated":true},{"name":"system_prompt","type":"string","kind":"scalar","description":"The system prompt to submit to the Ollama LLM.","is_advanced":true,"is_optional":true,"interpolated":true},{"name":"image","type":"string","kind":"scalar","description":"The image to submit along with the prompt to the model. The result should be a byte array.","is_optional":true,"bloblang":true,"examples":["root = this.image.decode(\"base64\") # decode base64 encoded image"],"version":"4.38.0"},{"name":"response_format","type":"string","kind":"scalar","description":"The format of the response that the Ollama model generates. If specifying JSON output, then the `prompt` should specify that the output should be in JSON as well.","default":"text","options":["text","json"],"linter":"\nlet options = {\n \"text\": true,\n \"json\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"max_tokens","type":"int","kind":"scalar","description":"The maximum number of tokens to predict and output. Limiting the amount of output means that requests are processed faster and have a fixed limit on the cost.","is_optional":true},{"name":"temperature","type":"int","kind":"scalar","description":"The temperature of the model. Increasing the temperature makes the model answer more creatively.","is_optional":true,"linter":"root = if this \u003e 2 || this \u003c 0 { [ \"field must be between 0.0 and 2.0\" ] }"},{"name":"num_keep","type":"int","kind":"scalar","description":"Specify the number of tokens from the initial prompt to retain when the model resets its internal context. By default, this value is set to `4`. Use `-1` to retain all tokens from the initial prompt.","is_advanced":true,"is_optional":true},{"name":"seed","type":"int","kind":"scalar","description":"Sets the random number seed to use for generation. Setting this to a specific number will make the model generate the same text for the same prompt.","is_advanced":true,"is_optional":true,"examples":[42]},{"name":"top_k","type":"int","kind":"scalar","description":"Reduces the probability of generating nonsense. A higher value, for example `100`, will give more diverse answers. A lower value, for example `10`, will be more conservative.","is_advanced":true,"is_optional":true},{"name":"top_p","type":"float","kind":"scalar","description":"Works together with `top-k`. A higher value, for example 0.95, will lead to more diverse text. A lower value, for example 0.5, will generate more focused and conservative text.","is_advanced":true,"is_optional":true,"linter":"root = if this \u003e 1 || this \u003c 0 { [ \"field must be between 0.0 and 1.0\" ] }"},{"name":"repeat_penalty","type":"float","kind":"scalar","description":"Sets how strongly to penalize repetitions. A higher value, for example 1.5, will penalize repetitions more strongly. A lower value, for example 0.9, will be more lenient.","is_advanced":true,"is_optional":true,"linter":"root = if this \u003e 2 || this \u003c -2 { [ \"field must be between -2.0 and 2.0\" ] }"},{"name":"presence_penalty","type":"float","kind":"scalar","description":"Positive values penalize new tokens if they have appeared in the text so far. This increases the model's likelihood to talk about new topics.","is_advanced":true,"is_optional":true,"linter":"root = if this \u003e 2 || this \u003c -2 { [ \"field must be between -2.0 and 2.0\" ] }"},{"name":"frequency_penalty","type":"float","kind":"scalar","description":"Positive values penalize new tokens based on the frequency of their appearance in the text so far. This decreases the model's likelihood to repeat the same line verbatim.","is_advanced":true,"is_optional":true,"linter":"root = if this \u003e 2 || this \u003c -2 { [ \"field must be between -2.0 and 2.0\" ] }"},{"name":"stop","type":"string","kind":"array","description":"Sets the stop sequences to use. When this pattern is encountered the LLM stops generating text and returns the final response.","is_advanced":true,"is_optional":true},{"name":"save_prompt_metadata","type":"bool","kind":"scalar","description":"If enabled the prompt is saved as @prompt metadata on the output message. If system_prompt is used it's also saved as @system_prompt","default":false},{"name":"history","type":"string","kind":"scalar","description":"Historical messages to include in the chat request. The result of the bloblang query should be an array of objects of the form of [{\"role\": \"\", \"content\":\"\"}].","is_optional":true,"bloblang":true},{"name":"max_tool_calls","type":"int","kind":"scalar","description":"The maximum number of sequential tool calls.","is_advanced":true,"default":3,"linter":"root = if this \u003c= 0 { [\"field must be greater than zero\"] }"},{"name":"tools","type":"object","kind":"array","description":"The tools to allow the LLM to invoke. This allows building subpipelines that the LLM can choose to invoke to execute agentic-like actions.","default":[],"children":[{"name":"name","type":"string","kind":"scalar","description":"The name of this tool."},{"name":"description","type":"string","kind":"scalar","description":"A description of this tool, the LLM uses this to decide if the tool should be used."},{"name":"parameters","type":"object","kind":"scalar","description":"The parameters the LLM needs to provide to invoke this tool.","children":[{"name":"required","type":"string","kind":"array","description":"The required parameters for this pipeline.","default":[]},{"name":"properties","type":"object","kind":"map","description":"The properties for the processor's input data","children":[{"name":"type","type":"string","kind":"scalar","description":"The type of this parameter."},{"name":"description","type":"string","kind":"scalar","description":"A description of this parameter."},{"name":"enum","type":"string","kind":"array","description":"Specifies that this parameter is an enum and only these specific values should be used.","default":[]}]}]},{"name":"processors","type":"processor","kind":"array","description":"The pipeline to execute when the LLM uses this tool.","is_optional":true}]},{"name":"runner","type":"object","kind":"scalar","description":"Options for the model runner that are used when the model is first loaded into memory.","is_optional":true,"children":[{"name":"context_size","type":"int","kind":"scalar","description":"Sets the size of the context window used to generate the next token. Using a larger context window uses more memory and takes longer to processor.","is_optional":true},{"name":"batch_size","type":"int","kind":"scalar","description":"The maximum number of requests to process in parallel.","is_optional":true},{"name":"gpu_layers","type":"int","kind":"scalar","description":"This option allows offloading some layers to the GPU for computation. This generally results in increased performance. By default, the runtime decides the number of layers dynamically.","is_advanced":true,"is_optional":true},{"name":"threads","type":"int","kind":"scalar","description":"Set the number of threads to use during generation. For optimal performance, it is recommended to set this value to the number of physical CPU cores your system has. By default, the runtime decides the optimal number of threads.","is_advanced":true,"is_optional":true},{"name":"use_mmap","type":"bool","kind":"scalar","description":"Map the model into memory. This is only support on unix systems and allows loading only the necessary parts of the model as needed.","is_advanced":true,"is_optional":true},{"name":"use_mlock","type":"bool","kind":"scalar","description":"Lock the model in memory, preventing it from being swapped out when memory-mapped. This option can improve performance but reduces some of the advantages of memory-mapping because it uses more RAM to run and can slow down load times as the model loads into RAM.","is_advanced":true,"is_optional":true}]},{"name":"server_address","type":"string","kind":"scalar","description":"The address of the Ollama server to use. Leave the field blank and the processor starts and runs a local Ollama server or specify the address of your own local or remote server.","is_optional":true,"examples":["http://127.0.0.1:11434"]},{"name":"cache_directory","type":"string","kind":"scalar","description":"If `server_address` is not set - the directory to download the ollama binary and use as a model cache.","is_advanced":true,"is_optional":true,"examples":["/opt/cache/connect/ollama"]},{"name":"download_url","type":"string","kind":"scalar","description":"If `server_address` is not set - the URL to download the ollama binary from. Defaults to the offical Ollama GitHub release for this platform.","is_advanced":true,"is_optional":true}]},"version":"4.32.0"},{"name":"ollama_embeddings","type":"processor","status":"experimental","plugin":true,"summary":"Generates vector embeddings from text, using the Ollama API.","description":"This processor sends text to your chosen Ollama large language model (LLM) and creates vector embeddings, using the Ollama API. Vector embeddings are long arrays of numbers that represent values or objects, in this case text. \n\nBy default, the processor starts and runs a locally installed Ollama server. Alternatively, to use an already running Ollama server, add your server details to the `server_address` field. You can https://ollama.com/download[download and install Ollama from the Ollama website^].\n\nFor more information, see the https://github.com/ollama/ollama/tree/main/docs[Ollama documentation^].","categories":["AI"],"examples":[{"title":"Store embedding vectors in Qdrant","summary":"Compute embeddings for some generated data and store it within xrefs:component:outputs/qdrant.adoc[Qdrant]","config":"input:\n generate:\n interval: 1s\n mapping: |\n root = {\"text\": fake(\"paragraph\")}\npipeline:\n processors:\n - ollama_embeddings:\n model: snowflake-artic-embed\n text: \"${!this.text}\"\noutput:\n qdrant:\n grpc_host: localhost:6334\n collection_name: \"example_collection\"\n id: \"root = uuid_v4()\"\n vector_mapping: \"root = this\"\n"},{"title":"Store embedding vectors in Clickhouse","summary":"Compute embeddings for some generated data and store it within https://clickhouse.com/[Clickhouse^]","config":"input:\n generate:\n interval: 1s\n mapping: |\n root = {\"text\": fake(\"paragraph\")}\npipeline:\n processors:\n - branch:\n processors:\n - ollama_embeddings:\n model: snowflake-artic-embed\n text: \"${!this.text}\"\n result_map: |\n root.embeddings = this\noutput:\n sql_insert:\n driver: clickhouse\n dsn: \"clickhouse://localhost:9000\"\n table: searchable_text\n columns: [\"id\", \"text\", \"vector\"]\n args_mapping: \"root = [uuid_v4(), this.text, this.embeddings]\"\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"model","type":"string","kind":"scalar","description":"The name of the Ollama LLM to use. For a full list of models, see the https://ollama.com/models[Ollama website].","examples":["nomic-embed-text","mxbai-embed-large","snowflake-artic-embed","all-minilm"]},{"name":"text","type":"string","kind":"scalar","description":"The text you want to create vector embeddings for. By default, the processor submits the entire payload as a string.","is_optional":true,"interpolated":true},{"name":"runner","type":"object","kind":"scalar","description":"Options for the model runner that are used when the model is first loaded into memory.","is_optional":true,"children":[{"name":"context_size","type":"int","kind":"scalar","description":"Sets the size of the context window used to generate the next token. Using a larger context window uses more memory and takes longer to processor.","is_optional":true},{"name":"batch_size","type":"int","kind":"scalar","description":"The maximum number of requests to process in parallel.","is_optional":true},{"name":"gpu_layers","type":"int","kind":"scalar","description":"This option allows offloading some layers to the GPU for computation. This generally results in increased performance. By default, the runtime decides the number of layers dynamically.","is_advanced":true,"is_optional":true},{"name":"threads","type":"int","kind":"scalar","description":"Set the number of threads to use during generation. For optimal performance, it is recommended to set this value to the number of physical CPU cores your system has. By default, the runtime decides the optimal number of threads.","is_advanced":true,"is_optional":true},{"name":"use_mmap","type":"bool","kind":"scalar","description":"Map the model into memory. This is only support on unix systems and allows loading only the necessary parts of the model as needed.","is_advanced":true,"is_optional":true},{"name":"use_mlock","type":"bool","kind":"scalar","description":"Lock the model in memory, preventing it from being swapped out when memory-mapped. This option can improve performance but reduces some of the advantages of memory-mapping because it uses more RAM to run and can slow down load times as the model loads into RAM.","is_advanced":true,"is_optional":true}]},{"name":"server_address","type":"string","kind":"scalar","description":"The address of the Ollama server to use. Leave the field blank and the processor starts and runs a local Ollama server or specify the address of your own local or remote server.","is_optional":true,"examples":["http://127.0.0.1:11434"]},{"name":"cache_directory","type":"string","kind":"scalar","description":"If `server_address` is not set - the directory to download the ollama binary and use as a model cache.","is_advanced":true,"is_optional":true,"examples":["/opt/cache/connect/ollama"]},{"name":"download_url","type":"string","kind":"scalar","description":"If `server_address` is not set - the URL to download the ollama binary from. Defaults to the offical Ollama GitHub release for this platform.","is_advanced":true,"is_optional":true}]},"version":"4.32.0"},{"name":"ollama_moderation","type":"processor","status":"experimental","plugin":true,"summary":"Generates responses to messages in a chat conversation, using the Ollama API.","description":"This processor checks LLM response safety using either `llama-guard3` or `shieldgemma`. If you want to check if a given prompt is safe, then that can be done with the `ollama_chat` processor - this processor is for response classification only.\n\nBy default, the processor starts and runs a locally installed Ollama server. Alternatively, to use an already running Ollama server, add your server details to the `server_address` field. You can https://ollama.com/download[download and install Ollama from the Ollama website^].\n\nFor more information, see the https://github.com/ollama/ollama/tree/main/docs[Ollama documentation^].","categories":["AI"],"examples":[{"title":"Use Llama Guard 3 classify a LLM response","summary":"This example uses Llama Guard 3 to check if another model responded with a safe or unsafe content.","config":"\ninput:\n stdin:\n scanner:\n lines: {}\npipeline:\n processors:\n - ollama_chat:\n model: llava\n prompt: \"${!content().string()}\"\n save_prompt_metadata: true\n - ollama_moderation:\n model: llama-guard3\n prompt: \"${!@prompt}\"\n response: \"${!content().string()}\"\n - mapping: |\n root.response = content().string()\n root.is_safe = @safe\noutput:\n stdout:\n codec: lines\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"model","type":"string","kind":"scalar","description":"The name of the Ollama LLM to use.","examples":["llama-guard3","shieldgemma"],"annotated_options":[["llama-guard3","When using llama-guard3, two pieces of metadata is added: @safe with the value of `yes` or `no` and the second being @category for the safety category violation. For more information see the https://ollama.com/library/llama-guard3[Llama Guard 3 Model Card]."],["shieldgemma","When using shieldgemma, the model output is a single piece of metadata of @safe with a value of `yes` or `no` if the response is not in violation of its defined safety policies."]],"linter":"\nlet options = {\n \"llama-guard3\": true,\n \"shieldgemma\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"prompt","type":"string","kind":"scalar","description":"The input prompt that was used with the LLM. If using `ollama_chat` the you can use `save_prompt_metadata` to safe the prompt as metadata.","interpolated":true},{"name":"response","type":"string","kind":"scalar","description":"The LLM's response to classify if it contains safe or unsafe content.","interpolated":true},{"name":"runner","type":"object","kind":"scalar","description":"Options for the model runner that are used when the model is first loaded into memory.","is_optional":true,"children":[{"name":"context_size","type":"int","kind":"scalar","description":"Sets the size of the context window used to generate the next token. Using a larger context window uses more memory and takes longer to processor.","is_optional":true},{"name":"batch_size","type":"int","kind":"scalar","description":"The maximum number of requests to process in parallel.","is_optional":true},{"name":"gpu_layers","type":"int","kind":"scalar","description":"This option allows offloading some layers to the GPU for computation. This generally results in increased performance. By default, the runtime decides the number of layers dynamically.","is_advanced":true,"is_optional":true},{"name":"threads","type":"int","kind":"scalar","description":"Set the number of threads to use during generation. For optimal performance, it is recommended to set this value to the number of physical CPU cores your system has. By default, the runtime decides the optimal number of threads.","is_advanced":true,"is_optional":true},{"name":"use_mmap","type":"bool","kind":"scalar","description":"Map the model into memory. This is only support on unix systems and allows loading only the necessary parts of the model as needed.","is_advanced":true,"is_optional":true},{"name":"use_mlock","type":"bool","kind":"scalar","description":"Lock the model in memory, preventing it from being swapped out when memory-mapped. This option can improve performance but reduces some of the advantages of memory-mapping because it uses more RAM to run and can slow down load times as the model loads into RAM.","is_advanced":true,"is_optional":true}]},{"name":"server_address","type":"string","kind":"scalar","description":"The address of the Ollama server to use. Leave the field blank and the processor starts and runs a local Ollama server or specify the address of your own local or remote server.","is_optional":true,"examples":["http://127.0.0.1:11434"]},{"name":"cache_directory","type":"string","kind":"scalar","description":"If `server_address` is not set - the directory to download the ollama binary and use as a model cache.","is_advanced":true,"is_optional":true,"examples":["/opt/cache/connect/ollama"]},{"name":"download_url","type":"string","kind":"scalar","description":"If `server_address` is not set - the URL to download the ollama binary from. Defaults to the offical Ollama GitHub release for this platform.","is_advanced":true,"is_optional":true}]},"version":"4.42.0"},{"name":"openai_chat_completion","type":"processor","status":"experimental","plugin":true,"summary":"Generates responses to messages in a chat conversation, using the OpenAI API.","description":"\nThis processor sends the contents of user prompts to the OpenAI API, which generates responses. By default, the processor submits the entire payload of each message as a string, unless you use the `prompt` configuration field to customize it.\n\nTo learn more about chat completion, see the https://platform.openai.com/docs/guides/chat-completions[OpenAI API documentation^].","categories":["AI"],"examples":[{"title":"Use GPT-4o analyze an image","summary":"This example fetches image URLs from stdin and has GPT-4o describe the image.","config":"\ninput:\n stdin:\n scanner:\n lines: {}\npipeline:\n processors:\n - http:\n verb: GET\n url: \"${!content().string()}\"\n - openai_chat_completion:\n model: gpt-4o\n api_key: TODO\n prompt: \"Describe the following image\"\n image: \"root = content()\"\noutput:\n stdout:\n codec: lines\n"},{"title":"Provide historical chat history","summary":"This pipeline provides a historical chat history to GPT-4o using a cache.","config":"\ninput:\n stdin:\n scanner:\n lines: {}\npipeline:\n processors:\n - mapping: |\n root.prompt = content().string()\n - branch:\n processors:\n - cache:\n resource: mem\n operator: get\n key: history\n - catch:\n - mapping: 'root = []'\n result_map: 'root.history = this'\n - branch:\n processors:\n - openai_chat_completion:\n model: gpt-4o\n api_key: TODO\n prompt: \"${!this.prompt}\"\n history: 'root = this.history'\n result_map: 'root.response = content().string()'\n - mutation: |\n root.history = this.history.concat([\n {\"role\": \"user\", \"content\": this.prompt},\n {\"role\": \"assistant\", \"content\": this.response},\n ])\n - cache:\n resource: mem\n operator: set\n key: history\n value: '${!this.history}'\n - mapping: |\n root = this.response\noutput:\n stdout:\n codec: lines\n\ncache_resources:\n - label: mem \n memory: {}\n"},{"title":"Use GPT-4o to call a tool","summary":"This example asks GPT-4o to respond with the weather by invoking an HTTP processor to get the forecast.","config":"\ninput:\n generate:\n count: 1\n mapping: |\n root = \"What is the weather like in Chicago?\"\npipeline:\n processors:\n - openai_chat_completion:\n model: gpt-4o\n api_key: \"${OPENAI_API_KEY}\"\n prompt: \"${!content().string()}\"\n tools:\n - name: GetWeather\n description: \"Retrieve the weather for a specific city\"\n parameters:\n required: [\"city\"]\n properties:\n city:\n type: string\n description: the city to look up the weather for\n processors:\n - http:\n verb: GET\n url: 'https://wttr.in/${!this.city}?T'\n headers:\n User-Agent: curl/8.11.1 # Returns a text string from the weather website\noutput:\n stdout: {}\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"server_address","type":"string","kind":"scalar","description":"The Open API endpoint that the processor sends requests to. Update the default value to use another OpenAI compatible service.","default":"https://api.openai.com/v1"},{"name":"api_key","type":"string","kind":"scalar","description":"The API key for OpenAI API.","is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"model","type":"string","kind":"scalar","description":"The name of the OpenAI model to use.","examples":["gpt-4o","gpt-4o-mini","gpt-4","gpt4-turbo"]},{"name":"prompt","type":"string","kind":"scalar","description":"The user prompt you want to generate a response for. By default, the processor submits the entire payload as a string.","is_optional":true,"interpolated":true},{"name":"system_prompt","type":"string","kind":"scalar","description":"The system prompt to submit along with the user prompt.","is_optional":true,"interpolated":true},{"name":"history","type":"string","kind":"scalar","description":"The history of the prior conversation. A bloblang query that should result in an array of objects of the form: [{\"role\": \"user\", \"content\": \"\u003ctext\u003e\"}, {\"role\":\"assistant\", \"content\":\"\u003ctext\u003e\"}]","is_optional":true,"bloblang":true},{"name":"image","type":"string","kind":"scalar","description":"An image to send along with the prompt. The mapping result must be a byte array.","is_optional":true,"bloblang":true,"examples":["root = this.image.decode(\"base64\") # decode base64 encoded image"],"version":"4.38.0"},{"name":"max_tokens","type":"int","kind":"scalar","description":"The maximum number of tokens that can be generated in the chat completion.","is_optional":true},{"name":"temperature","type":"float","kind":"scalar","description":"What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic.\n\nWe generally recommend altering this or top_p but not both.","is_optional":true,"linter":"root = if this \u003e 2 || this \u003c 0 { [ \"field must be between 0 and 2\" ] }"},{"name":"user","type":"string","kind":"scalar","description":"A unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse.","is_optional":true,"interpolated":true},{"name":"response_format","type":"string","kind":"scalar","description":"Specify the model's output format. If `json_schema` is specified, then additionally a `json_schema` or `schema_registry` must be configured.","default":"text","options":["text","json","json_schema"],"linter":"\nlet options = {\n \"text\": true,\n \"json\": true,\n \"json_schema\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"json_schema","type":"object","kind":"scalar","description":"The JSON schema to use when responding in `json_schema` format. To learn more about what JSON schema is supported see the https://platform.openai.com/docs/guides/structured-outputs/supported-schemas[OpenAI documentation^].","is_optional":true,"children":[{"name":"name","type":"string","kind":"scalar","description":"The name of the schema."},{"name":"description","type":"string","kind":"scalar","description":"Additional description of the schema for the LLM.","is_advanced":true,"is_optional":true},{"name":"schema","type":"string","kind":"scalar","description":"The JSON schema for the LLM to use when generating the output."}]},{"name":"schema_registry","type":"object","kind":"scalar","description":"The schema registry to dynamically load schemas from when responding in `json_schema` format. Schemas themselves must be in JSON format. To learn more about what JSON schema is supported see the https://platform.openai.com/docs/guides/structured-outputs/supported-schemas[OpenAI documentation^].","is_advanced":true,"is_optional":true,"children":[{"name":"url","type":"string","kind":"scalar","description":"The base URL of the schema registry service.","is_advanced":true,"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"name_prefix","type":"string","kind":"scalar","description":"The prefix of the name for this schema, the schema ID is used as a suffix.","is_advanced":true,"default":"schema_registry_id_"},{"name":"subject","type":"string","kind":"scalar","description":"The subject name to fetch the schema for.","is_advanced":true},{"name":"refresh_interval","type":"string","kind":"scalar","description":"The refresh rate for getting the latest schema. If not specified the schema does not refresh.","is_advanced":true,"is_optional":true},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"oauth","type":"object","kind":"scalar","description":"Allows you to specify open authentication via OAuth version 1.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use OAuth version 1 in requests.","is_advanced":true,"default":false},{"name":"consumer_key","type":"string","kind":"scalar","description":"A value used to identify the client to the service provider.","is_advanced":true,"default":""},{"name":"consumer_secret","type":"string","kind":"scalar","description":"A secret used to establish ownership of the consumer key.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"access_token","type":"string","kind":"scalar","description":"A value used to gain access to the protected resources on behalf of the user.","is_advanced":true,"default":""},{"name":"access_token_secret","type":"string","kind":"scalar","description":"A secret provided in order to establish ownership of a given access token.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"jwt","type":"object","kind":"scalar","description":"BETA: Allows you to specify JWT authentication.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use JWT authentication in requests.","is_advanced":true,"default":false},{"name":"private_key_file","type":"string","kind":"scalar","description":"A file with the PEM encoded via PKCS1 or PKCS8 as private key.","is_advanced":true,"default":""},{"name":"signing_method","type":"string","kind":"scalar","description":"A method used to sign the token such as RS256, RS384, RS512 or EdDSA.","is_advanced":true,"default":""},{"name":"claims","type":"unknown","kind":"map","description":"A value used to identify the claims that issued the JWT.","is_advanced":true,"default":{}},{"name":"headers","type":"unknown","kind":"map","description":"Add optional key/value headers to the JWT.","is_advanced":true,"default":{}}]}]},{"name":"top_p","type":"float","kind":"scalar","description":"An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.\n\nWe generally recommend altering this or temperature but not both.","is_advanced":true,"is_optional":true,"linter":"root = if this \u003e 1 || this \u003c 0 { [ \"field must be between 0 and 1\" ] }"},{"name":"frequency_penalty","type":"float","kind":"scalar","description":"Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim.","is_advanced":true,"is_optional":true,"linter":"root = if this \u003e 2 || this \u003c -2 { [ \"field must be less than 2 and greater than -2\" ] }"},{"name":"presence_penalty","type":"float","kind":"scalar","description":"Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics.","is_advanced":true,"is_optional":true,"linter":"root = if this \u003e 2 || this \u003c -2 { [ \"field must be less than 2 and greater than -2\" ] }"},{"name":"seed","type":"int","kind":"scalar","description":"If specified, our system will make a best effort to sample deterministically, such that repeated requests with the same seed and parameters should return the same result. Determinism is not guaranteed.","is_advanced":true,"is_optional":true},{"name":"stop","type":"string","kind":"array","description":"Up to 4 sequences where the API will stop generating further tokens.","is_advanced":true,"is_optional":true},{"name":"tools","type":"object","kind":"array","description":"The tools to allow the LLM to invoke. This allows building subpipelines that the LLM can choose to invoke to execute agentic-like actions.","children":[{"name":"name","type":"string","kind":"scalar","description":"The name of this tool."},{"name":"description","type":"string","kind":"scalar","description":"A description of this tool, the LLM uses this to decide if the tool should be used."},{"name":"parameters","type":"object","kind":"scalar","description":"The parameters the LLM needs to provide to invoke this tool.","default":[],"children":[{"name":"required","type":"string","kind":"array","description":"The required parameters for this pipeline.","default":[]},{"name":"properties","type":"object","kind":"map","description":"The properties for the processor's input data","children":[{"name":"type","type":"string","kind":"scalar","description":"The type of this parameter."},{"name":"description","type":"string","kind":"scalar","description":"A description of this parameter."},{"name":"enum","type":"string","kind":"array","description":"Specifies that this parameter is an enum and only these specific values should be used.","default":[]}]}]},{"name":"processors","type":"processor","kind":"array","description":"The pipeline to execute when the LLM uses this tool.","is_optional":true}]}],"linter":"\n root = match {\n this.exists(\"json_schema\") \u0026\u0026 this.exists(\"schema_registry\") =\u003e [\"cannot set both `json_schema` and `schema_registry`\"]\n this.response_format == \"json_schema\" \u0026\u0026 !this.exists(\"json_schema\") \u0026\u0026 !this.exists(\"schema_registry\") =\u003e [\"schema must be specified using either `json_schema` or `schema_registry`\"]\n }\n "},"version":"4.32.0"},{"name":"openai_embeddings","type":"processor","status":"experimental","plugin":true,"summary":"Generates vector embeddings to represent input text, using the OpenAI API.","description":"\nThis processor sends text strings to the OpenAI API, which generates vector embeddings. By default, the processor submits the entire payload of each message as a string, unless you use the `text_mapping` configuration field to customize it.\n\nTo learn more about vector embeddings, see the https://platform.openai.com/docs/guides/embeddings[OpenAI API documentation^].","categories":["AI"],"examples":[{"title":"Store embedding vectors in Pinecone","summary":"Compute embeddings for some generated data and store it within xrefs:component:outputs/pinecone.adoc[Pinecone]","config":"input:\n generate:\n interval: 1s\n mapping: |\n root = {\"text\": fake(\"paragraph\")}\npipeline:\n processors:\n - openai_embeddings:\n model: text-embedding-3-large\n api_key: \"${OPENAI_API_KEY}\"\n text_mapping: \"root = this.text\"\noutput:\n pinecone:\n host: \"${PINECONE_HOST}\"\n api_key: \"${PINECONE_API_KEY}\"\n id: \"root = uuid_v4()\"\n vector_mapping: \"root = this\""}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"server_address","type":"string","kind":"scalar","description":"The Open API endpoint that the processor sends requests to. Update the default value to use another OpenAI compatible service.","default":"https://api.openai.com/v1"},{"name":"api_key","type":"string","kind":"scalar","description":"The API key for OpenAI API.","is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"model","type":"string","kind":"scalar","description":"The name of the OpenAI model to use.","examples":["text-embedding-3-large","text-embedding-3-small","text-embedding-ada-002"]},{"name":"text_mapping","type":"string","kind":"scalar","description":"The text you want to generate a vector embedding for. By default, the processor submits the entire payload as a string.","is_optional":true,"bloblang":true},{"name":"dimensions","type":"int","kind":"scalar","description":"The number of dimensions the resulting output embeddings should have. Only supported in `text-embedding-3` and later models.","is_optional":true}]},"version":"4.32.0"},{"name":"openai_image_generation","type":"processor","status":"experimental","plugin":true,"summary":"Generates an image from a text description and other attributes, using OpenAI API.","description":"\nThis processor sends an image description and other attributes, such as image size and quality to the OpenAI API, which generates an image. By default, the processor submits the entire payload of each message as a string, unless you use the `prompt` configuration field to customize it.\n\nTo learn more about image generation, see the https://platform.openai.com/docs/guides/images[OpenAI API documentation^].","categories":["AI"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"server_address","type":"string","kind":"scalar","description":"The Open API endpoint that the processor sends requests to. Update the default value to use another OpenAI compatible service.","default":"https://api.openai.com/v1"},{"name":"api_key","type":"string","kind":"scalar","description":"The API key for OpenAI API.","is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"model","type":"string","kind":"scalar","description":"The name of the OpenAI model to use.","examples":["dall-e-3","dall-e-2"]},{"name":"prompt","type":"string","kind":"scalar","description":"A text description of the image you want to generate. The `prompt` field accepts a maximum of 1000 characters for `dall-e-2` and 4000 characters for `dall-e-3`.","is_optional":true,"bloblang":true},{"name":"quality","type":"string","kind":"scalar","description":"The quality of the image to generate. Use `hd` to create images with finer details and greater consistency across the image. This parameter is only supported for `dall-e-3` models.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["standard","hd"]},{"name":"size","type":"string","kind":"scalar","description":"The size of the generated image. Choose from `256x256`, `512x512`, or `1024x1024` for `dall-e-2`. Choose from `1024x1024`, `1792x1024`, or `1024x1792` for `dall-e-3` models.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["1024x1024","512x512","1792x1024","1024x1792"]},{"name":"style","type":"string","kind":"scalar","description":"The style of the generated image. Choose from `vivid` or `natural`. Vivid causes the model to lean towards generating hyperreal and dramatic images. Natural causes the model to produce more natural, less hyperreal looking images. This parameter is only supported for `dall-e-3`.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["vivid","natural"]}]},"version":"4.32.0"},{"name":"openai_speech","type":"processor","status":"experimental","plugin":true,"summary":"Generates audio from a text description and other attributes, using OpenAI API.","description":"\nThis processor sends a text description and other attributes, such as a voice type and format to the OpenAI API, which generates audio. By default, the processor submits the entire payload of each message as a string, unless you use the `input` configuration field to customize it.\n\nTo learn more about turning text into spoken audio, see the https://platform.openai.com/docs/guides/text-to-speech[OpenAI API documentation^].","categories":["AI"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"server_address","type":"string","kind":"scalar","description":"The Open API endpoint that the processor sends requests to. Update the default value to use another OpenAI compatible service.","default":"https://api.openai.com/v1"},{"name":"api_key","type":"string","kind":"scalar","description":"The API key for OpenAI API.","is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"model","type":"string","kind":"scalar","description":"The name of the OpenAI model to use.","examples":["tts-1","tts-1-hd"]},{"name":"input","type":"string","kind":"scalar","description":"A text description of the audio you want to generate. The `input` field accepts a maximum of 4096 characters.","is_optional":true,"bloblang":true},{"name":"voice","type":"string","kind":"scalar","description":"The type of voice to use when generating the audio.","interpolated":true,"examples":["alloy","echo","fable","onyx","nova","shimmer"]},{"name":"response_format","type":"string","kind":"scalar","description":"The format to generate audio in. Default is `mp3`.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["mp3","opus","aac","flac","wav","pcm"]}]},"version":"4.32.0"},{"name":"openai_transcription","type":"processor","status":"experimental","plugin":true,"summary":"Generates a transcription of spoken audio in the input language, using the OpenAI API.","description":"\nThis processor sends an audio file object along with the input language to OpenAI API to generate a transcription. By default, the processor submits the entire payload of each message as a string, unless you use the `file` configuration field to customize it.\n\nTo learn more about audio transcription, see the: https://platform.openai.com/docs/guides/speech-to-text[OpenAI API documentation^].","categories":["AI"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"server_address","type":"string","kind":"scalar","description":"The Open API endpoint that the processor sends requests to. Update the default value to use another OpenAI compatible service.","default":"https://api.openai.com/v1"},{"name":"api_key","type":"string","kind":"scalar","description":"The API key for OpenAI API.","is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"model","type":"string","kind":"scalar","description":"The name of the OpenAI model to use.","examples":["whisper-1"]},{"name":"file","type":"string","kind":"scalar","description":"The audio file object (not file name) to transcribe, in one of the following formats: `flac`, `mp3`, `mp4`, `mpeg`, `mpga`, `m4a`, `ogg`, `wav`, or `webm`.","bloblang":true},{"name":"language","type":"string","kind":"scalar","description":"The language of the input audio. Supplying the input language in ISO-639-1 format improves accuracy and latency.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["en","fr","de","zh"]},{"name":"prompt","type":"string","kind":"scalar","description":"Optional text to guide the model's style or continue a previous audio segment. The prompt should match the audio language.","is_advanced":true,"is_optional":true,"interpolated":true}]},"version":"4.32.0"},{"name":"openai_translation","type":"processor","status":"experimental","plugin":true,"summary":"Translates spoken audio into English, using the OpenAI API.","description":"\nThis processor sends an audio file object to OpenAI API to generate a translation. By default, the processor submits the entire payload of each message as a string, unless you use the `file` configuration field to customize it.\n\nTo learn more about translation, see the https://platform.openai.com/docs/guides/speech-to-text[OpenAI API documentation^].","categories":["AI"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"server_address","type":"string","kind":"scalar","description":"The Open API endpoint that the processor sends requests to. Update the default value to use another OpenAI compatible service.","default":"https://api.openai.com/v1"},{"name":"api_key","type":"string","kind":"scalar","description":"The API key for OpenAI API.","is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"model","type":"string","kind":"scalar","description":"The name of the OpenAI model to use.","examples":["whisper-1"]},{"name":"file","type":"string","kind":"scalar","description":"The audio file object (not file name) to translate, in one of the following formats: `flac`, `mp3`, `mp4`, `mpeg`, `mpga`, `m4a`, `ogg`, `wav`, or `webm`.","is_optional":true,"bloblang":true},{"name":"prompt","type":"string","kind":"scalar","description":"Optional text to guide the model's style or continue a previous audio segment. The prompt should match the audio language.","is_advanced":true,"is_optional":true,"interpolated":true}]},"version":"4.32.0"},{"name":"parallel","type":"processor","status":"stable","plugin":true,"summary":"A processor that applies a list of child processors to messages of a batch as though they were each a batch of one message (similar to the xref:components:processors/for_each.adoc[`for_each`] processor), but where each message is processed in parallel.","description":"\nThe field `cap`, if greater than zero, caps the maximum number of parallel processing threads.\n\nThe functionality of this processor depends on being applied across messages that are batched. You can find out more about batching in xref:configuration:batching.adoc[].","categories":["Composition"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"cap","type":"int","kind":"scalar","description":"The maximum number of messages to have processing at a given time.","default":0},{"name":"processors","type":"processor","kind":"array","description":"A list of child processors to apply."}]}},{"name":"parquet","type":"processor","status":"deprecated","plugin":true,"summary":"Converts batches of documents to or from https://parquet.apache.org/docs/[Parquet files^].","description":"\n== Alternatives\n\nThis processor is now deprecated, it's recommended that you use the new xref:components:processors/parquet_decode.adoc[`parquet_decode`] and xref:components:processors/parquet_encode.adoc[`parquet_encode`] processors as they provide a number of advantages, the most important of which is better error messages for when schemas are mismatched or files could not be consumed.\n\n== Troubleshooting\n\nThis processor is experimental and the error messages that it provides are often vague and unhelpful. An error message of the form `interface \\{} is nil, not \u003cvalue type\u003e` implies that a field of the given type was expected but not found in the processed message when writing parquet files.\n\nUnfortunately the name of the field will sometimes be missing from the error, in which case it's worth double checking the schema you provided to make sure that there are no typos in the field names, and if that doesn't reveal the issue it can help to mark fields as OPTIONAL in the schema and gradually change them back to REQUIRED until the error returns.\n\n== Define the schema\n\nThe schema must be specified as a JSON string, containing an object that describes the fields expected at the root of each document. Each field can itself have more fields defined, allowing for nested structures:\n\n```json\n{\n \"Tag\": \"name=root, repetitiontype=REQUIRED\",\n \"Fields\": [\n {\"Tag\": \"name=name, inname=NameIn, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=REQUIRED\"},\n {\"Tag\": \"name=age, inname=Age, type=INT32, repetitiontype=REQUIRED\"},\n {\"Tag\": \"name=id, inname=Id, type=INT64, repetitiontype=REQUIRED\"},\n {\"Tag\": \"name=weight, inname=Weight, type=FLOAT, repetitiontype=REQUIRED\"},\n {\n \"Tag\": \"name=favPokemon, inname=FavPokemon, type=LIST, repetitiontype=OPTIONAL\",\n \"Fields\": [\n {\"Tag\": \"name=name, inname=PokeName, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=REQUIRED\"},\n {\"Tag\": \"name=coolness, inname=Coolness, type=FLOAT, repetitiontype=REQUIRED\"}\n ]\n }\n ]\n}\n```\n\nA schema can be derived from a source file using https://github.com/xitongsys/parquet-go/tree/master/tool/parquet-tools:\n\n```sh\n./parquet-tools -cmd schema -file foo.parquet\n```","categories":["Parsing"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"operator","type":"string","kind":"scalar","description":"Determines whether the processor converts messages into a parquet file or expands parquet files into messages. Converting into JSON allows subsequent processors and mappings to convert the data into any other format.","annotated_options":[["from_json","Compress a batch of JSON documents into a file."],["to_json","Expand a file into one or more JSON messages."]],"linter":"\nlet options = {\n \"from_json\": true,\n \"to_json\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"compression","type":"string","kind":"scalar","description":"The type of compression to use when writing parquet files, this field is ignored when consuming parquet files.","default":"snappy","options":["uncompressed","snappy","gzip","lz4","zstd"],"linter":"\nlet options = {\n \"uncompressed\": true,\n \"snappy\": true,\n \"gzip\": true,\n \"lz4\": true,\n \"zstd\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"schema_file","type":"string","kind":"scalar","description":"A file path containing a schema used to describe the parquet files being generated or consumed, the format of the schema is a JSON document detailing the tag and fields of documents. The schema can be found at: https://pkg.go.dev/github.com/xitongsys/parquet-go#readme-json. Either a `schema_file` or `schema` field must be specified when creating Parquet files via the `from_json` operator.","is_optional":true,"examples":["schemas/foo.json"]},{"name":"schema","type":"string","kind":"scalar","description":"A schema used to describe the parquet files being generated or consumed, the format of the schema is a JSON document detailing the tag and fields of documents. The schema can be found at: https://pkg.go.dev/github.com/xitongsys/parquet-go#readme-json. Either a `schema_file` or `schema` field must be specified when creating Parquet files via the `from_json` operator.","is_optional":true,"examples":["{\n \"Tag\": \"name=root, repetitiontype=REQUIRED\",\n \"Fields\": [\n {\"Tag\":\"name=name,inname=NameIn,type=BYTE_ARRAY,convertedtype=UTF8, repetitiontype=REQUIRED\"},\n {\"Tag\":\"name=age,inname=Age,type=INT32,repetitiontype=REQUIRED\"}\n ]\n}"]}],"linter":"\nroot = if this.operator == \"from_json\" \u0026\u0026 (this.schema | this.schema_file | \"\") == \"\" {\n\t\"a schema or schema_file must be specified when the operator is set to from_json\"\n}"},"version":"3.62.0"},{"name":"parquet_decode","type":"processor","status":"experimental","plugin":true,"summary":"Decodes https://parquet.apache.org/docs/[Parquet files^] into a batch of structured messages.","description":"\nThis processor uses https://github.com/parquet-go/parquet-go[https://github.com/parquet-go/parquet-go^], which is itself experimental. Therefore changes could be made into how this processor functions outside of major version releases.","categories":["Parsing"],"examples":[{"title":"Reading Parquet Files from AWS S3","summary":"In this example we consume files from AWS S3 as they're written by listening onto an SQS queue for upload events. We make sure to use the `to_the_end` scanner which means files are read into memory in full, which then allows us to use a `parquet_decode` processor to expand each file into a batch of messages. Finally, we write the data out to local files as newline delimited JSON.","config":"\ninput:\n aws_s3:\n bucket: TODO\n prefix: foos/\n scanner:\n to_the_end: {}\n sqs:\n url: TODO\n processors:\n - parquet_decode: {}\n\noutput:\n file:\n codec: lines\n path: './foos/${! meta(\"s3_key\") }.jsonl'\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"byte_array_as_string","type":"bool","kind":"scalar","description":"Whether to extract BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY values as strings rather than byte slices in all cases. Values with a logical type of UTF8 will automatically be extracted as strings irrespective of this field. Enabling this field makes serializing the data as JSON more intuitive as `[]byte` values are serialized as base64 encoded strings by default.","is_deprecated":true,"default":false},{"name":"handle_logical_types","type":"string","kind":"scalar","description":"Whether to be smart about decoding logical types. In the Parquet format, logical types are stored as one of the standard physical types with some additional metadata describing the logical type. For example, UUIDs are stored in a FIXED_LEN_BYTE_ARRAY physical type, but there is metadata in the schema denoting that it is a UUID. By default, this logical type metadata will be ignored and values will be decoded directly from the physical type, which isn't always desirable. By enabling this option, logical types will be given special treatment and will decode into more useful values. The value for this field specifies a version, i.e. v0, v1... Any given version enables the logical type handling for that version and all versions below it, which allows the handling of new logical types to be introduced without breaking existing pipelines. We recommend enabling the newest version available of this feature when creating new pipelines.","default":"v1","examples":["v2"],"annotated_options":[["v1","No special handling of logical types"],["v2","\n- TIMESTAMP - decodes as an RFC3339 string describing the time. If the `isAdjustedToUTC` flag is set to true in the parquet file, the time zone will be set to UTC. If it is set to false the time zone will be set to local time.\n- UUID - decodes as a string, i.e. `00112233-4455-6677-8899-aabbccddeeff`."]],"linter":"\nlet options = {\n \"v1\": true,\n \"v2\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"}]},"version":"4.4.0"},{"name":"parquet_encode","type":"processor","status":"experimental","plugin":true,"summary":"Encodes https://parquet.apache.org/docs/[Parquet files^] from a batch of structured messages.","description":"\nThis processor uses https://github.com/parquet-go/parquet-go[https://github.com/parquet-go/parquet-go^], which is itself experimental. Therefore changes could be made into how this processor functions outside of major version releases.\n","categories":["Parsing"],"examples":[{"title":"Writing Parquet Files to AWS S3","summary":"In this example we use the batching mechanism of an `aws_s3` output to collect a batch of messages in memory, which then converts it to a parquet file and uploads it.","config":"\noutput:\n aws_s3:\n bucket: TODO\n path: 'stuff/${! timestamp_unix() }-${! uuid_v4() }.parquet'\n batching:\n count: 1000\n period: 10s\n processors:\n - parquet_encode:\n schema:\n - name: id\n type: INT64\n - name: weight\n type: DOUBLE\n - name: content\n type: BYTE_ARRAY\n default_compression: zstd\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"schema","type":"object","kind":"array","description":"Parquet schema.","children":[{"name":"name","type":"string","kind":"scalar","description":"The name of the column."},{"name":"type","type":"string","kind":"scalar","description":"The type of the column, only applicable for leaf columns with no child fields. Some logical types can be specified here such as UTF8.","is_optional":true,"options":["BOOLEAN","INT32","INT64","FLOAT","DOUBLE","BYTE_ARRAY","UTF8","TIMESTAMP","BSON","ENUM","JSON","UUID"],"linter":"\nlet options = {\n \"boolean\": true,\n \"int32\": true,\n \"int64\": true,\n \"float\": true,\n \"double\": true,\n \"byte_array\": true,\n \"utf8\": true,\n \"timestamp\": true,\n \"bson\": true,\n \"enum\": true,\n \"json\": true,\n \"uuid\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"repeated","type":"bool","kind":"scalar","description":"Whether the field is repeated.","default":false},{"name":"optional","type":"bool","kind":"scalar","description":"Whether the field is optional.","default":false},{"name":"fields","type":"unknown","kind":"array","description":"A list of child fields.","is_optional":true,"examples":[[{"name":"foo","type":"INT64"},{"name":"bar","type":"BYTE_ARRAY"}]]}]},{"name":"default_compression","type":"string","kind":"scalar","description":"The default compression type to use for fields.","default":"uncompressed","options":["uncompressed","snappy","gzip","brotli","zstd","lz4raw"],"linter":"\nlet options = {\n \"uncompressed\": true,\n \"snappy\": true,\n \"gzip\": true,\n \"brotli\": true,\n \"zstd\": true,\n \"lz4raw\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"default_encoding","type":"string","kind":"scalar","description":"The default encoding type to use for fields. A custom default encoding is only necessary when consuming data with libraries that do not support `DELTA_LENGTH_BYTE_ARRAY` and is therefore best left unset where possible.","is_advanced":true,"default":"DELTA_LENGTH_BYTE_ARRAY","options":["DELTA_LENGTH_BYTE_ARRAY","PLAIN"],"version":"4.11.0","linter":"\nlet options = {\n \"delta_length_byte_array\": true,\n \"plain\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"}]},"version":"4.4.0"},{"name":"parse_log","type":"processor","status":"stable","plugin":true,"summary":"Parses common log \u003c\u003cformats\u003e\u003e into \u003c\u003ccodecs, structured data\u003e\u003e. This is easier and often much faster than xref:components:processors/grok.adoc[`grok`].","categories":["Parsing"],"footnotes":"\n== Codecs\n\nCurrently the only supported structured data codec is `json`.\n\n== Formats\n\n=== `syslog_rfc5424`\n\nAttempts to parse a log following the https://tools.ietf.org/html/rfc5424[Syslog RFC5424^] spec. The resulting structured document may contain any of the following fields:\n\n- `message` (string)\n- `timestamp` (string, RFC3339)\n- `facility` (int)\n- `severity` (int)\n- `priority` (int)\n- `version` (int)\n- `hostname` (string)\n- `procid` (string)\n- `appname` (string)\n- `msgid` (string)\n- `structureddata` (object)\n\n=== `syslog_rfc3164`\n\nAttempts to parse a log following the https://tools.ietf.org/html/rfc3164[Syslog rfc3164] spec. The resulting structured document may contain any of the following fields:\n\n- `message` (string)\n- `timestamp` (string, RFC3339)\n- `facility` (int)\n- `severity` (int)\n- `priority` (int)\n- `hostname` (string)\n- `procid` (string)\n- `appname` (string)\n- `msgid` (string)\n","config":{"name":"","type":"object","kind":"scalar","children":[{"name":"format","type":"string","kind":"scalar","description":"A common log \u003c\u003cformats, format\u003e\u003e to parse.","options":["syslog_rfc5424","syslog_rfc3164"],"linter":"\nlet options = {\n \"syslog_rfc5424\": true,\n \"syslog_rfc3164\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"best_effort","type":"bool","kind":"scalar","description":"Still returns partially parsed messages even if an error occurs.","is_advanced":true,"default":true},{"name":"allow_rfc3339","type":"bool","kind":"scalar","description":"Also accept timestamps in rfc3339 format while parsing. Applicable to format `syslog_rfc3164`.","is_advanced":true,"default":true},{"name":"default_year","type":"string","kind":"scalar","description":"Sets the strategy used to set the year for rfc3164 timestamps. Applicable to format `syslog_rfc3164`. When set to `current` the current year will be set, when set to an integer that value will be used. Leave this field empty to not set a default year at all.","is_advanced":true,"default":"current"},{"name":"default_timezone","type":"string","kind":"scalar","description":"Sets the strategy to decide the timezone for rfc3164 timestamps. Applicable to format `syslog_rfc3164`. This value should follow the https://golang.org/pkg/time/#LoadLocation[time.LoadLocation^] format.","is_advanced":true,"default":"UTC"},{"name":"codec","type":"string","kind":"scalar","is_deprecated":true}]}},{"name":"processors","type":"processor","status":"stable","plugin":true,"summary":"A processor grouping several sub-processors.","description":"This processor is useful in situations where you want to collect several processors under a single resource identifier, whether it is for making your configuration easier to read and navigate, or for improving the testability of your configuration. The behavior of child processors will match exactly the behavior they would have under any other processors block.","categories":["Composition"],"examples":[{"title":"Grouped Processing","summary":"Imagine we have a collection of processors who cover a specific functionality. We could use this processor to group them together and make it easier to read and mock during testing by giving the whole block a label:","config":"\npipeline:\n processors:\n - label: my_super_feature\n processors:\n - log:\n message: \"Let's do something cool\"\n - archive:\n format: json_array\n - mapping: root.items = this\n"}],"config":{"name":"","type":"processor","kind":"array","default":[]}},{"name":"protobuf","type":"processor","status":"stable","plugin":true,"summary":"\nPerforms conversions to or from a protobuf message. This processor uses reflection, meaning conversions can be made directly from the target .proto files.\n","description":"\nThe main functionality of this processor is to map to and from JSON documents, you can read more about JSON mapping of protobuf messages here: https://developers.google.com/protocol-buffers/docs/proto3#json[https://developers.google.com/protocol-buffers/docs/proto3#json^]\n\nUsing reflection for processing protobuf messages in this way is less performant than generating and using native code. Therefore when performance is critical it is recommended that you use Redpanda Connect plugins instead for processing protobuf messages natively, you can find an example of Redpanda Connect plugins at https://github.com/benthosdev/benthos-plugin-example[https://github.com/benthosdev/benthos-plugin-example^]\n\n== Operators\n\n=== `to_json`\n\nConverts protobuf messages into a generic JSON structure. This makes it easier to manipulate the contents of the document within Benthos.\n\n=== `from_json`\n\nAttempts to create a target protobuf message from a generic JSON structure.\n","categories":["Parsing"],"examples":[{"title":"JSON to Protobuf","summary":"\nIf we have the following protobuf definition within a directory called `testing/schema`:\n\n```protobuf\nsyntax = \"proto3\";\npackage testing;\n\nimport \"google/protobuf/timestamp.proto\";\n\nmessage Person {\n string first_name = 1;\n string last_name = 2;\n string full_name = 3;\n int32 age = 4;\n int32 id = 5; // Unique ID number for this person.\n string email = 6;\n\n google.protobuf.Timestamp last_updated = 7;\n}\n```\n\nAnd a stream of JSON documents of the form:\n\n```json\n{\n\t\"firstName\": \"caleb\",\n\t\"lastName\": \"quaye\",\n\t\"email\": \"caleb@myspace.com\"\n}\n```\n\nWe can convert the documents into protobuf messages with the following config:","config":"\npipeline:\n processors:\n - protobuf:\n operator: from_json\n message: testing.Person\n import_paths: [ testing/schema ]\n"},{"title":"Protobuf to JSON","summary":"\nIf we have the following protobuf definition within a directory called `testing/schema`:\n\n```protobuf\nsyntax = \"proto3\";\npackage testing;\n\nimport \"google/protobuf/timestamp.proto\";\n\nmessage Person {\n string first_name = 1;\n string last_name = 2;\n string full_name = 3;\n int32 age = 4;\n int32 id = 5; // Unique ID number for this person.\n string email = 6;\n\n google.protobuf.Timestamp last_updated = 7;\n}\n```\n\nAnd a stream of protobuf messages of the type `Person`, we could convert them into JSON documents of the format:\n\n```json\n{\n\t\"firstName\": \"caleb\",\n\t\"lastName\": \"quaye\",\n\t\"email\": \"caleb@myspace.com\"\n}\n```\n\nWith the following config:","config":"\npipeline:\n processors:\n - protobuf:\n operator: to_json\n message: testing.Person\n import_paths: [ testing/schema ]\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"operator","type":"string","kind":"scalar","description":"The \u003c\u003coperators, operator\u003e\u003e to execute","options":["to_json","from_json"],"linter":"\nlet options = {\n \"to_json\": true,\n \"from_json\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"message","type":"string","kind":"scalar","description":"The fully qualified name of the protobuf message to convert to/from."},{"name":"discard_unknown","type":"bool","kind":"scalar","description":"If `true`, the `from_json` operator discards fields that are unknown to the schema.","default":false},{"name":"use_proto_names","type":"bool","kind":"scalar","description":"If `true`, the `to_json` operator deserializes fields exactly as named in schema file.","default":false},{"name":"import_paths","type":"string","kind":"array","description":"A list of directories containing .proto files, including all definitions required for parsing the target message. If left empty the current directory is used. Each directory listed will be walked with all found .proto files imported.","default":[]},{"name":"use_enum_numbers","type":"bool","kind":"scalar","description":"If `true`, the `to_json` operator deserializes enums as numerical values instead of string names.","default":false}]}},{"name":"rate_limit","type":"processor","status":"stable","plugin":true,"summary":"Throttles the throughput of a pipeline according to a specified xref:components:rate_limits/about.adoc[`rate_limit`] resource. Rate limits are shared across components and therefore apply globally to all processing pipelines.","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"resource","type":"string","kind":"scalar","description":"The target xref:components:rate_limits/about.adoc[`rate_limit` resource]."}]}},{"name":"redis","type":"processor","status":"stable","plugin":true,"summary":"Performs actions against Redis that aren't possible using a xref:components:processors/cache.adoc[`cache`] processor. Actions are\nperformed for each message and the message contents are replaced with the result. In order to merge the result into the original message compose this processor within a xref:components:processors/branch.adoc[`branch` processor].","categories":["Integration"],"examples":[{"title":"Querying Cardinality","summary":"If given payloads containing a metadata field `set_key` it's possible to query and store the cardinality of the set for each message using a xref:components:processors/branch.adoc[`branch` processor] in order to augment rather than replace the message contents:","config":"\npipeline:\n processors:\n - branch:\n processors:\n - redis:\n url: TODO\n command: scard\n args_mapping: 'root = [ meta(\"set_key\") ]'\n result_map: 'root.cardinality = this'\n"},{"title":"Running Total","summary":"If we have JSON data containing number of friends visited during covid 19:\n\n```json\n{\"name\":\"ash\",\"month\":\"feb\",\"year\":2019,\"friends_visited\":10}\n{\"name\":\"ash\",\"month\":\"apr\",\"year\":2019,\"friends_visited\":-2}\n{\"name\":\"bob\",\"month\":\"feb\",\"year\":2019,\"friends_visited\":3}\n{\"name\":\"bob\",\"month\":\"apr\",\"year\":2019,\"friends_visited\":1}\n```\n\nWe can add a field that contains the running total number of friends visited:\n\n```json\n{\"name\":\"ash\",\"month\":\"feb\",\"year\":2019,\"friends_visited\":10,\"total\":10}\n{\"name\":\"ash\",\"month\":\"apr\",\"year\":2019,\"friends_visited\":-2,\"total\":8}\n{\"name\":\"bob\",\"month\":\"feb\",\"year\":2019,\"friends_visited\":3,\"total\":3}\n{\"name\":\"bob\",\"month\":\"apr\",\"year\":2019,\"friends_visited\":1,\"total\":4}\n```\n\nUsing the `incrby` command:","config":"\npipeline:\n processors:\n - branch:\n processors:\n - redis:\n url: TODO\n command: incrby\n args_mapping: 'root = [ this.name, this.friends_visited ]'\n result_map: 'root.total = this'\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target Redis server. Database is optional and is supplied as the URL path.","examples":["redis://:6379","redis://localhost:6379","redis://foousername:foopassword@redisplace:6379","redis://:foopassword@redisplace:6379","redis://localhost:6379/1","redis://localhost:6379/1,redis://localhost:6380/1"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"kind","type":"string","kind":"scalar","description":"Specifies a simple, cluster-aware, or failover-aware redis client.","is_advanced":true,"default":"simple","options":["simple","cluster","failover"],"linter":"\nlet options = {\n \"simple\": true,\n \"cluster\": true,\n \"failover\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"master","type":"string","kind":"scalar","description":"Name of the redis master when `kind` is `failover`","is_advanced":true,"default":"","examples":["mymaster"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"command","type":"string","kind":"scalar","description":"The command to execute.","is_optional":true,"interpolated":true,"examples":["scard","incrby","${! meta(\"command\") }"],"version":"4.3.0"},{"name":"args_mapping","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of arguments required for the specified Redis command.","is_optional":true,"bloblang":true,"examples":["root = [ this.key ]","root = [ meta(\"kafka_key\"), this.count ]"],"version":"4.3.0"},{"name":"operator","type":"string","kind":"scalar","description":"The operator to apply.","is_deprecated":true,"is_optional":true,"annotated_options":[["incrby","Increments the number stored at `key` by the message content. If the key does not exist, it is set to `0` before performing the operation. Returns the value of `key` after the increment."],["keys","Returns an array of strings containing all the keys that match the pattern specified by the `key` field."],["sadd","Adds a new member to a set. Returns `1` if the member was added."],["scard","Returns the cardinality of a set, or `0` if the key does not exist."]],"linter":"\nlet options = {\n \"incrby\": true,\n \"keys\": true,\n \"sadd\": true,\n \"scard\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"key","type":"string","kind":"scalar","description":"A key to use for the target operator.","is_deprecated":true,"is_optional":true,"interpolated":true},{"name":"retries","type":"int","kind":"scalar","description":"The maximum number of retries before abandoning a request.","is_advanced":true,"default":3},{"name":"retry_period","type":"string","kind":"scalar","description":"The time to wait before consecutive retry attempts.","is_advanced":true,"default":"500ms"}],"linter":"root = match {\n this.exists(\"operator\") == this.exists(\"command\") =\u003e [ \"one of 'operator' (old style) or 'command' (new style) fields must be specified\" ]\n this.exists(\"args_mapping\") \u0026\u0026 this.exists(\"operator\") =\u003e [ \"field args_mapping is invalid with an operator set\" ],\n}"}},{"name":"redis_script","type":"processor","status":"beta","plugin":true,"summary":"Performs actions against Redis using https://redis.io/docs/manual/programmability/eval-intro/[LUA scripts^].","description":"Actions are performed for each message and the message contents are replaced with the result.\n\nIn order to merge the result into the original message compose this processor within a xref:components:processors/branch.adoc[`branch` processor].","categories":["Integration"],"examples":[{"title":"Running a script","summary":"The following example will use a script execution to get next element from a sorted set and set its score with timestamp unix nano value.","config":"\npipeline:\n processors:\n - redis_script:\n url: TODO\n script: |\n local value = redis.call(\"ZRANGE\", KEYS[1], '0', '0')\n\n if next(elements) == nil then\n return ''\n end\n\n redis.call(\"ZADD\", \"XX\", KEYS[1], ARGV[1], value)\n\n return value\n keys_mapping: 'root = [ meta(\"key\") ]'\n args_mapping: 'root = [ timestamp_unix_nano() ]'\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target Redis server. Database is optional and is supplied as the URL path.","examples":["redis://:6379","redis://localhost:6379","redis://foousername:foopassword@redisplace:6379","redis://:foopassword@redisplace:6379","redis://localhost:6379/1","redis://localhost:6379/1,redis://localhost:6380/1"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"kind","type":"string","kind":"scalar","description":"Specifies a simple, cluster-aware, or failover-aware redis client.","is_advanced":true,"default":"simple","options":["simple","cluster","failover"],"linter":"\nlet options = {\n \"simple\": true,\n \"cluster\": true,\n \"failover\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"master","type":"string","kind":"scalar","description":"Name of the redis master when `kind` is `failover`","is_advanced":true,"default":"","examples":["mymaster"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"script","type":"string","kind":"scalar","description":"A script to use for the target operator. It has precedence over the 'command' field.","examples":["return redis.call('set', KEYS[1], ARGV[1])"]},{"name":"args_mapping","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of arguments required for the specified Redis script.","bloblang":true,"examples":["root = [ this.key ]","root = [ meta(\"kafka_key\"), \"hardcoded_value\" ]"]},{"name":"keys_mapping","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of keys matching in size to the number of arguments required for the specified Redis script.","bloblang":true,"examples":["root = [ this.key ]","root = [ meta(\"kafka_key\"), this.count ]"]},{"name":"retries","type":"int","kind":"scalar","description":"The maximum number of retries before abandoning a request.","is_advanced":true,"default":3},{"name":"retry_period","type":"string","kind":"scalar","description":"The time to wait before consecutive retry attempts.","is_advanced":true,"default":"500ms"}]},"version":"4.11.0"},{"name":"redpanda_data_transform","type":"processor","status":"experimental","plugin":true,"summary":"Executes a Redpanda Data Transform as a processor","description":"\nThis processor executes a Redpanda Data Transform WebAssembly module, calling OnRecordWritten for each message being processed.\n\nYou can find out about how transforms work here: https://docs.redpanda.com/current/develop/data-transforms/how-transforms-work/[https://docs.redpanda.com/current/develop/data-transforms/how-transforms-work/^]\n","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"module_path","type":"string","kind":"scalar","description":"The path of the target WASM module to execute."},{"name":"input_key","type":"string","kind":"scalar","description":"An optional key to populate for each message.","is_optional":true,"interpolated":true},{"name":"output_key","type":"string","kind":"scalar","description":"An optional name of metadata for an output message key.","is_optional":true},{"name":"input_headers","type":"object","kind":"scalar","description":"Determine which (if any) metadata values should be added to messages as headers.","is_optional":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"output_metadata","type":"object","kind":"scalar","description":"Determine which (if any) message headers should be added to the output as metadata.","is_optional":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"timestamp","type":"string","kind":"scalar","description":"An optional timestamp to set for each message. When left empty, the current timestamp is used.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["${! timestamp_unix() }","${! metadata(\"kafka_timestamp_ms\") }"]},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period of time for a message to be processed","is_advanced":true,"default":"10s"},{"name":"max_memory_pages","type":"int","kind":"scalar","description":"The maximum amount of wasm memory pages (64KiB) that an individual wasm module instance can use","is_advanced":true,"default":1600}]},"version":"4.31.0"},{"name":"resource","type":"processor","status":"stable","plugin":true,"summary":"Resource is a processor type that runs a processor resource identified by its label.","description":"\nThis processor allows you to reference the same configured processor resource in multiple places, and can also tidy up large nested configs. For example, the config:\n\n```yaml\npipeline:\n processors:\n - mapping: |\n root.message = this\n root.meta.link_count = this.links.length()\n root.user.age = this.user.age.number()\n```\n\nIs equivalent to:\n\n```yaml\npipeline:\n processors:\n - resource: foo_proc\n\nprocessor_resources:\n - label: foo_proc\n mapping: |\n root.message = this\n root.meta.link_count = this.links.length()\n root.user.age = this.user.age.number()\n```\n\nYou can find out more about resources in xref:configuration:resources.adoc[]","categories":["Utility"],"config":{"name":"","type":"string","kind":"scalar","default":""}},{"name":"retry","type":"processor","status":"beta","plugin":true,"summary":"Attempts to execute a series of child processors until success.","description":"\nExecutes child processors and if a resulting message is errored then, after a specified backoff period, the same original message will be attempted again through those same processors. If the child processors result in more than one message then the retry mechanism will kick in if _any_ of the resulting messages are errored.\n\nIt is important to note that any mutations performed on the message during these child processors will be discarded for the next retry, and therefore it is safe to assume that each execution of the child processors will always be performed on the data as it was when it first reached the retry processor.\n\nBy default the retry backoff has a specified \u003c\u003cbackoffmax_elapsed_time,`max_elapsed_time`\u003e\u003e, if this time period is reached during retries and an error still occurs these errored messages will proceed through to the next processor after the retry (or your outputs). Normal xref:configuration:error_handling.adoc[error handling patterns] can be used on these messages.\n\nIn order to avoid permanent loops any error associated with messages as they first enter a retry processor will be cleared.\n\n== Metadata\n\nThis processor adds the following metadata fields to each message:\n\n```text\n- retry_count - The number of retry attempts.\n- backoff_duration - The total time elapsed while performing retries.\n```\n\n[CAUTION]\n.Batching\n====\nIf you wish to wrap a batch-aware series of processors then take a look at the \u003c\u003cbatching, batching section\u003e\u003e.\n====\n","categories":["Composition"],"footnotes":"\n== Batching\n\nWhen messages are batched the child processors of a retry are executed for each individual message in isolation, performed serially by default but in parallel when the field \u003c\u003cparallel, `parallel`\u003e\u003e is set to `true`. This is an intentional limitation of the retry processor and is done in order to ensure that errors are correctly associated with a given input message. Otherwise, the archiving, expansion, grouping, filtering and so on of the child processors could obfuscate this relationship.\n\nIf the target behavior of your retried processors is \"batch aware\", in that you wish to perform some processing across the entire batch of messages and repeat it in the event of errors, you can use an xref:components:processors/archive.adoc[`archive` processor] to collapse the batch into an individual message. Then, within these child processors either perform your batch aware processing on the archive, or use an xref:components:processors/unarchive.adoc[`unarchive` processor] in order to expand the single message back out into a batch.\n\nFor example, if the retry processor were being used to wrap an HTTP request where the payload data is a batch archived into a JSON array it should look something like this:\n\n```yaml\npipeline:\n processors:\n - archive:\n format: json_array\n - retry:\n processors:\n - http:\n url: example.com/nope\n verb: POST\n - unarchive:\n format: json_array\n```\n","examples":[{"title":"Stop ignoring me Taz","summary":"\nHere we have a config where I generate animal noises and send them to Taz via HTTP. Taz has a tendency to stop his servers whenever I dispatch my animals upon him, and therefore these HTTP requests sometimes fail. However, I have the retry processor and with this super power I can specify a back off policy and it will ensure that for each animal noise the HTTP processor is attempted until either it succeeds or my Redpanda Connect instance is stopped.\n\nI even go as far as to zero-out the maximum elapsed time field, which means that for each animal noise I will wait indefinitely, because I really really want Taz to receive every single animal noise that he is entitled to.","config":"\ninput:\n generate:\n interval: 1s\n mapping: 'root.noise = [ \"woof\", \"meow\", \"moo\", \"quack\" ].index(random_int(min: 0, max: 3))'\n\npipeline:\n processors:\n - retry:\n backoff:\n initial_interval: 100ms\n max_interval: 5s\n max_elapsed_time: 0s\n processors:\n - http:\n url: 'http://example.com/try/not/to/dox/taz'\n verb: POST\n\noutput:\n # Drop everything because it's junk data, I don't want it lol\n drop: {}\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"backoff","type":"object","kind":"scalar","description":"Determine time intervals and cut offs for retry attempts.","children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","default":"500ms","examples":["50ms","1s"]},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts","default":"10s","examples":["5s","1m"]},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum overall period of time to spend on retry attempts before the request is aborted. Setting this value to a zeroed duration (such as `0s`) will result in unbounded retries.","default":"1m","examples":["1m","1h"]}]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to execute on each message."},{"name":"parallel","type":"bool","kind":"scalar","description":"When processing batches of messages these batches are ignored and the processors apply to each message sequentially. However, when this field is set to `true` each message will be processed in parallel. Caution should be made to ensure that batch sizes do not surpass a point where this would cause resource (CPU, memory, API limits) contention.","default":false},{"name":"max_retries","type":"int","kind":"scalar","description":"The maximum number of retry attempts before the request is aborted. Setting this value to `0` will result in unbounded number of retries.","default":0}]},"version":"4.27.0"},{"name":"schema_registry_decode","type":"processor","status":"beta","plugin":true,"summary":"Automatically decodes and validates messages with schemas from a Confluent Schema Registry service.","description":"\nDecodes messages automatically from a schema stored within a https://docs.confluent.io/platform/current/schema-registry/index.html[Confluent Schema Registry service^] by extracting a schema ID from the message and obtaining the associated schema from the registry. If a message fails to match against the schema then it will remain unchanged and the error can be caught using xref:configuration:error_handling.adoc[error handling methods].\n\nAvro, Protobuf and Json schemas are supported, all are capable of expanding from schema references as of v4.22.0.\n\n== Avro JSON format\n\nThis processor creates documents formatted as https://avro.apache.org/docs/current/specification/_print/#json-encoding[Avro JSON^] when decoding with Avro schemas. In this format the value of a union is encoded in JSON as follows:\n\n- if its type is `null`, then it is encoded as a JSON `null`;\n- otherwise it is encoded as a JSON object with one name/value pair whose name is the type's name and whose value is the recursively encoded value. For Avro's named types (record, fixed or enum) the user-specified name is used, for other types the type name is used.\n\nFor example, the union schema `[\"null\",\"string\",\"Foo\"]`, where `Foo` is a record name, would encode:\n\n- `null` as `null`;\n- the string `\"a\"` as `{\"string\": \"a\"}`; and\n- a `Foo` instance as `{\"Foo\": {...}}`, where `{...}` indicates the JSON encoding of a `Foo` instance.\n\nHowever, it is possible to instead create documents in https://pkg.go.dev/github.com/linkedin/goavro/v2#NewCodecForStandardJSONFull[standard/raw JSON format^] by setting the field \u003c\u003cavro_raw_json, `avro_raw_json`\u003e\u003e to `true`.\n\n== Protobuf format\n\nThis processor decodes protobuf messages to JSON documents, you can read more about JSON mapping of protobuf messages here: https://developers.google.com/protocol-buffers/docs/proto3#json\n\n== Metadata\n\nThis processor also adds the following metadata to each outgoing message:\n\nschema_id: the ID of the schema in the schema registry that was associated with the message.\n","categories":["Parsing","Integration"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"avro_raw_json","type":"bool","kind":"scalar","description":"Whether Avro messages should be decoded into normal JSON (\"json that meets the expectations of regular internet json\") rather than https://avro.apache.org/docs/current/specification/_print/#json-encoding[Avro JSON^]. If `true` the schema returned from the subject should be decoded as https://pkg.go.dev/github.com/linkedin/goavro/v2#NewCodecForStandardJSONFull[standard json^] instead of as https://pkg.go.dev/github.com/linkedin/goavro/v2#NewCodec[avro json^]. There is a https://github.com/linkedin/goavro/blob/5ec5a5ee7ec82e16e6e2b438d610e1cab2588393/union.go#L224-L249[comment in goavro^], the https://github.com/linkedin/goavro[underlining library used for avro serialization^], that explains in more detail the difference between the standard json and avro json.","is_advanced":true,"is_deprecated":true,"default":false},{"name":"avro","type":"object","kind":"scalar","description":"Configuration for how to decode schemas that are of type AVRO.","children":[{"name":"raw_unions","type":"bool","kind":"scalar","description":"Whether avro messages should be decoded into normal JSON (\"json that meets the expectations of regular internet json\") rather than https://avro.apache.org/docs/current/specification/_print/#json-encoding[JSON as specified in the Avro Spec^].\n\nFor example, if there is a union schema `[\"null\", \"string\", \"Foo\"]` where `Foo` is a record name, with raw_unions as false (the default) you get:\n- `null` as `null`;\n- the string `\"a\"` as `{\"string\": \"a\"}`; and\n- a `Foo` instance as `{\"Foo\": {...}}`, where `{...}` indicates the JSON encoding of a `Foo` instance.\n\nWhen raw_unions is set to true then the above union schema is decoded as the following:\n- `null` as `null`;\n- the string `\"a\"` as `\"a\"`; and\n- a `Foo` instance as `{...}`, where `{...}` indicates the JSON encoding of a `Foo` instance.\n","is_optional":true},{"name":"preserve_logical_types","type":"bool","kind":"scalar","description":"Whether logical types should be preserved or transformed back into their primitive type. By default, decimals are decoded as raw bytes and timestamps are decoded as plain integers. Setting this field to true keeps decimal types as numbers in bloblang and timestamps as time values.","default":false},{"name":"mapping","type":"string","kind":"scalar","description":"A custom mapping to apply to Avro schemas JSON representation. This is useful to transform custom types emitted by other tools into standard avro.","is_advanced":true,"is_optional":true,"bloblang":true,"examples":["\nmap isDebeziumTimestampType {\n root = this.type == \"long\" \u0026\u0026 this.\"connect.name\" == \"io.debezium.time.Timestamp\" \u0026\u0026 !this.exists(\"logicalType\")\n}\nmap debeziumTimestampToAvroTimestamp {\n let mapped_fields = this.fields.or([]).map_each(item -\u003e item.apply(\"debeziumTimestampToAvroTimestamp\"))\n root = match {\n this.type == \"record\" =\u003e this.assign({\"fields\": $mapped_fields})\n this.type.type() == \"array\" =\u003e this.assign({\"type\": this.type.map_each(item -\u003e item.apply(\"debeziumTimestampToAvroTimestamp\"))})\n # Add a logical type so that it's decoded as a timestamp instead of a long.\n this.type.type() == \"object\" \u0026\u0026 this.type.apply(\"isDebeziumTimestampType\") =\u003e this.merge({\"type\":{\"logicalType\": \"timestamp-millis\"}})\n _ =\u003e this\n }\n}\nroot = this.apply(\"debeziumTimestampToAvroTimestamp\")\n"]}]},{"name":"url","type":"string","kind":"scalar","description":"The base URL of the schema registry service.","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"oauth","type":"object","kind":"scalar","description":"Allows you to specify open authentication via OAuth version 1.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use OAuth version 1 in requests.","is_advanced":true,"default":false},{"name":"consumer_key","type":"string","kind":"scalar","description":"A value used to identify the client to the service provider.","is_advanced":true,"default":""},{"name":"consumer_secret","type":"string","kind":"scalar","description":"A secret used to establish ownership of the consumer key.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"access_token","type":"string","kind":"scalar","description":"A value used to gain access to the protected resources on behalf of the user.","is_advanced":true,"default":""},{"name":"access_token_secret","type":"string","kind":"scalar","description":"A secret provided in order to establish ownership of a given access token.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}],"version":"4.7.0"},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}],"version":"4.7.0"},{"name":"jwt","type":"object","kind":"scalar","description":"BETA: Allows you to specify JWT authentication.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use JWT authentication in requests.","is_advanced":true,"default":false},{"name":"private_key_file","type":"string","kind":"scalar","description":"A file with the PEM encoded via PKCS1 or PKCS8 as private key.","is_advanced":true,"default":""},{"name":"signing_method","type":"string","kind":"scalar","description":"A method used to sign the token such as RS256, RS384, RS512 or EdDSA.","is_advanced":true,"default":""},{"name":"claims","type":"unknown","kind":"map","description":"A value used to identify the claims that issued the JWT.","is_advanced":true,"default":{}},{"name":"headers","type":"unknown","kind":"map","description":"Add optional key/value headers to the JWT.","is_advanced":true,"default":{}}],"version":"4.7.0"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]}]}},{"name":"schema_registry_encode","type":"processor","status":"beta","plugin":true,"summary":"Automatically encodes and validates messages with schemas from a Confluent Schema Registry service.","description":"\nEncodes messages automatically from schemas obtains from a https://docs.confluent.io/platform/current/schema-registry/index.html[Confluent Schema Registry service^] by polling the service for the latest schema version for target subjects.\n\nIf a message fails to encode under the schema then it will remain unchanged and the error can be caught using xref:configuration:error_handling.adoc[error handling methods].\n\nAvro, Protobuf and Json schemas are supported, all are capable of expanding from schema references as of v4.22.0.\n\n== Avro JSON format\n\nBy default this processor expects documents formatted as https://avro.apache.org/docs/current/specification/_print/#json-encoding[Avro JSON^] when encoding with Avro schemas. In this format the value of a union is encoded in JSON as follows:\n\n- if its type is `null`, then it is encoded as a JSON `null`;\n- otherwise it is encoded as a JSON object with one name/value pair whose name is the type's name and whose value is the recursively encoded value. For Avro's named types (record, fixed or enum) the user-specified name is used, for other types the type name is used.\n\nFor example, the union schema `[\"null\",\"string\",\"Foo\"]`, where `Foo` is a record name, would encode:\n\n- `null` as `null`;\n- the string `\"a\"` as `\\{\"string\": \"a\"}`; and\n- a `Foo` instance as `\\{\"Foo\": {...}}`, where `{...}` indicates the JSON encoding of a `Foo` instance.\n\nHowever, it is possible to instead consume documents in https://pkg.go.dev/github.com/linkedin/goavro/v2#NewCodecForStandardJSONFull[standard/raw JSON format^] by setting the field \u003c\u003cavro_raw_json, `avro_raw_json`\u003e\u003e to `true`.\n\n=== Known issues\n\nImportant! There is an outstanding issue in the https://github.com/linkedin/goavro[avro serializing library^] that Redpanda Connect uses which means it https://github.com/linkedin/goavro/issues/252[doesn't encode logical types correctly^]. It's still possible to encode logical types that are in-line with the spec if `avro_raw_json` is set to true, though now of course non-logical types will not be in-line with the spec.\n\n== Protobuf format\n\nThis processor encodes protobuf messages either from any format parsed within Redpanda Connect (encoded as JSON by default), or from raw JSON documents, you can read more about JSON mapping of protobuf messages here: https://developers.google.com/protocol-buffers/docs/proto3#json\n\n=== Multiple message support\n\nWhen a target subject presents a protobuf schema that contains multiple messages it becomes ambiguous which message definition a given input data should be encoded against. In such scenarios Redpanda Connect will attempt to encode the data against each of them and select the first to successfully match against the data, this process currently *ignores all nested message definitions*. In order to speed up this exhaustive search the last known successful message will be attempted first for each subsequent input.\n\nWe will be considering alternative approaches in future so please https://redpanda.com/slack[get in touch^] with thoughts and feedback.\n","categories":["Parsing","Integration"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The base URL of the schema registry service.","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"subject","type":"string","kind":"scalar","description":"The schema subject to derive schemas from.","interpolated":true,"examples":["foo","${! meta(\"kafka_topic\") }"]},{"name":"refresh_period","type":"string","kind":"scalar","description":"The period after which a schema is refreshed for each subject, this is done by polling the schema registry service.","default":"10m","examples":["60s","1h"]},{"name":"avro_raw_json","type":"bool","kind":"scalar","description":"Whether messages encoded in Avro format should be parsed as normal JSON (\"json that meets the expectations of regular internet json\") rather than https://avro.apache.org/docs/current/specification/_print/#json-encoding[Avro JSON^]. If `true` the schema returned from the subject should be parsed as https://pkg.go.dev/github.com/linkedin/goavro/v2#NewCodecForStandardJSONFull[standard json^] instead of as https://pkg.go.dev/github.com/linkedin/goavro/v2#NewCodec[avro json^]. There is a https://github.com/linkedin/goavro/blob/5ec5a5ee7ec82e16e6e2b438d610e1cab2588393/union.go#L224-L249[comment in goavro^], the https://github.com/linkedin/goavro[underlining library used for avro serialization^], that explains in more detail the difference between standard json and avro json.","is_advanced":true,"default":false,"version":"3.59.0"},{"name":"oauth","type":"object","kind":"scalar","description":"Allows you to specify open authentication via OAuth version 1.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use OAuth version 1 in requests.","is_advanced":true,"default":false},{"name":"consumer_key","type":"string","kind":"scalar","description":"A value used to identify the client to the service provider.","is_advanced":true,"default":""},{"name":"consumer_secret","type":"string","kind":"scalar","description":"A secret used to establish ownership of the consumer key.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"access_token","type":"string","kind":"scalar","description":"A value used to gain access to the protected resources on behalf of the user.","is_advanced":true,"default":""},{"name":"access_token_secret","type":"string","kind":"scalar","description":"A secret provided in order to establish ownership of a given access token.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}],"version":"4.7.0"},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}],"version":"4.7.0"},{"name":"jwt","type":"object","kind":"scalar","description":"BETA: Allows you to specify JWT authentication.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use JWT authentication in requests.","is_advanced":true,"default":false},{"name":"private_key_file","type":"string","kind":"scalar","description":"A file with the PEM encoded via PKCS1 or PKCS8 as private key.","is_advanced":true,"default":""},{"name":"signing_method","type":"string","kind":"scalar","description":"A method used to sign the token such as RS256, RS384, RS512 or EdDSA.","is_advanced":true,"default":""},{"name":"claims","type":"unknown","kind":"map","description":"A value used to identify the claims that issued the JWT.","is_advanced":true,"default":{}},{"name":"headers","type":"unknown","kind":"map","description":"Add optional key/value headers to the JWT.","is_advanced":true,"default":{}}],"version":"4.7.0"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]}]},"version":"3.58.0"},{"name":"select_parts","type":"processor","status":"stable","plugin":true,"summary":"Cherry pick a set of messages from a batch by their index. Indexes larger than the number of messages are simply ignored.","description":"\nThe selected parts are added to the new message batch in the same order as the selection array. E.g. with 'parts' set to [ 2, 0, 1 ] and the message parts [ '0', '1', '2', '3' ], the output will be [ '2', '0', '1' ].\n\nIf none of the selected parts exist in the input batch (resulting in an empty output message) the batch is dropped entirely.\n\nMessage indexes can be negative, and if so the part will be selected from the end counting backwards starting from -1. E.g. if index = -1 then the selected part will be the last part of the message, if index = -2 then the part before the last element with be selected, and so on.\n\nThis processor is only applicable to xref:configuration:batching.adoc[batched messages].","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"parts","type":"int","kind":"array","description":"An array of message indexes of a batch. Indexes can be negative, and if so the part will be selected from the end counting backwards starting from -1.","default":[]}]}},{"name":"sentry_capture","type":"processor","status":"experimental","plugin":true,"summary":"Captures log events from messages and submits them to https://sentry.io/[Sentry^].","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"dsn","type":"string","kind":"scalar","description":"The DSN address to send sentry events to. If left empty, then SENTRY_DSN is used.","default":""},{"name":"message","type":"string","kind":"scalar","description":"A message to set on the sentry event","interpolated":true,"examples":["webhook event received","failed to find product in database: ${! error() }"]},{"name":"context","type":"string","kind":"scalar","description":"A mapping that must evaluate to an object-of-objects or `deleted()`. If this mapping produces a value, then it is set on a sentry event as additional context.","is_optional":true,"bloblang":true,"examples":["root = {\"order\": {\"product_id\": \"P93174\", \"quantity\": 5}}","root = deleted()"]},{"name":"tags","type":"string","kind":"map","description":"Sets key/value string tags on an event. Unlike context, these are indexed and searchable on Sentry but have length limitations.","is_optional":true,"interpolated":true},{"name":"environment","type":"string","kind":"scalar","description":"The environment to be sent with events. If left empty, then SENTRY_ENVIRONMENT is used.","default":""},{"name":"release","type":"string","kind":"scalar","description":"The version of the code deployed to an environment. If left empty, then the Sentry client will attempt to detect the release from the environment.","default":""},{"name":"level","type":"string","kind":"scalar","description":"Sets the level on sentry events similar to logging levels.","default":"INFO","options":["DEBUG","INFO","WARN","ERROR","FATAL"],"linter":"\nlet options = {\n \"debug\": true,\n \"info\": true,\n \"warn\": true,\n \"error\": true,\n \"fatal\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"transport_mode","type":"string","kind":"scalar","description":"Determines how events are sent. A sync transport will block when sending each event until a response is received from the Sentry server. The recommended async transport will enqueue events in a buffer and send them in the background.","default":"async","options":["async","sync"],"linter":"\nlet options = {\n \"async\": true,\n \"sync\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"flush_timeout","type":"string","kind":"scalar","description":"The duration to wait when closing the processor to flush any remaining enqueued events.","default":"5s"},{"name":"sampling_rate","type":"float","kind":"scalar","description":"The rate at which events are sent to the server. A value of 0 disables capturing sentry events entirely. A value of 1 results in sending all events to Sentry. Any value in between results sending some percentage of events.","default":1,"linter":"root = if this \u003c 0 || this \u003e 1 { [\"sampling rate must be between 0.0 and 1.0\" ] }"}]},"version":"4.16.0"},{"name":"slack_thread","type":"processor","status":"experimental","plugin":true,"description":"Read a thread using the https://api.slack.com/methods/conversations.replies[^Slack API]","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"bot_token","type":"string","kind":"scalar","description":"The Slack Bot User OAuth token to use.","linter":"\n root = if !this.has_prefix(\"xoxb-\") { [ \"field must start with xoxb-\" ] }\n "},{"name":"channel_id","type":"string","kind":"scalar","description":"The channel ID to read messages from.","interpolated":true},{"name":"thread_ts","type":"string","kind":"scalar","description":"The thread timestamp to read the full thread of.","interpolated":true}]}},{"name":"sleep","type":"processor","status":"stable","plugin":true,"summary":"Sleep for a period of time specified as a duration string for each message. This processor will interpolate functions within the `duration` field, you can find a list of functions xref:configuration:interpolation.adoc#bloblang-queries[here].","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"duration","type":"string","kind":"scalar","description":"The duration of time to sleep for each execution.","interpolated":true}]}},{"name":"split","type":"processor","status":"stable","plugin":true,"summary":"Breaks message batches (synonymous with multiple part messages) into smaller batches. The size of the resulting batches are determined either by a discrete size or, if the field `byte_size` is non-zero, then by total size in bytes (which ever limit is reached first).","description":"\nThis processor is for breaking batches down into smaller ones. In order to break a single message out into multiple messages use the xref:components:processors/unarchive.adoc[`unarchive` processor].\n\nIf there is a remainder of messages after splitting a batch the remainder is also sent as a single batch. For example, if your target size was 10, and the processor received a batch of 95 message parts, the result would be 9 batches of 10 messages followed by a batch of 5 messages.","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"size","type":"int","kind":"scalar","description":"The target number of messages.","default":1},{"name":"byte_size","type":"int","kind":"scalar","description":"An optional target of total message bytes.","default":0}]}},{"name":"sql","type":"processor","status":"deprecated","plugin":true,"summary":"Runs an arbitrary SQL query against a database and (optionally) returns the result as an array of objects, one for each row returned.","description":"\nIf the query fails to execute then the message will remain unchanged and the error can be caught using xref:configuration:error_handling.adoc[error handling methods].\n\n== Alternatives\n\nFor basic inserts or select queries use either the xref:components:processors/sql_insert.adoc[`sql_insert`] or the xref:components:processors/sql_select.adoc[`sql_select`] processor. For more complex queries use the xref:components:processors/sql_raw.adoc[`sql_raw`] processor.","categories":["Integration"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"driver","type":"string","kind":"scalar","description":"A database \u003c\u003cdrivers, driver\u003e\u003e to use.","options":["mysql","postgres","clickhouse","mssql","sqlite","oracle","snowflake","trino","gocosmos","spanner"],"linter":"\nlet options = {\n \"mysql\": true,\n \"postgres\": true,\n \"clickhouse\": true,\n \"mssql\": true,\n \"sqlite\": true,\n \"oracle\": true,\n \"snowflake\": true,\n \"trino\": true,\n \"gocosmos\": true,\n \"spanner\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"data_source_name","type":"string","kind":"scalar","description":"Data source name."},{"name":"query","type":"string","kind":"scalar","description":"The query to execute. The style of placeholder to use depends on the driver, some drivers require question marks (`?`) whereas others expect incrementing dollar signs (`$1`, `$2`, and so on) or colons (`:1`, `:2` and so on). The style to use is outlined in this table:\n\n| Driver | Placeholder Style |\n|---|---|\n| `clickhouse` | Dollar sign |\n| `mysql` | Question mark |\n| `postgres` | Dollar sign |\n| `mssql` | Question mark |\n| `sqlite` | Question mark |\n| `oracle` | Colon |\n| `snowflake` | Question mark |\n| `trino` | Question mark |\n| `gocosmos` | Colon |\n","examples":["INSERT INTO footable (foo, bar, baz) VALUES (?, ?, ?);"]},{"name":"unsafe_dynamic_query","type":"bool","kind":"scalar","description":"Whether to enable xref:configuration:interpolation.adoc#bloblang-queries[interpolation functions] in the query. Great care should be made to ensure your queries are defended against injection attacks.","is_advanced":true,"default":false},{"name":"args_mapping","type":"string","kind":"scalar","description":"An optional xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `query`.","is_optional":true,"bloblang":true,"examples":["root = [ this.cat.meow, this.doc.woofs[0] ]","root = [ meta(\"user.id\") ]"]},{"name":"result_codec","type":"string","kind":"scalar","description":"Result codec.","default":"none"}]},"version":"3.65.0"},{"name":"sql_insert","type":"processor","status":"stable","plugin":true,"summary":"Inserts rows into an SQL database for each message, and leaves the message unchanged.","description":"\nIf the insert fails to execute then the message will still remain unchanged and the error can be caught using xref:configuration:error_handling.adoc[error handling methods].","categories":["Integration"],"examples":[{"title":"Table Insert (MySQL)","summary":"\nHere we insert rows into a database by populating the columns id, name and topic with values extracted from messages and metadata:","config":"\npipeline:\n processors:\n - sql_insert:\n driver: mysql\n dsn: foouser:foopassword@tcp(localhost:3306)/foodb\n table: footable\n columns: [ id, name, topic ]\n args_mapping: |\n root = [\n this.user.id,\n this.user.name,\n meta(\"kafka_topic\"),\n ]\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"driver","type":"string","kind":"scalar","description":"A database \u003c\u003cdrivers, driver\u003e\u003e to use.","options":["mysql","postgres","clickhouse","mssql","sqlite","oracle","snowflake","trino","gocosmos","spanner"],"linter":"\nlet options = {\n \"mysql\": true,\n \"postgres\": true,\n \"clickhouse\": true,\n \"mssql\": true,\n \"sqlite\": true,\n \"oracle\": true,\n \"snowflake\": true,\n \"trino\": true,\n \"gocosmos\": true,\n \"spanner\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"dsn","type":"string","kind":"scalar","description":"A Data Source Name to identify the target database.\n\n==== Drivers\n\n:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community, spanner=community\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n|===\n| Driver | Data Source Name Format\n\n| `clickhouse` \n| https://github.com/ClickHouse/clickhouse-go#dsn[`clickhouse://[username[:password\\]@\\][netloc\\][:port\\]/dbname[?param1=value1\u0026...\u0026paramN=valueN\\]`^] \n\n| `mysql` \n| `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` \n\n| `postgres` \n| `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` \n\n| `mssql` \n| `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` \n\n| `sqlite` \n| `file:/path/to/filename.db[?param\u0026=value1\u0026...]` \n\n| `oracle` \n| `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` \n\n| `snowflake` \n| `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` \n\n| `trino` \n| https://github.com/trinodb/trino-go-client#dsn-data-source-name[`http[s\\]://user[:pass\\]@host[:port\\][?parameters\\]`^] \n\n| `gocosmos` \n| https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage[`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e\\][;Version=\u003ccosmosdb-api-version\u003e\\][;DefaultDb/Db=\u003cdb-name\u003e\\][;AutoId=\u003ctrue/false\u003e\\][;InsecureSkipVerify=\u003ctrue/false\u003e\\]`^] \n\n| `spanner` \n| projects/[PROJECT]/instances/[INSTANCE]/databases/[DATABASE] \n|===\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String[the docs^] for more details. For https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication[key pair authentication^], the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe https://pkg.go.dev/github.com/microsoft/gocosmos[`gocosmos`^] driver is still experimental, but it has support for https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys[hierarchical partition keys^] as well as https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query[cross-partition queries^]. Please refer to the https://github.com/microsoft/gocosmos/blob/main/SQL.md[SQL notes^] for details.","examples":["clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60","foouser:foopassword@tcp(localhost:3306)/foodb","postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable","oracle://foouser:foopass@localhost:1521/service_name"]},{"name":"table","type":"string","kind":"scalar","description":"The table to insert to.","examples":["foo"]},{"name":"columns","type":"string","kind":"array","description":"A list of columns to insert.","examples":[["foo","bar","baz"]]},{"name":"args_mapping","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of columns specified.","bloblang":true,"examples":["root = [ this.cat.meow, this.doc.woofs[0] ]","root = [ meta(\"user.id\") ]"]},{"name":"prefix","type":"string","kind":"scalar","description":"An optional prefix to prepend to the insert query (before INSERT).","is_advanced":true,"is_optional":true},{"name":"suffix","type":"string","kind":"scalar","description":"An optional suffix to append to the insert query.","is_advanced":true,"is_optional":true,"examples":["ON CONFLICT (name) DO NOTHING"]},{"name":"options","type":"string","kind":"array","description":"A list of keyword options to add before the INTO clause of the query.","is_advanced":true,"is_optional":true,"examples":[["DELAYED","IGNORE"]]},{"name":"init_files","type":"string","kind":"array","description":"\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":[["./init/*.sql"],["./foo.sql","./bar.sql"]],"version":"4.10.0"},{"name":"init_statement","type":"string","kind":"scalar","description":"\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":["\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n"],"version":"4.10.0"},{"name":"conn_max_idle_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time.","is_advanced":true,"is_optional":true},{"name":"conn_max_life_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age.","is_advanced":true,"is_optional":true},{"name":"conn_max_idle","type":"int","kind":"scalar","description":"An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release.","is_advanced":true,"is_optional":true,"default":2},{"name":"conn_max_open","type":"int","kind":"scalar","description":"An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited).","is_advanced":true,"is_optional":true}]},"version":"3.59.0"},{"name":"sql_raw","type":"processor","status":"stable","plugin":true,"summary":"Runs an arbitrary SQL query against a database and (optionally) returns the result as an array of objects, one for each row returned.","description":"\nIf the query fails to execute then the message will remain unchanged and the error can be caught using xref:configuration:error_handling.adoc[error handling methods].","categories":["Integration"],"examples":[{"title":"Table Insert (MySQL)","summary":"The following example inserts rows into the table footable with the columns foo, bar and baz populated with values extracted from messages.","config":"\npipeline:\n processors:\n - sql_raw:\n driver: mysql\n dsn: foouser:foopassword@tcp(localhost:3306)/foodb\n query: \"INSERT INTO footable (foo, bar, baz) VALUES (?, ?, ?);\"\n args_mapping: '[ document.foo, document.bar, meta(\"kafka_topic\") ]'\n exec_only: true\n"},{"title":"Table Query (PostgreSQL)","summary":"Here we query a database for columns of footable that share a `user_id` with the message field `user.id`. A xref:components:processors/branch.adoc[`branch` processor] is used in order to insert the resulting array into the original message at the path `foo_rows`.","config":"\npipeline:\n processors:\n - branch:\n processors:\n - sql_raw:\n driver: postgres\n dsn: postgres://foouser:foopass@localhost:5432/testdb?sslmode=disable\n query: \"SELECT * FROM footable WHERE user_id = $1;\"\n args_mapping: '[ this.user.id ]'\n result_map: 'root.foo_rows = this'\n"},{"title":"Dynamically Creating Tables (PostgreSQL)","summary":"Here we query a database for columns of footable that share a `user_id` with the message field `user.id`. A xref:components:processors/branch.adoc[`branch` processor] is used in order to insert the resulting array into the original message at the path `foo_rows`.","config":"\npipeline:\n processors:\n - mapping: |\n root = this\n # Prevent SQL injection when using unsafe_dynamic_query\n meta table_name = \"\\\"\" + metadata(\"table_name\").replace_all(\"\\\"\", \"\\\"\\\"\") + \"\\\"\"\n - sql_raw:\n driver: postgres\n dsn: postgres://localhost/postgres\n unsafe_dynamic_query: true\n queries:\n - query: |\n CREATE TABLE IF NOT EXISTS ${!metadata(\"table_name\")} (id varchar primary key, document jsonb);\n - query: |\n INSERT INTO ${!metadata(\"table_name\")} (id, document) VALUES ($1, $2)\n ON CONFLICT (id) DO UPDATE SET document = EXCLUDED.document;\n args_mapping: |\n root = [ this.id, this.document.string() ]\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"driver","type":"string","kind":"scalar","description":"A database \u003c\u003cdrivers, driver\u003e\u003e to use.","options":["mysql","postgres","clickhouse","mssql","sqlite","oracle","snowflake","trino","gocosmos","spanner"],"linter":"\nlet options = {\n \"mysql\": true,\n \"postgres\": true,\n \"clickhouse\": true,\n \"mssql\": true,\n \"sqlite\": true,\n \"oracle\": true,\n \"snowflake\": true,\n \"trino\": true,\n \"gocosmos\": true,\n \"spanner\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"dsn","type":"string","kind":"scalar","description":"A Data Source Name to identify the target database.\n\n==== Drivers\n\n:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community, spanner=community\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n|===\n| Driver | Data Source Name Format\n\n| `clickhouse` \n| https://github.com/ClickHouse/clickhouse-go#dsn[`clickhouse://[username[:password\\]@\\][netloc\\][:port\\]/dbname[?param1=value1\u0026...\u0026paramN=valueN\\]`^] \n\n| `mysql` \n| `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` \n\n| `postgres` \n| `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` \n\n| `mssql` \n| `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` \n\n| `sqlite` \n| `file:/path/to/filename.db[?param\u0026=value1\u0026...]` \n\n| `oracle` \n| `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` \n\n| `snowflake` \n| `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` \n\n| `trino` \n| https://github.com/trinodb/trino-go-client#dsn-data-source-name[`http[s\\]://user[:pass\\]@host[:port\\][?parameters\\]`^] \n\n| `gocosmos` \n| https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage[`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e\\][;Version=\u003ccosmosdb-api-version\u003e\\][;DefaultDb/Db=\u003cdb-name\u003e\\][;AutoId=\u003ctrue/false\u003e\\][;InsecureSkipVerify=\u003ctrue/false\u003e\\]`^] \n\n| `spanner` \n| projects/[PROJECT]/instances/[INSTANCE]/databases/[DATABASE] \n|===\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String[the docs^] for more details. For https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication[key pair authentication^], the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe https://pkg.go.dev/github.com/microsoft/gocosmos[`gocosmos`^] driver is still experimental, but it has support for https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys[hierarchical partition keys^] as well as https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query[cross-partition queries^]. Please refer to the https://github.com/microsoft/gocosmos/blob/main/SQL.md[SQL notes^] for details.","examples":["clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60","foouser:foopassword@tcp(localhost:3306)/foodb","postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable","oracle://foouser:foopass@localhost:1521/service_name"]},{"name":"query","type":"string","kind":"scalar","description":"The query to execute. The style of placeholder to use depends on the driver, some drivers require question marks (`?`) whereas others expect incrementing dollar signs (`$1`, `$2`, and so on) or colons (`:1`, `:2` and so on). The style to use is outlined in this table:\n\n| Driver | Placeholder Style |\n|---|---|\n| `clickhouse` | Dollar sign |\n| `mysql` | Question mark |\n| `postgres` | Dollar sign |\n| `mssql` | Question mark |\n| `sqlite` | Question mark |\n| `oracle` | Colon |\n| `snowflake` | Question mark |\n| `trino` | Question mark |\n| `gocosmos` | Colon |\n","is_optional":true,"examples":["INSERT INTO footable (foo, bar, baz) VALUES (?, ?, ?);","SELECT * FROM footable WHERE user_id = $1;"]},{"name":"unsafe_dynamic_query","type":"bool","kind":"scalar","description":"Whether to enable xref:configuration:interpolation.adoc#bloblang-queries[interpolation functions] in the query. Great care should be made to ensure your queries are defended against injection attacks.","is_advanced":true,"default":false},{"name":"args_mapping","type":"string","kind":"scalar","description":"An optional xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `query`.","is_optional":true,"bloblang":true,"examples":["root = [ this.cat.meow, this.doc.woofs[0] ]","root = [ meta(\"user.id\") ]"]},{"name":"exec_only","type":"bool","kind":"scalar","description":"Whether the query result should be discarded. When set to `true` the message contents will remain unchanged, which is useful in cases where you are executing inserts, updates, etc. By default this is true for the last query, and previous queries don't change the results. If set to true for any query but the last one, the subsequent `args_mappings` input is overwritten.","is_optional":true},{"name":"queries","type":"object","kind":"array","description":"A list of statements to run in addition to `query`. When specifying multiple statements, they are all executed within a transaction. The output of the processor is always the last query that runs, unless `exec_only` is used.","is_optional":true,"children":[{"name":"query","type":"string","kind":"scalar","description":"The query to execute. The style of placeholder to use depends on the driver, some drivers require question marks (`?`) whereas others expect incrementing dollar signs (`$1`, `$2`, and so on) or colons (`:1`, `:2` and so on). The style to use is outlined in this table:\n\n| Driver | Placeholder Style |\n|---|---|\n| `clickhouse` | Dollar sign |\n| `mysql` | Question mark |\n| `postgres` | Dollar sign |\n| `mssql` | Question mark |\n| `sqlite` | Question mark |\n| `oracle` | Colon |\n| `snowflake` | Question mark |\n| `trino` | Question mark |\n| `gocosmos` | Colon |\n"},{"name":"args_mapping","type":"string","kind":"scalar","description":"An optional xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `query`.","is_optional":true,"bloblang":true,"examples":["root = [ this.cat.meow, this.doc.woofs[0] ]","root = [ meta(\"user.id\") ]"]},{"name":"exec_only","type":"bool","kind":"scalar","description":"Whether the query result should be discarded. When set to `true` the message contents will remain unchanged, which is useful in cases where you are executing inserts, updates, etc. By default this is true for the last query, and previous queries don't change the results. If set to true for any query but the last one, the subsequent `args_mappings` input is overwritten.","is_optional":true}]},{"name":"init_files","type":"string","kind":"array","description":"\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":[["./init/*.sql"],["./foo.sql","./bar.sql"]],"version":"4.10.0"},{"name":"init_statement","type":"string","kind":"scalar","description":"\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":["\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n"],"version":"4.10.0"},{"name":"conn_max_idle_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time.","is_advanced":true,"is_optional":true},{"name":"conn_max_life_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age.","is_advanced":true,"is_optional":true},{"name":"conn_max_idle","type":"int","kind":"scalar","description":"An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release.","is_advanced":true,"is_optional":true,"default":2},{"name":"conn_max_open","type":"int","kind":"scalar","description":"An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited).","is_advanced":true,"is_optional":true}],"linter":"root = match {\n !this.exists(\"queries\") \u0026\u0026 !this.exists(\"query\") =\u003e [ \"either `query` or `queries` is required\" ],\n }"},"version":"3.65.0"},{"name":"sql_select","type":"processor","status":"stable","plugin":true,"summary":"Runs an SQL select query against a database and returns the result as an array of objects, one for each row returned, containing a key for each column queried and its value.","description":"\nIf the query fails to execute then the message will remain unchanged and the error can be caught using xref:configuration:error_handling.adoc[error handling methods].","categories":["Integration"],"examples":[{"title":"Table Query (PostgreSQL)","summary":"\nHere we query a database for columns of footable that share a `user_id`\nwith the message `user.id`. A xref:components:processors/branch.adoc[`branch` processor]\nis used in order to insert the resulting array into the original message at the\npath `foo_rows`:","config":"\npipeline:\n processors:\n - branch:\n processors:\n - sql_select:\n driver: postgres\n dsn: postgres://foouser:foopass@localhost:5432/testdb?sslmode=disable\n table: footable\n columns: [ '*' ]\n where: user_id = ?\n args_mapping: '[ this.user.id ]'\n result_map: 'root.foo_rows = this'\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"driver","type":"string","kind":"scalar","description":"A database \u003c\u003cdrivers, driver\u003e\u003e to use.","options":["mysql","postgres","clickhouse","mssql","sqlite","oracle","snowflake","trino","gocosmos","spanner"],"linter":"\nlet options = {\n \"mysql\": true,\n \"postgres\": true,\n \"clickhouse\": true,\n \"mssql\": true,\n \"sqlite\": true,\n \"oracle\": true,\n \"snowflake\": true,\n \"trino\": true,\n \"gocosmos\": true,\n \"spanner\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"dsn","type":"string","kind":"scalar","description":"A Data Source Name to identify the target database.\n\n==== Drivers\n\n:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community, spanner=community\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n|===\n| Driver | Data Source Name Format\n\n| `clickhouse` \n| https://github.com/ClickHouse/clickhouse-go#dsn[`clickhouse://[username[:password\\]@\\][netloc\\][:port\\]/dbname[?param1=value1\u0026...\u0026paramN=valueN\\]`^] \n\n| `mysql` \n| `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` \n\n| `postgres` \n| `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` \n\n| `mssql` \n| `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` \n\n| `sqlite` \n| `file:/path/to/filename.db[?param\u0026=value1\u0026...]` \n\n| `oracle` \n| `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` \n\n| `snowflake` \n| `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` \n\n| `trino` \n| https://github.com/trinodb/trino-go-client#dsn-data-source-name[`http[s\\]://user[:pass\\]@host[:port\\][?parameters\\]`^] \n\n| `gocosmos` \n| https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage[`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e\\][;Version=\u003ccosmosdb-api-version\u003e\\][;DefaultDb/Db=\u003cdb-name\u003e\\][;AutoId=\u003ctrue/false\u003e\\][;InsecureSkipVerify=\u003ctrue/false\u003e\\]`^] \n\n| `spanner` \n| projects/[PROJECT]/instances/[INSTANCE]/databases/[DATABASE] \n|===\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String[the docs^] for more details. For https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication[key pair authentication^], the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe https://pkg.go.dev/github.com/microsoft/gocosmos[`gocosmos`^] driver is still experimental, but it has support for https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys[hierarchical partition keys^] as well as https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query[cross-partition queries^]. Please refer to the https://github.com/microsoft/gocosmos/blob/main/SQL.md[SQL notes^] for details.","examples":["clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60","foouser:foopassword@tcp(localhost:3306)/foodb","postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable","oracle://foouser:foopass@localhost:1521/service_name"]},{"name":"table","type":"string","kind":"scalar","description":"The table to query.","examples":["foo"]},{"name":"columns","type":"string","kind":"array","description":"A list of columns to query.","examples":[["*"],["foo","bar","baz"]]},{"name":"where","type":"string","kind":"scalar","description":"An optional where clause to add. Placeholder arguments are populated with the `args_mapping` field. Placeholders should always be question marks, and will automatically be converted to dollar syntax when the postgres or clickhouse drivers are used.","is_optional":true,"examples":["meow = ? and woof = ?","user_id = ?"]},{"name":"args_mapping","type":"string","kind":"scalar","description":"An optional xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `where`.","is_optional":true,"bloblang":true,"examples":["root = [ this.cat.meow, this.doc.woofs[0] ]","root = [ meta(\"user.id\") ]"]},{"name":"prefix","type":"string","kind":"scalar","description":"An optional prefix to prepend to the query (before SELECT).","is_advanced":true,"is_optional":true},{"name":"suffix","type":"string","kind":"scalar","description":"An optional suffix to append to the select query.","is_advanced":true,"is_optional":true},{"name":"init_files","type":"string","kind":"array","description":"\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":[["./init/*.sql"],["./foo.sql","./bar.sql"]],"version":"4.10.0"},{"name":"init_statement","type":"string","kind":"scalar","description":"\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":["\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n"],"version":"4.10.0"},{"name":"conn_max_idle_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time.","is_advanced":true,"is_optional":true},{"name":"conn_max_life_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age.","is_advanced":true,"is_optional":true},{"name":"conn_max_idle","type":"int","kind":"scalar","description":"An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release.","is_advanced":true,"is_optional":true,"default":2},{"name":"conn_max_open","type":"int","kind":"scalar","description":"An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited).","is_advanced":true,"is_optional":true}]},"version":"3.59.0"},{"name":"subprocess","type":"processor","status":"stable","plugin":true,"summary":"Executes a command as a subprocess and, for each message, will pipe its contents to the stdin stream of the process followed by a newline.","description":"\n[NOTE]\n====\nThis processor keeps the subprocess alive and requires very specific behavior from the command executed. If you wish to simply execute a command for each message take a look at the xref:components:processors/command.adoc[`command` processor] instead.\n====\n\nThe subprocess must then either return a line over stdout or stderr. If a response is returned over stdout then its contents will replace the message. If a response is instead returned from stderr it will be logged and the message will continue unchanged and will be xref:configuration:error_handling.adoc[marked as failed].\n\nRather than separating data by a newline it's possible to specify alternative \u003c\u003ccodec_send,`codec_send`\u003e\u003e and \u003c\u003ccodec_recv,`codec_recv`\u003e\u003e values, which allow binary messages to be encoded for logical separation.\n\nThe execution environment of the subprocess is the same as the Redpanda Connect instance, including environment variables and the current working directory.\n\nThe field `max_buffer` defines the maximum response size able to be read from the subprocess. This value should be set significantly above the real expected maximum response size.\n\n== Subprocess requirements\n\nIt is required that subprocesses flush their stdout and stderr pipes for each line. Redpanda Connect will attempt to keep the process alive for as long as the pipeline is running. If the process exits early it will be restarted.\n\n== Messages containing line breaks\n\nIf a message contains line breaks each line of the message is piped to the subprocess and flushed, and a response is expected from the subprocess before another line is fed in.","categories":["Integration"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"name","type":"string","kind":"scalar","description":"The command to execute as a subprocess.","examples":["cat","sed","awk"]},{"name":"args","type":"string","kind":"array","description":"A list of arguments to provide the command.","default":[]},{"name":"max_buffer","type":"int","kind":"scalar","description":"The maximum expected response size.","is_advanced":true,"default":65536},{"name":"codec_send","type":"string","kind":"scalar","description":"Determines how messages written to the subprocess are encoded, which allows them to be logically separated.","is_advanced":true,"default":"lines","options":["lines","length_prefixed_uint32_be","netstring"],"version":"3.37.0","linter":"\nlet options = {\n \"lines\": true,\n \"length_prefixed_uint32_be\": true,\n \"netstring\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"codec_recv","type":"string","kind":"scalar","description":"Determines how messages read from the subprocess are decoded, which allows them to be logically separated.","is_advanced":true,"default":"lines","options":["lines","length_prefixed_uint32_be","netstring"],"version":"3.37.0","linter":"\nlet options = {\n \"lines\": true,\n \"length_prefixed_uint32_be\": true,\n \"netstring\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"}]}},{"name":"switch","type":"processor","status":"stable","plugin":true,"summary":"Conditionally processes messages based on their contents.","description":"For each switch case a xref:guides:bloblang/about.adoc[Bloblang query] is checked and, if the result is true (or the check is empty) the child processors are executed on the message.","categories":["Composition"],"footnotes":"\n== Batching\n\nWhen a switch processor executes on a xref:configuration:batching.adoc[batch of messages] they are checked individually and can be matched independently against cases. During processing the messages matched against a case are processed as a batch, although the ordering of messages during case processing cannot be guaranteed to match the order as received.\n\nAt the end of switch processing the resulting batch will follow the same ordering as the batch was received. If any child processors have split or otherwise grouped messages this grouping will be lost as the result of a switch is always a single batch. In order to perform conditional grouping and/or splitting use the xref:components:processors/group_by.adoc[`group_by` processor].","examples":[{"title":"Ignore George","summary":"\nWe have a system where we're counting a metric for all messages that pass through our system. However, occasionally we get messages from George that we don't care about.\n\nFor George's messages we want to instead emit a metric that gauges how angry he is about being ignored and then we drop it.","config":"\npipeline:\n processors:\n - switch:\n - check: this.user.name.first != \"George\"\n processors:\n - metric:\n type: counter\n name: MessagesWeCareAbout\n\n - processors:\n - metric:\n type: gauge\n name: GeorgesAnger\n value: ${! json(\"user.anger\") }\n - mapping: root = deleted()\n"}],"config":{"name":"","type":"object","kind":"array","children":[{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should have the processors of this case executed on it. If left empty the case always passes. If the check mapping throws an error the message will be flagged xref:configuration:error_handling.adoc[as having failed] and will not be tested against any other cases.","default":"","bloblang":true,"examples":["this.type == \"foo\"","this.contents.urls.contains(\"https://benthos.dev/\")"]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to execute on a message.","default":[]},{"name":"fallthrough","type":"bool","kind":"scalar","description":"Indicates whether, if this case passes for a message, the next case should also be executed.","is_advanced":true,"default":false}]}},{"name":"sync_response","type":"processor","status":"stable","plugin":true,"summary":"Adds the payload in its current state as a synchronous response to the input source, where it is dealt with according to that specific input type.","description":"\nFor most inputs this mechanism is ignored entirely, in which case the sync response is dropped without penalty. It is therefore safe to use this processor even when combining input types that might not have support for sync responses. An example of an input able to utilize this is the `http_server`.\n\nFor more information please read xref:guides:sync_responses.adoc[synchronous responses].","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","default":{}}},{"name":"text_chunker","type":"processor","status":"experimental","plugin":true,"summary":"A processor that allows chunking and splitting text based on some strategy. Usually used for creating vector embeddings of large documents.","description":"A processor allowing splitting text into chunks based on several different strategies.","categories":["AI"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"strategy","type":"string","kind":"scalar","annotated_options":[["markdown","Split text by markdown headers."],["recursive_character","Split text recursively by characters (defined in `separators`)."],["token","Split text by tokens."]],"linter":"\nlet options = {\n \"markdown\": true,\n \"recursive_character\": true,\n \"token\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"chunk_size","type":"int","kind":"scalar","description":"The maximum size of each chunk.","default":512},{"name":"chunk_overlap","type":"int","kind":"scalar","description":"The number of characters to overlap between chunks.","default":100},{"name":"separators","type":"string","kind":"array","description":"A list of strings that should be considered as separators between chunks.","default":["\n\n","\n"," ",""]},{"name":"length_measure","type":"string","kind":"scalar","description":"The method for measuring the length of a string.","default":"runes","annotated_options":[["graphemes","Use unicode graphemes to determine the length of a string."],["runes","Use the number of codepoints to determine the length of a string."],["token","Use the number of tokens (using the `token_encoding` tokenizer) to determine the length of a string."],["utf8","Determine the length of text using the number of utf8 bytes."]],"linter":"\nlet options = {\n \"graphemes\": true,\n \"runes\": true,\n \"token\": true,\n \"utf8\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"token_encoding","type":"string","kind":"scalar","description":"The encoding to use for tokenization.","is_advanced":true,"is_optional":true,"examples":["cl100k_base","r50k_base"]},{"name":"allowed_special","type":"string","kind":"array","description":"A list of special tokens that are allowed in the output.","is_advanced":true,"default":[]},{"name":"disallowed_special","type":"string","kind":"array","description":"A list of special tokens that are disallowed in the output.","is_advanced":true,"default":["all"]},{"name":"include_code_blocks","type":"bool","kind":"scalar","description":"Whether to include code blocks in the output.","default":false},{"name":"keep_reference_links","type":"bool","kind":"scalar","description":"Whether to keep reference links in the output.","default":false}]}},{"name":"try","type":"processor","status":"stable","plugin":true,"summary":"Executes a list of child processors on messages only if no prior processors have failed (or the errors have been cleared).","description":"\nThis processor behaves similarly to the xref:components:processors/for_each.adoc[`for_each`] processor, where a list of child processors are applied to individual messages of a batch. However, if a message has failed any prior processor (before or during the try block) then that message will skip all following processors.\n\nFor example, with the following config:\n\n```yaml\npipeline:\n processors:\n - resource: foo\n - try:\n - resource: bar\n - resource: baz\n - resource: buz\n```\n\nIf the processor `bar` fails for a particular message, that message will skip the processors `baz` and `buz`. Similarly, if `bar` succeeds but `baz` does not then `buz` will be skipped. If the processor `foo` fails for a message then none of `bar`, `baz` or `buz` are executed on that message.\n\nThis processor is useful for when child processors depend on the successful output of previous processors. This processor can be followed with a xref:components:processors/catch.adoc[catch] processor for defining child processors to be applied only to failed messages.\n\nMore information about error handing can be found in xref:configuration:error_handling.adoc[].\n\n== Nest within a catch block\n\nIn some cases it might be useful to nest a try block within a catch block, since the xref:components:processors/catch.adoc[`catch` processor] only clears errors _after_ executing its child processors this means a nested try processor will not execute unless the errors are explicitly cleared beforehand.\n\nThis can be done by inserting an empty catch block before the try block like as follows:\n\n```yaml\npipeline:\n processors:\n - resource: foo\n - catch:\n - log:\n level: ERROR\n message: \"Foo failed due to: ${! error() }\"\n - catch: [] # Clear prior error\n - try:\n - resource: bar\n - resource: baz\n```","categories":["Composition"],"config":{"name":"","type":"processor","kind":"array","default":[]}},{"name":"unarchive","type":"processor","status":"stable","plugin":true,"summary":"Unarchives messages according to the selected archive format into multiple messages within a xref:configuration:batching.adoc[batch].","description":"\nWhen a message is unarchived the new messages replace the original message in the batch. Messages that are selected but fail to unarchive (invalid format) will remain unchanged in the message batch but will be flagged as having failed, allowing you to xref:configuration:error_handling.adoc[error handle them].\n\n== Metadata\n\nThe metadata found on the messages handled by this processor will be copied into the resulting messages. For the unarchive formats that contain file information (tar, zip), a metadata field is also added to each message called `archive_filename` with the extracted filename.\n","categories":["Parsing","Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"format","type":"string","kind":"scalar","description":"The unarchiving format to apply.","annotated_options":[["binary","Extract messages from a https://github.com/redpanda-data/benthos/blob/main/internal/message/message.go#L96[binary blob format^]."],["csv","Attempt to parse the message as a csv file (header required) and for each row in the file expands its contents into a json object in a new message."],["csv:x","Attempt to parse the message as a csv file (header required) and for each row in the file expands its contents into a json object in a new message using a custom delimiter. The custom delimiter must be a single character, e.g. the format \"csv:\\t\" would consume a tab delimited file."],["json_array","Attempt to parse a message as a JSON array, and extract each element into its own message."],["json_documents","Attempt to parse a message as a stream of concatenated JSON documents. Each parsed document is expanded into a new message."],["json_map","Attempt to parse the message as a JSON map and for each element of the map expands its contents into a new message. A metadata field is added to each message called `archive_key` with the relevant key from the top-level map."],["lines","Extract the lines of a message each into their own message."],["tar","Extract messages from a unix standard tape archive."],["zip","Extract messages from a zip file."]]}]}},{"name":"wasm","type":"processor","status":"experimental","plugin":true,"summary":"Executes a function exported by a WASM module for each message.","description":"\nThis processor uses https://github.com/tetratelabs/wazero[Wazero^] to execute a WASM module (with support for WASI), calling a specific function for each message being processed. From within the WASM module it is possible to query and mutate the message being processed via a suite of functions exported to the module.\n\nThis ecosystem is delicate as WASM doesn't have a single clearly defined way to pass strings back and forth between the host and the module. In order to remedy this we're gradually working on introducing libraries and examples for multiple languages which can be found in https://github.com/redpanda-data/benthos/tree/main/public/wasm/README.md[the codebase^].\n\nThese examples, as well as the processor itself, is a work in progress.\n\n== Parallelism\n\nIt's not currently possible to execute a single WASM runtime across parallel threads with this processor. Therefore, in order to support parallel processing this processor implements pooling of module runtimes. Ideally your WASM module shouldn't depend on any global state, but if it does then you need to ensure the processor xref:configuration:processing_pipelines.adoc[is only run on a single thread].\n","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"module_path","type":"string","kind":"scalar","description":"The path of the target WASM module to execute."},{"name":"function","type":"string","kind":"scalar","description":"The name of the function exported by the target WASM module to run for each message.","default":"process"}]},"version":"4.11.0"},{"name":"while","type":"processor","status":"stable","plugin":true,"summary":"A processor that checks a xref:guides:bloblang/about.adoc[Bloblang query] against each batch of messages and executes child processors on them for as long as the query resolves to true.","description":"\nThe field `at_least_once`, if true, ensures that the child processors are always executed at least one time (like a do .. while loop.)\n\nThe field `max_loops`, if greater than zero, caps the number of loops for a message batch to this value.\n\nIf following a loop execution the number of messages in a batch is reduced to zero the loop is exited regardless of the condition result. If following a loop execution there are more than 1 message batches the query is checked against the first batch only.\n\nThe conditions of this processor are applied across entire message batches. You can find out more about batching xref:configuration:batching.adoc[in this doc].","categories":["Composition"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"at_least_once","type":"bool","kind":"scalar","description":"Whether to always run the child processors at least one time.","default":false},{"name":"max_loops","type":"int","kind":"scalar","description":"An optional maximum number of loops to execute. Helps protect against accidentally creating infinite loops.","is_advanced":true,"default":0},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether the while loop should execute again.","default":"","bloblang":true,"examples":["errored()","this.urls.unprocessed.length() \u003e 0"]},{"name":"processors","type":"processor","kind":"array","description":"A list of child processors to execute on each loop."}]}},{"name":"workflow","type":"processor","status":"stable","plugin":true,"summary":"Executes a topology of xref:components:processors/branch.adoc[`branch` processors], performing them in parallel where possible.","description":"\n== Why use a workflow\n\n=== Performance\n\nMost of the time the best way to compose processors is also the simplest, just configure them in series. This is because processors are often CPU bound, low-latency, and you can gain vertical scaling by increasing the number of processor pipeline threads, allowing Redpanda Connect to process xref:configuration:processing_pipelines.adoc[multiple messages in parallel].\n\nHowever, some processors such as xref:components:processors/http.adoc[`http`], xref:components:processors/aws_lambda.adoc[`aws_lambda`] or xref:components:processors/cache.adoc[`cache`] interact with external services and therefore spend most of their time waiting for a response. These processors tend to be high-latency and low CPU activity, which causes messages to process slowly.\n\nWhen a processing pipeline contains multiple network processors that aren't dependent on each other we can benefit from performing these processors in parallel for each individual message, reducing the overall message processing latency.\n\n=== Simplifying processor topology\n\nA workflow is often expressed as a https://en.wikipedia.org/wiki/Directed_acyclic_graph[DAG^] of processing stages, where each stage can result in N possible next stages, until finally the flow ends at an exit node.\n\nFor example, if we had processing stages A, B, C and D, where stage A could result in either stage B or C being next, always followed by D, it might look something like this:\n\n```text\n /--\u003e B --\\\nA --| |--\u003e D\n \\--\u003e C --/\n```\n\nThis flow would be easy to express in a standard Redpanda Connect config, we could simply use a xref:components:processors/switch.adoc[`switch` processor] to route to either B or C depending on a condition on the result of A. However, this method of flow control quickly becomes unfeasible as the DAG gets more complicated, imagine expressing this flow using switch processors:\n\n```text\n /--\u003e B -------------|--\u003e D\n / /\nA --| /--\u003e E --|\n \\--\u003e C --| \\\n \\----------|--\u003e F\n```\n\nAnd imagine doing so knowing that the diagram is subject to change over time. Yikes! Instead, with a workflow we can either trust it to automatically resolve the DAG or express it manually as simply as `order: [ [ A ], [ B, C ], [ E ], [ D, F ] ]`, and the conditional logic for determining if a stage is executed is defined as part of the branch itself.","categories":["Composition"],"footnotes":"\n== Structured metadata\n\nWhen the field `meta_path` is non-empty the workflow processor creates an object describing which workflows were successful, skipped or failed for each message and stores the object within the message at the end.\n\nThe object is of the following form:\n\n```json\n{\n\t\"succeeded\": [ \"foo\" ],\n\t\"skipped\": [ \"bar\" ],\n\t\"failed\": {\n\t\t\"baz\": \"the error message from the branch\"\n\t}\n}\n```\n\nIf a message already has a meta object at the given path when it is processed then the object is used in order to determine which branches have already been performed on the message (or skipped) and can therefore be skipped on this run.\n\nThis is a useful pattern when replaying messages that have failed some branches previously. For example, given the above example object the branches foo and bar would automatically be skipped, and baz would be reattempted.\n\nThe previous meta object will also be preserved in the field `\u003cmeta_path\u003e.previous` when the new meta object is written, preserving a full record of all workflow executions.\n\nIf a field `\u003cmeta_path\u003e.apply` exists in the meta object for a message and is an array then it will be used as an explicit list of stages to apply, all other stages will be skipped.\n\n== Resources\n\nIt's common to configure processors (and other components) xref:configuration:resources.adoc[as resources] in order to keep the pipeline configuration cleaner. With the workflow processor you can include branch processors configured as resources within your workflow either by specifying them by name in the field `order`, if Redpanda Connect doesn't find a branch within the workflow configuration of that name it'll refer to the resources.\n\nAlternatively, if you do not wish to have an explicit ordering, you can add resource names to the field `branch_resources` and they will be included in the workflow with automatic DAG resolution along with any branches configured in the `branches` field.\n\n=== Resource error conditions\n\nThere are two error conditions that could potentially occur when resources included in your workflow are mutated, and if you are planning to mutate resources in your workflow it is important that you understand them.\n\nThe first error case is that a resource in the workflow is removed and not replaced, when this happens the workflow will still be executed but the individual branch will fail. This should only happen if you explicitly delete a branch resource, as any mutation operation will create the new resource before removing the old one.\n\nThe second error case is when automatic DAG resolution is being used and a resource in the workflow is changed in a way that breaks the DAG (circular dependencies, etc). When this happens it is impossible to execute the workflow and therefore the processor will fail, which is possible to capture and handle using xref:configuration:error_handling.adoc[standard error handling patterns].\n\n== Error handling\n\nThe recommended approach to handle failures within a workflow is to query against the \u003c\u003cstructured-metadata, structured metadata\u003e\u003e it provides, as it provides granular information about exactly which branches failed and which ones succeeded and therefore aren't necessary to perform again.\n\nFor example, if our meta object is stored at the path `meta.workflow` and we wanted to check whether a message has failed for any branch we can do that using a xref:guides:bloblang/about.adoc[Bloblang query] like `this.meta.workflow.failed.length() | 0 \u003e 0`, or to check whether a specific branch failed we can use `this.exists(\"meta.workflow.failed.foo\")`.\n\nHowever, if structured metadata is disabled by setting the field `meta_path` to empty then the workflow processor instead adds a general error flag to messages when any executed branch fails. In this case it's possible to handle failures using xref:configuration:error_handling.adoc[standard error handling patterns].\n\n","examples":[{"title":"Automatic Ordering","summary":"\nWhen the field `order` is omitted a best attempt is made to determine a dependency tree between branches based on their request and result mappings. In the following example the branches foo and bar will be executed first in parallel, and afterwards the branch baz will be executed.","config":"\npipeline:\n processors:\n - workflow:\n meta_path: meta.workflow\n branches:\n foo:\n request_map: 'root = \"\"'\n processors:\n - http:\n url: TODO\n result_map: 'root.foo = this'\n\n bar:\n request_map: 'root = this.body'\n processors:\n - aws_lambda:\n function: TODO\n result_map: 'root.bar = this'\n\n baz:\n request_map: |\n root.fooid = this.foo.id\n root.barstuff = this.bar.content\n processors:\n - cache:\n resource: TODO\n operator: set\n key: ${! json(\"fooid\") }\n value: ${! json(\"barstuff\") }\n"},{"title":"Conditional Branches","summary":"\nBranches of a workflow are skipped when the `request_map` assigns `deleted()` to the root. In this example the branch A is executed when the document type is \"foo\", and branch B otherwise. Branch C is executed afterwards and is skipped unless either A or B successfully provided a result at `tmp.result`.","config":"\npipeline:\n processors:\n - workflow:\n branches:\n A:\n request_map: |\n root = if this.document.type != \"foo\" {\n deleted()\n }\n processors:\n - http:\n url: TODO\n result_map: 'root.tmp.result = this'\n\n B:\n request_map: |\n root = if this.document.type == \"foo\" {\n deleted()\n }\n processors:\n - aws_lambda:\n function: TODO\n result_map: 'root.tmp.result = this'\n\n C:\n request_map: |\n root = if this.tmp.result != null {\n deleted()\n }\n processors:\n - http:\n url: TODO_SOMEWHERE_ELSE\n result_map: 'root.tmp.result = this'\n"},{"title":"Resources","summary":"\nThe `order` field can be used in order to refer to \u003c\u003cresources, branch processor resources\u003e\u003e, this can sometimes make your pipeline configuration cleaner, as well as allowing you to reuse branch configurations in order places. It's also possible to mix and match branches configured within the workflow and configured as resources.","config":"\npipeline:\n processors:\n - workflow:\n order: [ [ foo, bar ], [ baz ] ]\n branches:\n bar:\n request_map: 'root = this.body'\n processors:\n - aws_lambda:\n function: TODO\n result_map: 'root.bar = this'\n\nprocessor_resources:\n - label: foo\n branch:\n request_map: 'root = \"\"'\n processors:\n - http:\n url: TODO\n result_map: 'root.foo = this'\n\n - label: baz\n branch:\n request_map: |\n root.fooid = this.foo.id\n root.barstuff = this.bar.content\n processors:\n - cache:\n resource: TODO\n operator: set\n key: ${! json(\"fooid\") }\n value: ${! json(\"barstuff\") }\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"meta_path","type":"string","kind":"scalar","description":"A xref:configuration:field_paths.adoc[dot path] indicating where to store and reference \u003c\u003cstructured-metadata, structured metadata\u003e\u003e about the workflow execution.","default":"meta.workflow"},{"name":"order","type":"string","kind":"2darray","description":"An explicit declaration of branch ordered tiers, which describes the order in which parallel tiers of branches should be executed. Branches should be identified by the name as they are configured in the field `branches`. It's also possible to specify branch processors configured \u003c\u003cresources, as a resource\u003e\u003e.","default":[],"examples":[[["foo","bar"],["baz"]],[["foo"],["bar"],["baz"]]]},{"name":"branch_resources","type":"string","kind":"array","description":"An optional list of xref:components:processors/branch.adoc[`branch` processor] names that are configured as \u003c\u003cresources\u003e\u003e. These resources will be included in the workflow with any branches configured inline within the \u003c\u003cbranches, `branches`\u003e\u003e field. The order and parallelism in which branches are executed is automatically resolved based on the mappings of each branch. When using resources with an explicit order it is not necessary to list resources in this field.","is_advanced":true,"default":[],"version":"3.38.0"},{"name":"branches","type":"object","kind":"map","description":"An object of named xref:components:processors/branch.adoc[`branch` processors] that make up the workflow. The order and parallelism in which branches are executed can either be made explicit with the field `order`, or if omitted an attempt is made to automatically resolve an ordering based on the mappings of each branch.","default":{},"children":[{"name":"request_map","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] that describes how to create a request payload suitable for the child processors of this branch. If left empty then the branch will begin with an exact copy of the origin message (including metadata).","default":"","bloblang":true,"examples":["root = {\n\t\"id\": this.doc.id,\n\t\"content\": this.doc.body.text\n}","root = if this.type == \"foo\" {\n\tthis.foo.request\n} else {\n\tdeleted()\n}"]},{"name":"processors","type":"processor","kind":"array","description":"A list of processors to apply to mapped requests. When processing message batches the resulting batch must match the size and ordering of the input batch, therefore filtering, grouping should not be performed within these processors."},{"name":"result_map","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] that describes how the resulting messages from branched processing should be mapped back into the original payload. If left empty the origin message will remain unchanged (including metadata).","default":"","bloblang":true,"examples":["meta foo_code = metadata(\"code\")\nroot.foo_result = this","meta = metadata()\nroot.bar.body = this.body\nroot.bar.id = this.user.id","root.raw_result = content().string()","root.enrichments.foo = if metadata(\"request_failed\") != null {\n throw(metadata(\"request_failed\"))\n} else {\n this\n}","# Retain only the updated metadata fields which were present in the origin message\nmeta = metadata().filter(v -\u003e @.get(v.key) != null)"]}]}]}},{"name":"xml","type":"processor","status":"beta","plugin":true,"summary":"Parses messages as an XML document, performs a mutation on the data, and then overwrites the previous contents with the new value.","description":"\n== Operators\n\n=== `to_json`\n\nConverts an XML document into a JSON structure, where elements appear as keys of an object according to the following rules:\n\n- If an element contains attributes they are parsed by prefixing a hyphen, `-`, to the attribute label.\n- If the element is a simple element and has attributes, the element value is given the key `#text`.\n- XML comments, directives, and process instructions are ignored.\n- When elements are repeated the resulting JSON value is an array.\n\nFor example, given the following XML:\n\n```xml\n\u003croot\u003e\n \u003ctitle\u003eThis is a title\u003c/title\u003e\n \u003cdescription tone=\"boring\"\u003eThis is a description\u003c/description\u003e\n \u003celements id=\"1\"\u003efoo1\u003c/elements\u003e\n \u003celements id=\"2\"\u003efoo2\u003c/elements\u003e\n \u003celements\u003efoo3\u003c/elements\u003e\n\u003c/root\u003e\n```\n\nThe resulting JSON structure would look like this:\n\n```json\n{\n \"root\":{\n \"title\":\"This is a title\",\n \"description\":{\n \"#text\":\"This is a description\",\n \"-tone\":\"boring\"\n },\n \"elements\":[\n {\"#text\":\"foo1\",\"-id\":\"1\"},\n {\"#text\":\"foo2\",\"-id\":\"2\"},\n \"foo3\"\n ]\n }\n}\n```\n\nWith cast set to true, the resulting JSON structure would look like this:\n\n```json\n{\n \"root\":{\n \"title\":\"This is a title\",\n \"description\":{\n \"#text\":\"This is a description\",\n \"-tone\":\"boring\"\n },\n \"elements\":[\n {\"#text\":\"foo1\",\"-id\":1},\n {\"#text\":\"foo2\",\"-id\":2},\n \"foo3\"\n ]\n }\n}\n```","categories":["Parsing"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"operator","type":"string","kind":"scalar","description":"An XML \u003c\u003coperators, operation\u003e\u003e to apply to messages.","default":"","options":["to_json"],"linter":"\nlet options = {\n \"to_json\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"cast","type":"bool","kind":"scalar","description":"Whether to try to cast values that are numbers and booleans to the right type. Default: all values are strings.","default":false}]}}],"rate-limits":[{"name":"local","type":"rate_limit","status":"stable","plugin":true,"summary":"The local rate limit is a simple X every Y type rate limit that can be shared across any number of components within the pipeline but does not support distributed rate limits across multiple running instances of Benthos.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"count","type":"int","kind":"scalar","description":"The maximum number of requests to allow for a given period of time.","default":1000},{"name":"interval","type":"string","kind":"scalar","description":"The time window to limit requests by.","default":"1s"}]}},{"name":"redis","type":"rate_limit","status":"experimental","plugin":true,"summary":"A rate limit implementation using Redis. It works by using a simple token bucket algorithm to limit the number of requests to a given count within a given time period. The rate limit is shared across all instances of Redpanda Connect that use the same Redis instance, which must all have a consistent count and interval.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target Redis server. Database is optional and is supplied as the URL path.","examples":["redis://:6379","redis://localhost:6379","redis://foousername:foopassword@redisplace:6379","redis://:foopassword@redisplace:6379","redis://localhost:6379/1","redis://localhost:6379/1,redis://localhost:6380/1"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"kind","type":"string","kind":"scalar","description":"Specifies a simple, cluster-aware, or failover-aware redis client.","is_advanced":true,"default":"simple","options":["simple","cluster","failover"],"linter":"\nlet options = {\n \"simple\": true,\n \"cluster\": true,\n \"failover\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"master","type":"string","kind":"scalar","description":"Name of the redis master when `kind` is `failover`","is_advanced":true,"default":"","examples":["mymaster"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"count","type":"int","kind":"scalar","description":"The maximum number of messages to allow for a given period of time.","default":1000,"linter":"root = if this \u003c= 0 { [ \"count must be larger than zero\" ] }"},{"name":"interval","type":"string","kind":"scalar","description":"The time window to limit requests by.","default":"1s"},{"name":"key","type":"string","kind":"scalar","description":"The key to use for the rate limit."}]},"version":"4.12.0"}],"metrics":[{"name":"aws_cloudwatch","type":"metrics","status":"stable","plugin":true,"summary":"Send metrics to AWS CloudWatch using the PutMetricData endpoint.","description":"\n== Timing metrics\n\nThe smallest timing unit that CloudWatch supports is microseconds, therefore timing metrics are automatically downgraded to microseconds (by dividing delta values by 1000). This conversion will also apply to custom timing metrics produced with a `metric` processor.\n\n== Billing\n\nAWS bills per metric series exported, it is therefore STRONGLY recommended that you reduce the metrics that are exposed with a `mapping` like this:\n\n```yaml\nmetrics:\n mapping: |\n if ![\n \"input_received\",\n \"input_latency\",\n \"output_sent\",\n ].contains(this) { deleted() }\n aws_cloudwatch:\n namespace: Foo\n```","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"namespace","type":"string","kind":"scalar","description":"The namespace used to distinguish metrics from other services.","default":"Benthos"},{"name":"flush_period","type":"string","kind":"scalar","description":"The period of time between PutMetricData requests.","is_advanced":true,"default":"100ms"},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]},"version":"3.36.0"},{"name":"influxdb","type":"metrics","status":"beta","plugin":true,"summary":"Send metrics to InfluxDB 1.x using the `/write` endpoint.","description":"See https://docs.influxdata.com/influxdb/v1.8/tools/api/#write-http-endpoint for further details on the write API.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"A URL of the format `[https|http|udp]://host:port` to the InfluxDB host.","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"db","type":"string","kind":"scalar","description":"The name of the database to use."},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"username","type":"string","kind":"scalar","description":"A username (when applicable).","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password (when applicable).","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"include","type":"object","kind":"scalar","description":"Optional additional metrics to collect, enabling these metrics may have some performance implications as it acquires a global semaphore and does `stoptheworld()`.","is_advanced":true,"children":[{"name":"runtime","type":"string","kind":"scalar","description":"A duration string indicating how often to poll and collect runtime metrics. Leave empty to disable this metric","is_advanced":true,"default":"","examples":["1m"]},{"name":"debug_gc","type":"string","kind":"scalar","description":"A duration string indicating how often to poll and collect GC metrics. Leave empty to disable this metric.","is_advanced":true,"default":"","examples":["1m"]}]},{"name":"interval","type":"string","kind":"scalar","description":"A duration string indicating how often metrics should be flushed.","is_advanced":true,"default":"1m"},{"name":"ping_interval","type":"string","kind":"scalar","description":"A duration string indicating how often to ping the host.","is_advanced":true,"default":"20s"},{"name":"precision","type":"string","kind":"scalar","description":"[ns|us|ms|s] timestamp precision passed to write api.","is_advanced":true,"default":"s"},{"name":"timeout","type":"string","kind":"scalar","description":"How long to wait for response for both ping and writing metrics.","is_advanced":true,"default":"5s"},{"name":"tags","type":"string","kind":"map","description":"Global tags added to each metric.","is_advanced":true,"default":{},"examples":[{"hostname":"localhost","zone":"danger"}]},{"name":"retention_policy","type":"string","kind":"scalar","description":"Sets the retention policy for each write.","is_advanced":true,"is_optional":true},{"name":"write_consistency","type":"string","kind":"scalar","description":"[any|one|quorum|all] sets write consistency when available.","is_advanced":true,"is_optional":true}]},"version":"3.36.0"},{"name":"json_api","type":"metrics","status":"stable","plugin":true,"summary":"Serves metrics as JSON object with the service wide HTTP service at the endpoints `/stats` and `/metrics`.","description":"This metrics type is useful for debugging as it provides a human readable format that you can parse with tools such as `jq`","categories":null,"config":{"name":"","type":"object","kind":"scalar","default":{}}},{"name":"logger","type":"metrics","status":"beta","plugin":true,"summary":"Prints aggregated metrics through the logger.","description":"\nPrints each metric produced by Redpanda Connect as a log event (level `info` by default) during shutdown, and optionally on an interval.\n\nThis metrics type is useful for debugging pipelines when you only have access to the logger output and not the service-wide server. Otherwise it's recommended that you use either the `prometheus` or `json_api`types.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"push_interval","type":"string","kind":"scalar","description":"An optional period of time to continuously print all metrics.","is_optional":true},{"name":"flush_metrics","type":"bool","kind":"scalar","description":"Whether counters and timing metrics should be reset to 0 each time metrics are printed.","default":false}]}},{"name":"none","type":"metrics","status":"stable","plugin":true,"summary":"Disable metrics entirely.","categories":null,"config":{"name":"","type":"object","kind":"scalar","default":{}}},{"name":"prometheus","type":"metrics","status":"stable","plugin":true,"summary":"Host endpoints (`/metrics` and `/stats`) for Prometheus scraping.","categories":null,"footnotes":"\n== Push gateway\n\nThe field `push_url` is optional and when set will trigger a push of metrics to a https://prometheus.io/docs/instrumenting/pushing/[Prometheus Push Gateway^] once Redpanda Connect shuts down. It is also possible to specify a `push_interval` which results in periodic pushes.\n\nThe Push Gateway is useful for when Redpanda Connect instances are short lived. Do not include the \"/metrics/jobs/...\" path in the push URL.\n\nIf the Push Gateway requires HTTP Basic Authentication it can be configured with `push_basic_auth`.","config":{"name":"","type":"object","kind":"scalar","children":[{"name":"use_histogram_timing","type":"bool","kind":"scalar","description":"Whether to export timing metrics as a histogram, if `false` a summary is used instead. When exporting histogram timings the delta values are converted from nanoseconds into seconds in order to better fit within bucket definitions. For more information on histograms and summaries refer to: https://prometheus.io/docs/practices/histograms/.","is_advanced":true,"default":false,"version":"3.63.0"},{"name":"histogram_buckets","type":"float","kind":"array","description":"Timing metrics histogram buckets (in seconds). If left empty defaults to DefBuckets (https://pkg.go.dev/github.com/prometheus/client_golang/prometheus#pkg-variables). Applicable when `use_histogram_timing` is set to `true`.","is_advanced":true,"default":[],"version":"3.63.0"},{"name":"summary_quantiles_objectives","type":"object","kind":"array","description":"A list of timing metrics summary buckets (as quantiles). Applicable when `use_histogram_timing` is set to `false`.","is_advanced":true,"default":[{"error":0.05,"quantile":0.5},{"error":0.01,"quantile":0.9},{"error":0.001,"quantile":0.99}],"examples":[[{"error":0.05,"quantile":0.5},{"error":0.01,"quantile":0.9},{"error":0.001,"quantile":0.99}]],"children":[{"name":"quantile","type":"float","kind":"scalar","description":"Quantile value.","is_advanced":true,"default":0},{"name":"error","type":"float","kind":"scalar","description":"Permissible margin of error for quantile calculations. Precise calculations in a streaming context (without prior knowledge of the full dataset) can be resource-intensive. To balance accuracy with computational efficiency, an error margin is introduced. For instance, if the 90th quantile (`0.9`) is determined to be `100ms` with a 1% error margin (`0.01`), the true value will fall within the `[99ms, 101ms]` range.)","is_advanced":true,"default":0}],"version":"4.23.0"},{"name":"add_process_metrics","type":"bool","kind":"scalar","description":"Whether to export process metrics such as CPU and memory usage in addition to Redpanda Connect metrics.","is_advanced":true,"default":false},{"name":"add_go_metrics","type":"bool","kind":"scalar","description":"Whether to export Go runtime metrics such as GC pauses in addition to Redpanda Connect metrics.","is_advanced":true,"default":false},{"name":"push_url","type":"string","kind":"scalar","description":"An optional \u003c\u003cpush-gateway, Push Gateway URL\u003e\u003e to push metrics to.","is_advanced":true,"is_optional":true,"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"push_interval","type":"string","kind":"scalar","description":"The period of time between each push when sending metrics to a Push Gateway.","is_advanced":true,"is_optional":true},{"name":"push_job_name","type":"string","kind":"scalar","description":"An identifier for push jobs.","is_advanced":true,"default":"benthos_push"},{"name":"push_basic_auth","type":"object","kind":"scalar","description":"The Basic Authentication credentials.","is_advanced":true,"children":[{"name":"username","type":"string","kind":"scalar","description":"The Basic Authentication username.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"The Basic Authentication password.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"file_output_path","type":"string","kind":"scalar","description":"An optional file path to write all prometheus metrics on service shutdown.","is_advanced":true,"default":""}]}},{"name":"statsd","type":"metrics","status":"stable","plugin":true,"summary":"Pushes metrics using the https://github.com/statsd/statsd[StatsD protocol^]. Supported tagging formats are 'none', 'datadog' and 'influxdb'.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"address","type":"string","kind":"scalar","description":"The address to send metrics to."},{"name":"flush_period","type":"string","kind":"scalar","description":"The time interval between metrics flushes.","default":"100ms"},{"name":"tag_format","type":"string","kind":"scalar","description":"Metrics tagging is supported in a variety of formats.","default":"none","options":["none","datadog","influxdb"],"linter":"\nlet options = {\n \"none\": true,\n \"datadog\": true,\n \"influxdb\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"}]}}],"tracers":[{"name":"gcp_cloudtrace","type":"tracer","status":"experimental","plugin":true,"summary":"Send tracing events to a https://cloud.google.com/trace[Google Cloud Trace^].","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"project","type":"string","kind":"scalar","description":"The google project with Cloud Trace API enabled. If this is omitted then the Google Cloud SDK will attempt auto-detect it from the environment."},{"name":"sampling_ratio","type":"float","kind":"scalar","description":"Sets the ratio of traces to sample. Tuning the sampling ratio is recommended for high-volume production workloads.","default":1,"examples":[1]},{"name":"tags","type":"string","kind":"map","description":"A map of tags to add to tracing spans.","is_advanced":true,"default":{}},{"name":"flush_interval","type":"string","kind":"scalar","description":"The period of time between each flush of tracing spans.","is_optional":true}]},"version":"4.2.0"},{"name":"jaeger","type":"tracer","status":"stable","plugin":true,"summary":"Send tracing events to a https://www.jaegertracing.io/[Jaeger^] agent or collector.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"agent_address","type":"string","kind":"scalar","description":"The address of a Jaeger agent to send tracing events to.","default":"","examples":["jaeger-agent:6831"]},{"name":"collector_url","type":"string","kind":"scalar","description":"The URL of a Jaeger collector to send tracing events to. If set, this will override `agent_address`.","default":"","examples":["https://jaeger-collector:14268/api/traces"],"version":"3.38.0"},{"name":"sampler_type","type":"string","kind":"scalar","description":"The sampler type to use.","default":"const","annotated_options":[["const","Sample a percentage of traces. 1 or more means all traces are sampled, 0 means no traces are sampled and anything in between means a percentage of traces are sampled. Tuning the sampling rate is recommended for high-volume production workloads."]],"linter":"\nlet options = {\n \"const\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"sampler_param","type":"float","kind":"scalar","description":"A parameter to use for sampling. This field is unused for some sampling types.","is_advanced":true,"default":1},{"name":"tags","type":"string","kind":"map","description":"A map of tags to add to tracing spans.","is_advanced":true,"default":{}},{"name":"flush_interval","type":"string","kind":"scalar","description":"The period of time between each flush of tracing spans.","is_optional":true}]}},{"name":"none","type":"tracer","status":"stable","plugin":true,"summary":"Do not send tracing events anywhere.","categories":null,"config":{"name":"","type":"object","kind":"scalar","default":{}}},{"name":"open_telemetry_collector","type":"tracer","status":"experimental","plugin":true,"summary":"Send tracing events to an https://opentelemetry.io/docs/collector/[Open Telemetry collector^].","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"http","type":"object","kind":"array","description":"A list of http collectors.","children":[{"name":"address","type":"string","kind":"scalar","description":"The endpoint of a collector to send tracing events to.","is_optional":true,"examples":["localhost:4318"]},{"name":"url","type":"string","kind":"scalar","description":"The URL of a collector to send tracing events to.","is_deprecated":true,"default":"localhost:4318"},{"name":"secure","type":"bool","kind":"scalar","description":"Connect to the collector over HTTPS","default":false}]},{"name":"grpc","type":"object","kind":"array","description":"A list of grpc collectors.","children":[{"name":"address","type":"string","kind":"scalar","description":"The endpoint of a collector to send tracing events to.","is_optional":true,"examples":["localhost:4317"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"url","type":"string","kind":"scalar","description":"The URL of a collector to send tracing events to.","is_deprecated":true,"default":"localhost:4317","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"secure","type":"bool","kind":"scalar","description":"Connect to the collector with client transport security","default":false}]},{"name":"tags","type":"string","kind":"map","description":"A map of tags to add to all tracing spans.","is_advanced":true,"default":{}},{"name":"sampling","type":"object","kind":"scalar","description":"Settings for trace sampling. Sampling is recommended for high-volume production workloads.","children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to enable sampling.","default":false},{"name":"ratio","type":"float","kind":"scalar","description":"Sets the ratio of traces to sample.","is_optional":true,"examples":[0.85,0.5]}],"version":"4.25.0"}]}}],"scanners":[{"name":"avro","type":"scanner","status":"stable","plugin":true,"summary":"Consume a stream of Avro OCF datum.","description":"\n== Avro JSON format\n\nThis scanner yields documents formatted as https://avro.apache.org/docs/current/specification/_print/#json-encoding[Avro JSON^] when decoding with Avro schemas. In this format the value of a union is encoded in JSON as follows:\n\n- if its type is `null`, then it is encoded as a JSON `null`;\n- otherwise it is encoded as a JSON object with one name/value pair whose name is the type's name and whose value is the recursively encoded value. For Avro's named types (record, fixed or enum) the user-specified name is used, for other types the type name is used.\n\nFor example, the union schema `[\"null\",\"string\",\"Foo\"]`, where `Foo` is a record name, would encode:\n\n- `null` as `null`;\n- the string `\"a\"` as `{\"string\": \"a\"}`; and\n- a `Foo` instance as `{\"Foo\": {...}}`, where `{...}` indicates the JSON encoding of a `Foo` instance.\n\nHowever, it is possible to instead create documents in https://pkg.go.dev/github.com/linkedin/goavro/v2#NewCodecForStandardJSONFull[standard/raw JSON format^] by setting the field \u003c\u003cavro_raw_json,`avro_raw_json`\u003e\u003e to `true`.\n\nThis scanner also emits the canonical Avro schema as `@avro_schema` metadata, along with the schema's fingerprint available via `@avro_schema_fingerprint`.\n","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"raw_json","type":"bool","kind":"scalar","description":"Whether messages should be decoded into normal JSON (\"json that meets the expectations of regular internet json\") rather than https://avro.apache.org/docs/current/specification/_print/#json-encoding[Avro JSON^]. If `true` the schema returned from the subject should be decoded as https://pkg.go.dev/github.com/linkedin/goavro/v2#NewCodecForStandardJSONFull[standard json^] instead of as https://pkg.go.dev/github.com/linkedin/goavro/v2#NewCodec[avro json^]. There is a https://github.com/linkedin/goavro/blob/5ec5a5ee7ec82e16e6e2b438d610e1cab2588393/union.go#L224-L249[comment in goavro^], the https://github.com/linkedin/goavro[underlining library used for avro serialization^], that explains in more detail the difference between the standard json and avro json.","is_advanced":true,"default":false}]}},{"name":"chunker","type":"scanner","status":"stable","plugin":true,"summary":"Split an input stream into chunks of a given number of bytes.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"size","type":"int","kind":"scalar","description":"The size of each chunk in bytes."}]}},{"name":"csv","type":"scanner","status":"stable","plugin":true,"summary":"Consume comma-separated values row by row, including support for custom delimiters.","description":"\n== Metadata\n\nThis scanner adds the following metadata to each message:\n\n- `csv_row` The index of each row, beginning at 0.\n\n","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"custom_delimiter","type":"string","kind":"scalar","description":"Use a provided custom delimiter instead of the default comma.","is_optional":true},{"name":"parse_header_row","type":"bool","kind":"scalar","description":"Whether to reference the first row as a header row. If set to true the output structure for messages will be an object where field keys are determined by the header row. Otherwise, each message will consist of an array of values from the corresponding CSV row.","default":true},{"name":"lazy_quotes","type":"bool","kind":"scalar","description":"If set to `true`, a quote may appear in an unquoted field and a non-doubled quote may appear in a quoted field.","default":false},{"name":"continue_on_error","type":"bool","kind":"scalar","description":"If a row fails to parse due to any error emit an empty message marked with the error and then continue consuming subsequent rows when possible. This can sometimes be useful in situations where input data contains individual rows which are malformed. However, when a row encounters a parsing error it is impossible to guarantee that following rows are valid, as this indicates that the input data is unreliable and could potentially emit misaligned rows.","default":false}]}},{"name":"decompress","type":"scanner","status":"stable","plugin":true,"summary":"Decompress the stream of bytes according to an algorithm, before feeding it into a child scanner.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"algorithm","type":"string","kind":"scalar","description":"One of `gzip`, `pgzip`, `zlib`, `bzip2`, `flate`, `snappy`, `lz4`, `zstd`."},{"name":"into","type":"scanner","kind":"scalar","description":"The child scanner to feed the decompressed stream into.","default":{"to_the_end":{}}}]}},{"name":"json_documents","type":"scanner","status":"stable","plugin":true,"summary":"Consumes a stream of one or more JSON documents.","categories":null,"config":{"name":"","type":"object","kind":"scalar","default":{}},"version":"4.27.0"},{"name":"lines","type":"scanner","status":"stable","plugin":true,"summary":"Split an input stream into a message per line of data.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"custom_delimiter","type":"string","kind":"scalar","description":"Use a provided custom delimiter for detecting the end of a line rather than a single line break.","is_optional":true},{"name":"max_buffer_size","type":"int","kind":"scalar","description":"Set the maximum buffer size for storing line data, this limits the maximum size that a line can be without causing an error.","default":65536},{"name":"omit_empty","type":"bool","kind":"scalar","description":"Omit empty lines.","default":false}]}},{"name":"re_match","type":"scanner","status":"stable","plugin":true,"summary":"Split an input stream into segments matching against a regular expression.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"pattern","type":"string","kind":"scalar","description":"The pattern to match against.","examples":["(?m)^\\d\\d:\\d\\d:\\d\\d"]},{"name":"max_buffer_size","type":"int","kind":"scalar","description":"Set the maximum buffer size for storing line data, this limits the maximum size that a message can be without causing an error.","default":65536}]}},{"name":"skip_bom","type":"scanner","status":"stable","plugin":true,"summary":"Skip one or more byte order marks for each opened child scanner.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"into","type":"scanner","kind":"scalar","description":"The child scanner to feed the resulting stream into.","default":{"to_the_end":{}}}]}},{"name":"switch","type":"scanner","status":"stable","plugin":true,"summary":"Select a child scanner dynamically for source data based on factors such as the filename.","description":"This scanner outlines a list of potential child scanner candidates to be chosen, and for each source of data the first candidate to pass will be selected. A candidate without any conditions acts as a catch-all and will pass for every source, it is recommended to always have a catch-all scanner at the end of your list. If a given source of data does not pass a candidate an error is returned and the data is rejected.","categories":null,"examples":[{"title":"Switch based on file name","summary":"In this example a file input chooses a scanner based on the extension of each file","config":"\ninput:\n file:\n paths: [ ./data/* ]\n scanner:\n switch:\n - re_match_name: '\\.avro$'\n scanner: { avro: {} }\n\n - re_match_name: '\\.csv$'\n scanner: { csv: {} }\n\n - re_match_name: '\\.csv.gz$'\n scanner:\n decompress:\n algorithm: gzip\n into:\n csv: {}\n\n - re_match_name: '\\.tar$'\n scanner: { tar: {} }\n\n - re_match_name: '\\.tar.gz$'\n scanner:\n decompress:\n algorithm: gzip\n into:\n tar: {}\n\n - scanner: { to_the_end: {} }\n"}],"config":{"name":"","type":"object","kind":"array","children":[{"name":"re_match_name","type":"string","kind":"scalar","description":"A regular expression to test against the name of each source of data fed into the scanner (filename or equivalent). If this pattern matches the child scanner is selected.","is_optional":true},{"name":"scanner","type":"scanner","kind":"scalar","description":"The scanner to activate if this candidate passes."}]}},{"name":"tar","type":"scanner","status":"stable","plugin":true,"summary":"Consume a tar archive file by file.","description":"\n== Metadata\n\nThis scanner adds the following metadata to each message:\n\n- `tar_name`\n\n","categories":null,"config":{"name":"","type":"object","kind":"scalar","default":{}}},{"name":"to_the_end","type":"scanner","status":"stable","plugin":true,"summary":"Read the input stream all the way until the end and deliver it as a single message.","description":"\n[CAUTION]\n====\nSome sources of data may not have a logical end, therefore caution should be made to exclusively use this scanner when the end of an input stream is clearly defined (and well within memory).\n====\n","categories":null,"config":{"name":"","type":"object","kind":"scalar","default":{}}}],"bloblang-functions":[{"status":"stable","category":"Message Info","name":"batch_index","description":"Returns the index of the mapped message within a batch. This is useful for applying maps only on certain messages of a batch.","params":{},"examples":[{"mapping":"root = if batch_index() \u003e 0 { deleted() }","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"Message Info","name":"batch_size","description":"Returns the size of the message batch.","params":{},"examples":[{"mapping":"root.foo = batch_size()","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"Message Info","name":"content","description":"Returns the full raw contents of the mapping target message as a byte array. When mapping to a JSON field the value should be encoded using the method xref:guides:bloblang/methods.adoc#encode[`encode`], or cast to a string directly using the method xref:guides:bloblang/methods.adoc#string[`string`], otherwise it will be base64 encoded by default.","params":{},"examples":[{"mapping":"root.doc = content().string()","summary":"","results":[["{\"foo\":\"bar\"}","{\"doc\":\"{\\\"foo\\\":\\\"bar\\\"}\"}"]],"skip_testing":false}],"impure":false},{"status":"deprecated","category":"Deprecated","name":"count","description":"The `count` function is a counter starting at 1 which increments after each time it is called. Count takes an argument which is an identifier for the counter, allowing you to specify multiple unique counters in your configuration.","params":{"named":[{"name":"name","description":"An identifier for the counter.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"examples":[{"mapping":"root = this\nroot.id = count(\"bloblang_function_example\")","summary":"","results":[["{\"message\":\"foo\"}","{\"id\":1,\"message\":\"foo\"}"],["{\"message\":\"bar\"}","{\"id\":2,\"message\":\"bar\"}"]],"skip_testing":false}],"impure":true},{"status":"experimental","category":"General","name":"counter","description":"Returns a non-negative integer that increments each time it is resolved, yielding the minimum (`1` by default) as the first value. Each instantiation of `counter` has its own independent count. Once the maximum integer (or `max` argument) is reached the counter resets back to the minimum.","params":{"named":[{"name":"min","description":"The minimum value of the counter, this is the first value that will be yielded. If this parameter is dynamic it will be resolved only once during the lifetime of the mapping.","type":"query expression","no_dynamic":false,"scalars_to_literal":true,"default":1},{"name":"max","description":"The maximum value of the counter, once this value is yielded the counter will reset back to the min. If this parameter is dynamic it will be resolved only once during the lifetime of the mapping.","type":"query expression","no_dynamic":false,"scalars_to_literal":true,"default":9223372036854775807},{"name":"set","description":"An optional mapping that when specified will be executed each time the counter is resolved. When this mapping resolves to a non-negative integer value it will cause the counter to reset to this value and yield it. If this mapping is omitted or doesn't resolve to anything then the counter will increment and yield the value as normal. If this mapping resolves to `null` then the counter is not incremented and the current value is yielded. If this mapping resolves to a deletion then the counter is reset to the `min` value.","type":"query expression","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"examples":[{"mapping":"root.id = counter()","summary":"","results":[["{}","{\"id\":1}"],["{}","{\"id\":2}"]],"skip_testing":false},{"mapping":"\nmap foos {\n root = counter()\n}\n\nroot.meow_id = null.apply(\"foos\")\nroot.woof_id = null.apply(\"foos\")\n","summary":"It's possible to increment a counter multiple times within a single mapping invocation using a map.","results":[["{}","{\"meow_id\":1,\"woof_id\":2}"],["{}","{\"meow_id\":3,\"woof_id\":4}"]],"skip_testing":false},{"mapping":"root.consecutive_doggos = counter(min: 1, set: if !this.sound.lowercase().contains(\"woof\") { 0 })","summary":"By specifying an optional `set` parameter it is possible to dynamically reset the counter based on input data.","results":[["{\"sound\":\"woof woof\"}","{\"consecutive_doggos\":1}"],["{\"sound\":\"woofer wooooo\"}","{\"consecutive_doggos\":2}"],["{\"sound\":\"meow\"}","{\"consecutive_doggos\":0}"],["{\"sound\":\"uuuuh uh uh woof uhhhhhh\"}","{\"consecutive_doggos\":1}"]],"skip_testing":false},{"mapping":"root.things = counter(set: if this.id == null { null })","summary":"The `set` parameter can also be utilized to peek at the counter without mutating it by returning `null`.","results":[["{\"id\":\"a\"}","{\"things\":1}"],["{\"id\":\"b\"}","{\"things\":2}"],["{\"what\":\"just checking\"}","{\"things\":2}"],["{\"id\":\"c\"}","{\"things\":3}"]],"skip_testing":false}],"impure":false},{"status":"stable","category":"General","name":"deleted","description":"A function that returns a result indicating that the mapping target should be deleted. Deleting, also known as dropping, messages will result in them being acknowledged as successfully processed to inputs in a Redpanda Connect pipeline. For more information about error handling patterns read xref:configuration:error_handling.adoc[].","params":{},"examples":[{"mapping":"root = this\nroot.bar = deleted()","summary":"","results":[["{\"bar\":\"bar_value\",\"baz\":\"baz_value\",\"foo\":\"foo value\"}","{\"baz\":\"baz_value\",\"foo\":\"foo value\"}"]],"skip_testing":false},{"mapping":"root.new_nums = this.nums.map_each(num -\u003e if num \u003c 10 { deleted() } else { num - 10 })","summary":"Since the result is a value it can be used to do things like remove elements of an array within `map_each`.","results":[["{\"nums\":[3,11,4,17]}","{\"new_nums\":[1,7]}"]],"skip_testing":false}],"impure":false},{"status":"stable","category":"Environment","name":"env","description":"Returns the value of an environment variable, or `null` if the environment variable does not exist.","params":{"named":[{"name":"name","description":"The name of an environment variable.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"no_cache","description":"Force the variable lookup to occur for each mapping invocation.","type":"bool","no_dynamic":false,"scalars_to_literal":false,"default":false}]},"examples":[{"mapping":"root.thing.key = env(\"key\").or(\"default value\")","summary":"","results":[],"skip_testing":false},{"mapping":"root.thing.key = env(this.thing.key_name)","summary":"","results":[],"skip_testing":false},{"mapping":"root.thing.key = env(name: \"key\", no_cache: true)","summary":"When the name parameter is static this function will only resolve once and yield the same result for each invocation as an optimization, this means that updates to env vars during runtime will not be reflected. You can disable this cache with the optional parameter `no_cache`, which when set to `true` will cause the variable lookup to be performed for each execution of the mapping.","results":[],"skip_testing":false}],"impure":true},{"status":"stable","category":"Message Info","name":"error","description":"If an error has occurred during the processing of a message this function returns the reported cause of the error as a string, otherwise `null`. For more information about error handling patterns read xref:configuration:error_handling.adoc[].","params":{},"examples":[{"mapping":"root.doc.error = error()","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"Message Info","name":"error_source_label","description":"Returns the label of the source component which raised the error during the processing of a message or an empty string if not set. `null` is returned when the error is null or no source component is associated with it. For more information about error handling patterns read xref:configuration:error_handling.adoc[].","params":{},"examples":[{"mapping":"root.doc.error_source_label = error_source_label()","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"Message Info","name":"error_source_name","description":"Returns the name of the source component which raised the error during the processing of a message. `null` is returned when the error is null or no source component is associated with it. For more information about error handling patterns read xref:configuration:error_handling.adoc[].","params":{},"examples":[{"mapping":"root.doc.error_source_name = error_source_name()","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"Message Info","name":"error_source_path","description":"Returns the path of the source component which raised the error during the processing of a message. `null` is returned when the error is null or no source component is associated with it. For more information about error handling patterns read xref:configuration:error_handling.adoc[].","params":{},"examples":[{"mapping":"root.doc.error_source_path = error_source_path()","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"Message Info","name":"errored","description":"Returns a boolean value indicating whether an error has occurred during the processing of a message. For more information about error handling patterns read xref:configuration:error_handling.adoc[].","params":{},"examples":[{"mapping":"root.doc.status = if errored() { 400 } else { 200 }","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"beta","category":"Fake Data Generation","name":"fake","description":"Takes in a string that maps to a https://github.com/go-faker/faker[faker^] function and returns the result from that faker function. Returns an error if the given string doesn't match a supported faker function. Supported functions: `latitude`, `longitude`, `unix_time`, `date`, `time_string`, `month_name`, `year_string`, `day_of_week`, `day_of_month`, `timestamp`, `century`, `timezone`, `time_period`, `email`, `mac_address`, `domain_name`, `url`, `username`, `ipv4`, `ipv6`, `password`, `jwt`, `word`, `sentence`, `paragraph`, `cc_type`, `cc_number`, `currency`, `amount_with_currency`, `title_male`, `title_female`, `first_name`, `first_name_male`, `first_name_female`, `last_name`, `name`, `gender`, `chinese_first_name`, `chinese_last_name`, `chinese_name`, `phone_number`, `toll_free_phone_number`, `e164_phone_number`, `uuid_hyphenated`, `uuid_digit`. Refer to the https://github.com/go-faker/faker[faker^] docs for details on these functions.","params":{"named":[{"name":"function","description":"The name of the function to use to generate the value.","type":"string","no_dynamic":false,"scalars_to_literal":false,"default":""}]},"examples":[{"mapping":"root.time = fake(\"time_string\")","summary":"Use `time_string` to generate a time in the format `00:00:00`:","results":[],"skip_testing":false},{"mapping":"root.email = fake(\"email\")","summary":"Use `email` to generate a string in email address format:","results":[],"skip_testing":false},{"mapping":"root.jwt = fake(\"jwt\")","summary":"Use `jwt` to generate a JWT token:","results":[],"skip_testing":false},{"mapping":"root.uuid = fake(\"uuid_hyphenated\")","summary":"Use `uuid_hyphenated` to generate a hyphenated UUID:","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"Environment","name":"file","description":"Reads a file and returns its contents. Relative paths are resolved from the directory of the process executing the mapping. In order to read files relative to the mapping file use the newer \u003c\u003cfile_rel, `file_rel` function\u003e\u003e","params":{"named":[{"name":"path","description":"The path of the target file.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"no_cache","description":"Force the file to be read for each mapping invocation.","type":"bool","no_dynamic":false,"scalars_to_literal":false,"default":false}]},"examples":[{"mapping":"root.doc = file(env(\"BENTHOS_TEST_BLOBLANG_FILE\")).parse_json()","summary":"","results":[["{}","{\"doc\":{\"foo\":\"bar\"}}"]],"skip_testing":false},{"mapping":"root.doc = file(path: env(\"BENTHOS_TEST_BLOBLANG_FILE\"), no_cache: true).parse_json()","summary":"When the path parameter is static this function will only read the specified file once and yield the same result for each invocation as an optimization, this means that updates to files during runtime will not be reflected. You can disable this cache with the optional parameter `no_cache`, which when set to `true` will cause the file to be read for each execution of the mapping.","results":[["{}","{\"doc\":{\"foo\":\"bar\"}}"]],"skip_testing":false}],"impure":true},{"status":"stable","category":"Environment","name":"file_rel","description":"Reads a file and returns its contents. Relative paths are resolved from the directory of the mapping.","params":{"named":[{"name":"path","description":"The path of the target file.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"no_cache","description":"Force the file to be read for each mapping invocation.","type":"bool","no_dynamic":false,"scalars_to_literal":false,"default":false}]},"examples":[{"mapping":"root.doc = file_rel(env(\"BENTHOS_TEST_BLOBLANG_FILE\")).parse_json()","summary":"","results":[["{}","{\"doc\":{\"foo\":\"bar\"}}"]],"skip_testing":false},{"mapping":"root.doc = file_rel(path: env(\"BENTHOS_TEST_BLOBLANG_FILE\"), no_cache: true).parse_json()","summary":"When the path parameter is static this function will only read the specified file once and yield the same result for each invocation as an optimization, this means that updates to files during runtime will not be reflected. You can disable this cache with the optional parameter `no_cache`, which when set to `true` will cause the file to be read for each execution of the mapping.","results":[["{}","{\"doc\":{\"foo\":\"bar\"}}"]],"skip_testing":false}],"impure":true},{"status":"stable","category":"Environment","name":"hostname","description":"Returns a string matching the hostname of the machine running Benthos.","params":{},"examples":[{"mapping":"root.thing.host = hostname()","summary":"","results":[],"skip_testing":false}],"impure":true},{"status":"stable","category":"Message Info","name":"json","description":"Returns the value of a field within a JSON message located by a [dot path][field_paths] argument. This function always targets the entire source JSON document regardless of the mapping context.","params":{"named":[{"name":"path","description":"An optional [dot path][field_paths] identifying a field to obtain.","type":"string","no_dynamic":false,"scalars_to_literal":false,"default":""}]},"examples":[{"mapping":"root.mapped = json(\"foo.bar\")","summary":"","results":[["{\"foo\":{\"bar\":\"hello world\"}}","{\"mapped\":\"hello world\"}"]],"skip_testing":false},{"mapping":"root.doc = json()","summary":"The path argument is optional and if omitted the entire JSON payload is returned.","results":[["{\"foo\":{\"bar\":\"hello world\"}}","{\"doc\":{\"foo\":{\"bar\":\"hello world\"}}}"]],"skip_testing":false}],"impure":false},{"status":"stable","category":"General","name":"ksuid","description":"Generates a new ksuid each time it is invoked and prints a string representation.","params":{},"examples":[{"mapping":"root.id = ksuid()","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"deprecated","category":"Deprecated","name":"meta","description":"Returns the value of a metadata key from the input message as a string, or `null` if the key does not exist. Since values are extracted from the read-only input message they do NOT reflect changes made from within the map. In order to query metadata mutations made within a mapping use the \u003c\u003croot_meta, `root_meta` function\u003e\u003e. This function supports extracting metadata from other messages of a batch with the `from` method.","params":{"named":[{"name":"key","description":"An optional key of a metadata value to obtain.","type":"string","no_dynamic":false,"scalars_to_literal":false,"default":""}]},"examples":[{"mapping":"root.topic = meta(\"kafka_topic\")","summary":"","results":[],"skip_testing":false},{"mapping":"root.all_metadata = meta()","summary":"The key parameter is optional and if omitted the entire metadata contents are returned as an object.","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"Message Info","name":"metadata","description":"Returns the value of a metadata key from the input message, or `null` if the key does not exist. Since values are extracted from the read-only input message they do NOT reflect changes made from within the map, in order to query metadata mutations made within a mapping use the xref:guides:bloblang/about.adoc#metadata[`@` operator]. This function supports extracting metadata from other messages of a batch with the `from` method.","params":{"named":[{"name":"key","description":"An optional key of a metadata value to obtain.","type":"string","no_dynamic":false,"scalars_to_literal":false,"default":""}]},"examples":[{"mapping":"root.topic = metadata(\"kafka_topic\")","summary":"","results":[],"skip_testing":false},{"mapping":"root.all_metadata = metadata()","summary":"The key parameter is optional and if omitted the entire metadata contents are returned as an object.","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"General","name":"nanoid","description":"Generates a new nanoid each time it is invoked and prints a string representation.","params":{"named":[{"name":"length","description":"An optional length.","type":"integer","no_dynamic":false,"scalars_to_literal":false,"is_optional":true},{"name":"alphabet","description":"An optional custom alphabet to use for generating IDs. When specified the field `length` must also be present.","type":"string","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"examples":[{"mapping":"root.id = nanoid()","summary":"","results":[],"skip_testing":false},{"mapping":"root.id = nanoid(54)","summary":"It is possible to specify an optional length parameter.","results":[],"skip_testing":false},{"mapping":"root.id = nanoid(54, \"abcde\")","summary":"It is also possible to specify an optional custom alphabet after the length parameter.","results":[],"skip_testing":false}],"impure":false},{"status":"hidden","category":"","name":"nothing","params":{},"impure":false},{"status":"stable","category":"Environment","name":"now","description":"Returns the current timestamp as a string in RFC 3339 format with the local timezone. Use the method `ts_format` in order to change the format and timezone.","params":{},"examples":[{"mapping":"root.received_at = now()","summary":"","results":[],"skip_testing":false},{"mapping":"root.received_at = now().ts_format(\"Mon Jan 2 15:04:05 -0700 MST 2006\", \"UTC\")","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"General","name":"pi","description":"Returns the value of the mathematical constant Pi.","params":{},"examples":[{"mapping":"root.radians = this.degrees * (pi() / 180)","summary":"","results":[["{\"degrees\":45}","{\"radians\":0.7853981633974483}"]],"skip_testing":false},{"mapping":"root.degrees = this.radians * (180 / pi())","summary":"","results":[["{\"radians\":0.78540}","{\"degrees\":45.00010522957486}"]],"skip_testing":false}],"impure":false},{"status":"stable","category":"General","name":"random_int","description":"\nGenerates a non-negative pseudo-random 64-bit integer. An optional integer argument can be provided in order to seed the random number generator.\n\nOptional `min` and `max` arguments can be provided in order to only generate numbers within a range. Neither of these parameters can be set via a dynamic expression (i.e. from values taken from mapped data). Instead, for dynamic ranges extract a min and max manually using a modulo operator (`random_int() % a + b`).","params":{"named":[{"name":"seed","description":"A seed to use, if a query is provided it will only be resolved once during the lifetime of the mapping.","type":"query expression","no_dynamic":false,"scalars_to_literal":true,"default":{"Value":0}},{"name":"min","description":"The minimum value the random generated number will have. The default value is 0.","type":"integer","no_dynamic":true,"scalars_to_literal":false,"default":0},{"name":"max","description":"The maximum value the random generated number will have. The default value is 9223372036854775806 (math.MaxInt64 - 1).","type":"integer","no_dynamic":true,"scalars_to_literal":false,"default":9223372036854775806}]},"examples":[{"mapping":"root.first = random_int()\nroot.second = random_int(1)\nroot.third = random_int(max:20)\nroot.fourth = random_int(min:10, max:20)\nroot.fifth = random_int(timestamp_unix_nano(), 5, 20)\nroot.sixth = random_int(seed:timestamp_unix_nano(), max:20)\n","summary":"","results":[],"skip_testing":false},{"mapping":"root.first = random_int(timestamp_unix_nano())","summary":"It is possible to specify a dynamic seed argument, in which case the argument will only be resolved once during the lifetime of the mapping.","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"General","name":"range","description":"The `range` function creates an array of integers following a range between a start, stop and optional step integer argument. If the step argument is omitted then it defaults to 1. A negative step can be provided as long as stop \u003c start.","params":{"named":[{"name":"start","description":"The start value.","type":"integer","no_dynamic":false,"scalars_to_literal":false},{"name":"stop","description":"The stop value.","type":"integer","no_dynamic":false,"scalars_to_literal":false},{"name":"step","description":"The step value.","type":"integer","no_dynamic":false,"scalars_to_literal":false,"default":1}]},"examples":[{"mapping":"root.a = range(0, 10)\nroot.b = range(start: 0, stop: this.max, step: 2) # Using named params\nroot.c = range(0, -this.max, -2)","summary":"","results":[["{\"max\":10}","{\"a\":[0,1,2,3,4,5,6,7,8,9],\"b\":[0,2,4,6,8],\"c\":[0,-2,-4,-6,-8]}"]],"skip_testing":false}],"impure":false},{"status":"deprecated","category":"Deprecated","name":"root_meta","description":"Returns the value of a metadata key from the new message being created as a string, or `null` if the key does not exist. Changes made to metadata during a mapping will be reflected by this function.","params":{"named":[{"name":"key","description":"An optional key of a metadata value to obtain.","type":"string","no_dynamic":false,"scalars_to_literal":false,"default":""}]},"examples":[{"mapping":"root.topic = root_meta(\"kafka_topic\")","summary":"","results":[],"skip_testing":false},{"mapping":"root.all_metadata = root_meta()","summary":"The key parameter is optional and if omitted the entire metadata contents are returned as an object.","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"General","name":"snowflake_id","description":"Generate a new snowflake ID each time it is invoked and prints a string representation. I.e.: 1559229974454472704","params":{"named":[{"name":"node_id","description":"It is possible to specify the node_id.","type":"integer","no_dynamic":false,"scalars_to_literal":false,"default":1}]},"examples":[{"mapping":"root.id = snowflake_id()","summary":"","results":[],"skip_testing":false},{"mapping":"root.id = snowflake_id(2)","summary":"It is possible to specify the node_id.","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"General","name":"throw","description":"Throws an error similar to a regular mapping error. This is useful for abandoning a mapping entirely given certain conditions.","params":{"named":[{"name":"why","description":"A string explanation for why an error was thrown, this will be added to the resulting error message.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"examples":[{"mapping":"root.doc.type = match {\n this.exists(\"header.id\") =\u003e \"foo\"\n this.exists(\"body.data\") =\u003e \"bar\"\n _ =\u003e throw(\"unknown type\")\n}\nroot.doc.contents = (this.body.content | this.thing.body)","summary":"","results":[["{\"header\":{\"id\":\"first\"},\"thing\":{\"body\":\"hello world\"}}","{\"doc\":{\"contents\":\"hello world\",\"type\":\"foo\"}}"],["{\"nothing\":\"matches\"}","Error(\"failed assignment (line 1): unknown type\")"]],"skip_testing":false}],"impure":false},{"status":"stable","category":"Environment","name":"timestamp_unix","description":"Returns the current unix timestamp in seconds.","params":{},"examples":[{"mapping":"root.received_at = timestamp_unix()","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"Environment","name":"timestamp_unix_micro","description":"Returns the current unix timestamp in microseconds.","params":{},"examples":[{"mapping":"root.received_at = timestamp_unix_micro()","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"Environment","name":"timestamp_unix_milli","description":"Returns the current unix timestamp in milliseconds.","params":{},"examples":[{"mapping":"root.received_at = timestamp_unix_milli()","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"Environment","name":"timestamp_unix_nano","description":"Returns the current unix timestamp in nanoseconds.","params":{},"examples":[{"mapping":"root.received_at = timestamp_unix_nano()","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"experimental","category":"Message Info","name":"tracing_id","description":"Provides the message trace id. The returned value will be zeroed if the message does not contain a span.","params":{},"examples":[{"mapping":"meta trace_id = tracing_id()","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"experimental","category":"Message Info","name":"tracing_span","description":"Provides the message tracing span xref:components:tracers/about.adoc[(created via Open Telemetry APIs)] as an object serialized via text map formatting. The returned value will be `null` if the message does not have a span.","params":{},"examples":[{"mapping":"root.headers.traceparent = tracing_span().traceparent","summary":"","results":[["{\"some_stuff\":\"just can't be explained by science\"}","{\"headers\":{\"traceparent\":\"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\"}}"]],"skip_testing":false}],"impure":false},{"status":"experimental","category":"General","name":"ulid","description":"Generate a random ULID.","params":{"named":[{"name":"encoding","description":"The format to encode a ULID into. Valid options are: crockford, hex","type":"string","no_dynamic":false,"scalars_to_literal":false,"default":"crockford"},{"name":"random_source","description":"The source of randomness to use for generating ULIDs. \"secure_random\" is recommended for most use cases. \"fast_random\" can be used if security is not a concern.","type":"string","no_dynamic":false,"scalars_to_literal":false,"default":"secure_random"}]},"examples":[{"mapping":"root.id = ulid()","summary":"Using the defaults of Crockford Base32 encoding and secure random source","results":[],"skip_testing":false},{"mapping":"root.id = ulid(\"hex\")","summary":"ULIDs can be hex-encoded too.","results":[],"skip_testing":false},{"mapping":"root.id = ulid(\"crockford\", \"fast_random\")","summary":"They can be generated using a fast, but unsafe, random source for use cases that are not security-sensitive.","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"General","name":"uuid_v4","description":"Generates a new RFC-4122 UUID each time it is invoked and prints a string representation.","params":{},"examples":[{"mapping":"root.id = uuid_v4()","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"General","name":"uuid_v7","description":"Generates a new time ordered UUID each time it is invoked and prints a string representation.","params":{"named":[{"name":"time","description":"An optional timestamp to use for the time ordered portion of the UUID.","type":"timestamp","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"examples":[{"mapping":"root.id = uuid_v7()","summary":"","results":[],"skip_testing":false},{"mapping":"root.id = uuid_v7(now().ts_sub_iso8601(\"PT1M\"))","summary":"It is also possible to specify the timestamp for the uuid_v7","results":[],"skip_testing":false}],"impure":false},{"status":"hidden","category":"","name":"var","params":{"named":[{"name":"name","description":"The name of the target variable.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"impure":false}],"bloblang-methods":[{"status":"stable","name":"abs","description":"Returns the absolute value of an int64 or float64 number. As a special case, when an integer is provided that is the minimum value it is converted to the maximum value.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"\nroot.outs = this.ins.map_each(ele -\u003e ele.abs())\n","summary":"","results":[["{\"ins\":[9,-18,1.23,-4.56]}","{\"outs\":[9,18,1.23,4.56]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"all","description":"Checks each element of an array against a query and returns true if all elements passed. An error occurs if the target is not an array, or if any element results in the provided query returning a non-boolean result. Returns false if the target array is empty.","params":{"named":[{"name":"test","description":"A test query to apply to each element.","type":"query expression","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.all_over_21 = this.patrons.all(patron -\u003e patron.age \u003e= 21)","summary":"","results":[["{\"patrons\":[{\"id\":\"1\",\"age\":18},{\"id\":\"2\",\"age\":23}]}","{\"all_over_21\":false}"],["{\"patrons\":[{\"id\":\"1\",\"age\":45},{\"id\":\"2\",\"age\":23}]}","{\"all_over_21\":true}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"any","description":"Checks the elements of an array against a query and returns true if any element passes. An error occurs if the target is not an array, or if an element results in the provided query returning a non-boolean result. Returns false if the target array is empty.","params":{"named":[{"name":"test","description":"A test query to apply to each element.","type":"query expression","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.any_over_21 = this.patrons.any(patron -\u003e patron.age \u003e= 21)","summary":"","results":[["{\"patrons\":[{\"id\":\"1\",\"age\":18},{\"id\":\"2\",\"age\":23}]}","{\"any_over_21\":true}"],["{\"patrons\":[{\"id\":\"1\",\"age\":10},{\"id\":\"2\",\"age\":12}]}","{\"any_over_21\":false}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"append","description":"Returns an array with new elements appended to the end.","params":{"variadic":true},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.foo = this.foo.append(\"and\", \"this\")","summary":"","results":[["{\"foo\":[\"bar\",\"baz\"]}","{\"foo\":[\"bar\",\"baz\",\"and\",\"this\"]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"apply","description":"Apply a declared mapping to a target value.","params":{"named":[{"name":"mapping","description":"The mapping to apply.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"examples":[{"mapping":"map thing {\n root.inner = this.first\n}\n\nroot.foo = this.doc.apply(\"thing\")","summary":"","results":[["{\"doc\":{\"first\":\"hello world\"}}","{\"foo\":{\"inner\":\"hello world\"}}"]],"skip_testing":false},{"mapping":"map create_foo {\n root.name = \"a foo\"\n root.purpose = \"to be a foo\"\n}\n\nroot = this\nroot.foo = null.apply(\"create_foo\")","summary":"","results":[["{\"id\":\"1234\"}","{\"foo\":{\"name\":\"a foo\",\"purpose\":\"to be a foo\"},\"id\":\"1234\"}"]],"skip_testing":false}],"impure":false},{"status":"stable","name":"array","params":{},"categories":[{"Category":"Type Coercion","Description":"Return an array containing the target value. If the value is already an array it is unchanged.","Examples":[{"mapping":"root.my_array = this.name.array()","summary":"","results":[["{\"name\":\"foobar bazson\"}","{\"my_array\":[\"foobar bazson\"]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"assign","description":"Merge a source object into an existing destination object. When a collision is found within the merged structures (both a source and destination object contain the same non-object keys) the value in the destination object will be overwritten by that of source object. In order to preserve both values on collision use the \u003c\u003cmerge, `merge`\u003e\u003e method.","params":{"named":[{"name":"with","description":"A value to merge the target value with.","type":"unknown","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root = this.foo.assign(this.bar)","summary":"","results":[["{\"foo\":{\"first_name\":\"fooer\",\"likes\":\"bars\"},\"bar\":{\"second_name\":\"barer\",\"likes\":\"foos\"}}","{\"first_name\":\"fooer\",\"likes\":\"foos\",\"second_name\":\"barer\"}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"bloblang","description":"Executes an argument Bloblang mapping on the target. This method can be used in order to execute dynamic mappings. Imports and functions that interact with the environment, such as `file` and `env`, or that access message information directly, such as `content` or `json`, are not enabled for dynamic Bloblang mappings.","params":{"named":[{"name":"mapping","description":"The mapping to execute.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Parsing","Description":"","Examples":[{"mapping":"root.body = this.body.bloblang(this.mapping)","summary":"","results":[["{\"body\":{\"foo\":\"hello world\"},\"mapping\":\"root.foo = this.foo.uppercase()\"}","{\"body\":{\"foo\":\"HELLO WORLD\"}}"],["{\"body\":{\"foo\":\"hello world 2\"},\"mapping\":\"root.foo = this.foo.capitalize()\"}","{\"body\":{\"foo\":\"Hello World 2\"}}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"bool","params":{"named":[{"name":"default","description":"An optional value to yield if the target cannot be parsed as a boolean.","type":"bool","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"categories":[{"Category":"Type Coercion","Description":"Attempt to parse a value into a boolean. An optional argument can be provided, in which case if the value cannot be parsed the argument will be returned instead. If the value is a number then any non-zero value will resolve to `true`, if the value is a string then any of the following values are considered valid: `1, t, T, TRUE, true, True, 0, f, F, FALSE`.","Examples":[{"mapping":"root.foo = this.thing.bool()\nroot.bar = this.thing.bool(true)","summary":"","results":[],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"bytes","params":{},"categories":[{"Category":"Type Coercion","Description":"Marshal a value into a byte array. If the value is already a byte array it is unchanged.","Examples":[{"mapping":"root.first_byte = this.name.bytes().index(0)","summary":"","results":[["{\"name\":\"foobar bazson\"}","{\"first_byte\":102}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"capitalize","params":{},"categories":[{"Category":"String Manipulation","Description":"Takes a string value and returns a copy with all Unicode letters that begin words mapped to their Unicode title case.","Examples":[{"mapping":"root.title = this.title.capitalize()","summary":"","results":[["{\"title\":\"the foo bar\"}","{\"title\":\"The Foo Bar\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"catch","description":"If the result of a target query fails (due to incorrect types, failed parsing, etc) the argument is returned instead.","params":{"named":[{"name":"fallback","description":"A value to yield, or query to execute, if the target query fails.","type":"query expression","no_dynamic":false,"scalars_to_literal":true}]},"examples":[{"mapping":"root.doc.id = this.thing.id.string().catch(uuid_v4())","summary":"","results":[],"skip_testing":false},{"mapping":"root.url = this.url.parse_url().catch(err -\u003e {\"error\":err,\"input\":this.url})","summary":"The fallback argument can be a mapping, allowing you to capture the error string and yield structured data back.","results":[["{\"url\":\"invalid %\u0026# url\"}","{\"url\":{\"error\":\"field `this.url`: parse \\\"invalid %\u0026\\\": invalid URL escape \\\"%\u0026\\\"\",\"input\":\"invalid %\u0026# url\"}}"]],"skip_testing":false},{"mapping":"root = this.catch(deleted())","summary":"When the input document is not structured attempting to reference structured fields with `this` will result in an error. Therefore, a convenient way to delete non-structured data is with a catch.","results":[["{\"doc\":{\"foo\":\"bar\"}}","{\"doc\":{\"foo\":\"bar\"}}"],["not structured data","\u003cMessage deleted\u003e"]],"skip_testing":false}],"impure":false},{"status":"stable","name":"ceil","description":"Returns the least integer value greater than or equal to a number. If the resulting value fits within a 64-bit integer then that is returned, otherwise a new floating point number is returned.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"root.new_value = this.value.ceil()","summary":"","results":[["{\"value\":5.3}","{\"new_value\":6}"],["{\"value\":-5.9}","{\"new_value\":-5}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"collapse","params":{"named":[{"name":"include_empty","description":"Whether to include empty objects and arrays in the resulting object.","type":"bool","no_dynamic":false,"scalars_to_literal":false,"default":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"Collapse an array or object into an object of key/value pairs for each field, where the key is the full path of the structured field in dot path notation. Empty arrays an objects are ignored by default.","Examples":[{"mapping":"root.result = this.collapse()","summary":"","results":[["{\"foo\":[{\"bar\":\"1\"},{\"bar\":{}},{\"bar\":\"2\"},{\"bar\":[]}]}","{\"result\":{\"foo.0.bar\":\"1\",\"foo.2.bar\":\"2\"}}"]],"skip_testing":false},{"mapping":"root.result = this.collapse(include_empty: true)","summary":"An optional boolean parameter can be set to true in order to include empty objects and arrays.","results":[["{\"foo\":[{\"bar\":\"1\"},{\"bar\":{}},{\"bar\":\"2\"},{\"bar\":[]}]}","{\"result\":{\"foo.0.bar\":\"1\",\"foo.1.bar\":{},\"foo.2.bar\":\"2\",\"foo.3.bar\":[]}}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"compare_argon2","description":"Checks whether a string matches a hashed secret using Argon2.","params":{"named":[{"name":"hashed_secret","description":"The hashed secret to compare with the input. This must be a fully-qualified string which encodes the Argon2 options used to generate the hash.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"String Manipulation","Description":"","Examples":[{"mapping":"root.match = this.secret.compare_argon2(\"$argon2id$v=19$m=4096,t=3,p=1$c2FsdHktbWNzYWx0ZmFjZQ$RMUMwgtS32/mbszd+ke4o4Ej1jFpYiUqY6MHWa69X7Y\")","summary":"","results":[["{\"secret\":\"there-are-many-blobs-in-the-sea\"}","{\"match\":true}"]],"skip_testing":false},{"mapping":"root.match = this.secret.compare_argon2(\"$argon2id$v=19$m=4096,t=3,p=1$c2FsdHktbWNzYWx0ZmFjZQ$RMUMwgtS32/mbszd+ke4o4Ej1jFpYiUqY6MHWa69X7Y\")","summary":"","results":[["{\"secret\":\"will-i-ever-find-love\"}","{\"match\":false}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"compare_bcrypt","description":"Checks whether a string matches a hashed secret using bcrypt.","params":{"named":[{"name":"hashed_secret","description":"The hashed secret value to compare with the input.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"String Manipulation","Description":"","Examples":[{"mapping":"root.match = this.secret.compare_bcrypt(\"$2y$10$Dtnt5NNzVtMCOZONT705tOcS8It6krJX8bEjnDJnwxiFKsz1C.3Ay\")","summary":"","results":[["{\"secret\":\"there-are-many-blobs-in-the-sea\"}","{\"match\":true}"]],"skip_testing":false},{"mapping":"root.match = this.secret.compare_bcrypt(\"$2y$10$Dtnt5NNzVtMCOZONT705tOcS8It6krJX8bEjnDJnwxiFKsz1C.3Ay\")","summary":"","results":[["{\"secret\":\"will-i-ever-find-love\"}","{\"match\":false}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"compress","description":"Compresses a string or byte array value according to a specified algorithm.","params":{"named":[{"name":"algorithm","description":"One of `flate`, `gzip`, `pgzip`, `lz4`, `snappy`, `zlib`, `zstd`.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"level","description":"The level of compression to use. May not be applicable to all algorithms.","type":"integer","no_dynamic":false,"scalars_to_literal":false,"default":-1}]},"categories":[{"Category":"Encoding and Encryption","Description":"","Examples":[{"mapping":"let long_content = range(0, 1000).map_each(content()).join(\" \")\nroot.a_len = $long_content.length()\nroot.b_len = $long_content.compress(\"gzip\").length()\n","summary":"","results":[["hello world this is some content","{\"a_len\":32999,\"b_len\":161}"]],"skip_testing":false},{"mapping":"root.compressed = content().compress(\"lz4\").encode(\"base64\")","summary":"","results":[["hello world I love space","{\"compressed\":\"BCJNGGRwuRgAAIBoZWxsbyB3b3JsZCBJIGxvdmUgc3BhY2UAAAAAGoETLg==\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"concat","description":"Concatenates an array value with one or more argument arrays.","params":{"variadic":true},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.foo = this.foo.concat(this.bar, this.baz)","summary":"","results":[["{\"foo\":[\"a\",\"b\"],\"bar\":[\"c\"],\"baz\":[\"d\",\"e\",\"f\"]}","{\"foo\":[\"a\",\"b\",\"c\",\"d\",\"e\",\"f\"]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"contains","params":{"named":[{"name":"value","description":"A value to test against elements of the target.","type":"unknown","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"Checks whether an array contains an element matching the argument, or an object contains a value matching the argument, and returns a boolean result. Numerical comparisons are made irrespective of the representation type (float versus integer).","Examples":[{"mapping":"root.has_foo = this.thing.contains(\"foo\")","summary":"","results":[["{\"thing\":[\"this\",\"foo\",\"that\"]}","{\"has_foo\":true}"],["{\"thing\":[\"this\",\"bar\",\"that\"]}","{\"has_foo\":false}"]],"skip_testing":false},{"mapping":"root.has_bar = this.thing.contains(20)","summary":"","results":[["{\"thing\":[10.3,20.0,\"huh\",3]}","{\"has_bar\":true}"],["{\"thing\":[2,3,40,67]}","{\"has_bar\":false}"]],"skip_testing":false}]},{"Category":"String Manipulation","Description":"Checks whether a string contains a substring and returns a boolean result.","Examples":[{"mapping":"root.has_foo = this.thing.contains(\"foo\")","summary":"","results":[["{\"thing\":\"this foo that\"}","{\"has_foo\":true}"],["{\"thing\":\"this bar that\"}","{\"has_foo\":false}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"cos","description":"Calculates the cosine of a given angle specified in radians.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"root.new_value = (this.value * (pi() / 180)).cos()","summary":"","results":[["{\"value\":45}","{\"new_value\":0.7071067811865476}"],["{\"value\":0}","{\"new_value\":1}"],["{\"value\":180}","{\"new_value\":-1}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"decode","params":{"named":[{"name":"scheme","description":"The decoding scheme to use.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Encoding and Encryption","Description":"Decodes an encoded string target according to a chosen scheme and returns the result as a byte array. When mapping the result to a JSON field the value should be cast to a string using the method `string`, or encoded using the method `encode`, otherwise it will be base64 encoded by default.\n\nAvailable schemes are: `base64`, `base64url` https://rfc-editor.org/rfc/rfc4648.html[(RFC 4648 with padding characters)], `base64rawurl` https://rfc-editor.org/rfc/rfc4648.html[(RFC 4648 without padding characters)], `hex`, `ascii85`.","Examples":[{"mapping":"root.decoded = this.value.decode(\"hex\").string()","summary":"","results":[["{\"value\":\"68656c6c6f20776f726c64\"}","{\"decoded\":\"hello world\"}"]],"skip_testing":false},{"mapping":"root = this.encoded.decode(\"ascii85\")","summary":"","results":[["{\"encoded\":\"FD,B0+DGm\u003eFDl80Ci\\\"A\u003eF`)8BEckl6F`M\u0026(+Cno\u0026@/\"}","this is totally unstructured data"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"decompress","description":"Decompresses a string or byte array value according to a specified algorithm. The result of decompression ","params":{"named":[{"name":"algorithm","description":"One of `gzip`, `pgzip`, `zlib`, `bzip2`, `flate`, `snappy`, `lz4`, `zstd`.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Encoding and Encryption","Description":"","Examples":[{"mapping":"root = this.compressed.decode(\"base64\").decompress(\"lz4\")","summary":"","results":[["{\"compressed\":\"BCJNGGRwuRgAAIBoZWxsbyB3b3JsZCBJIGxvdmUgc3BhY2UAAAAAGoETLg==\"}","hello world I love space"]],"skip_testing":false},{"mapping":"root.result = this.compressed.decode(\"base64\").decompress(\"lz4\").string()","summary":"Use the `.string()` method in order to coerce the result into a string, this makes it possible to place the data within a JSON document without automatic base64 encoding.","results":[["{\"compressed\":\"BCJNGGRwuRgAAIBoZWxsbyB3b3JsZCBJIGxvdmUgc3BhY2UAAAAAGoETLg==\"}","{\"result\":\"hello world I love space\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"decrypt_aes","params":{"named":[{"name":"scheme","description":"The scheme to use for decryption, one of `ctr`, `gcm`, `ofb`, `cbc`.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"key","description":"A key to decrypt with.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"iv","description":"An initialization vector / nonce.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Encoding and Encryption","Description":"Decrypts an encrypted string or byte array target according to a chosen AES encryption method and returns the result as a byte array. The algorithms require a key and an initialization vector / nonce. Available schemes are: `ctr`, `gcm`, `ofb`, `cbc`.","Examples":[{"mapping":"let key = \"2b7e151628aed2a6abf7158809cf4f3c\".decode(\"hex\")\nlet vector = \"f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff\".decode(\"hex\")\nroot.decrypted = this.value.decode(\"hex\").decrypt_aes(\"ctr\", $key, $vector).string()","summary":"","results":[["{\"value\":\"84e9b31ff7400bdf80be7254\"}","{\"decrypted\":\"hello world!\"}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"diff","description":"Create a diff by comparing the current value with the given one. Wraps the github.com/r3labs/diff/v3 package. See its https://pkg.go.dev/github.com/r3labs/diff/v3[docs^] for more information.","params":{"named":[{"name":"other","description":"The value to compare against.","type":"unknown","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":null}],"impure":false,"version":"4.25.0"},{"status":"stable","name":"encode","params":{"named":[{"name":"scheme","description":"The encoding scheme to use.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Encoding and Encryption","Description":"Encodes a string or byte array target according to a chosen scheme and returns a string result. Available schemes are: `base64`, `base64url` https://rfc-editor.org/rfc/rfc4648.html[(RFC 4648 with padding characters)], `base64rawurl` https://rfc-editor.org/rfc/rfc4648.html[(RFC 4648 without padding characters)], `hex`, `ascii85`.","Examples":[{"mapping":"root.encoded = this.value.encode(\"hex\")","summary":"","results":[["{\"value\":\"hello world\"}","{\"encoded\":\"68656c6c6f20776f726c64\"}"]],"skip_testing":false},{"mapping":"root.encoded = content().encode(\"ascii85\")","summary":"","results":[["this is totally unstructured data","{\"encoded\":\"FD,B0+DGm\u003eFDl80Ci\\\"A\u003eF`)8BEckl6F`M\u0026(+Cno\u0026@/\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"encrypt_aes","params":{"named":[{"name":"scheme","description":"The scheme to use for encryption, one of `ctr`, `gcm`, `ofb`, `cbc`.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"key","description":"A key to encrypt with.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"iv","description":"An initialization vector / nonce.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Encoding and Encryption","Description":"Encrypts a string or byte array target according to a chosen AES encryption method and returns a string result. The algorithms require a key and an initialization vector / nonce. Available schemes are: `ctr`, `gcm`, `ofb`, `cbc`.","Examples":[{"mapping":"let key = \"2b7e151628aed2a6abf7158809cf4f3c\".decode(\"hex\")\nlet vector = \"f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff\".decode(\"hex\")\nroot.encrypted = this.value.encrypt_aes(\"ctr\", $key, $vector).encode(\"hex\")","summary":"","results":[["{\"value\":\"hello world!\"}","{\"encrypted\":\"84e9b31ff7400bdf80be7254\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"enumerated","description":"Converts an array into a new array of objects, where each object has a field index containing the `index` of the element and a field `value` containing the original value of the element.","params":{},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.foo = this.foo.enumerated()","summary":"","results":[["{\"foo\":[\"bar\",\"baz\"]}","{\"foo\":[{\"index\":0,\"value\":\"bar\"},{\"index\":1,\"value\":\"baz\"}]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"escape_html","params":{},"categories":[{"Category":"String Manipulation","Description":"Escapes a string so that special characters like `\u003c` to become `\u0026lt;`. It escapes only five such characters: `\u003c`, `\u003e`, `\u0026`, `'` and `\"` so that it can be safely placed within an HTML entity.","Examples":[{"mapping":"root.escaped = this.value.escape_html()","summary":"","results":[["{\"value\":\"foo \u0026 bar\"}","{\"escaped\":\"foo \u0026amp; bar\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"escape_url_query","params":{},"categories":[{"Category":"String Manipulation","Description":"Escapes a string so that it can be safely placed within a URL query.","Examples":[{"mapping":"root.escaped = this.value.escape_url_query()","summary":"","results":[["{\"value\":\"foo \u0026 bar\"}","{\"escaped\":\"foo+%26+bar\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"exists","description":"Checks that a field, identified via a xref:configuration:field_paths.adoc[dot path], exists in an object.","params":{"named":[{"name":"path","description":"A xref:configuration:field_paths.adoc[dot path] to a field.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"examples":[{"mapping":"root.result = this.foo.exists(\"bar.baz\")","summary":"","results":[["{\"foo\":{\"bar\":{\"baz\":\"yep, I exist\"}}}","{\"result\":true}"],["{\"foo\":{\"bar\":{}}}","{\"result\":false}"],["{\"foo\":{}}","{\"result\":false}"]],"skip_testing":false}],"impure":false},{"status":"stable","name":"explode","params":{"named":[{"name":"path","description":"A xref:configuration:field_paths.adoc[dot path] to a field to explode.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"Explodes an array or object at a xref:configuration:field_paths.adoc[field path].","Examples":[{"mapping":"root = this.explode(\"value\")","summary":"##### On arrays\n\nExploding arrays results in an array containing elements matching the original document, where the target field of each element is an element of the exploded array:","results":[["{\"id\":1,\"value\":[\"foo\",\"bar\",\"baz\"]}","[{\"id\":1,\"value\":\"foo\"},{\"id\":1,\"value\":\"bar\"},{\"id\":1,\"value\":\"baz\"}]"]],"skip_testing":false},{"mapping":"root = this.explode(\"value\")","summary":"##### On objects\n\nExploding objects results in an object where the keys match the target object, and the values match the original document but with the target field replaced by the exploded value:","results":[["{\"id\":1,\"value\":{\"foo\":2,\"bar\":[3,4],\"baz\":{\"bev\":5}}}","{\"bar\":{\"id\":1,\"value\":[3,4]},\"baz\":{\"id\":1,\"value\":{\"bev\":5}},\"foo\":{\"id\":1,\"value\":2}}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"filepath_join","params":{},"categories":[{"Category":"String Manipulation","Description":"Joins an array of path elements into a single file path. The separator depends on the operating system of the machine.","Examples":[{"mapping":"root.path = this.path_elements.filepath_join()","summary":"","results":[["{\"path_elements\":[\"/foo/\",\"bar.txt\"]}","{\"path\":\"/foo/bar.txt\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"filepath_split","params":{},"categories":[{"Category":"String Manipulation","Description":"Splits a file path immediately following the final Separator, separating it into a directory and file name component returned as a two element array of strings. If there is no Separator in the path, the first element will be empty and the second will contain the path. The separator depends on the operating system of the machine.","Examples":[{"mapping":"root.path_sep = this.path.filepath_split()","summary":"","results":[["{\"path\":\"/foo/bar.txt\"}","{\"path_sep\":[\"/foo/\",\"bar.txt\"]}"],["{\"path\":\"baz.txt\"}","{\"path_sep\":[\"\",\"baz.txt\"]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"filter","params":{"named":[{"name":"test","description":"A query to apply to each element, if this query resolves to any value other than a boolean `true` the element will be removed from the result.","type":"query expression","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"Executes a mapping query argument for each element of an array or key/value pair of an object. If the query returns `false` the item is removed from the resulting array or object. The item will also be removed if the query returns any non-boolean value.","Examples":[{"mapping":"root.new_nums = this.nums.filter(num -\u003e num \u003e 10)","summary":"","results":[["{\"nums\":[3,11,4,17]}","{\"new_nums\":[11,17]}"]],"skip_testing":false},{"mapping":"root.new_dict = this.dict.filter(item -\u003e item.value.contains(\"foo\"))","summary":"##### On objects\n\nWhen filtering objects the mapping query argument is provided a context with a field `key` containing the value key, and a field `value` containing the value.","results":[["{\"dict\":{\"first\":\"hello foo\",\"second\":\"world\",\"third\":\"this foo is great\"}}","{\"new_dict\":{\"first\":\"hello foo\",\"third\":\"this foo is great\"}}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"find","description":"Returns the index of the first occurrence of a value in an array. `-1` is returned if there are no matches. Numerical comparisons are made irrespective of the representation type (float versus integer).","params":{"named":[{"name":"value","description":"A value to find.","type":"unknown","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.index = this.find(\"bar\")","summary":"","results":[["[\"foo\", \"bar\", \"baz\"]","{\"index\":1}"]],"skip_testing":false},{"mapping":"root.index = this.things.find(this.goal)","summary":"","results":[["{\"goal\":\"bar\",\"things\":[\"foo\", \"bar\", \"baz\"]}","{\"index\":1}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"find_all","description":"Returns an array containing the indexes of all occurrences of a value in an array. An empty array is returned if there are no matches. Numerical comparisons are made irrespective of the representation type (float versus integer).","params":{"named":[{"name":"value","description":"A value to find.","type":"unknown","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.index = this.find_all(\"bar\")","summary":"","results":[["[\"foo\", \"bar\", \"baz\", \"bar\"]","{\"index\":[1,3]}"]],"skip_testing":false},{"mapping":"root.indexes = this.things.find_all(this.goal)","summary":"","results":[["{\"goal\":\"bar\",\"things\":[\"foo\", \"bar\", \"baz\", \"bar\", \"buz\"]}","{\"indexes\":[1,3]}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"find_all_by","description":"Returns an array containing the indexes of all occurrences of an array where the provided query resolves to a boolean `true`. An empty array is returned if there are no matches. Numerical comparisons are made irrespective of the representation type (float versus integer).","params":{"named":[{"name":"query","description":"A query to execute for each element.","type":"query expression","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.index = this.find_all_by(v -\u003e v != \"bar\")","summary":"","results":[["[\"foo\", \"bar\", \"baz\"]","{\"index\":[0,2]}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"find_by","description":"Returns the index of the first occurrence of an array where the provided query resolves to a boolean `true`. `-1` is returned if there are no matches.","params":{"named":[{"name":"query","description":"A query to execute for each element.","type":"query expression","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.index = this.find_by(v -\u003e v != \"bar\")","summary":"","results":[["[\"foo\", \"bar\", \"baz\"]","{\"index\":0}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"flatten","description":"Iterates an array and any element that is itself an array is removed and has its elements inserted directly in the resulting array.","params":{},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.result = this.flatten()","summary":"","results":[["[\"foo\",[\"bar\",\"baz\"],\"buz\"]","{\"result\":[\"foo\",\"bar\",\"baz\",\"buz\"]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"float32","description":"\nConverts a numerical type into a 32-bit floating point number, this is for advanced use cases where a specific data type is needed for a given component (such as the ClickHouse SQL driver).\n\nIf the value is a string then an attempt will be made to parse it as a 32-bit floating point number. Please refer to the https://pkg.go.dev/strconv#ParseFloat[`strconv.ParseFloat` documentation] for details regarding the supported formats.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"\nroot.out = this.in.float32()\n","summary":"","results":[["{\"in\":\"6.674282313423543523453425345e-11\"}","{\"out\":6.674283e-11}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"float64","description":"\nConverts a numerical type into a 64-bit floating point number, this is for advanced use cases where a specific data type is needed for a given component (such as the ClickHouse SQL driver).\n\nIf the value is a string then an attempt will be made to parse it as a 64-bit floating point number. Please refer to the https://pkg.go.dev/strconv#ParseFloat[`strconv.ParseFloat` documentation] for details regarding the supported formats.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"\nroot.out = this.in.float64()\n","summary":"","results":[["{\"in\":\"6.674282313423543523453425345e-11\"}","{\"out\":6.674282313423544e-11}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"floor","description":"Returns the greatest integer value less than or equal to the target number. If the resulting value fits within a 64-bit integer then that is returned, otherwise a new floating point number is returned.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"root.new_value = this.value.floor()","summary":"","results":[["{\"value\":5.7}","{\"new_value\":5}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"fold","description":"Takes two arguments: an initial value, and a mapping query. For each element of an array the mapping context is an object with two fields `tally` and `value`, where `tally` contains the current accumulated value and `value` is the value of the current element. The mapping must return the result of adding the value to the tally.\n\nThe first argument is the value that `tally` will have on the first call.","params":{"named":[{"name":"initial","description":"The initial value to start the fold with. For example, an empty object `{}`, a zero count `0`, or an empty string `\"\"`.","type":"unknown","no_dynamic":false,"scalars_to_literal":false},{"name":"query","description":"A query to apply for each element. The query is provided an object with two fields; `tally` containing the current tally, and `value` containing the value of the current element. The query should result in a new tally to be passed to the next element query.","type":"query expression","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.sum = this.foo.fold(0, item -\u003e item.tally + item.value)","summary":"","results":[["{\"foo\":[3,8,11]}","{\"sum\":22}"]],"skip_testing":false},{"mapping":"root.result = this.foo.fold(\"\", item -\u003e \"%v%v\".format(item.tally, item.value))","summary":"","results":[["{\"foo\":[\"hello \", \"world\"]}","{\"result\":\"hello world\"}"]],"skip_testing":false},{"mapping":"root.smoothie = this.fruits.fold({}, item -\u003e item.tally.merge(item.value))","summary":"You can use fold to merge an array of objects together:","results":[["{\"fruits\":[{\"apple\":5},{\"banana\":3},{\"orange\":8}]}","{\"smoothie\":{\"apple\":5,\"banana\":3,\"orange\":8}}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"format","params":{"variadic":true},"categories":[{"Category":"String Manipulation","Description":"Use a value string as a format specifier in order to produce a new string, using any number of provided arguments. Please refer to the Go https://pkg.go.dev/fmt[`fmt` package documentation^] for the list of valid format verbs.","Examples":[{"mapping":"root.foo = \"%s(%v): %v\".format(this.name, this.age, this.fingers)","summary":"","results":[["{\"name\":\"lance\",\"age\":37,\"fingers\":13}","{\"foo\":\"lance(37): 13\"}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"format_json","params":{"named":[{"name":"indent","description":"Indentation string. Each element in a JSON object or array will begin on a new, indented line followed by one or more copies of indent according to the indentation nesting.","type":"string","no_dynamic":false,"scalars_to_literal":false,"default":" "},{"name":"no_indent","description":"Disable indentation.","type":"bool","no_dynamic":false,"scalars_to_literal":false,"default":false},{"name":"escape_html","description":"Escape problematic HTML characters.","type":"bool","no_dynamic":false,"scalars_to_literal":false,"default":true}]},"categories":[{"Category":"Parsing","Description":"Serializes a target value into a pretty-printed JSON byte array (with 4 space indentation by default).","Examples":[{"mapping":"root = this.doc.format_json()","summary":"","results":[["{\"doc\":{\"foo\":\"bar\"}}","{\n \"foo\": \"bar\"\n}"]],"skip_testing":false},{"mapping":"root = this.format_json(\" \")","summary":"Pass a string to the `indent` parameter in order to customise the indentation.","results":[["{\"doc\":{\"foo\":\"bar\"}}","{\n \"doc\": {\n \"foo\": \"bar\"\n }\n}"]],"skip_testing":false},{"mapping":"root.doc = this.doc.format_json().string()","summary":"Use the `.string()` method in order to coerce the result into a string.","results":[["{\"doc\":{\"foo\":\"bar\"}}","{\"doc\":\"{\\n \\\"foo\\\": \\\"bar\\\"\\n}\"}"]],"skip_testing":false},{"mapping":"root = this.doc.format_json(no_indent: true)","summary":"Set the `no_indent` parameter to true to disable indentation. The result is equivalent to calling `bytes()`.","results":[["{\"doc\":{\"foo\":\"bar\"}}","{\"foo\":\"bar\"}"]],"skip_testing":false},{"mapping":"root = this.doc.format_json()","summary":"Escapes problematic HTML characters.","results":[["{\"doc\":{\"email\":\"foo\u0026bar@benthos.dev\",\"name\":\"foo\u003ebar\"}}","{\n \"email\": \"foo\\u0026bar@benthos.dev\",\n \"name\": \"foo\\u003ebar\"\n}"]],"skip_testing":false},{"mapping":"root = this.doc.format_json(escape_html: false)","summary":"Set the `escape_html` parameter to false to disable escaping of problematic HTML characters.","results":[["{\"doc\":{\"email\":\"foo\u0026bar@benthos.dev\",\"name\":\"foo\u003ebar\"}}","{\n \"email\": \"foo\u0026bar@benthos.dev\",\n \"name\": \"foo\u003ebar\"\n}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"format_msgpack","description":"Formats data as a https://msgpack.org/[MessagePack^] message in bytes format.","params":{},"categories":[{"Category":"Parsing","Description":"","Examples":[{"mapping":"root = this.format_msgpack().encode(\"hex\")","summary":"","results":[["{\"foo\":\"bar\"}","81a3666f6fa3626172"]],"skip_testing":false},{"mapping":"root.encoded = this.format_msgpack().encode(\"base64\")","summary":"","results":[["{\"foo\":\"bar\"}","{\"encoded\":\"gaNmb2+jYmFy\"}"]],"skip_testing":false}]}],"impure":false},{"status":"deprecated","name":"format_timestamp","description":"Attempts to format a timestamp value as a string according to a specified format, or RFC 3339 by default. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format.\n\nThe output format is defined by showing how the reference time, defined to be Mon Jan 2 15:04:05 -0700 MST 2006, would be displayed if it were the value. For an alternative way to specify formats check out the \u003c\u003cts_strftime, `ts_strftime`\u003e\u003e method.","params":{"named":[{"name":"format","description":"The output format to use.","type":"string","no_dynamic":false,"scalars_to_literal":false,"default":"2006-01-02T15:04:05.999999999Z07:00"},{"name":"tz","description":"An optional timezone to use, otherwise the timezone of the input string is used, or in the case of unix timestamps the local timezone is used.","type":"string","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"categories":[{"Category":"Deprecated","Description":"","Examples":null}],"impure":false},{"status":"deprecated","name":"format_timestamp_strftime","description":"Attempts to format a timestamp value as a string according to a specified strftime-compatible format. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format.","params":{"named":[{"name":"format","description":"The output format to use.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"tz","description":"An optional timezone to use, otherwise the timezone of the input string is used.","type":"string","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"categories":[{"Category":"Deprecated","Description":"","Examples":null}],"impure":false},{"status":"deprecated","name":"format_timestamp_unix","description":"Attempts to format a timestamp value as a unix timestamp. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format. The \u003c\u003cts_parse, `ts_parse`\u003e\u003e method can be used in order to parse different timestamp formats.","params":{},"categories":[{"Category":"Deprecated","Description":"","Examples":null}],"impure":false},{"status":"deprecated","name":"format_timestamp_unix_micro","description":"Attempts to format a timestamp value as a unix timestamp with microsecond precision. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format. The \u003c\u003cts_parse, `ts_parse`\u003e\u003e method can be used in order to parse different timestamp formats.","params":{},"categories":[{"Category":"Deprecated","Description":"","Examples":null}],"impure":false},{"status":"deprecated","name":"format_timestamp_unix_milli","description":"Attempts to format a timestamp value as a unix timestamp with millisecond precision. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format. The \u003c\u003cts_parse, `ts_parse`\u003e\u003e method can be used in order to parse different timestamp formats.","params":{},"categories":[{"Category":"Deprecated","Description":"","Examples":null}],"impure":false},{"status":"deprecated","name":"format_timestamp_unix_nano","description":"Attempts to format a timestamp value as a unix timestamp with nanosecond precision. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format. The \u003c\u003cts_parse, `ts_parse`\u003e\u003e method can be used in order to parse different timestamp formats.","params":{},"categories":[{"Category":"Deprecated","Description":"","Examples":null}],"impure":false},{"status":"stable","name":"format_xml","description":"\nSerializes a target value into an XML byte array.\n","params":{"named":[{"name":"indent","description":"Indentation string. Each element in an XML object or array will begin on a new, indented line followed by one or more copies of indent according to the indentation nesting.","type":"string","no_dynamic":false,"scalars_to_literal":false,"default":" "},{"name":"no_indent","description":"Disable indentation.","type":"bool","no_dynamic":false,"scalars_to_literal":false,"default":false}]},"categories":[{"Category":"Parsing","Description":"","Examples":[{"mapping":"root = this.format_xml()","summary":"Serializes a target value into a pretty-printed XML byte array (with 4 space indentation by default).","results":[["{\"foo\":{\"bar\":{\"baz\":\"foo bar baz\"}}}","\u003cfoo\u003e\n \u003cbar\u003e\n \u003cbaz\u003efoo bar baz\u003c/baz\u003e\n \u003c/bar\u003e\n\u003c/foo\u003e"]],"skip_testing":false},{"mapping":"root = this.format_xml(\" \")","summary":"Pass a string to the `indent` parameter in order to customise the indentation.","results":[["{\"foo\":{\"bar\":{\"baz\":\"foo bar baz\"}}}","\u003cfoo\u003e\n \u003cbar\u003e\n \u003cbaz\u003efoo bar baz\u003c/baz\u003e\n \u003c/bar\u003e\n\u003c/foo\u003e"]],"skip_testing":false},{"mapping":"root.doc = this.format_xml(\"\").string()","summary":"Use the `.string()` method in order to coerce the result into a string.","results":[["{\"foo\":{\"bar\":{\"baz\":\"foo bar baz\"}}}","{\"doc\":\"\u003cfoo\u003e\\n\u003cbar\u003e\\n\u003cbaz\u003efoo bar baz\u003c/baz\u003e\\n\u003c/bar\u003e\\n\u003c/foo\u003e\"}"]],"skip_testing":false},{"mapping":"root = this.format_xml(no_indent: true)","summary":"Set the `no_indent` parameter to true to disable indentation.","results":[["{\"foo\":{\"bar\":{\"baz\":\"foo bar baz\"}}}","\u003cfoo\u003e\u003cbar\u003e\u003cbaz\u003efoo bar baz\u003c/baz\u003e\u003c/bar\u003e\u003c/foo\u003e"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"format_yaml","params":{},"categories":[{"Category":"Parsing","Description":"Serializes a target value into a YAML byte array.","Examples":[{"mapping":"root = this.doc.format_yaml()","summary":"","results":[["{\"doc\":{\"foo\":\"bar\"}}","foo: bar\n"]],"skip_testing":false},{"mapping":"root.doc = this.doc.format_yaml().string()","summary":"Use the `.string()` method in order to coerce the result into a string.","results":[["{\"doc\":{\"foo\":\"bar\"}}","{\"doc\":\"foo: bar\\n\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"from","description":"Modifies a target query such that certain functions are executed from the perspective of another message in the batch. This allows you to mutate events based on the contents of other messages. Functions that support this behavior are `content`, `json` and `meta`.","params":{"named":[{"name":"index","description":"The message index to use as a perspective.","type":"integer","no_dynamic":false,"scalars_to_literal":false}]},"examples":[{"mapping":"root = this\nroot.foo = json(\"foo\").from(1)","summary":"For example, the following map extracts the contents of the JSON field `foo` specifically from message index `1` of a batch, effectively overriding the field `foo` for all messages of a batch to that of message 1:","results":[],"skip_testing":false}],"impure":false},{"status":"stable","name":"from_all","description":"Modifies a target query such that certain functions are executed from the perspective of each message in the batch, and returns the set of results as an array. Functions that support this behavior are `content`, `json` and `meta`.","params":{},"examples":[{"mapping":"root = this\nroot.foo_summed = json(\"foo\").from_all().sum()","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"experimental","name":"geoip_anonymous_ip","description":"Looks up an IP address against a https://www.maxmind.com/en/home[MaxMind database file^] and, if found, returns an object describing the anonymous IP associated with it.","params":{"named":[{"name":"path","description":"A path to an mmdb (maxmind) file.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"GeoIP","Description":"","Examples":null}],"impure":false},{"status":"experimental","name":"geoip_asn","description":"Looks up an IP address against a https://www.maxmind.com/en/home[MaxMind database file^] and, if found, returns an object describing the ASN associated with it.","params":{"named":[{"name":"path","description":"A path to an mmdb (maxmind) file.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"GeoIP","Description":"","Examples":null}],"impure":false},{"status":"experimental","name":"geoip_city","description":"Looks up an IP address against a https://www.maxmind.com/en/home[MaxMind database file^] and, if found, returns an object describing the city associated with it.","params":{"named":[{"name":"path","description":"A path to an mmdb (maxmind) file.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"GeoIP","Description":"","Examples":null}],"impure":false},{"status":"experimental","name":"geoip_connection_type","description":"Looks up an IP address against a https://www.maxmind.com/en/home[MaxMind database file^] and, if found, returns an object describing the connection type associated with it.","params":{"named":[{"name":"path","description":"A path to an mmdb (maxmind) file.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"GeoIP","Description":"","Examples":null}],"impure":false},{"status":"experimental","name":"geoip_country","description":"Looks up an IP address against a https://www.maxmind.com/en/home[MaxMind database file^] and, if found, returns an object describing the country associated with it.","params":{"named":[{"name":"path","description":"A path to an mmdb (maxmind) file.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"GeoIP","Description":"","Examples":null}],"impure":false},{"status":"experimental","name":"geoip_domain","description":"Looks up an IP address against a https://www.maxmind.com/en/home[MaxMind database file^] and, if found, returns an object describing the domain associated with it.","params":{"named":[{"name":"path","description":"A path to an mmdb (maxmind) file.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"GeoIP","Description":"","Examples":null}],"impure":false},{"status":"experimental","name":"geoip_enterprise","description":"Looks up an IP address against a https://www.maxmind.com/en/home[MaxMind database file^] and, if found, returns an object describing the enterprise associated with it.","params":{"named":[{"name":"path","description":"A path to an mmdb (maxmind) file.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"GeoIP","Description":"","Examples":null}],"impure":false},{"status":"experimental","name":"geoip_isp","description":"Looks up an IP address against a https://www.maxmind.com/en/home[MaxMind database file^] and, if found, returns an object describing the ISP associated with it.","params":{"named":[{"name":"path","description":"A path to an mmdb (maxmind) file.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"GeoIP","Description":"","Examples":null}],"impure":false},{"status":"stable","name":"get","description":"Extract a field value, identified via a xref:configuration:field_paths.adoc[dot path], from an object.","params":{"named":[{"name":"path","description":"A xref:configuration:field_paths.adoc[dot path] identifying a field to obtain.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.result = this.foo.get(this.target)","summary":"","results":[["{\"foo\":{\"bar\":\"from bar\",\"baz\":\"from baz\"},\"target\":\"bar\"}","{\"result\":\"from bar\"}"],["{\"foo\":{\"bar\":\"from bar\",\"baz\":\"from baz\"},\"target\":\"baz\"}","{\"result\":\"from baz\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"has_prefix","params":{"named":[{"name":"value","description":"The string to test.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"String Manipulation","Description":"Checks whether a string has a prefix argument and returns a bool.","Examples":[{"mapping":"root.t1 = this.v1.has_prefix(\"foo\")\nroot.t2 = this.v2.has_prefix(\"foo\")","summary":"","results":[["{\"v1\":\"foobar\",\"v2\":\"barfoo\"}","{\"t1\":true,\"t2\":false}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"has_suffix","params":{"named":[{"name":"value","description":"The string to test.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"String Manipulation","Description":"Checks whether a string has a suffix argument and returns a bool.","Examples":[{"mapping":"root.t1 = this.v1.has_suffix(\"foo\")\nroot.t2 = this.v2.has_suffix(\"foo\")","summary":"","results":[["{\"v1\":\"foobar\",\"v2\":\"barfoo\"}","{\"t1\":false,\"t2\":true}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"hash","params":{"named":[{"name":"algorithm","description":"The hasing algorithm to use.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"key","description":"An optional key to use.","type":"string","no_dynamic":false,"scalars_to_literal":false,"is_optional":true},{"name":"polynomial","description":"An optional polynomial key to use when selecting the `crc32` algorithm, otherwise ignored. Options are `IEEE` (default), `Castagnoli` and `Koopman`","type":"string","no_dynamic":false,"scalars_to_literal":false,"default":"IEEE"}]},"categories":[{"Category":"Encoding and Encryption","Description":"\nHashes a string or byte array according to a chosen algorithm and returns the result as a byte array. When mapping the result to a JSON field the value should be cast to a string using the method xref:guides:bloblang/methods.adoc#string[`string`], or encoded using the method xref:guides:bloblang/methods.adoc#encode[`encode`], otherwise it will be base64 encoded by default.\n\nAvailable algorithms are: `hmac_sha1`, `hmac_sha256`, `hmac_sha512`, `md5`, `sha1`, `sha256`, `sha512`, `xxhash64`, `crc32`, `fnv32`.\n\nThe following algorithms require a key, which is specified as a second argument: `hmac_sha1`, `hmac_sha256`, `hmac_sha512`.","Examples":[{"mapping":"root.h1 = this.value.hash(\"sha1\").encode(\"hex\")\nroot.h2 = this.value.hash(\"hmac_sha1\",\"static-key\").encode(\"hex\")","summary":"","results":[["{\"value\":\"hello world\"}","{\"h1\":\"2aae6c35c94fcfb415dbe95f408b9ce91ee846ed\",\"h2\":\"d87e5f068fa08fe90bb95bc7c8344cb809179d76\"}"]],"skip_testing":false},{"mapping":"root.h1 = this.value.hash(algorithm: \"crc32\", polynomial: \"Castagnoli\").encode(\"hex\")\nroot.h2 = this.value.hash(algorithm: \"crc32\", polynomial: \"Koopman\").encode(\"hex\")","summary":"The `crc32` algorithm supports options for the polynomial.","results":[["{\"value\":\"hello world\"}","{\"h1\":\"c99465aa\",\"h2\":\"df373d3c\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"index","description":"Extract an element from an array by an index. The index can be negative, and if so the element will be selected from the end counting backwards starting from -1. E.g. an index of -1 returns the last element, an index of -2 returns the element before the last, and so on.","params":{"named":[{"name":"index","description":"The index to obtain from an array.","type":"integer","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.last_name = this.names.index(-1)","summary":"","results":[["{\"names\":[\"rachel\",\"stevens\"]}","{\"last_name\":\"stevens\"}"]],"skip_testing":false},{"mapping":"root.last_byte = this.name.bytes().index(-1)","summary":"It is also possible to use this method on byte arrays, in which case the selected element will be returned as an integer.","results":[["{\"name\":\"foobar bazson\"}","{\"last_byte\":110}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"index_of","params":{"named":[{"name":"value","description":"A string to search for.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"String Manipulation","Description":"Returns the starting index of the argument substring in a string target, or `-1` if the target doesn't contain the argument.","Examples":[{"mapping":"root.index = this.thing.index_of(\"bar\")","summary":"","results":[["{\"thing\":\"foobar\"}","{\"index\":3}"]],"skip_testing":false},{"mapping":"root.index = content().index_of(\"meow\")","summary":"","results":[["the cat meowed, the dog woofed","{\"index\":8}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"int16","description":"\nConverts a numerical type into a 16-bit signed integer, this is for advanced use cases where a specific data type is needed for a given component (such as the ClickHouse SQL driver).\n\nIf the value is a string then an attempt will be made to parse it as a 16-bit signed integer. If the target value exceeds the capacity of an integer or contains decimal values then this method will throw an error. In order to convert a floating point number containing decimals first use \u003c\u003cround, `.round()`\u003e\u003e on the value. Please refer to the https://pkg.go.dev/strconv#ParseInt[`strconv.ParseInt` documentation] for details regarding the supported formats.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"\nroot.a = this.a.int16()\nroot.b = this.b.round().int16()\nroot.c = this.c.int16()\nroot.d = this.d.int16().catch(0)\n","summary":"","results":[["{\"a\":12,\"b\":12.34,\"c\":\"12\",\"d\":-12}","{\"a\":12,\"b\":12,\"c\":12,\"d\":-12}"]],"skip_testing":false},{"mapping":"\nroot = this.int16()\n","summary":"","results":[["\"0xDE\"","222"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"int32","description":"\nConverts a numerical type into a 32-bit signed integer, this is for advanced use cases where a specific data type is needed for a given component (such as the ClickHouse SQL driver).\n\nIf the value is a string then an attempt will be made to parse it as a 32-bit signed integer. If the target value exceeds the capacity of an integer or contains decimal values then this method will throw an error. In order to convert a floating point number containing decimals first use \u003c\u003cround, `.round()`\u003e\u003e on the value. Please refer to the https://pkg.go.dev/strconv#ParseInt[`strconv.ParseInt` documentation] for details regarding the supported formats.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"\nroot.a = this.a.int32()\nroot.b = this.b.round().int32()\nroot.c = this.c.int32()\nroot.d = this.d.int32().catch(0)\n","summary":"","results":[["{\"a\":12,\"b\":12.34,\"c\":\"12\",\"d\":-12}","{\"a\":12,\"b\":12,\"c\":12,\"d\":-12}"]],"skip_testing":false},{"mapping":"\nroot = this.int32()\n","summary":"","results":[["\"0xDEAD\"","57005"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"int64","description":"\nConverts a numerical type into a 64-bit signed integer, this is for advanced use cases where a specific data type is needed for a given component (such as the ClickHouse SQL driver).\n\nIf the value is a string then an attempt will be made to parse it as a 64-bit signed integer. If the target value exceeds the capacity of an integer or contains decimal values then this method will throw an error. In order to convert a floating point number containing decimals first use \u003c\u003cround, `.round()`\u003e\u003e on the value. Please refer to the https://pkg.go.dev/strconv#ParseInt[`strconv.ParseInt` documentation] for details regarding the supported formats.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"\nroot.a = this.a.int64()\nroot.b = this.b.round().int64()\nroot.c = this.c.int64()\nroot.d = this.d.int64().catch(0)\n","summary":"","results":[["{\"a\":12,\"b\":12.34,\"c\":\"12\",\"d\":-12}","{\"a\":12,\"b\":12,\"c\":12,\"d\":-12}"]],"skip_testing":false},{"mapping":"\nroot = this.int64()\n","summary":"","results":[["\"0xDEADBEEF\"","3735928559"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"int8","description":"\nConverts a numerical type into a 8-bit signed integer, this is for advanced use cases where a specific data type is needed for a given component (such as the ClickHouse SQL driver).\n\nIf the value is a string then an attempt will be made to parse it as a 8-bit signed integer. If the target value exceeds the capacity of an integer or contains decimal values then this method will throw an error. In order to convert a floating point number containing decimals first use \u003c\u003cround, `.round()`\u003e\u003e on the value. Please refer to the https://pkg.go.dev/strconv#ParseInt[`strconv.ParseInt` documentation] for details regarding the supported formats.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"\nroot.a = this.a.int8()\nroot.b = this.b.round().int8()\nroot.c = this.c.int8()\nroot.d = this.d.int8().catch(0)\n","summary":"","results":[["{\"a\":12,\"b\":12.34,\"c\":\"12\",\"d\":-12}","{\"a\":12,\"b\":12,\"c\":12,\"d\":-12}"]],"skip_testing":false},{"mapping":"\nroot = this.int8()\n","summary":"","results":[["\"0xD\"","13"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"join","params":{"named":[{"name":"delimiter","description":"An optional delimiter to add between each string.","type":"string","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"Join an array of strings with an optional delimiter into a single string.","Examples":[{"mapping":"root.joined_words = this.words.join()\nroot.joined_numbers = this.numbers.map_each(this.string()).join(\",\")","summary":"","results":[["{\"words\":[\"hello\",\"world\"],\"numbers\":[3,8,11]}","{\"joined_numbers\":\"3,8,11\",\"joined_words\":\"helloworld\"}"]],"skip_testing":false}]}],"impure":false},{"status":"experimental","name":"json_path","description":"Executes the given JSONPath expression on an object or array and returns the result. The JSONPath expression syntax can be found at https://goessner.net/articles/JsonPath/. For more complex logic, you can use Gval expressions (https://github.com/PaesslerAG/gval).","params":{"named":[{"name":"expression","description":"The JSONPath expression to execute.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.all_names = this.json_path(\"$..name\")","summary":"","results":[["{\"name\":\"alice\",\"foo\":{\"name\":\"bob\"}}","{\"all_names\":[\"alice\",\"bob\"]}"],["{\"thing\":[\"this\",\"bar\",{\"name\":\"alice\"}]}","{\"all_names\":[\"alice\"]}"]],"skip_testing":false},{"mapping":"root.text_objects = this.json_path(\"$.body[?(@.type=='text')]\")","summary":"","results":[["{\"body\":[{\"type\":\"image\",\"id\":\"foo\"},{\"type\":\"text\",\"id\":\"bar\"}]}","{\"text_objects\":[{\"id\":\"bar\",\"type\":\"text\"}]}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"json_schema","description":"Checks a https://json-schema.org/[JSON schema^] against a value and returns the value if it matches or throws and error if it does not.","params":{"named":[{"name":"schema","description":"The schema to check values against.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root = this.json_schema(\"\"\"{\n \"type\":\"object\",\n \"properties\":{\n \"foo\":{\n \"type\":\"string\"\n }\n }\n}\"\"\")","summary":"","results":[["{\"foo\":\"bar\"}","{\"foo\":\"bar\"}"],["{\"foo\":5}","Error(\"failed assignment (line 1): field `this`: foo invalid type. expected: string, given: integer\")"]],"skip_testing":false},{"mapping":"root = this.json_schema(file(env(\"BENTHOS_TEST_BLOBLANG_SCHEMA_FILE\")))","summary":"In order to load a schema from a file use the `file` function.","results":[],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"key_values","description":"Returns the key/value pairs of an object as an array, where each element is an object with a `key` field and a `value` field. The order of the resulting array will be random.","params":{},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.foo_key_values = this.foo.key_values().sort_by(pair -\u003e pair.key)","summary":"","results":[["{\"foo\":{\"bar\":1,\"baz\":2}}","{\"foo_key_values\":[{\"key\":\"bar\",\"value\":1},{\"key\":\"baz\",\"value\":2}]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"keys","description":"Returns the keys of an object as an array.","params":{},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.foo_keys = this.foo.keys()","summary":"","results":[["{\"foo\":{\"bar\":1,\"baz\":2}}","{\"foo_keys\":[\"bar\",\"baz\"]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"length","params":{},"categories":[{"Category":"String Manipulation","Description":"Returns the length of a string.","Examples":[{"mapping":"root.foo_len = this.foo.length()","summary":"","results":[["{\"foo\":\"hello world\"}","{\"foo_len\":11}"]],"skip_testing":false}]},{"Category":"Object \u0026 Array Manipulation","Description":"Returns the length of an array or object (number of keys).","Examples":[{"mapping":"root.foo_len = this.foo.length()","summary":"","results":[["{\"foo\":[\"first\",\"second\"]}","{\"foo_len\":2}"],["{\"foo\":{\"first\":\"bar\",\"second\":\"baz\"}}","{\"foo_len\":2}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"log","description":"Returns the natural logarithm of a number.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"root.new_value = this.value.log().round()","summary":"","results":[["{\"value\":1}","{\"new_value\":0}"],["{\"value\":2.7183}","{\"new_value\":1}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"log10","description":"Returns the decimal logarithm of a number.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"root.new_value = this.value.log10()","summary":"","results":[["{\"value\":100}","{\"new_value\":2}"],["{\"value\":1000}","{\"new_value\":3}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"lowercase","params":{},"categories":[{"Category":"String Manipulation","Description":"Convert a string value into lowercase.","Examples":[{"mapping":"root.foo = this.foo.lowercase()","summary":"","results":[["{\"foo\":\"HELLO WORLD\"}","{\"foo\":\"hello world\"}"]],"skip_testing":false}]}],"impure":false},{"status":"hidden","name":"map","params":{"named":[{"name":"query","description":"A query to execute on the target.","type":"query expression","no_dynamic":false,"scalars_to_literal":false}]},"impure":false},{"status":"stable","name":"map_each","params":{"named":[{"name":"query","description":"A query that will be used to map each element.","type":"query expression","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.new_nums = this.nums.map_each(num -\u003e if num \u003c 10 {\n deleted()\n} else {\n num - 10\n})","summary":"##### On arrays\n\nApply a mapping to each element of an array and replace the element with the result. Within the argument mapping the context is the value of the element being mapped.","results":[["{\"nums\":[3,11,4,17]}","{\"new_nums\":[1,7]}"]],"skip_testing":false},{"mapping":"root.new_dict = this.dict.map_each(item -\u003e item.value.uppercase())","summary":"##### On objects\n\nApply a mapping to each value of an object and replace the value with the result. Within the argument mapping the context is an object with a field `key` containing the value key, and a field `value`.","results":[["{\"dict\":{\"foo\":\"hello\",\"bar\":\"world\"}}","{\"new_dict\":{\"bar\":\"WORLD\",\"foo\":\"HELLO\"}}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"map_each_key","params":{"named":[{"name":"query","description":"A query that will be used to map each key.","type":"query expression","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"Apply a mapping to each key of an object, and replace the key with the result, which must be a string.","Examples":[{"mapping":"root.new_dict = this.dict.map_each_key(key -\u003e key.uppercase())","summary":"","results":[["{\"dict\":{\"keya\":\"hello\",\"keyb\":\"world\"}}","{\"new_dict\":{\"KEYA\":\"hello\",\"KEYB\":\"world\"}}"]],"skip_testing":false},{"mapping":"root = this.map_each_key(key -\u003e if key.contains(\"kafka\") { \"_\" + key })","summary":"","results":[["{\"amqp_key\":\"foo\",\"kafka_key\":\"bar\",\"kafka_topic\":\"baz\"}","{\"_kafka_key\":\"bar\",\"_kafka_topic\":\"baz\",\"amqp_key\":\"foo\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"max","description":"Returns the largest numerical value found within an array. All values must be numerical and the array must not be empty, otherwise an error is returned.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"root.biggest = this.values.max()","summary":"","results":[["{\"values\":[0,3,2.5,7,5]}","{\"biggest\":7}"]],"skip_testing":false},{"mapping":"root.new_value = [0,this.value].max()","summary":"","results":[["{\"value\":-1}","{\"new_value\":0}"],["{\"value\":7}","{\"new_value\":7}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"merge","description":"Merge a source object into an existing destination object. When a collision is found within the merged structures (both a source and destination object contain the same non-object keys) the result will be an array containing both values, where values that are already arrays will be expanded into the resulting array. In order to simply override destination fields on collision use the \u003c\u003cassign, `assign`\u003e\u003e method.","params":{"named":[{"name":"with","description":"A value to merge the target value with.","type":"unknown","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root = this.foo.merge(this.bar)","summary":"","results":[["{\"foo\":{\"first_name\":\"fooer\",\"likes\":\"bars\"},\"bar\":{\"second_name\":\"barer\",\"likes\":\"foos\"}}","{\"first_name\":\"fooer\",\"likes\":[\"bars\",\"foos\"],\"second_name\":\"barer\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"min","description":"Returns the smallest numerical value found within an array. All values must be numerical and the array must not be empty, otherwise an error is returned.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"root.smallest = this.values.min()","summary":"","results":[["{\"values\":[0,3,-2.5,7,5]}","{\"smallest\":-2.5}"]],"skip_testing":false},{"mapping":"root.new_value = [10,this.value].min()","summary":"","results":[["{\"value\":2}","{\"new_value\":2}"],["{\"value\":23}","{\"new_value\":10}"]],"skip_testing":false}]}],"impure":false},{"status":"hidden","name":"not","params":{},"impure":false},{"status":"stable","name":"not_empty","params":{},"categories":[{"Category":"Type Coercion","Description":"Ensures that the given string, array or object value is not empty, and if so returns it, otherwise an error is returned.","Examples":[{"mapping":"root.a = this.a.not_empty()","summary":"","results":[["{\"a\":\"foo\"}","{\"a\":\"foo\"}"],["{\"a\":\"\"}","Error(\"failed assignment (line 1): field `this.a`: string value is empty\")"],["{\"a\":[\"foo\",\"bar\"]}","{\"a\":[\"foo\",\"bar\"]}"],["{\"a\":[]}","Error(\"failed assignment (line 1): field `this.a`: array value is empty\")"],["{\"a\":{\"b\":\"foo\",\"c\":\"bar\"}}","{\"a\":{\"b\":\"foo\",\"c\":\"bar\"}}"],["{\"a\":{}}","Error(\"failed assignment (line 1): field `this.a`: object value is empty\")"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"not_null","params":{},"categories":[{"Category":"Type Coercion","Description":"Ensures that the given value is not `null`, and if so returns it, otherwise an error is returned.","Examples":[{"mapping":"root.a = this.a.not_null()","summary":"","results":[["{\"a\":\"foobar\",\"b\":\"barbaz\"}","{\"a\":\"foobar\"}"],["{\"b\":\"barbaz\"}","Error(\"failed assignment (line 1): field `this.a`: value is null\")"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"number","params":{"named":[{"name":"default","description":"An optional value to yield if the target cannot be parsed as a number.","type":"float","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"categories":[{"Category":"Type Coercion","Description":"Attempt to parse a value into a number. An optional argument can be provided, in which case if the value cannot be parsed into a number the argument will be returned instead.","Examples":[{"mapping":"root.foo = this.thing.number() + 10\nroot.bar = this.thing.number(5) * 10","summary":"","results":[],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"or","description":"If the result of the target query fails or resolves to `null`, returns the argument instead. This is an explicit method alternative to the coalesce pipe operator `|`.","params":{"named":[{"name":"fallback","description":"A value to yield, or query to execute, if the target query fails or resolves to `null`.","type":"query expression","no_dynamic":false,"scalars_to_literal":true}]},"examples":[{"mapping":"root.doc.id = this.thing.id.or(uuid_v4())","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"stable","name":"parse_csv","params":{"named":[{"name":"parse_header_row","description":"Whether to reference the first row as a header row. If set to true the output structure for messages will be an object where field keys are determined by the header row. Otherwise, the output will be an array of row arrays.","type":"bool","no_dynamic":false,"scalars_to_literal":false,"default":true},{"name":"delimiter","description":"The delimiter to use for splitting values in each record. It must be a single character.","type":"string","no_dynamic":false,"scalars_to_literal":false,"default":","},{"name":"lazy_quotes","description":"If set to `true`, a quote may appear in an unquoted field and a non-doubled quote may appear in a quoted field.","type":"bool","no_dynamic":false,"scalars_to_literal":false,"default":false}]},"categories":[{"Category":"Parsing","Description":"Attempts to parse a string into an array of objects by following the CSV format described in RFC 4180.","Examples":[{"mapping":"root.orders = this.orders.parse_csv()","summary":"Parses CSV data with a header row","results":[["{\"orders\":\"foo,bar\\nfoo 1,bar 1\\nfoo 2,bar 2\"}","{\"orders\":[{\"bar\":\"bar 1\",\"foo\":\"foo 1\"},{\"bar\":\"bar 2\",\"foo\":\"foo 2\"}]}"]],"skip_testing":false},{"mapping":"root.orders = this.orders.parse_csv(false)","summary":"Parses CSV data without a header row","results":[["{\"orders\":\"foo 1,bar 1\\nfoo 2,bar 2\"}","{\"orders\":[[\"foo 1\",\"bar 1\"],[\"foo 2\",\"bar 2\"]]}"]],"skip_testing":false},{"mapping":"root.orders = this.orders.parse_csv(delimiter:\".\")","summary":"Parses CSV data delimited by dots","results":[["{\"orders\":\"foo.bar\\nfoo 1.bar 1\\nfoo 2.bar 2\"}","{\"orders\":[{\"bar\":\"bar 1\",\"foo\":\"foo 1\"},{\"bar\":\"bar 2\",\"foo\":\"foo 2\"}]}"]],"skip_testing":false},{"mapping":"root.orders = this.orders.parse_csv(lazy_quotes:true)","summary":"Parses CSV data containing a quote in an unquoted field","results":[["{\"orders\":\"foo,bar\\nfoo 1,bar 1\\nfoo\\\" \\\"2,bar\\\" \\\"2\"}","{\"orders\":[{\"bar\":\"bar 1\",\"foo\":\"foo 1\"},{\"bar\":\"bar\\\" \\\"2\",\"foo\":\"foo\\\" \\\"2\"}]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"parse_duration","description":"Attempts to parse a string as a duration and returns an integer of nanoseconds. A duration string is a possibly signed sequence of decimal numbers, each with an optional fraction and a unit suffix, such as \"300ms\", \"-1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"Β΅s\"), \"ms\", \"s\", \"m\", \"h\".","params":{},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":[{"mapping":"root.delay_for_ns = this.delay_for.parse_duration()","summary":"","results":[["{\"delay_for\":\"50us\"}","{\"delay_for_ns\":50000}"]],"skip_testing":false},{"mapping":"root.delay_for_s = this.delay_for.parse_duration() / 1000000000","summary":"","results":[["{\"delay_for\":\"2h\"}","{\"delay_for_s\":7200}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"parse_duration_iso8601","description":"Attempts to parse a string using ISO-8601 rules as a duration and returns an integer of nanoseconds. A duration string is represented by the format \"P[n]Y[n]M[n]DT[n]H[n]M[n]S\" or \"P[n]W\". In these representations, the \"[n]\" is replaced by the value for each of the date and time elements that follow the \"[n]\". For example, \"P3Y6M4DT12H30M5S\" represents a duration of \"three years, six months, four days, twelve hours, thirty minutes, and five seconds\". The last field of the format allows fractions with one decimal place, so \"P3.5S\" will return 3500000000ns. Any additional decimals will be truncated.","params":{},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":[{"mapping":"root.delay_for_ns = this.delay_for.parse_duration_iso8601()","summary":"Arbitrary ISO-8601 duration string to nanoseconds:","results":[["{\"delay_for\":\"P3Y6M4DT12H30M5S\"}","{\"delay_for_ns\":110839937000000000}"]],"skip_testing":false},{"mapping":"root.delay_for_s = this.delay_for.parse_duration_iso8601() / 1000000000","summary":"Two hours ISO-8601 duration string to seconds:","results":[["{\"delay_for\":\"PT2H\"}","{\"delay_for_s\":7200}"]],"skip_testing":false},{"mapping":"root.delay_for_s = this.delay_for.parse_duration_iso8601() / 1000000000","summary":"Two and a half seconds ISO-8601 duration string to seconds:","results":[["{\"delay_for\":\"PT2.5S\"}","{\"delay_for_s\":2.5}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"parse_form_url_encoded","description":"Attempts to parse a url-encoded query string (from an x-www-form-urlencoded request body) and returns a structured result.","params":{},"categories":[{"Category":"Parsing","Description":"","Examples":[{"mapping":"root.values = this.body.parse_form_url_encoded()","summary":"","results":[["{\"body\":\"noise=meow\u0026animal=cat\u0026fur=orange\u0026fur=fluffy\"}","{\"values\":{\"animal\":\"cat\",\"fur\":[\"orange\",\"fluffy\"],\"noise\":\"meow\"}}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"parse_json","params":{"named":[{"name":"use_number","description":"An optional flag that when set makes parsing numbers as json.Number instead of the default float64.","type":"bool","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"categories":[{"Category":"Parsing","Description":"Attempts to parse a string as a JSON document and returns the result.","Examples":[{"mapping":"root.doc = this.doc.parse_json()","summary":"","results":[["{\"doc\":\"{\\\"foo\\\":\\\"bar\\\"}\"}","{\"doc\":{\"foo\":\"bar\"}}"]],"skip_testing":false},{"mapping":"root.doc = this.doc.parse_json(use_number: true)","summary":"","results":[["{\"doc\":\"{\\\"foo\\\":\\\"11380878173205700000000000000000000000000000000\\\"}\"}","{\"doc\":{\"foo\":\"11380878173205700000000000000000000000000000000\"}}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"parse_jwt_es256","description":"Parses a claims object from a JWT string encoded with ES256. This method does not validate JWT claims.","params":{"named":[{"name":"signing_secret","description":"The ES256 secret that was used for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.claims = this.signed.parse_jwt_es256(\"\"\"-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEGtLqIBePHmIhQcf0JLgc+F/4W/oI\ndp0Gta53G35VerNDgUUXmp78J2kfh4qLdh0XtmOMI587tCaqjvDAXfs//w==\n-----END PUBLIC KEY-----\"\"\")","summary":"","results":[["{\"signed\":\"eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE1MTYyMzkwMjIsIm1vb2QiOiJEaXNkYWluZnVsIiwic3ViIjoiMTIzNDU2Nzg5MCJ9.GIRajP9JJbpTlqSCdNEz4qpQkRvzX4Q51YnTwVyxLDM9tKjR_a8ggHWn9CWj7KG0x8J56OWtmUxn112SRTZVhQ\"}","{\"claims\":{\"iat\":1516239022,\"mood\":\"Disdainful\",\"sub\":\"1234567890\"}}"]],"skip_testing":false}]}],"impure":false,"version":"v4.20.0"},{"status":"stable","name":"parse_jwt_es384","description":"Parses a claims object from a JWT string encoded with ES384. This method does not validate JWT claims.","params":{"named":[{"name":"signing_secret","description":"The ES384 secret that was used for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.claims = this.signed.parse_jwt_es384(\"\"\"-----BEGIN PUBLIC KEY-----\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAERoz74/B6SwmLhs8X7CWhnrWyRrB13AuU\n8OYeqy0qHRu9JWNw8NIavqpTmu6XPT4xcFanYjq8FbeuM11eq06C52mNmS4LLwzA\n2imlFEgn85bvJoC3bnkuq4mQjwt9VxdH\n-----END PUBLIC KEY-----\"\"\")","summary":"","results":[["{\"signed\":\"eyJhbGciOiJFUzM4NCIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE1MTYyMzkwMjIsIm1vb2QiOiJEaXNkYWluZnVsIiwic3ViIjoiMTIzNDU2Nzg5MCJ9.H2HBSlrvQBaov2tdreGonbBexxtQB-xzaPL4-tNQZ6TVh7VH8VBcSwcWHYa1lBAHmdsKOFcB2Wk0SB7QWeGT3ptSgr-_EhDMaZ8bA5spgdpq5DsKfaKHrd7DbbQlmxNq\"}","{\"claims\":{\"iat\":1516239022,\"mood\":\"Disdainful\",\"sub\":\"1234567890\"}}"]],"skip_testing":false}]}],"impure":false,"version":"v4.20.0"},{"status":"stable","name":"parse_jwt_es512","description":"Parses a claims object from a JWT string encoded with ES512. This method does not validate JWT claims.","params":{"named":[{"name":"signing_secret","description":"The ES512 secret that was used for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.claims = this.signed.parse_jwt_es512(\"\"\"-----BEGIN PUBLIC KEY-----\nMIGbMBAGByqGSM49AgEGBSuBBAAjA4GGAAQAkHLdts9P56fFkyhpYQ31M/Stwt3w\nvpaxhlfudxnXgTO1IP4RQRgryRxZ19EUzhvWDcG3GQIckoNMY5PelsnCGnIBT2Xh\n9NQkjWF5K6xS4upFsbGSAwQ+GIyyk5IPJ2LHgOyMSCVh5gRZXV3CZLzXujx/umC9\nUeYyTt05zRRWuD+p5bY=\n-----END PUBLIC KEY-----\"\"\")","summary":"","results":[["{\"signed\":\"eyJhbGciOiJFUzUxMiIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE1MTYyMzkwMjIsIm1vb2QiOiJEaXNkYWluZnVsIiwic3ViIjoiMTIzNDU2Nzg5MCJ9.ACrpLuU7TKpAnncDCpN9m85nkL55MJ45NFOBl6-nEXmNT1eIxWjiP4pwWVbFH9et_BgN14119jbL_KqEJInPYc9nAXC6dDLq0aBU-dalvNl4-O5YWpP43-Y-TBGAsWnbMTrchILJ4-AEiICe73Ck5yWPleKg9c3LtkEFWfGs7BoPRguZ\"}","{\"claims\":{\"iat\":1516239022,\"mood\":\"Disdainful\",\"sub\":\"1234567890\"}}"]],"skip_testing":false}]}],"impure":false,"version":"v4.20.0"},{"status":"stable","name":"parse_jwt_hs256","description":"Parses a claims object from a JWT string encoded with HS256. This method does not validate JWT claims.","params":{"named":[{"name":"signing_secret","description":"The HS256 secret that was used for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.claims = this.signed.parse_jwt_hs256(\"\"\"dont-tell-anyone\"\"\")","summary":"","results":[["{\"signed\":\"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE1MTYyMzkwMjIsIm1vb2QiOiJEaXNkYWluZnVsIiwic3ViIjoiMTIzNDU2Nzg5MCJ9.YwXOM8v3gHVWcQRRRQc_zDlhmLnM62fwhFYGpiA0J1A\"}","{\"claims\":{\"iat\":1516239022,\"mood\":\"Disdainful\",\"sub\":\"1234567890\"}}"]],"skip_testing":false}]}],"impure":false,"version":"v4.12.0"},{"status":"stable","name":"parse_jwt_hs384","description":"Parses a claims object from a JWT string encoded with HS384. This method does not validate JWT claims.","params":{"named":[{"name":"signing_secret","description":"The HS384 secret that was used for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.claims = this.signed.parse_jwt_hs384(\"\"\"dont-tell-anyone\"\"\")","summary":"","results":[["{\"signed\":\"eyJhbGciOiJIUzM4NCIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE1MTYyMzkwMjIsIm1vb2QiOiJEaXNkYWluZnVsIiwic3ViIjoiMTIzNDU2Nzg5MCJ9.2Y8rf_ijwN4t8hOGGViON_GrirLkCQVbCOuax6EoZ3nluX0tCGezcJxbctlIfsQ2\"}","{\"claims\":{\"iat\":1516239022,\"mood\":\"Disdainful\",\"sub\":\"1234567890\"}}"]],"skip_testing":false}]}],"impure":false,"version":"v4.12.0"},{"status":"stable","name":"parse_jwt_hs512","description":"Parses a claims object from a JWT string encoded with HS512. This method does not validate JWT claims.","params":{"named":[{"name":"signing_secret","description":"The HS512 secret that was used for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.claims = this.signed.parse_jwt_hs512(\"\"\"dont-tell-anyone\"\"\")","summary":"","results":[["{\"signed\":\"eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE1MTYyMzkwMjIsIm1vb2QiOiJEaXNkYWluZnVsIiwic3ViIjoiMTIzNDU2Nzg5MCJ9.utRb0urG6LGGyranZJVo5Dk0Fns1QNcSUYPN0TObQ-YzsGGB8jrxHwM5NAJccjJZzKectEUqmmKCaETZvuX4Fg\"}","{\"claims\":{\"iat\":1516239022,\"mood\":\"Disdainful\",\"sub\":\"1234567890\"}}"]],"skip_testing":false}]}],"impure":false,"version":"v4.12.0"},{"status":"stable","name":"parse_jwt_rs256","description":"Parses a claims object from a JWT string encoded with RS256. This method does not validate JWT claims.","params":{"named":[{"name":"signing_secret","description":"The RS256 secret that was used for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.claims = this.signed.parse_jwt_rs256(\"\"\"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAs/ibN8r68pLMR6gRzg4S\n8v8l6Q7yi8qURjkEbcNeM1rkokC7xh0I4JVTwxYSVv/JIW8qJdyspl5NIfuAVi32\nWfKvSAs+NIs+DMsNPYw3yuQals4AX8hith1YDvYpr8SD44jxhz/DR9lYKZFGhXGB\n+7NqQ7vpTWp3BceLYocazWJgusZt7CgecIq57ycM5hjM93BvlrUJ8nQ1a46wfL/8\nCy4P0et70hzZrsjjN41KFhKY0iUwlyU41yEiDHvHDDsTMBxAZosWjSREGfJL6Mfp\nXOInTHs/Gg6DZMkbxjQu6L06EdJ+Q/NwglJdAXM7Zo9rNELqRig6DdvG5JesdMsO\n+QIDAQAB\n-----END PUBLIC KEY-----\"\"\")","summary":"","results":[["{\"signed\":\"eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE1MTYyMzkwMjIsIm1vb2QiOiJEaXNkYWluZnVsIiwic3ViIjoiMTIzNDU2Nzg5MCJ9.b0lH3jEupZZ4zoaly4Y_GCvu94HH6UKdKY96zfGNsIkPZpQLHIkZ7jMWlLlNOAd8qXlsBGP_i8H2qCKI4zlWJBGyPZgxXDzNRPVrTDfFpn4t4nBcA1WK2-ntXP3ehQxsaHcQU8Z_nsogId7Pme5iJRnoHWEnWtbwz5DLSXL3ZZNnRdrHM9MdI7QSDz9mojKDCaMpGN9sG7Xl-tGdBp1XzXuUOzG8S03mtZ1IgVR1uiBL2N6oohHIAunk8DIAmNWI-zgycTgzUGU7mvPkKH43qO8Ua1-13tCUBKKa8VxcotZ67Mxm1QAvBGoDnTKwWMwghLzs6d6WViXQg6eWlJcpBA\"}","{\"claims\":{\"iat\":1516239022,\"mood\":\"Disdainful\",\"sub\":\"1234567890\"}}"]],"skip_testing":false}]}],"impure":false,"version":"v4.20.0"},{"status":"stable","name":"parse_jwt_rs384","description":"Parses a claims object from a JWT string encoded with RS384. This method does not validate JWT claims.","params":{"named":[{"name":"signing_secret","description":"The RS384 secret that was used for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.claims = this.signed.parse_jwt_rs384(\"\"\"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAs/ibN8r68pLMR6gRzg4S\n8v8l6Q7yi8qURjkEbcNeM1rkokC7xh0I4JVTwxYSVv/JIW8qJdyspl5NIfuAVi32\nWfKvSAs+NIs+DMsNPYw3yuQals4AX8hith1YDvYpr8SD44jxhz/DR9lYKZFGhXGB\n+7NqQ7vpTWp3BceLYocazWJgusZt7CgecIq57ycM5hjM93BvlrUJ8nQ1a46wfL/8\nCy4P0et70hzZrsjjN41KFhKY0iUwlyU41yEiDHvHDDsTMBxAZosWjSREGfJL6Mfp\nXOInTHs/Gg6DZMkbxjQu6L06EdJ+Q/NwglJdAXM7Zo9rNELqRig6DdvG5JesdMsO\n+QIDAQAB\n-----END PUBLIC KEY-----\"\"\")","summary":"","results":[["{\"signed\":\"eyJhbGciOiJSUzM4NCIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE1MTYyMzkwMjIsIm1vb2QiOiJEaXNkYWluZnVsIiwic3ViIjoiMTIzNDU2Nzg5MCJ9.orcXYBcjVE5DU7mvq4KKWFfNdXR4nEY_xupzWoETRpYmQZIozlZnM_nHxEk2dySvpXlAzVm7kgOPK2RFtGlOVaNRIa3x-pMMr-bhZTno4L8Hl4sYxOks3bWtjK7wql4uqUbqThSJB12psAXw2-S-I_FMngOPGIn4jDT9b802ottJSvTpXcy0-eKTjrV2PSkRRu-EYJh0CJZW55MNhqlt6kCGhAXfbhNazN3ASX-dmpd_JixyBKphrngr_zRA-FCn_Xf3QQDA-5INopb4Yp5QiJ7UxVqQEKI80X_JvJqz9WE1qiAw8pq5-xTen1t7zTP-HT1NbbD3kltcNa3G8acmNg\"}","{\"claims\":{\"iat\":1516239022,\"mood\":\"Disdainful\",\"sub\":\"1234567890\"}}"]],"skip_testing":false}]}],"impure":false,"version":"v4.20.0"},{"status":"stable","name":"parse_jwt_rs512","description":"Parses a claims object from a JWT string encoded with RS512. This method does not validate JWT claims.","params":{"named":[{"name":"signing_secret","description":"The RS512 secret that was used for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.claims = this.signed.parse_jwt_rs512(\"\"\"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAs/ibN8r68pLMR6gRzg4S\n8v8l6Q7yi8qURjkEbcNeM1rkokC7xh0I4JVTwxYSVv/JIW8qJdyspl5NIfuAVi32\nWfKvSAs+NIs+DMsNPYw3yuQals4AX8hith1YDvYpr8SD44jxhz/DR9lYKZFGhXGB\n+7NqQ7vpTWp3BceLYocazWJgusZt7CgecIq57ycM5hjM93BvlrUJ8nQ1a46wfL/8\nCy4P0et70hzZrsjjN41KFhKY0iUwlyU41yEiDHvHDDsTMBxAZosWjSREGfJL6Mfp\nXOInTHs/Gg6DZMkbxjQu6L06EdJ+Q/NwglJdAXM7Zo9rNELqRig6DdvG5JesdMsO\n+QIDAQAB\n-----END PUBLIC KEY-----\"\"\")","summary":"","results":[["{\"signed\":\"eyJhbGciOiJSUzUxMiIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE1MTYyMzkwMjIsIm1vb2QiOiJEaXNkYWluZnVsIiwic3ViIjoiMTIzNDU2Nzg5MCJ9.rsMp_X5HMrUqKnZJIxo27aAoscovRA6SSQYR9rq7pifIj0YHXxMyNyOBDGnvVALHKTi25VUGHpfNUW0VVMmae0A4t_ObNU6hVZHguWvetKZZq4FZpW1lgWHCMqgPGwT5_uOqwYCH6r8tJuZT3pqXeL0CY4putb1AN2w6CVp620nh3l8d3XWb4jaifycd_4CEVCqHuWDmohfug4VhmoVKlIXZkYoAQowgHlozATDssBSWdYtv107Wd2AzEoiXPu6e3pflsuXULlyqQnS4ELEKPYThFLafh1NqvZDPddqozcPZ-iODBW-xf3A4DYDdivnMYLrh73AZOGHexxu8ay6nDA\"}","{\"claims\":{\"iat\":1516239022,\"mood\":\"Disdainful\",\"sub\":\"1234567890\"}}"]],"skip_testing":false}]}],"impure":false,"version":"v4.20.0"},{"status":"stable","name":"parse_msgpack","description":"Parses a https://msgpack.org/[MessagePack^] message into a structured document.","params":{},"categories":[{"Category":"Parsing","Description":"","Examples":[{"mapping":"root = content().decode(\"hex\").parse_msgpack()","summary":"","results":[["81a3666f6fa3626172","{\"foo\":\"bar\"}"]],"skip_testing":false},{"mapping":"root = this.encoded.decode(\"base64\").parse_msgpack()","summary":"","results":[["{\"encoded\":\"gaNmb2+jYmFy\"}","{\"foo\":\"bar\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"parse_parquet","description":"Decodes a https://parquet.apache.org/docs/[Parquet file^] into an array of objects, one for each row within the file.","params":{"named":[{"name":"byte_array_as_string","description":"Deprecated: This parameter is no longer used.","type":"bool","no_dynamic":false,"scalars_to_literal":false,"default":false}]},"categories":[{"Category":"Parsing","Description":"","Examples":[{"mapping":"root = content().parse_parquet()","summary":"","results":[],"skip_testing":false}]}],"impure":false},{"status":"deprecated","name":"parse_timestamp","description":"Attempts to parse a string as a timestamp following a specified format and outputs a timestamp, which can then be fed into methods such as \u003c\u003cts_format, `ts_format`\u003e\u003e.\n\nThe input format is defined by showing how the reference time, defined to be Mon Jan 2 15:04:05 -0700 MST 2006, would be displayed if it were the value. For an alternative way to specify formats check out the \u003c\u003cts_strptime, `ts_strptime`\u003e\u003e method.","params":{"named":[{"name":"format","description":"The format of the target string.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Deprecated","Description":"","Examples":null}],"impure":false},{"status":"deprecated","name":"parse_timestamp_strptime","description":"Attempts to parse a string as a timestamp following a specified strptime-compatible format and outputs a timestamp, which can then be fed into \u003c\u003cts_format, `ts_format`\u003e\u003e.","params":{"named":[{"name":"format","description":"The format of the target string.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Deprecated","Description":"","Examples":null}],"impure":false},{"status":"stable","name":"parse_url","description":"Attempts to parse a URL from a string value, returning a structured result that describes the various facets of the URL. The fields returned within the structured result roughly follow https://pkg.go.dev/net/url#URL, and may be expanded in future in order to present more information.","params":{},"categories":[{"Category":"Parsing","Description":"","Examples":[{"mapping":"root.foo_url = this.foo_url.parse_url()","summary":"","results":[["{\"foo_url\":\"https://docs.redpanda.com/redpanda-connect/guides/bloblang/about/\"}","{\"foo_url\":{\"fragment\":\"\",\"host\":\"docs.redpanda.com\",\"opaque\":\"\",\"path\":\"/redpanda-connect/guides/bloblang/about/\",\"raw_fragment\":\"\",\"raw_path\":\"\",\"raw_query\":\"\",\"scheme\":\"https\"}}"]],"skip_testing":false},{"mapping":"root.username = this.url.parse_url().user.name | \"unknown\"","summary":"","results":[["{\"url\":\"amqp://foo:bar@127.0.0.1:5672/\"}","{\"username\":\"foo\"}"],["{\"url\":\"redis://localhost:6379\"}","{\"username\":\"unknown\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"parse_xml","description":"\nAttempts to parse a string as an XML document and returns a structured result, where elements appear as keys of an object according to the following rules:\n\n- If an element contains attributes they are parsed by prefixing a hyphen, `-`, to the attribute label.\n- If the element is a simple element and has attributes, the element value is given the key `#text`.\n- XML comments, directives, and process instructions are ignored.\n- When elements are repeated the resulting JSON value is an array.\n- If cast is true, try to cast values to numbers and booleans instead of returning strings.\n","params":{"named":[{"name":"cast","description":"whether to try to cast values that are numbers and booleans to the right type.","type":"bool","no_dynamic":false,"scalars_to_literal":false,"is_optional":true,"default":false}]},"categories":[{"Category":"Parsing","Description":"","Examples":[{"mapping":"root.doc = this.doc.parse_xml()","summary":"","results":[["{\"doc\":\"\u003croot\u003e\u003ctitle\u003eThis is a title\u003c/title\u003e\u003ccontent\u003eThis is some content\u003c/content\u003e\u003c/root\u003e\"}","{\"doc\":{\"root\":{\"content\":\"This is some content\",\"title\":\"This is a title\"}}}"]],"skip_testing":false},{"mapping":"root.doc = this.doc.parse_xml(cast: false)","summary":"","results":[["{\"doc\":\"\u003croot\u003e\u003ctitle\u003eThis is a title\u003c/title\u003e\u003cnumber id=99\u003e123\u003c/number\u003e\u003cbool\u003eTrue\u003c/bool\u003e\u003c/root\u003e\"}","{\"doc\":{\"root\":{\"bool\":\"True\",\"number\":{\"#text\":\"123\",\"-id\":\"99\"},\"title\":\"This is a title\"}}}"]],"skip_testing":false},{"mapping":"root.doc = this.doc.parse_xml(cast: true)","summary":"","results":[["{\"doc\":\"\u003croot\u003e\u003ctitle\u003eThis is a title\u003c/title\u003e\u003cnumber id=99\u003e123\u003c/number\u003e\u003cbool\u003eTrue\u003c/bool\u003e\u003c/root\u003e\"}","{\"doc\":{\"root\":{\"bool\":true,\"number\":{\"#text\":123,\"-id\":99},\"title\":\"This is a title\"}}}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"parse_yaml","params":{},"categories":[{"Category":"Parsing","Description":"Attempts to parse a string as a single YAML document and returns the result.","Examples":[{"mapping":"root.doc = this.doc.parse_yaml()","summary":"","results":[["{\"doc\":\"foo: bar\"}","{\"doc\":{\"foo\":\"bar\"}}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"patch","description":"Create a diff by comparing the current value with the given one. Wraps the github.com/r3labs/diff/v3 package. See its https://pkg.go.dev/github.com/r3labs/diff/v3[docs^] for more information.","params":{"named":[{"name":"changelog","description":"The changelog to apply.","type":"unknown","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":null}],"impure":false,"version":"4.25.0"},{"status":"stable","name":"pow","description":"Returns the number raised to the specified exponent.","params":{"named":[{"name":"exponent","description":"The exponent you want to raise to the power of.","type":"float","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"root.new_value = this.value * 10.pow(-2)","summary":"","results":[["{\"value\":2}","{\"new_value\":0.02}"]],"skip_testing":false},{"mapping":"root.new_value = this.value.pow(-2)","summary":"","results":[["{\"value\":2}","{\"new_value\":0.25}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"quote","params":{},"categories":[{"Category":"String Manipulation","Description":"Quotes a target string using escape sequences (`\\t`, `\\n`, `\\xFF`, `\\u0100`) for control characters and non-printable characters.","Examples":[{"mapping":"root.quoted = this.thing.quote()","summary":"","results":[["{\"thing\":\"foo\\nbar\"}","{\"quoted\":\"\\\"foo\\\\nbar\\\"\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"re_find_all","params":{"named":[{"name":"pattern","description":"The pattern to match against.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Regular Expressions","Description":"Returns an array containing all successive matches of a regular expression in a string.","Examples":[{"mapping":"root.matches = this.value.re_find_all(\"a.\")","summary":"","results":[["{\"value\":\"paranormal\"}","{\"matches\":[\"ar\",\"an\",\"al\"]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"re_find_all_object","params":{"named":[{"name":"pattern","description":"The pattern to match against.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Regular Expressions","Description":"Returns an array of objects containing all matches of the regular expression and the matches of its subexpressions. The key of each match value is the name of the group when specified, otherwise it is the index of the matching group, starting with the expression as a whole at 0.","Examples":[{"mapping":"root.matches = this.value.re_find_all_object(\"a(?P\u003cfoo\u003ex*)b\")","summary":"","results":[["{\"value\":\"-axxb-ab-\"}","{\"matches\":[{\"0\":\"axxb\",\"foo\":\"xx\"},{\"0\":\"ab\",\"foo\":\"\"}]}"]],"skip_testing":false},{"mapping":"root.matches = this.value.re_find_all_object(\"(?m)(?P\u003ckey\u003e\\\\w+):\\\\s+(?P\u003cvalue\u003e\\\\w+)$\")","summary":"","results":[["{\"value\":\"option1: value1\\noption2: value2\\noption3: value3\"}","{\"matches\":[{\"0\":\"option1: value1\",\"key\":\"option1\",\"value\":\"value1\"},{\"0\":\"option2: value2\",\"key\":\"option2\",\"value\":\"value2\"},{\"0\":\"option3: value3\",\"key\":\"option3\",\"value\":\"value3\"}]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"re_find_all_submatch","params":{"named":[{"name":"pattern","description":"The pattern to match against.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Regular Expressions","Description":"Returns an array of arrays containing all successive matches of the regular expression in a string and the matches, if any, of its subexpressions.","Examples":[{"mapping":"root.matches = this.value.re_find_all_submatch(\"a(x*)b\")","summary":"","results":[["{\"value\":\"-axxb-ab-\"}","{\"matches\":[[\"axxb\",\"xx\"],[\"ab\",\"\"]]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"re_find_object","params":{"named":[{"name":"pattern","description":"The pattern to match against.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Regular Expressions","Description":"Returns an object containing the first match of the regular expression and the matches of its subexpressions. The key of each match value is the name of the group when specified, otherwise it is the index of the matching group, starting with the expression as a whole at 0.","Examples":[{"mapping":"root.matches = this.value.re_find_object(\"a(?P\u003cfoo\u003ex*)b\")","summary":"","results":[["{\"value\":\"-axxb-ab-\"}","{\"matches\":{\"0\":\"axxb\",\"foo\":\"xx\"}}"]],"skip_testing":false},{"mapping":"root.matches = this.value.re_find_object(\"(?P\u003ckey\u003e\\\\w+):\\\\s+(?P\u003cvalue\u003e\\\\w+)\")","summary":"","results":[["{\"value\":\"option1: value1\"}","{\"matches\":{\"0\":\"option1: value1\",\"key\":\"option1\",\"value\":\"value1\"}}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"re_match","params":{"named":[{"name":"pattern","description":"The pattern to match against.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Regular Expressions","Description":"Checks whether a regular expression matches against any part of a string and returns a boolean.","Examples":[{"mapping":"root.matches = this.value.re_match(\"[0-9]\")","summary":"","results":[["{\"value\":\"there are 10 puppies\"}","{\"matches\":true}"],["{\"value\":\"there are ten puppies\"}","{\"matches\":false}"]],"skip_testing":false}]}],"impure":false},{"status":"hidden","name":"re_replace","params":{"named":[{"name":"pattern","description":"The pattern to match against.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"value","description":"The value to replace with.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"impure":false},{"status":"stable","name":"re_replace_all","params":{"named":[{"name":"pattern","description":"The pattern to match against.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"value","description":"The value to replace with.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Regular Expressions","Description":"Replaces all occurrences of the argument regular expression in a string with a value. Inside the value $ signs are interpreted as submatch expansions, e.g. `$1` represents the text of the first submatch.","Examples":[{"mapping":"root.new_value = this.value.re_replace_all(\"ADD ([0-9]+)\",\"+($1)\")","summary":"","results":[["{\"value\":\"foo ADD 70\"}","{\"new_value\":\"foo +(70)\"}"]],"skip_testing":false}]}],"impure":false},{"status":"hidden","name":"replace","params":{"named":[{"name":"old","description":"A string to match against.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"new","description":"A string to replace with.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"impure":false},{"status":"stable","name":"replace_all","params":{"named":[{"name":"old","description":"A string to match against.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"new","description":"A string to replace with.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"String Manipulation","Description":"Replaces all occurrences of the first argument in a target string with the second argument.","Examples":[{"mapping":"root.new_value = this.value.replace_all(\"foo\",\"dog\")","summary":"","results":[["{\"value\":\"The foo ate my homework\"}","{\"new_value\":\"The dog ate my homework\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"replace_all_many","params":{"named":[{"name":"values","description":"An array of values, each even value will be replaced with the following odd value.","type":"array","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"String Manipulation","Description":"For each pair of strings in an argument array, replaces all occurrences of the first item of the pair with the second. This is a more compact way of chaining a series of `replace_all` methods.","Examples":[{"mapping":"root.new_value = this.value.replace_all_many([\n \"\u003cb\u003e\", \"\u0026lt;b\u0026gt;\",\n \"\u003c/b\u003e\", \"\u0026lt;/b\u0026gt;\",\n \"\u003ci\u003e\", \"\u0026lt;i\u0026gt;\",\n \"\u003c/i\u003e\", \"\u0026lt;/i\u0026gt;\",\n])","summary":"","results":[["{\"value\":\"\u003ci\u003eHello\u003c/i\u003e \u003cb\u003eWorld\u003c/b\u003e\"}","{\"new_value\":\"\u0026lt;i\u0026gt;Hello\u0026lt;/i\u0026gt; \u0026lt;b\u0026gt;World\u0026lt;/b\u0026gt;\"}"]],"skip_testing":false}]}],"impure":false},{"status":"hidden","name":"replace_many","params":{"named":[{"name":"values","description":"An array of values, each even value will be replaced with the following odd value.","type":"array","no_dynamic":false,"scalars_to_literal":false}]},"impure":false},{"status":"stable","name":"reverse","params":{},"categories":[{"Category":"String Manipulation","Description":"Returns the target string in reverse order.","Examples":[{"mapping":"root.reversed = this.thing.reverse()","summary":"","results":[["{\"thing\":\"backwards\"}","{\"reversed\":\"sdrawkcab\"}"]],"skip_testing":false},{"mapping":"root = content().reverse()","summary":"","results":[["{\"thing\":\"backwards\"}","}\"sdrawkcab\":\"gniht\"{"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"round","description":"Rounds numbers to the nearest integer, rounding half away from zero. If the resulting value fits within a 64-bit integer then that is returned, otherwise a new floating point number is returned.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"root.new_value = this.value.round()","summary":"","results":[["{\"value\":5.3}","{\"new_value\":5}"],["{\"value\":5.9}","{\"new_value\":6}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"sign_jwt_es256","description":"Hash and sign an object representing JSON Web Token (JWT) claims using ES256.","params":{"named":[{"name":"signing_secret","description":"The secret to use for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.signed = this.claims.sign_jwt_es256(\"\"\"-----BEGIN EC PRIVATE KEY-----\n... signature data ...\n-----END EC PRIVATE KEY-----\"\"\")","summary":"","results":[["{\"claims\":{\"sub\":\"user123\"}}","{\"signed\":\"eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE1MTYyMzkwMjIsIm1vb2QiOiJEaXNkYWluZnVsIiwic3ViIjoiMTIzNDU2Nzg5MCJ9.-8LrOdkEiv_44ADWW08lpbq41ZmHCel58NMORPq1q4Dyw0zFhqDVLrRoSvCvuyyvgXAFb9IHfR-9MlJ_2ShA9A\"}"]],"skip_testing":true}]}],"impure":false,"version":"v4.20.0"},{"status":"stable","name":"sign_jwt_es384","description":"Hash and sign an object representing JSON Web Token (JWT) claims using ES384.","params":{"named":[{"name":"signing_secret","description":"The secret to use for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.signed = this.claims.sign_jwt_es384(\"\"\"-----BEGIN EC PRIVATE KEY-----\n... signature data ...\n-----END EC PRIVATE KEY-----\"\"\")","summary":"","results":[["{\"claims\":{\"sub\":\"user123\"}}","{\"signed\":\"eyJhbGciOiJFUzM4NCIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJ1c2VyMTIzIn0.8FmTKH08dl7dyxrNu0rmvhegiIBCy-O9cddGco2e9lpZtgv5mS5qHgPkgBC5eRw1d7SRJsHwHZeehzdqT5Ba7aZJIhz9ds0sn37YQ60L7jT0j2gxCzccrt4kECHnUnLw\"}"]],"skip_testing":true}]}],"impure":false,"version":"v4.20.0"},{"status":"stable","name":"sign_jwt_es512","description":"Hash and sign an object representing JSON Web Token (JWT) claims using ES512.","params":{"named":[{"name":"signing_secret","description":"The secret to use for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.signed = this.claims.sign_jwt_es512(\"\"\"-----BEGIN EC PRIVATE KEY-----\n... signature data ...\n-----END EC PRIVATE KEY-----\"\"\")","summary":"","results":[["{\"claims\":{\"sub\":\"user123\"}}","{\"signed\":\"eyJhbGciOiJFUzUxMiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJ1c2VyMTIzIn0.AQbEWymoRZxDJEJtKSFFG2k2VbDCTYSuBwAZyMqexCspr3If8aERTVGif8HXG3S7TzMBCCzxkcKr3eIU441l3DlpAMNfQbkcOlBqMvNBn-CX481WyKf3K5rFHQ-6wRonz05aIsWAxCDvAozI_9J0OWllxdQ2MBAuTPbPJ38OqXsYkCQs\"}"]],"skip_testing":true}]}],"impure":false,"version":"v4.20.0"},{"status":"stable","name":"sign_jwt_hs256","description":"Hash and sign an object representing JSON Web Token (JWT) claims using HS256.","params":{"named":[{"name":"signing_secret","description":"The secret to use for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.signed = this.claims.sign_jwt_hs256(\"\"\"dont-tell-anyone\"\"\")","summary":"","results":[["{\"claims\":{\"sub\":\"user123\"}}","{\"signed\":\"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJ1c2VyMTIzIn0.hUl-nngPMY_3h9vveWJUPsCcO5PeL6k9hWLnMYeFbFQ\"}"]],"skip_testing":true}]}],"impure":false,"version":"v4.12.0"},{"status":"stable","name":"sign_jwt_hs384","description":"Hash and sign an object representing JSON Web Token (JWT) claims using HS384.","params":{"named":[{"name":"signing_secret","description":"The secret to use for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.signed = this.claims.sign_jwt_hs384(\"\"\"dont-tell-anyone\"\"\")","summary":"","results":[["{\"claims\":{\"sub\":\"user123\"}}","{\"signed\":\"eyJhbGciOiJIUzM4NCIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJ1c2VyMTIzIn0.zGYLr83aToon1efUNq-hw7XgT20lPvZb8sYei8x6S6mpHwb433SJdXJXx0Oio8AZ\"}"]],"skip_testing":true}]}],"impure":false,"version":"v4.12.0"},{"status":"stable","name":"sign_jwt_hs512","description":"Hash and sign an object representing JSON Web Token (JWT) claims using HS512.","params":{"named":[{"name":"signing_secret","description":"The secret to use for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.signed = this.claims.sign_jwt_hs512(\"\"\"dont-tell-anyone\"\"\")","summary":"","results":[["{\"claims\":{\"sub\":\"user123\"}}","{\"signed\":\"eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJ1c2VyMTIzIn0.zBNR9o_6EDwXXKkpKLNJhG26j8Dc-mV-YahBwmEdCrmiWt5les8I9rgmNlWIowpq6Yxs4kLNAdFhqoRz3NXT3w\"}"]],"skip_testing":true}]}],"impure":false,"version":"v4.12.0"},{"status":"stable","name":"sign_jwt_rs256","description":"Hash and sign an object representing JSON Web Token (JWT) claims using RS256.","params":{"named":[{"name":"signing_secret","description":"The secret to use for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.signed = this.claims.sign_jwt_rs256(\"\"\"-----BEGIN RSA PRIVATE KEY-----\n... signature data ...\n-----END RSA PRIVATE KEY-----\"\"\")","summary":"","results":[["{\"claims\":{\"sub\":\"user123\"}}","{\"signed\":\"eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE1MTYyMzkwMjIsIm1vb2QiOiJEaXNkYWluZnVsIiwic3ViIjoiMTIzNDU2Nzg5MCJ9.b0lH3jEupZZ4zoaly4Y_GCvu94HH6UKdKY96zfGNsIkPZpQLHIkZ7jMWlLlNOAd8qXlsBGP_i8H2qCKI4zlWJBGyPZgxXDzNRPVrTDfFpn4t4nBcA1WK2-ntXP3ehQxsaHcQU8Z_nsogId7Pme5iJRnoHWEnWtbwz5DLSXL3ZZNnRdrHM9MdI7QSDz9mojKDCaMpGN9sG7Xl-tGdBp1XzXuUOzG8S03mtZ1IgVR1uiBL2N6oohHIAunk8DIAmNWI-zgycTgzUGU7mvPkKH43qO8Ua1-13tCUBKKa8VxcotZ67Mxm1QAvBGoDnTKwWMwghLzs6d6WViXQg6eWlJcpBA\"}"]],"skip_testing":true}]}],"impure":false,"version":"v4.18.0"},{"status":"stable","name":"sign_jwt_rs384","description":"Hash and sign an object representing JSON Web Token (JWT) claims using RS384.","params":{"named":[{"name":"signing_secret","description":"The secret to use for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.signed = this.claims.sign_jwt_rs384(\"\"\"-----BEGIN RSA PRIVATE KEY-----\n... signature data ...\n-----END RSA PRIVATE KEY-----\"\"\")","summary":"","results":[["{\"claims\":{\"sub\":\"user123\"}}","{\"signed\":\"eyJhbGciOiJSUzM4NCIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE1MTYyMzkwMjIsIm1vb2QiOiJEaXNkYWluZnVsIiwic3ViIjoiMTIzNDU2Nzg5MCJ9.orcXYBcjVE5DU7mvq4KKWFfNdXR4nEY_xupzWoETRpYmQZIozlZnM_nHxEk2dySvpXlAzVm7kgOPK2RFtGlOVaNRIa3x-pMMr-bhZTno4L8Hl4sYxOks3bWtjK7wql4uqUbqThSJB12psAXw2-S-I_FMngOPGIn4jDT9b802ottJSvTpXcy0-eKTjrV2PSkRRu-EYJh0CJZW55MNhqlt6kCGhAXfbhNazN3ASX-dmpd_JixyBKphrngr_zRA-FCn_Xf3QQDA-5INopb4Yp5QiJ7UxVqQEKI80X_JvJqz9WE1qiAw8pq5-xTen1t7zTP-HT1NbbD3kltcNa3G8acmNg\"}"]],"skip_testing":true}]}],"impure":false,"version":"v4.18.0"},{"status":"stable","name":"sign_jwt_rs512","description":"Hash and sign an object representing JSON Web Token (JWT) claims using RS512.","params":{"named":[{"name":"signing_secret","description":"The secret to use for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.signed = this.claims.sign_jwt_rs512(\"\"\"-----BEGIN RSA PRIVATE KEY-----\n... signature data ...\n-----END RSA PRIVATE KEY-----\"\"\")","summary":"","results":[["{\"claims\":{\"sub\":\"user123\"}}","{\"signed\":\"eyJhbGciOiJSUzUxMiIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE1MTYyMzkwMjIsIm1vb2QiOiJEaXNkYWluZnVsIiwic3ViIjoiMTIzNDU2Nzg5MCJ9.rsMp_X5HMrUqKnZJIxo27aAoscovRA6SSQYR9rq7pifIj0YHXxMyNyOBDGnvVALHKTi25VUGHpfNUW0VVMmae0A4t_ObNU6hVZHguWvetKZZq4FZpW1lgWHCMqgPGwT5_uOqwYCH6r8tJuZT3pqXeL0CY4putb1AN2w6CVp620nh3l8d3XWb4jaifycd_4CEVCqHuWDmohfug4VhmoVKlIXZkYoAQowgHlozATDssBSWdYtv107Wd2AzEoiXPu6e3pflsuXULlyqQnS4ELEKPYThFLafh1NqvZDPddqozcPZ-iODBW-xf3A4DYDdivnMYLrh73AZOGHexxu8ay6nDA\"}"]],"skip_testing":true}]}],"impure":false,"version":"v4.18.0"},{"status":"stable","name":"sin","description":"Calculates the sine of a given angle specified in radians.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"root.new_value = (this.value * (pi() / 180)).sin()","summary":"","results":[["{\"value\":45}","{\"new_value\":0.7071067811865475}"],["{\"value\":0}","{\"new_value\":0}"],["{\"value\":90}","{\"new_value\":1}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"slice","params":{"named":[{"name":"low","description":"The low bound, which is the first element of the selection, or if negative selects from the end.","type":"integer","no_dynamic":false,"scalars_to_literal":false},{"name":"high","description":"An optional high bound.","type":"integer","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"categories":[{"Category":"String Manipulation","Description":"Extract a slice from a string by specifying two indices, a low and high bound, which selects a half-open range that includes the first character, but excludes the last one. If the second index is omitted then it defaults to the length of the input sequence.","Examples":[{"mapping":"root.beginning = this.value.slice(0, 2)\nroot.end = this.value.slice(4)","summary":"","results":[["{\"value\":\"foo bar\"}","{\"beginning\":\"fo\",\"end\":\"bar\"}"]],"skip_testing":false},{"mapping":"root.last_chunk = this.value.slice(-4)\nroot.the_rest = this.value.slice(0, -4)","summary":"A negative low index can be used, indicating an offset from the end of the sequence. If the low index is greater than the length of the sequence then an empty result is returned.","results":[["{\"value\":\"foo bar\"}","{\"last_chunk\":\" bar\",\"the_rest\":\"foo\"}"]],"skip_testing":false}]},{"Category":"Object \u0026 Array Manipulation","Description":"Extract a slice from an array by specifying two indices, a low and high bound, which selects a half-open range that includes the first element, but excludes the last one. If the second index is omitted then it defaults to the length of the input sequence.","Examples":[{"mapping":"root.beginning = this.value.slice(0, 2)\nroot.end = this.value.slice(4)","summary":"","results":[["{\"value\":[\"foo\",\"bar\",\"baz\",\"buz\",\"bev\"]}","{\"beginning\":[\"foo\",\"bar\"],\"end\":[\"bev\"]}"]],"skip_testing":false},{"mapping":"root.last_chunk = this.value.slice(-2)\nroot.the_rest = this.value.slice(0, -2)","summary":"A negative low index can be used, indicating an offset from the end of the sequence. If the low index is greater than the length of the sequence then an empty result is returned.","results":[["{\"value\":[\"foo\",\"bar\",\"baz\",\"buz\",\"bev\"]}","{\"last_chunk\":[\"buz\",\"bev\"],\"the_rest\":[\"foo\",\"bar\",\"baz\"]}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"slug","description":"Creates a \"slug\" from a given string. Wraps the github.com/gosimple/slug package. See its https://pkg.go.dev/github.com/gosimple/slug[docs^] for more information.","params":{"named":[{"name":"lang","type":"string","no_dynamic":false,"scalars_to_literal":false,"is_optional":true,"default":"en"}]},"categories":[{"Category":"String Manipulation","Description":"","Examples":[{"mapping":"root.slug = this.value.slug()","summary":"Creates a slug from an English string","results":[["{\"value\":\"Gopher \u0026 Benthos\"}","{\"slug\":\"gopher-and-benthos\"}"]],"skip_testing":false},{"mapping":"root.slug = this.value.slug(\"fr\")","summary":"Creates a slug from a French string","results":[["{\"value\":\"Gaufre \u0026 Poisson d'Eau Profonde\"}","{\"slug\":\"gaufre-et-poisson-deau-profonde\"}"]],"skip_testing":false}]}],"impure":false,"version":"4.2.0"},{"status":"stable","name":"sort","params":{"named":[{"name":"compare","description":"An optional query that should explicitly compare elements `left` and `right` and provide a boolean result.","type":"query expression","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"Attempts to sort the values of an array in increasing order. The type of all values must match in order for the ordering to succeed. Supports string and number values.","Examples":[{"mapping":"root.sorted = this.foo.sort()","summary":"","results":[["{\"foo\":[\"bbb\",\"ccc\",\"aaa\"]}","{\"sorted\":[\"aaa\",\"bbb\",\"ccc\"]}"]],"skip_testing":false},{"mapping":"root.sorted = this.foo.sort(item -\u003e item.left.v \u003c item.right.v)","summary":"It's also possible to specify a mapping argument, which is provided an object context with fields `left` and `right`, the mapping must return a boolean indicating whether the `left` value is less than `right`. This allows you to sort arrays containing non-string or non-number values.","results":[["{\"foo\":[{\"id\":\"foo\",\"v\":\"bbb\"},{\"id\":\"bar\",\"v\":\"ccc\"},{\"id\":\"baz\",\"v\":\"aaa\"}]}","{\"sorted\":[{\"id\":\"baz\",\"v\":\"aaa\"},{\"id\":\"foo\",\"v\":\"bbb\"},{\"id\":\"bar\",\"v\":\"ccc\"}]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"sort_by","params":{"named":[{"name":"query","description":"A query to apply to each element that yields a value used for sorting.","type":"query expression","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"Attempts to sort the elements of an array, in increasing order, by a value emitted by an argument query applied to each element. The type of all values must match in order for the ordering to succeed. Supports string and number values.","Examples":[{"mapping":"root.sorted = this.foo.sort_by(ele -\u003e ele.id)","summary":"","results":[["{\"foo\":[{\"id\":\"bbb\",\"message\":\"bar\"},{\"id\":\"aaa\",\"message\":\"foo\"},{\"id\":\"ccc\",\"message\":\"baz\"}]}","{\"sorted\":[{\"id\":\"aaa\",\"message\":\"foo\"},{\"id\":\"bbb\",\"message\":\"bar\"},{\"id\":\"ccc\",\"message\":\"baz\"}]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"split","params":{"named":[{"name":"delimiter","description":"The delimiter to split with.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"String Manipulation","Description":"Split a string value into an array of strings by splitting it on a string separator.","Examples":[{"mapping":"root.new_value = this.value.split(\",\")","summary":"","results":[["{\"value\":\"foo,bar,baz\"}","{\"new_value\":[\"foo\",\"bar\",\"baz\"]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"squash","description":"Squashes an array of objects into a single object, where key collisions result in the values being merged (following similar rules as the `.merge()` method)","params":{},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.locations = this.locations.map_each(loc -\u003e {loc.state: [loc.name]}).squash()","summary":"","results":[["{\"locations\":[{\"name\":\"Seattle\",\"state\":\"WA\"},{\"name\":\"New York\",\"state\":\"NY\"},{\"name\":\"Bellevue\",\"state\":\"WA\"},{\"name\":\"Olympia\",\"state\":\"WA\"}]}","{\"locations\":{\"NY\":[\"New York\"],\"WA\":[\"Seattle\",\"Bellevue\",\"Olympia\"]}}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"string","params":{},"categories":[{"Category":"Type Coercion","Description":"Marshal a value into a string. If the value is already a string it is unchanged.","Examples":[{"mapping":"root.nested_json = this.string()","summary":"","results":[["{\"foo\":\"bar\"}","{\"nested_json\":\"{\\\"foo\\\":\\\"bar\\\"}\"}"]],"skip_testing":false},{"mapping":"root.id = this.id.string()","summary":"","results":[["{\"id\":228930314431312345}","{\"id\":\"228930314431312345\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"strip_html","description":"Attempts to remove all HTML tags from a target string.","params":{"named":[{"name":"preserve","description":"An optional array of element types to preserve in the output.","type":"unknown","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"categories":[{"Category":"String Manipulation","Description":"","Examples":[{"mapping":"root.stripped = this.value.strip_html()","summary":"","results":[["{\"value\":\"\u003cp\u003ethe plain \u003cstrong\u003eold text\u003c/strong\u003e\u003c/p\u003e\"}","{\"stripped\":\"the plain old text\"}"]],"skip_testing":false},{"mapping":"root.stripped = this.value.strip_html([\"article\"])","summary":"It's also possible to provide an explicit list of element types to preserve in the output.","results":[["{\"value\":\"\u003carticle\u003e\u003cp\u003ethe plain \u003cstrong\u003eold text\u003c/strong\u003e\u003c/p\u003e\u003c/article\u003e\"}","{\"stripped\":\"\u003carticle\u003ethe plain old text\u003c/article\u003e\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"sum","params":{},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"Sum the numerical values of an array.","Examples":[{"mapping":"root.sum = this.foo.sum()","summary":"","results":[["{\"foo\":[3,8,4]}","{\"sum\":15}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"tan","description":"Calculates the tangent of a given angle specified in radians.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"root.new_value = \"%f\".format((this.value * (pi() / 180)).tan())","summary":"","results":[["{\"value\":0}","{\"new_value\":\"0.000000\"}"],["{\"value\":45}","{\"new_value\":\"1.000000\"}"],["{\"value\":180}","{\"new_value\":\"-0.000000\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"timestamp","params":{"named":[{"name":"default","description":"An optional value to yield if the target cannot be parsed as a timestamp.","type":"timestamp","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"categories":[{"Category":"Type Coercion","Description":"Attempt to parse a value into a timestamp. An optional argument can be provided, in which case if the value cannot be parsed into a timestamp the argument will be returned instead.","Examples":[{"mapping":"root.foo = this.ts.timestamp()\nroot.bar = this.none.timestamp(1234567890.timestamp())","summary":"","results":[],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"trim","params":{"named":[{"name":"cutset","description":"An optional string of characters to trim from the target value.","type":"string","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"categories":[{"Category":"String Manipulation","Description":"Remove all leading and trailing characters from a string that are contained within an argument cutset. If no arguments are provided then whitespace is removed.","Examples":[{"mapping":"root.title = this.title.trim(\"!?\")\nroot.description = this.description.trim()","summary":"","results":[["{\"description\":\" something happened and its amazing! \",\"title\":\"!!!watch out!?\"}","{\"description\":\"something happened and its amazing!\",\"title\":\"watch out\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"trim_prefix","params":{"named":[{"name":"prefix","description":"The leading prefix substring to trim from the string.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"String Manipulation","Description":"Remove the provided leading prefix substring from a string. If the string does not have the prefix substring, it is returned unchanged.","Examples":[{"mapping":"root.name = this.name.trim_prefix(\"foobar_\")\nroot.description = this.description.trim_prefix(\"foobar_\")","summary":"","results":[["{\"description\":\"unchanged\",\"name\":\"foobar_blobton\"}","{\"description\":\"unchanged\",\"name\":\"blobton\"}"]],"skip_testing":false}]}],"impure":false,"version":"4.12.0"},{"status":"stable","name":"trim_suffix","params":{"named":[{"name":"suffix","description":"The trailing suffix substring to trim from the string.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"String Manipulation","Description":"Remove the provided trailing suffix substring from a string. If the string does not have the suffix substring, it is returned unchanged.","Examples":[{"mapping":"root.name = this.name.trim_suffix(\"_foobar\")\nroot.description = this.description.trim_suffix(\"_foobar\")","summary":"","results":[["{\"description\":\"unchanged\",\"name\":\"blobton_foobar\"}","{\"description\":\"unchanged\",\"name\":\"blobton\"}"]],"skip_testing":false}]}],"impure":false,"version":"4.12.0"},{"status":"beta","name":"ts_add_iso8601","description":"Parse parameter string as ISO 8601 period and add it to value with high precision for units larger than an hour.","params":{"named":[{"name":"duration","description":"Duration in ISO 8601 format","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":null}],"impure":false},{"status":"beta","name":"ts_format","description":"Attempts to format a timestamp value as a string according to a specified format, or RFC 3339 by default. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format.\n\nThe output format is defined by showing how the reference time, defined to be Mon Jan 2 15:04:05 -0700 MST 2006, would be displayed if it were the value. For an alternative way to specify formats check out the \u003c\u003cts_strftime, `ts_strftime`\u003e\u003e method.","params":{"named":[{"name":"format","description":"The output format to use.","type":"string","no_dynamic":false,"scalars_to_literal":false,"default":"2006-01-02T15:04:05.999999999Z07:00"},{"name":"tz","description":"An optional timezone to use, otherwise the timezone of the input string is used, or in the case of unix timestamps the local timezone is used.","type":"string","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":[{"mapping":"root.something_at = (this.created_at + 300).ts_format()","summary":"","results":[],"skip_testing":false},{"mapping":"root.something_at = (this.created_at + 300).ts_format(\"2006-Jan-02 15:04:05\")","summary":"An optional string argument can be used in order to specify the output format of the timestamp. The format is defined by showing how the reference time, defined to be Mon Jan 2 15:04:05 -0700 MST 2006, would be displayed if it were the value.","results":[],"skip_testing":false},{"mapping":"root.something_at = this.created_at.ts_format(format: \"2006-Jan-02 15:04:05\", tz: \"UTC\")","summary":"A second optional string argument can also be used in order to specify a timezone, otherwise the timezone of the input string is used, or in the case of unix timestamps the local timezone is used.","results":[["{\"created_at\":1597405526}","{\"something_at\":\"2020-Aug-14 11:45:26\"}"],["{\"created_at\":\"2020-08-14T11:50:26.371Z\"}","{\"something_at\":\"2020-Aug-14 11:50:26\"}"]],"skip_testing":false},{"mapping":"root.something_at = this.created_at.ts_format(\"2006-Jan-02 15:04:05.999999\", \"UTC\")","summary":"And `ts_format` supports up to nanosecond precision with floating point timestamp values.","results":[["{\"created_at\":1597405526.123456}","{\"something_at\":\"2020-Aug-14 11:45:26.123456\"}"],["{\"created_at\":\"2020-08-14T11:50:26.371Z\"}","{\"something_at\":\"2020-Aug-14 11:50:26.371\"}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"ts_parse","description":"Attempts to parse a string as a timestamp following a specified format and outputs a timestamp, which can then be fed into methods such as \u003c\u003cts_format, `ts_format`\u003e\u003e.\n\nThe input format is defined by showing how the reference time, defined to be Mon Jan 2 15:04:05 -0700 MST 2006, would be displayed if it were the value. For an alternative way to specify formats check out the \u003c\u003cts_strptime, `ts_strptime`\u003e\u003e method.","params":{"named":[{"name":"format","description":"The format of the target string.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":[{"mapping":"root.doc.timestamp = this.doc.timestamp.ts_parse(\"2006-Jan-02\")","summary":"","results":[["{\"doc\":{\"timestamp\":\"2020-Aug-14\"}}","{\"doc\":{\"timestamp\":\"2020-08-14T00:00:00Z\"}}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"ts_round","description":"Returns the result of rounding a timestamp to the nearest multiple of the argument duration (nanoseconds). The rounding behavior for halfway values is to round up. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format. The \u003c\u003cts_parse, `ts_parse`\u003e\u003e method can be used in order to parse different timestamp formats.","params":{"named":[{"name":"duration","description":"A duration measured in nanoseconds to round by.","type":"integer","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":[{"mapping":"root.created_at_hour = this.created_at.ts_round(\"1h\".parse_duration())","summary":"Use the method `parse_duration` to convert a duration string into an integer argument.","results":[["{\"created_at\":\"2020-08-14T05:54:23Z\"}","{\"created_at_hour\":\"2020-08-14T06:00:00Z\"}"]],"skip_testing":false}]}],"impure":false,"version":"4.2.0"},{"status":"beta","name":"ts_strftime","description":"Attempts to format a timestamp value as a string according to a specified strftime-compatible format. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format.","params":{"named":[{"name":"format","description":"The output format to use.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"tz","description":"An optional timezone to use, otherwise the timezone of the input string is used.","type":"string","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":[{"mapping":"root.something_at = (this.created_at + 300).ts_strftime(\"%Y-%b-%d %H:%M:%S\")","summary":"The format consists of zero or more conversion specifiers and ordinary characters (except `%`). All ordinary characters are copied to the output string without modification. Each conversion specification begins with `%` character followed by the character that determines the behavior of the specifier. Please refer to https://linux.die.net/man/3/strftime[man 3 strftime] for the list of format specifiers.","results":[],"skip_testing":false},{"mapping":"root.something_at = this.created_at.ts_strftime(\"%Y-%b-%d %H:%M:%S\", \"UTC\")","summary":"A second optional string argument can also be used in order to specify a timezone, otherwise the timezone of the input string is used, or in the case of unix timestamps the local timezone is used.","results":[["{\"created_at\":1597405526}","{\"something_at\":\"2020-Aug-14 11:45:26\"}"],["{\"created_at\":\"2020-08-14T11:50:26.371Z\"}","{\"something_at\":\"2020-Aug-14 11:50:26\"}"]],"skip_testing":false},{"mapping":"root.something_at = this.created_at.ts_strftime(\"%Y-%b-%d %H:%M:%S.%f\", \"UTC\")","summary":"As an extension provided by the underlying formatting library, https://github.com/itchyny/timefmt-go[itchyny/timefmt-go], the `%f` directive is supported for zero-padded microseconds, which originates from Python. Note that E and O modifier characters are not supported.","results":[["{\"created_at\":1597405526}","{\"something_at\":\"2020-Aug-14 11:45:26.000000\"}"],["{\"created_at\":\"2020-08-14T11:50:26.371Z\"}","{\"something_at\":\"2020-Aug-14 11:50:26.371000\"}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"ts_strptime","description":"Attempts to parse a string as a timestamp following a specified strptime-compatible format and outputs a timestamp, which can then be fed into \u003c\u003cts_format, `ts_format`\u003e\u003e.","params":{"named":[{"name":"format","description":"The format of the target string.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":[{"mapping":"root.doc.timestamp = this.doc.timestamp.ts_strptime(\"%Y-%b-%d\")","summary":"The format consists of zero or more conversion specifiers and ordinary characters (except `%`). All ordinary characters are copied to the output string without modification. Each conversion specification begins with a `%` character followed by the character that determines the behavior of the specifier. Please refer to https://linux.die.net/man/3/strptime[man 3 strptime] for the list of format specifiers.","results":[["{\"doc\":{\"timestamp\":\"2020-Aug-14\"}}","{\"doc\":{\"timestamp\":\"2020-08-14T00:00:00Z\"}}"]],"skip_testing":false},{"mapping":"root.doc.timestamp = this.doc.timestamp.ts_strptime(\"%Y-%b-%d %H:%M:%S.%f\")","summary":"As an extension provided by the underlying formatting library, https://github.com/itchyny/timefmt-go[itchyny/timefmt-go], the `%f` directive is supported for zero-padded microseconds, which originates from Python. Note that E and O modifier characters are not supported.","results":[["{\"doc\":{\"timestamp\":\"2020-Aug-14 11:50:26.371000\"}}","{\"doc\":{\"timestamp\":\"2020-08-14T11:50:26.371Z\"}}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"ts_sub","description":"Returns the difference in nanoseconds between the target timestamp (t1) and the timestamp provided as a parameter (t2). The \u003c\u003cts_parse, `ts_parse`\u003e\u003e method can be used in order to parse different timestamp formats.","params":{"named":[{"name":"t2","description":"The second timestamp to be subtracted from the method target.","type":"timestamp","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":[{"mapping":"root.between = this.started_at.ts_sub(\"2020-08-14T05:54:23Z\").abs()","summary":"Use the `.abs()` method in order to calculate an absolute duration between two timestamps.","results":[["{\"started_at\":\"2020-08-13T05:54:23Z\"}","{\"between\":86400000000000}"]],"skip_testing":false}]}],"impure":false,"version":"4.23.0"},{"status":"beta","name":"ts_sub_iso8601","description":"Parse parameter string as ISO 8601 period and subtract it from value with high precision for units larger than an hour.","params":{"named":[{"name":"duration","description":"Duration in ISO 8601 format","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":null}],"impure":false},{"status":"beta","name":"ts_tz","description":"Returns the result of converting a timestamp to a specified timezone. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format. The \u003c\u003cts_parse, `ts_parse`\u003e\u003e method can be used in order to parse different timestamp formats.","params":{"named":[{"name":"tz","description":"The timezone to change to. If set to \"UTC\" then the timezone will be UTC. If set to \"Local\" then the local timezone will be used. Otherwise, the argument is taken to be a location name corresponding to a file in the IANA Time Zone database, such as \"America/New_York\".","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":[{"mapping":"root.created_at_utc = this.created_at.ts_tz(\"UTC\")","summary":"","results":[["{\"created_at\":\"2021-02-03T17:05:06+01:00\"}","{\"created_at_utc\":\"2021-02-03T16:05:06Z\"}"]],"skip_testing":false}]}],"impure":false,"version":"4.3.0"},{"status":"beta","name":"ts_unix","description":"Attempts to format a timestamp value as a unix timestamp. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format. The \u003c\u003cts_parse, `ts_parse`\u003e\u003e method can be used in order to parse different timestamp formats.","params":{},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":[{"mapping":"root.created_at_unix = this.created_at.ts_unix()","summary":"","results":[["{\"created_at\":\"2009-11-10T23:00:00Z\"}","{\"created_at_unix\":1257894000}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"ts_unix_micro","description":"Attempts to format a timestamp value as a unix timestamp with microsecond precision. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format. The \u003c\u003cts_parse, `ts_parse`\u003e\u003e method can be used in order to parse different timestamp formats.","params":{},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":[{"mapping":"root.created_at_unix = this.created_at.ts_unix_micro()","summary":"","results":[["{\"created_at\":\"2009-11-10T23:00:00Z\"}","{\"created_at_unix\":1257894000000000}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"ts_unix_milli","description":"Attempts to format a timestamp value as a unix timestamp with millisecond precision. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format. The \u003c\u003cts_parse, `ts_parse`\u003e\u003e method can be used in order to parse different timestamp formats.","params":{},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":[{"mapping":"root.created_at_unix = this.created_at.ts_unix_milli()","summary":"","results":[["{\"created_at\":\"2009-11-10T23:00:00Z\"}","{\"created_at_unix\":1257894000000}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"ts_unix_nano","description":"Attempts to format a timestamp value as a unix timestamp with nanosecond precision. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format. The \u003c\u003cts_parse, `ts_parse`\u003e\u003e method can be used in order to parse different timestamp formats.","params":{},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":[{"mapping":"root.created_at_unix = this.created_at.ts_unix_nano()","summary":"","results":[["{\"created_at\":\"2009-11-10T23:00:00Z\"}","{\"created_at_unix\":1257894000000000000}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"type","params":{},"categories":[{"Category":"Type Coercion","Description":"Returns the type of a value as a string, providing one of the following values: `string`, `bytes`, `number`, `bool`, `timestamp`, `array`, `object` or `null`.","Examples":[{"mapping":"root.bar_type = this.bar.type()\nroot.foo_type = this.foo.type()","summary":"","results":[["{\"bar\":10,\"foo\":\"is a string\"}","{\"bar_type\":\"number\",\"foo_type\":\"string\"}"]],"skip_testing":false},{"mapping":"root.type = this.type()","summary":"","results":[["\"foobar\"","{\"type\":\"string\"}"],["666","{\"type\":\"number\"}"],["false","{\"type\":\"bool\"}"],["[\"foo\", \"bar\"]","{\"type\":\"array\"}"],["{\"foo\": \"bar\"}","{\"type\":\"object\"}"],["null","{\"type\":\"null\"}"]],"skip_testing":false},{"mapping":"root.type = content().type()","summary":"","results":[["foobar","{\"type\":\"bytes\"}"]],"skip_testing":false},{"mapping":"root.type = this.ts_parse(\"2006-01-02\").type()","summary":"","results":[["\"2022-06-06\"","{\"type\":\"timestamp\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"uint16","description":"\nConverts a numerical type into a 16-bit unsigned integer, this is for advanced use cases where a specific data type is needed for a given component (such as the ClickHouse SQL driver).\n\nIf the value is a string then an attempt will be made to parse it as a 16-bit unsigned integer. If the target value exceeds the capacity of an integer or contains decimal values then this method will throw an error. In order to convert a floating point number containing decimals first use \u003c\u003cround, `.round()`\u003e\u003e on the value. Please refer to the https://pkg.go.dev/strconv#ParseInt[`strconv.ParseInt` documentation] for details regarding the supported formats.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"\nroot.a = this.a.uint16()\nroot.b = this.b.round().uint16()\nroot.c = this.c.uint16()\nroot.d = this.d.uint16().catch(0)\n","summary":"","results":[["{\"a\":12,\"b\":12.34,\"c\":\"12\",\"d\":-12}","{\"a\":12,\"b\":12,\"c\":12,\"d\":0}"]],"skip_testing":false},{"mapping":"\nroot = this.uint16()\n","summary":"","results":[["\"0xDE\"","222"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"uint32","description":"\nConverts a numerical type into a 32-bit unsigned integer, this is for advanced use cases where a specific data type is needed for a given component (such as the ClickHouse SQL driver).\n\nIf the value is a string then an attempt will be made to parse it as a 32-bit unsigned integer. If the target value exceeds the capacity of an integer or contains decimal values then this method will throw an error. In order to convert a floating point number containing decimals first use \u003c\u003cround, `.round()`\u003e\u003e on the value. Please refer to the https://pkg.go.dev/strconv#ParseInt[`strconv.ParseInt` documentation] for details regarding the supported formats.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"\nroot.a = this.a.uint32()\nroot.b = this.b.round().uint32()\nroot.c = this.c.uint32()\nroot.d = this.d.uint32().catch(0)\n","summary":"","results":[["{\"a\":12,\"b\":12.34,\"c\":\"12\",\"d\":-12}","{\"a\":12,\"b\":12,\"c\":12,\"d\":0}"]],"skip_testing":false},{"mapping":"\nroot = this.uint32()\n","summary":"","results":[["\"0xDEAD\"","57005"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"uint64","description":"\nConverts a numerical type into a 64-bit unsigned integer, this is for advanced use cases where a specific data type is needed for a given component (such as the ClickHouse SQL driver).\n\nIf the value is a string then an attempt will be made to parse it as a 64-bit unsigned integer. If the target value exceeds the capacity of an integer or contains decimal values then this method will throw an error. In order to convert a floating point number containing decimals first use \u003c\u003cround, `.round()`\u003e\u003e on the value. Please refer to the https://pkg.go.dev/strconv#ParseInt[`strconv.ParseInt` documentation] for details regarding the supported formats.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"\nroot.a = this.a.uint64()\nroot.b = this.b.round().uint64()\nroot.c = this.c.uint64()\nroot.d = this.d.uint64().catch(0)\n","summary":"","results":[["{\"a\":12,\"b\":12.34,\"c\":\"12\",\"d\":-12}","{\"a\":12,\"b\":12,\"c\":12,\"d\":0}"]],"skip_testing":false},{"mapping":"\nroot = this.uint64()\n","summary":"","results":[["\"0xDEADBEEF\"","3735928559"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"uint8","description":"\nConverts a numerical type into a 8-bit unsigned integer, this is for advanced use cases where a specific data type is needed for a given component (such as the ClickHouse SQL driver).\n\nIf the value is a string then an attempt will be made to parse it as a 8-bit unsigned integer. If the target value exceeds the capacity of an integer or contains decimal values then this method will throw an error. In order to convert a floating point number containing decimals first use \u003c\u003cround, `.round()`\u003e\u003e on the value. Please refer to the https://pkg.go.dev/strconv#ParseInt[`strconv.ParseInt` documentation] for details regarding the supported formats.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"\nroot.a = this.a.uint8()\nroot.b = this.b.round().uint8()\nroot.c = this.c.uint8()\nroot.d = this.d.uint8().catch(0)\n","summary":"","results":[["{\"a\":12,\"b\":12.34,\"c\":\"12\",\"d\":-12}","{\"a\":12,\"b\":12,\"c\":12,\"d\":0}"]],"skip_testing":false},{"mapping":"\nroot = this.uint8()\n","summary":"","results":[["\"0xD\"","13"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"unescape_html","params":{},"categories":[{"Category":"String Manipulation","Description":"Unescapes a string so that entities like `\u0026lt;` become `\u003c`. It unescapes a larger range of entities than `escape_html` escapes. For example, `\u0026aacute;` unescapes to `Γ‘`, as does `\u0026#225;` and `\u0026xE1;`.","Examples":[{"mapping":"root.unescaped = this.value.unescape_html()","summary":"","results":[["{\"value\":\"foo \u0026amp; bar\"}","{\"unescaped\":\"foo \u0026 bar\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"unescape_url_query","params":{},"categories":[{"Category":"String Manipulation","Description":"Expands escape sequences from a URL query string.","Examples":[{"mapping":"root.unescaped = this.value.unescape_url_query()","summary":"","results":[["{\"value\":\"foo+%26+bar\"}","{\"unescaped\":\"foo \u0026 bar\"}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"unicode_segments","description":"Splits text into segments from a given string based on the unicode text segmentation rules.","params":{"named":[{"name":"segmentation_type","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"String Manipulation","Description":"","Examples":[{"mapping":"root.sentences = this.value.unicode_segments(\"sentence\")","summary":"Splits a string into different sentences","results":[["{\"value\":\"This is sentence 1.0. And this is sentence two.\"}","{\"sentences\":[\"This is sentence 1.0. \",\"And this is sentence two.\"]}"]],"skip_testing":false},{"mapping":"root.graphemes = this.value.unicode_segments(\"grapheme\")","summary":"Splits a string into different graphemes","results":[["{\"value\":\"πŸ•β€πŸ¦Ί 🫠\"}","{\"graphemes\":[\"πŸ•β€πŸ¦Ί\",\" \",\"🫠\"]}"]],"skip_testing":false},{"mapping":"root.words = this.value.unicode_segments(\"word\")","summary":"Splits text into words","results":[["{\"value\":\"Hello, world!\"}","{\"words\":[\"Hello\",\",\",\" \",\"world\",\"!\"]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"unique","params":{"named":[{"name":"emit","description":"An optional query that can be used in order to yield a value for each element to determine uniqueness.","type":"query expression","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"Attempts to remove duplicate values from an array. The array may contain a combination of different value types, but numbers and strings are checked separately (`\"5\"` is a different element to `5`).","Examples":[{"mapping":"root.uniques = this.foo.unique()","summary":"","results":[["{\"foo\":[\"a\",\"b\",\"a\",\"c\"]}","{\"uniques\":[\"a\",\"b\",\"c\"]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"unquote","params":{},"categories":[{"Category":"String Manipulation","Description":"Unquotes a target string, expanding any escape sequences (`\\t`, `\\n`, `\\xFF`, `\\u0100`) for control characters and non-printable characters.","Examples":[{"mapping":"root.unquoted = this.thing.unquote()","summary":"","results":[["{\"thing\":\"\\\"foo\\\\nbar\\\"\"}","{\"unquoted\":\"foo\\nbar\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"uppercase","params":{},"categories":[{"Category":"String Manipulation","Description":"Convert a string value into uppercase.","Examples":[{"mapping":"root.foo = this.foo.uppercase()","summary":"","results":[["{\"foo\":\"hello world\"}","{\"foo\":\"HELLO WORLD\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"values","params":{},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"Returns the values of an object as an array. The order of the resulting array will be random.","Examples":[{"mapping":"root.foo_vals = this.foo.values().sort()","summary":"","results":[["{\"foo\":{\"bar\":1,\"baz\":2}}","{\"foo_vals\":[1,2]}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"vector","description":"Creates a vector from a given array of floating point numbers.\n\nThis vector can be inserted into various SQL databases if they have support for embeddings vectors (for example `pgvector`).","params":{},"categories":[{"Category":"SQL","Description":"","Examples":[{"mapping":"root.embeddings = [1.2, 0.6, 0.9].vector()","summary":"Create a vector from an array literal","results":[],"skip_testing":false},{"mapping":"root.embedding_vector = this.embedding_array.vector()","summary":"Create a vector from an array","results":[],"skip_testing":false}]}],"impure":false,"version":"4.33.0"},{"status":"stable","name":"with","description":"Returns an object where all but one or more xref:configuration:field_paths.adoc[field path] arguments are removed. Each path specifies a specific field to be retained from the input object, allowing for nested fields.\n\nIf a key within a nested path does not exist then it is ignored.","params":{"variadic":true},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root = this.with(\"inner.a\",\"inner.c\",\"d\")","summary":"","results":[["{\"inner\":{\"a\":\"first\",\"b\":\"second\",\"c\":\"third\"},\"d\":\"fourth\",\"e\":\"fifth\"}","{\"d\":\"fourth\",\"inner\":{\"a\":\"first\",\"c\":\"third\"}}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"without","params":{"variadic":true},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"Returns an object where one or more xref:configuration:field_paths.adoc[field path] arguments are removed. Each path specifies a specific field to be deleted from the input object, allowing for nested fields.\n\nIf a key within a nested path does not exist or is not an object then it is not removed.","Examples":[{"mapping":"root = this.without(\"inner.a\",\"inner.c\",\"d\")","summary":"","results":[["{\"inner\":{\"a\":\"first\",\"b\":\"second\",\"c\":\"third\"},\"d\":\"fourth\",\"e\":\"fifth\"}","{\"e\":\"fifth\",\"inner\":{\"b\":\"second\"}}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"zip","description":"Zip an array value with one or more argument arrays. Each array must match in length.","params":{"variadic":true},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.foo = this.foo.zip(this.bar, this.baz)","summary":"","results":[["{\"foo\":[\"a\",\"b\",\"c\"],\"bar\":[1,2,3],\"baz\":[4,5,6]}","{\"foo\":[[\"a\",1,4],[\"b\",2,5],[\"c\",3,6]]}"]],"skip_testing":false}]}],"impure":false}]} +{"version":"4.53.0","date":"2025-04-18T17:49:53Z","config":[{"name":"http","type":"object","kind":"scalar","description":"Configures the service-wide HTTP server.","children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to enable to HTTP server.","default":true},{"name":"address","type":"string","kind":"scalar","description":"The address to bind to.","default":"0.0.0.0:4195"},{"name":"root_path","type":"string","kind":"scalar","description":"Specifies a general prefix for all endpoints, this can help isolate the service endpoints when using a reverse proxy with other shared services. All endpoints will still be registered at the root as well as behind the prefix, e.g. with a root_path set to `/foo` the endpoint `/version` will be accessible from both `/version` and `/foo/version`.","default":"/benthos"},{"name":"debug_endpoints","type":"bool","kind":"scalar","description":"Whether to register a few extra endpoints that can be useful for debugging performance or behavioral problems.","default":false},{"name":"cert_file","type":"string","kind":"scalar","description":"An optional certificate file for enabling TLS.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"An optional key file for enabling TLS.","is_advanced":true,"default":""},{"name":"cors","type":"object","kind":"scalar","description":"Adds Cross-Origin Resource Sharing headers.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to allow CORS requests.","is_advanced":true,"default":false},{"name":"allowed_origins","type":"string","kind":"array","description":"An explicit list of origins that are allowed for CORS requests.","is_advanced":true,"default":[]}],"version":"3.63.0"},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to enforce and customise basic authentication for requests to the HTTP server.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Enable basic authentication","is_advanced":true,"default":false},{"name":"realm","type":"string","kind":"scalar","description":"Custom realm name","is_advanced":true,"default":"restricted"},{"name":"username","type":"string","kind":"scalar","description":"Username required to authenticate.","is_advanced":true,"default":""},{"name":"password_hash","type":"string","kind":"scalar","description":"Hashed password required to authenticate. (base64 encoded)","is_advanced":true,"default":""},{"name":"algorithm","type":"string","kind":"scalar","description":"Encryption algorithm used to generate `password_hash`.","is_advanced":true,"default":"sha256","examples":["md5","sha256","bcrypt","scrypt"]},{"name":"salt","type":"string","kind":"scalar","description":"Salt for scrypt algorithm. (base64 encoded)","is_advanced":true,"default":""}]}]},{"name":"input","type":"input","kind":"scalar","description":"An input to source messages from.","default":{"stdin":{}}},{"name":"buffer","type":"buffer","kind":"scalar","description":"An optional buffer to store messages during transit.","default":{"none":{}}},{"name":"pipeline","type":"object","kind":"scalar","description":"Describes optional processing pipelines used for mutating messages.","children":[{"name":"threads","type":"int","kind":"scalar","description":"The number of threads to execute processing pipelines across.","default":-1},{"name":"processors","type":"processor","kind":"array","description":"A list of processors to apply to messages.","default":[]}]},{"name":"output","type":"output","kind":"scalar","description":"An output to sink messages to.","default":{"stdout":{}}},{"name":"input_resources","type":"input","kind":"array","description":"A list of input resources, each must have a unique label.","is_advanced":true,"default":[]},{"name":"processor_resources","type":"processor","kind":"array","description":"A list of processor resources, each must have a unique label.","is_advanced":true,"default":[]},{"name":"output_resources","type":"output","kind":"array","description":"A list of output resources, each must have a unique label.","is_advanced":true,"default":[]},{"name":"cache_resources","type":"cache","kind":"array","description":"A list of cache resources, each must have a unique label.","is_advanced":true,"default":[]},{"name":"rate_limit_resources","type":"rate_limit","kind":"array","description":"A list of rate limit resources, each must have a unique label.","is_advanced":true,"default":[]},{"name":"logger","type":"object","kind":"scalar","description":"Describes how operational logs should be emitted.","children":[{"name":"level","type":"string","kind":"scalar","description":"Set the minimum severity level for emitting logs.","default":"INFO","options":["OFF","FATAL","ERROR","WARN","INFO","DEBUG","TRACE","ALL","NONE"]},{"name":"format","type":"string","kind":"scalar","description":"Set the format of emitted logs.","default":"logfmt","options":["json","logfmt"],"linter":"\nlet options = {\n \"json\": true,\n \"logfmt\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"add_timestamp","type":"bool","kind":"scalar","description":"Whether to include timestamps in logs.","default":false},{"name":"level_name","type":"string","kind":"scalar","description":"The name of the level field added to logs when the `format` is `json`.","is_advanced":true,"default":"level"},{"name":"timestamp_name","type":"string","kind":"scalar","description":"The name of the timestamp field added to logs when `add_timestamp` is set to `true` and the `format` is `json`.","is_advanced":true,"default":"time"},{"name":"message_name","type":"string","kind":"scalar","description":"The name of the message field added to logs when the `format` is `json`.","is_advanced":true,"default":"msg"},{"name":"static_fields","type":"string","kind":"map","description":"A map of key/value pairs to add to each structured log.","default":{"@service":"redpanda-connect"}},{"name":"file","type":"object","kind":"scalar","description":"Experimental: Specify fields for optionally writing logs to a file.","is_advanced":true,"children":[{"name":"path","type":"string","kind":"scalar","description":"The file path to write logs to, if the file does not exist it will be created. Leave this field empty or unset to disable file based logging.","is_advanced":true,"default":""},{"name":"rotate","type":"bool","kind":"scalar","description":"Whether to rotate log files automatically.","is_advanced":true,"default":false},{"name":"rotate_max_age_days","type":"int","kind":"scalar","description":"The maximum number of days to retain old log files based on the timestamp encoded in their filename, after which they are deleted. Setting to zero disables this mechanism.","is_advanced":true,"default":0}]}]},{"name":"metrics","type":"metrics","kind":"scalar","description":"A mechanism for exporting metrics.","default":{"mapping":"","prometheus":{}}},{"name":"tracer","type":"tracer","kind":"scalar","description":"A mechanism for exporting traces.","default":{"none":{}}},{"name":"shutdown_delay","type":"string","kind":"scalar","description":"A period of time to wait for metrics and traces to be pulled or pushed from the process.","default":"0s"},{"name":"shutdown_timeout","type":"string","kind":"scalar","description":"The maximum period of time to wait for a clean shutdown. If this time is exceeded Redpanda Connect will forcefully close.","default":"20s"},{"name":"tests","type":"object","kind":"array","description":"A list of one or more unit tests to execute.","is_advanced":true,"is_optional":true,"children":[{"name":"name","type":"string","kind":"scalar","description":"The name of the test, this should be unique and give a rough indication of what behavior is being tested.","is_advanced":true},{"name":"environment","type":"string","kind":"map","description":"An optional map of environment variables to set for the duration of the test.","is_advanced":true,"is_optional":true},{"name":"target_processors","type":"string","kind":"scalar","description":"\nA [JSON Pointer][json-pointer] that identifies the specific processors which should be executed by the test. The target can either be a single processor or an array of processors. Alternatively a resource label can be used to identify a processor.\n\nIt is also possible to target processors in a separate file by prefixing the target with a path relative to the test file followed by a # symbol.\n","is_advanced":true,"default":"/pipeline/processors","examples":["foo_processor","/pipeline/processors/0","target.yaml#/pipeline/processors","target.yaml#/pipeline/processors"]},{"name":"target_mapping","type":"string","kind":"scalar","description":"A file path relative to the test definition path of a Bloblang file to execute as an alternative to testing processors with the `target_processors` field. This allows you to define unit tests for Bloblang mappings directly.","is_advanced":true,"default":""},{"name":"mocks","type":"unknown","kind":"map","description":"An optional map of processors to mock. Keys should contain either a label or a JSON pointer of a processor that should be mocked. Values should contain a processor definition, which will replace the mocked processor. Most of the time you'll want to use a [`mapping` processor][processors.mapping] here, and use it to create a result that emulates the target processor.","is_advanced":true,"is_optional":true,"examples":[{"get_foobar_api":{"mapping":"root = content().string() + \" this is some mock content\""}},{"/pipeline/processors/1":{"mapping":"root = content().string() + \" this is some mock content\""}}]},{"name":"input_batch","type":"object","kind":"array","description":"Define a batch of messages to feed into your test, specify either an `input_batch` or a series of `input_batches`.","is_advanced":true,"is_optional":true,"children":[{"name":"content","type":"string","kind":"scalar","description":"The raw content of the input message.","is_advanced":true,"is_optional":true},{"name":"json_content","type":"unknown","kind":"scalar","description":"Sets the raw content of the message to a JSON document matching the structure of the value.","is_advanced":true,"is_optional":true,"examples":[{"bar":["element1",10],"foo":"foo value"}]},{"name":"file_content","type":"string","kind":"scalar","description":"Sets the raw content of the message by reading a file. The path of the file should be relative to the path of the test file.","is_advanced":true,"is_optional":true,"examples":["./foo/bar.txt"]},{"name":"metadata","type":"unknown","kind":"map","description":"A map of metadata key/values to add to the input message.","is_advanced":true,"is_optional":true}]},{"name":"input_batches","type":"object","kind":"2darray","description":"Define a series of batches of messages to feed into your test, specify either an `input_batch` or a series of `input_batches`.","is_advanced":true,"is_optional":true,"children":[{"name":"content","type":"string","kind":"scalar","description":"The raw content of the input message.","is_advanced":true,"is_optional":true},{"name":"json_content","type":"unknown","kind":"scalar","description":"Sets the raw content of the message to a JSON document matching the structure of the value.","is_advanced":true,"is_optional":true,"examples":[{"bar":["element1",10],"foo":"foo value"}]},{"name":"file_content","type":"string","kind":"scalar","description":"Sets the raw content of the message by reading a file. The path of the file should be relative to the path of the test file.","is_advanced":true,"is_optional":true,"examples":["./foo/bar.txt"]},{"name":"metadata","type":"unknown","kind":"map","description":"A map of metadata key/values to add to the input message.","is_advanced":true,"is_optional":true}]},{"name":"output_batches","type":"object","kind":"2darray","description":"List of output batches.","is_advanced":true,"is_optional":true,"children":[{"name":"bloblang","type":"string","kind":"scalar","description":"Executes a Bloblang mapping on the output message, if the result is anything other than a boolean equalling `true` the test fails.","is_advanced":true,"is_optional":true,"bloblang":true,"examples":["this.age \u003e 10 \u0026\u0026 @foo.length() \u003e 0"]},{"name":"content_equals","type":"string","kind":"scalar","description":"Checks the full raw contents of a message against a value.","is_advanced":true,"is_optional":true},{"name":"content_matches","type":"string","kind":"scalar","description":"Checks whether the full raw contents of a message matches a regular expression (re2).","is_advanced":true,"is_optional":true,"examples":["^foo [a-z]+ bar$"]},{"name":"metadata_equals","type":"unknown","kind":"map","description":"Checks a map of metadata keys to values against the metadata stored in the message. If there is a value mismatch between a key of the condition versus the message metadata this condition will fail.","is_advanced":true,"is_optional":true,"examples":[{"example_key":"example metadata value"}]},{"name":"file_equals","type":"string","kind":"scalar","description":"Checks that the contents of a message matches the contents of a file. The path of the file should be relative to the path of the test file.","is_advanced":true,"is_optional":true,"examples":["./foo/bar.txt"]},{"name":"file_json_equals","type":"string","kind":"scalar","description":"Checks that both the message and the file contents are valid JSON documents, and that they are structurally equivalent. Will ignore formatting and ordering differences. The path of the file should be relative to the path of the test file.","is_advanced":true,"is_optional":true,"examples":["./foo/bar.json"]},{"name":"json_equals","type":"unknown","kind":"scalar","description":"Checks that both the message and the condition are valid JSON documents, and that they are structurally equivalent. Will ignore formatting and ordering differences.","is_advanced":true,"is_optional":true,"examples":[{"key":"value"}]},{"name":"json_contains","type":"unknown","kind":"scalar","description":"Checks that both the message and the condition are valid JSON documents, and that the message is a superset of the condition.","is_advanced":true,"is_optional":true,"examples":[{"key":"value"}]},{"name":"file_json_contains","type":"string","kind":"scalar","description":"Checks that both the message and the file contents are valid JSON documents, and that the message is a superset of the condition. Will ignore formatting and ordering differences. The path of the file should be relative to the path of the test file.","is_advanced":true,"is_optional":true,"examples":["./foo/bar.json"]}]}]},{"name":"redpanda","type":"object","kind":"scalar","children":[{"name":"seed_brokers","type":"string","kind":"array","description":"A list of broker addresses to connect to in order to establish connections. If an item of the list contains commas it will be expanded into multiple addresses.","examples":[["localhost:9092"],["foo:9092","bar:9092"],["foo:9092,bar:9092"]]},{"name":"client_id","type":"string","kind":"scalar","description":"An identifier for the client connection.","is_advanced":true,"default":"benthos"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"sasl","type":"object","kind":"array","description":"Specify one or more methods of SASL authentication. SASL is tried in order; if the broker supports the first mechanism, all connections will use that mechanism. If the first mechanism fails, the client will pick the first supported mechanism. If the broker does not support any client mechanisms, connections will fail.","is_advanced":true,"is_optional":true,"examples":[[{"mechanism":"SCRAM-SHA-512","password":"bar","username":"foo"}]],"children":[{"name":"mechanism","type":"string","kind":"scalar","description":"The SASL mechanism to use.","is_advanced":true,"annotated_options":[["AWS_MSK_IAM","AWS IAM based authentication as specified by the 'aws-msk-iam-auth' java library."],["OAUTHBEARER","OAuth Bearer based authentication."],["PLAIN","Plain text authentication."],["SCRAM-SHA-256","SCRAM based authentication as specified in RFC5802."],["SCRAM-SHA-512","SCRAM based authentication as specified in RFC5802."],["none","Disable sasl authentication"]],"linter":"\nlet options = {\n \"aws_msk_iam\": true,\n \"oauthbearer\": true,\n \"plain\": true,\n \"scram-sha-256\": true,\n \"scram-sha-512\": true,\n \"none\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"username","type":"string","kind":"scalar","description":"A username to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token to use for a single session's OAUTHBEARER authentication.","is_advanced":true,"default":""},{"name":"extensions","type":"string","kind":"map","description":"Key/value pairs to add to OAUTHBEARER authentication requests.","is_advanced":true,"is_optional":true},{"name":"aws","type":"object","kind":"scalar","description":"Contains AWS specific fields for when the `mechanism` is set to `AWS_MSK_IAM`.","is_advanced":true,"is_optional":true,"children":[{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]}]},{"name":"metadata_max_age","type":"string","kind":"scalar","description":"The maximum age of metadata before it is refreshed.","is_advanced":true,"default":"5m"},{"name":"request_timeout_overhead","type":"string","kind":"scalar","description":"The request time overhead. Uses the given time as overhead while deadlining requests. Roughly equivalent to request.timeout.ms, but grants additional time to requests that have timeout fields.","is_advanced":true,"default":"10s"},{"name":"conn_idle_timeout","type":"string","kind":"scalar","description":"The rough amount of time to allow connections to idle before they are closed.","is_advanced":true,"default":"20s"},{"name":"pipeline_id","type":"string","kind":"scalar","description":"An optional identifier for the pipeline, this will be present in logs and status updates sent to topics.","default":""},{"name":"logs_topic","type":"string","kind":"scalar","description":"A topic to send process logs to.","default":"","examples":["__redpanda.connect.logs"]},{"name":"logs_level","type":"string","kind":"scalar","default":"info","options":["debug","info","warn","error"],"linter":"\nlet options = {\n \"debug\": true,\n \"info\": true,\n \"warn\": true,\n \"error\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"status_topic","type":"string","kind":"scalar","description":"A topic to send status updates to.","default":"","examples":["__redpanda.connect.status"]},{"name":"rack_id","type":"string","kind":"scalar","is_deprecated":true},{"name":"partitioner","type":"string","kind":"scalar","description":"Override the default murmur2 hashing partitioner.","is_advanced":true,"is_optional":true,"annotated_options":[["least_backup","Chooses the least backed up partition (the partition with the fewest amount of buffered records). Partitions are selected per batch."],["manual","Manually select a partition for each message, requires the field `partition` to be specified."],["murmur2_hash","Kafka's default hash algorithm that uses a 32-bit murmur2 hash of the key to compute which partition the record will be on."],["round_robin","Round-robin's messages through all available partitions. This algorithm has lower throughput and causes higher CPU load on brokers, but can be useful if you want to ensure an even distribution of records to partitions."]],"linter":"\nlet options = {\n \"least_backup\": true,\n \"manual\": true,\n \"murmur2_hash\": true,\n \"round_robin\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"idempotent_write","type":"bool","kind":"scalar","description":"Enable the idempotent write producer option. This requires the `IDEMPOTENT_WRITE` permission on `CLUSTER` and can be disabled if this permission is not available.","is_advanced":true,"default":true},{"name":"compression","type":"string","kind":"scalar","description":"Optionally set an explicit compression type. The default preference is to use snappy when the broker supports it, and fall back to none if not.","is_advanced":true,"is_optional":true,"options":["lz4","snappy","gzip","none","zstd"],"linter":"\nlet options = {\n \"lz4\": true,\n \"snappy\": true,\n \"gzip\": true,\n \"none\": true,\n \"zstd\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period of time to wait for message sends before abandoning the request and retrying","is_advanced":true,"default":"10s"},{"name":"max_message_bytes","type":"string","kind":"scalar","description":"The maximum space in bytes than an individual message may take, messages larger than this value will be rejected. This field corresponds to Kafka's `max.message.bytes`.","is_advanced":true,"default":"1MiB","examples":["100MB","50mib"]},{"name":"broker_write_max_bytes","type":"string","kind":"scalar","description":"The upper bound for the number of bytes written to a broker connection in a single write. This field corresponds to Kafka's `socket.request.max.bytes`.","is_advanced":true,"default":"100MiB","examples":["128MB","50mib"]}]}],"buffers":[{"name":"memory","type":"buffer","status":"stable","plugin":true,"summary":"Stores consumed messages in memory and acknowledges them at the input level. During shutdown Redpanda Connect will make a best attempt at flushing all remaining messages before exiting cleanly.","description":"\nThis buffer is appropriate when consuming messages from inputs that do not gracefully handle back pressure and where delivery guarantees aren't critical.\n\nThis buffer has a configurable limit, where consumption will be stopped with back pressure upstream if the total size of messages in the buffer reaches this amount. Since this calculation is only an estimate, and the real size of messages in RAM is always higher, it is recommended to set the limit significantly below the amount of RAM available.\n\n== Delivery guarantees\n\nThis buffer intentionally weakens the delivery guarantees of the pipeline and therefore should never be used in places where data loss is unacceptable.\n\n== Batching\n\nIt is possible to batch up messages sent from this buffer using a xref:configuration:batching.adoc#batch-policy[batch policy].","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"limit","type":"int","kind":"scalar","description":"The maximum buffer size (in bytes) to allow before applying backpressure upstream.","default":524288000},{"name":"batch_policy","type":"object","kind":"","description":"Optionally configure a policy to flush buffered messages in batches.","children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to batch messages as they are flushed.","default":false},{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]}},{"name":"none","type":"buffer","status":"stable","plugin":true,"summary":"Do not buffer messages. This is the default and most resilient configuration.","description":"Selecting no buffer means the output layer is directly coupled with the input layer. This is the safest and lowest latency option since acknowledgements from at-least-once protocols can be propagated all the way from the output protocol to the input protocol.\n\nIf the output layer is hit with back pressure it will propagate all the way to the input layer, and further up the data stream. If you need to relieve your pipeline of this back pressure consider using a more robust buffering solution such as Kafka before resorting to alternatives.","categories":null,"config":{"name":"","type":"object","kind":"scalar","default":{}}},{"name":"sqlite","type":"buffer","status":"stable","plugin":true,"summary":"Stores messages in an SQLite database and acknowledges them at the input level.","description":"\nStored messages are then consumed as a stream from the database and deleted only once they are successfully sent at the output level. If the service is restarted Redpanda Connect will make a best attempt to finish delivering messages that are already read from the database, and when it starts again it will consume from the oldest message that has not yet been delivered.\n\n== Delivery guarantees\n\nMessages are not acknowledged at the input level until they have been added to the SQLite database, and they are not removed from the SQLite database until they have been successfully delivered. This means at-least-once delivery guarantees are preserved in cases where the service is shut down unexpectedly. However, since this process relies on interaction with the disk (wherever the SQLite DB is stored) these delivery guarantees are not resilient to disk corruption or loss.\n\n== Batching\n\nMessages that are logically batched at the point where they are added to the buffer will continue to be associated with that batch when they are consumed. This buffer is also more efficient when storing messages within batches, and therefore it is recommended to use batching at the input level in high-throughput use cases even if they are not required for processing.\n","categories":["Utility"],"examples":[{"title":"Batching for optimization","summary":"Batching at the input level greatly increases the throughput of this buffer. If logical batches aren't needed for processing add a xref:components:processors/split.adoc[`split` processor] to the `post_processors`.","config":"\ninput:\n batched:\n child:\n sql_select:\n driver: postgres\n dsn: postgres://foouser:foopass@localhost:5432/testdb?sslmode=disable\n table: footable\n columns: [ '*' ]\n policy:\n count: 100\n period: 500ms\n\nbuffer:\n sqlite:\n path: ./foo.db\n post_processors:\n - split: {}\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"path","type":"string","kind":"scalar","description":"The path of the database file, which will be created if it does not already exist."},{"name":"pre_processors","type":"processor","kind":"array","description":"An optional list of processors to apply to messages before they are stored within the buffer. These processors are useful for compressing, archiving or otherwise reducing the data in size before it's stored on disk.","is_optional":true},{"name":"post_processors","type":"processor","kind":"array","description":"An optional list of processors to apply to messages after they are consumed from the buffer. These processors are useful for undoing any compression, archiving, etc that may have been done by your `pre_processors`.","is_optional":true}]}},{"name":"system_window","type":"buffer","status":"beta","plugin":true,"summary":"Chops a stream of messages into tumbling or sliding windows of fixed temporal size, following the system clock.","description":"\nA window is a grouping of messages that fit within a discrete measure of time following the system clock. Messages are allocated to a window either by the processing time (the time at which they're ingested) or by the event time, and this is controlled via the \u003c\u003ctimestamp_mapping, `timestamp_mapping` field\u003e\u003e.\n\nIn tumbling mode (default) the beginning of a window immediately follows the end of a prior window. When the buffer is initialized the first window to be created and populated is aligned against the zeroth minute of the zeroth hour of the day by default, and may therefore be open for a shorter period than the specified size.\n\nA window is flushed only once the system clock surpasses its scheduled end. If an \u003c\u003callowed_lateness, `allowed_lateness`\u003e\u003e is specified then the window will not be flushed until the scheduled end plus that length of time.\n\nWhen a message is added to a window it has a metadata field `window_end_timestamp` added to it containing the timestamp of the end of the window as an RFC3339 string.\n\n== Sliding windows\n\nSliding windows begin from an offset of the prior windows' beginning rather than its end, and therefore messages may belong to multiple windows. In order to produce sliding windows specify a \u003c\u003cslide, `slide` duration\u003e\u003e.\n\n== Back pressure\n\nIf back pressure is applied to this buffer either due to output services being unavailable or resources being saturated, windows older than the current and last according to the system clock will be dropped in order to prevent unbounded resource usage. This means you should ensure that under the worst case scenario you have enough system memory to store two windows' worth of data at a given time (plus extra for redundancy and other services).\n\nIf messages could potentially arrive with event timestamps in the future (according to the system clock) then you should also factor in these extra messages in memory usage estimates.\n\n== Delivery guarantees\n\nThis buffer honours the transaction model within Redpanda Connect in order to ensure that messages are not acknowledged until they are either intentionally dropped or successfully delivered to outputs. However, since messages belonging to an expired window are intentionally dropped there are circumstances where not all messages entering the system will be delivered.\n\nWhen this buffer is configured with a slide duration it is possible for messages to belong to multiple windows, and therefore be delivered multiple times. In this case the first time the message is delivered it will be acked (or nacked) and subsequent deliveries of the same message will be a \"best attempt\".\n\nDuring graceful termination if the current window is partially populated with messages they will be nacked such that they are re-consumed the next time the service starts.\n","categories":["Windowing"],"examples":[{"title":"Counting Passengers at Traffic","summary":"Given a stream of messages relating to cars passing through various traffic lights of the form:\n\n```json\n{\n \"traffic_light\": \"cbf2eafc-806e-4067-9211-97be7e42cee3\",\n \"created_at\": \"2021-08-07T09:49:35Z\",\n \"registration_plate\": \"AB1C DEF\",\n \"passengers\": 3\n}\n```\n\nWe can use a window buffer in order to create periodic messages summarizing the traffic for a period of time of this form:\n\n```json\n{\n \"traffic_light\": \"cbf2eafc-806e-4067-9211-97be7e42cee3\",\n \"created_at\": \"2021-08-07T10:00:00Z\",\n \"total_cars\": 15,\n \"passengers\": 43\n}\n```\n\nWith the following config:","config":"\nbuffer:\n system_window:\n timestamp_mapping: root = this.created_at\n size: 1h\n\npipeline:\n processors:\n # Group messages of the window into batches of common traffic light IDs\n - group_by_value:\n value: '${! json(\"traffic_light\") }'\n\n # Reduce each batch to a single message by deleting indexes \u003e 0, and\n # aggregate the car and passenger counts.\n - mapping: |\n root = if batch_index() == 0 {\n {\n \"traffic_light\": this.traffic_light,\n \"created_at\": meta(\"window_end_timestamp\"),\n \"total_cars\": json(\"registration_plate\").from_all().unique().length(),\n \"passengers\": json(\"passengers\").from_all().sum(),\n }\n } else { deleted() }\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"timestamp_mapping","type":"string","kind":"scalar","description":"\nA xref:guides:bloblang/about.adoc[Bloblang mapping] applied to each message during ingestion that provides the timestamp to use for allocating it a window. By default the function `now()` is used in order to generate a fresh timestamp at the time of ingestion (the processing time), whereas this mapping can instead extract a timestamp from the message itself (the event time).\n\nThe timestamp value assigned to `root` must either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in ISO 8601 format. If the mapping fails or provides an invalid result the message will be dropped (with logging to describe the problem).\n","default":"root = now()","bloblang":true,"examples":["root = this.created_at","root = meta(\"kafka_timestamp_unix\").number()"]},{"name":"size","type":"string","kind":"scalar","description":"A duration string describing the size of each window. By default windows are aligned to the zeroth minute and zeroth hour on the UTC clock, meaning windows of 1 hour duration will match the turn of each hour in the day, this can be adjusted with the `offset` field.","examples":["30s","10m"]},{"name":"slide","type":"string","kind":"scalar","description":"An optional duration string describing by how much time the beginning of each window should be offset from the beginning of the previous, and therefore creates sliding windows instead of tumbling. When specified this duration must be smaller than the `size` of the window.","default":"","examples":["30s","10m"]},{"name":"offset","type":"string","kind":"scalar","description":"An optional duration string to offset the beginning of each window by, otherwise they are aligned to the zeroth minute and zeroth hour on the UTC clock. The offset cannot be a larger or equal measure to the window size or the slide.","default":"","examples":["-6h","30m"]},{"name":"allowed_lateness","type":"string","kind":"scalar","description":"An optional duration string describing the length of time to wait after a window has ended before flushing it, allowing late arrivals to be included. Since this windowing buffer uses the system clock an allowed lateness can improve the matching of messages when using event time.","default":"","examples":["10s","1m"]}]},"version":"3.53.0"}],"caches":[{"name":"aws_dynamodb","type":"cache","status":"stable","plugin":true,"summary":"Stores key/value pairs as a single document in a DynamoDB table. The key is stored as a string value and used as the table hash key. The value is stored as\na binary value using the `data_key` field name.","description":"A prefix can be specified to allow multiple cache types to share a single DynamoDB table. An optional TTL duration (`ttl`) and field\n(`ttl_key`) can be specified if the backing table has TTL enabled.\n\nStrong read consistency can be enabled using the `consistent_read` configuration field.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"table","type":"string","kind":"scalar","description":"The table to store items in."},{"name":"hash_key","type":"string","kind":"scalar","description":"The key of the table column to store item keys within."},{"name":"data_key","type":"string","kind":"scalar","description":"The key of the table column to store item values within."},{"name":"consistent_read","type":"bool","kind":"scalar","description":"Whether to use strongly consistent reads on Get commands.","is_advanced":true,"default":false},{"name":"default_ttl","type":"string","kind":"scalar","description":"An optional default TTL to set for items, calculated from the moment the item is cached. A `ttl_key` must be specified in order to set item TTLs.","is_advanced":true,"is_optional":true},{"name":"ttl_key","type":"string","kind":"scalar","description":"The column key to place the TTL value within.","is_advanced":true,"is_optional":true},{"name":"retries","type":"object","kind":"scalar","description":"Determine time intervals and cut offs for retry attempts.","is_advanced":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"1s","examples":["50ms","1s"]},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts","is_advanced":true,"default":"5s","examples":["5s","1m"]},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum overall period of time to spend on retry attempts before the request is aborted.","is_advanced":true,"default":"30s","examples":["1m","1h"]}]},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]},"version":"3.36.0"},{"name":"aws_s3","type":"cache","status":"stable","plugin":true,"summary":"Stores each item in an S3 bucket as a file, where an item ID is the path of the item within the bucket.","description":"It is not possible to atomically upload S3 objects exclusively when the target does not already exist, therefore this cache is not suitable for deduplication.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"bucket","type":"string","kind":"scalar","description":"The S3 bucket to store items in."},{"name":"content_type","type":"string","kind":"scalar","description":"The content type to set for each item.","default":"application/octet-stream"},{"name":"force_path_style_urls","type":"bool","kind":"scalar","description":"Forces the client API to use path style URLs, which helps when connecting to custom endpoints.","is_advanced":true,"default":false},{"name":"retries","type":"object","kind":"scalar","description":"Determine time intervals and cut offs for retry attempts.","is_advanced":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"1s","examples":["50ms","1s"]},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts","is_advanced":true,"default":"5s","examples":["5s","1m"]},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum overall period of time to spend on retry attempts before the request is aborted.","is_advanced":true,"default":"30s","examples":["1m","1h"]}]},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]},"version":"3.36.0"},{"name":"couchbase","type":"cache","status":"experimental","plugin":true,"summary":"Use a Couchbase instance as a cache.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"Couchbase connection string.","examples":["couchbase://localhost:11210"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"username","type":"string","kind":"scalar","description":"Username to connect to the cluster.","is_optional":true},{"name":"password","type":"string","kind":"scalar","description":"Password to connect to the cluster.","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"bucket","type":"string","kind":"scalar","description":"Couchbase bucket."},{"name":"collection","type":"string","kind":"scalar","description":"Bucket collection.","is_advanced":true,"is_optional":true,"default":"_default"},{"name":"transcoder","type":"string","kind":"scalar","description":"Couchbase transcoder to use.","is_advanced":true,"default":"legacy","annotated_options":[["json","JSONTranscoder implements the default transcoding behavior and applies JSON transcoding to all values. This will apply the following behavior to the value: binary ([]byte) -\u003e error. default -\u003e JSON value, JSON Flags."],["legacy","LegacyTranscoder implements the behavior for a backward-compatible transcoder. This transcoder implements behavior matching that of gocb v1.This will apply the following behavior to the value: binary ([]byte) -\u003e binary bytes, Binary expectedFlags. string -\u003e string bytes, String expectedFlags. default -\u003e JSON value, JSON expectedFlags."],["raw","RawBinaryTranscoder implements passthrough behavior of raw binary data. This transcoder does not apply any serialization. This will apply the following behavior to the value: binary ([]byte) -\u003e binary bytes, binary expectedFlags. default -\u003e error."],["rawjson","RawJSONTranscoder implements passthrough behavior of JSON data. This transcoder does not apply any serialization. It will forward data across the network without incurring unnecessary parsing costs. This will apply the following behavior to the value: binary ([]byte) -\u003e JSON bytes, JSON expectedFlags. string -\u003e JSON bytes, JSON expectedFlags. default -\u003e error."],["rawstring","RawStringTranscoder implements passthrough behavior of raw string data. This transcoder does not apply any serialization. This will apply the following behavior to the value: string -\u003e string bytes, string expectedFlags. default -\u003e error."]],"linter":"\nlet options = {\n \"json\": true,\n \"legacy\": true,\n \"raw\": true,\n \"rawjson\": true,\n \"rawstring\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"timeout","type":"string","kind":"scalar","description":"Operation timeout.","is_advanced":true,"default":"15s"},{"name":"default_ttl","type":"string","kind":"scalar","description":"An optional default TTL to set for items, calculated from the moment the item is cached.","is_advanced":true,"is_optional":true}]},"version":"4.12.0"},{"name":"file","type":"cache","status":"stable","plugin":true,"summary":"Stores each item in a directory as a file, where an item ID is the path relative to the configured directory.","description":"This type currently offers no form of item expiry or garbage collection, and is intended to be used for development and debugging purposes only.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"directory","type":"string","kind":"scalar","description":"The directory within which to store items."}]}},{"name":"gcp_cloud_storage","type":"cache","status":"beta","plugin":true,"summary":"Use a Google Cloud Storage bucket as a cache.","description":"It is not possible to atomically upload cloud storage objects exclusively when the target does not already exist, therefore this cache is not suitable for deduplication.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"bucket","type":"string","kind":"scalar","description":"The Google Cloud Storage bucket to store items in."},{"name":"content_type","type":"string","kind":"scalar","description":"Optional field to explicitly set the Content-Type.","is_optional":true},{"name":"credentials_json","type":"string","kind":"scalar","description":"An optional field to set Google Service Account Credentials json.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}},{"name":"lru","type":"cache","status":"stable","plugin":true,"summary":"Stores key/value pairs in a lru in-memory cache. This cache is therefore reset every time the service restarts.","description":"This provides the lru package which implements a fixed-size thread safe LRU cache.\n\nIt uses the package https://github.com/hashicorp/golang-lru/v2[`lru`^]\n\nThe field init_values can be used to pre-populate the memory cache with any number of key/value pairs:\n\n```yaml\ncache_resources:\n - label: foocache\n lru:\n cap: 1024\n init_values:\n foo: bar\n```\n\nThese values can be overridden during execution.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"cap","type":"int","kind":"scalar","description":"The cache maximum capacity (number of entries)","default":1000},{"name":"init_values","type":"string","kind":"map","description":"A table of key/value pairs that should be present in the cache on initialization. This can be used to create static lookup tables.","default":{},"examples":[{"Nickelback":"1995","Spice Girls":"1994","The Human League":"1977"}]},{"name":"algorithm","type":"string","kind":"scalar","description":"the lru cache implementation","is_advanced":true,"default":"standard","annotated_options":[["arc","is an adaptive replacement cache. It tracks recent evictions as well as recent usage in both the frequent and recent caches. Its computational overhead is comparable to two_queues, but the memory overhead is linear with the size of the cache. ARC has been patented by IBM."],["standard","is a simple LRU cache. It is based on the LRU implementation in groupcache"],["two_queues","tracks frequently used and recently used entries separately. This avoids a burst of accesses from taking out frequently used entries, at the cost of about 2x computational overhead and some extra bookkeeping."]],"linter":"\nlet options = {\n \"arc\": true,\n \"standard\": true,\n \"two_queues\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"two_queues_recent_ratio","type":"float","kind":"scalar","description":"is the ratio of the two_queues cache dedicated to recently added entries that have only been accessed once.","is_advanced":true,"is_optional":true,"default":0.25},{"name":"two_queues_ghost_ratio","type":"float","kind":"scalar","description":"is the default ratio of ghost entries kept to track entries recently evicted on two_queues cache.","is_advanced":true,"is_optional":true,"default":0.5},{"name":"optimistic","type":"bool","kind":"scalar","description":"If true, we do not lock on read/write events. The lru package is thread-safe, however the ADD operation is not atomic.","is_advanced":true,"default":false}]}},{"name":"memcached","type":"cache","status":"stable","plugin":true,"summary":"Connects to a cluster of memcached services, a prefix can be specified to allow multiple cache types to share a memcached cluster under different namespaces.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"addresses","type":"string","kind":"array","description":"A list of addresses of memcached servers to use."},{"name":"prefix","type":"string","kind":"scalar","description":"An optional string to prefix item keys with in order to prevent collisions with similar services.","is_optional":true},{"name":"default_ttl","type":"string","kind":"scalar","description":"A default TTL to set for items, calculated from the moment the item is cached.","default":"300s"},{"name":"retries","type":"object","kind":"scalar","description":"Determine time intervals and cut offs for retry attempts.","is_advanced":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"1s","examples":["50ms","1s"]},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts","is_advanced":true,"default":"5s","examples":["5s","1m"]},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum overall period of time to spend on retry attempts before the request is aborted.","is_advanced":true,"default":"30s","examples":["1m","1h"]}]}]}},{"name":"memory","type":"cache","status":"stable","plugin":true,"summary":"Stores key/value pairs in a map held in memory. This cache is therefore reset every time the service restarts. Each item in the cache has a TTL set from the moment it was last edited, after which it will be removed during the next compaction.","description":"The compaction interval determines how often the cache is cleared of expired items, and this process is only triggered on writes to the cache. Access to the cache is blocked during this process.\n\nItem expiry can be disabled entirely by setting the `compaction_interval` to an empty string.\n\nThe field `init_values` can be used to prepopulate the memory cache with any number of key/value pairs which are exempt from TTLs:\n\n```yaml\ncache_resources:\n - label: foocache\n memory:\n default_ttl: 60s\n init_values:\n foo: bar\n```\n\nThese values can be overridden during execution, at which point the configured TTL is respected as usual.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"default_ttl","type":"string","kind":"scalar","description":"The default TTL of each item. After this period an item will be eligible for removal during the next compaction.","default":"5m"},{"name":"compaction_interval","type":"string","kind":"scalar","description":"The period of time to wait before each compaction, at which point expired items are removed. This field can be set to an empty string in order to disable compactions/expiry entirely.","default":"60s"},{"name":"init_values","type":"string","kind":"map","description":"A table of key/value pairs that should be present in the cache on initialization. This can be used to create static lookup tables.","default":{},"examples":[{"Nickelback":"1995","Spice Girls":"1994","The Human League":"1977"}]},{"name":"shards","type":"int","kind":"scalar","description":"A number of logical shards to spread keys across, increasing the shards can have a performance benefit when processing a large number of keys.","is_advanced":true,"default":1}]}},{"name":"mongodb","type":"cache","status":"experimental","plugin":true,"summary":"Use a MongoDB instance as a cache.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target MongoDB server.","examples":["mongodb://localhost:27017"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"database","type":"string","kind":"scalar","description":"The name of the target MongoDB database."},{"name":"username","type":"string","kind":"scalar","description":"The username to connect to the database.","default":""},{"name":"password","type":"string","kind":"scalar","description":"The password to connect to the database.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"app_name","type":"string","kind":"scalar","description":"The client application name.","is_advanced":true,"default":"benthos","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"collection","type":"string","kind":"scalar","description":"The name of the target collection."},{"name":"key_field","type":"string","kind":"scalar","description":"The field in the document that is used as the key."},{"name":"value_field","type":"string","kind":"scalar","description":"The field in the document that is used as the value."}]},"version":"3.43.0"},{"name":"multilevel","type":"cache","status":"stable","plugin":true,"summary":"Combines multiple caches as levels, performing read-through and write-through operations across them.","categories":null,"examples":[{"title":"Hot and cold cache","summary":"The multilevel cache is useful for reducing traffic against a remote cache by routing it through a local cache. In the following example requests will only go through to the memcached server if the local memory cache is missing the key.","config":"\npipeline:\n processors:\n - branch:\n processors:\n - cache:\n resource: leveled\n operator: get\n key: ${! json(\"key\") }\n - catch:\n - mapping: 'root = {\"err\":error()}'\n result_map: 'root.result = this'\n\ncache_resources:\n - label: leveled\n multilevel: [ hot, cold ]\n\n - label: hot\n memory:\n default_ttl: 60s\n\n - label: cold\n memcached:\n addresses: [ TODO:11211 ]\n default_ttl: 60s\n"}],"config":{"name":"","type":"string","kind":"array"}},{"name":"nats_kv","type":"cache","status":"experimental","plugin":true,"summary":"Cache key/values in a NATS key-value bucket.","description":"== Connection name\n\nWhen monitoring and managing a production NATS system, it is often useful to\nknow which connection a message was send/received from. This can be achieved by\nsetting the connection name option when creating a NATS connection.\n\nRedpanda Connect will automatically set the connection name based off the label of the given\nNATS component, so that monitoring tools between NATS and Redpanda Connect can stay in sync.\n\n\n== Authentication\n\nThere are several components within Redpanda Connect which uses NATS services. You will find that each of these components\nsupport optional advanced authentication parameters for https://docs.nats.io/nats-server/configuration/securing_nats/auth_intro/nkey_auth[NKeys^]\nand https://docs.nats.io/using-nats/developer/connecting/creds[User Credentials^].\n\nSee an https://docs.nats.io/running-a-nats-service/nats_admin/security/jwt[in-depth tutorial^].\n\n=== NKey file\n\nThe NATS server can use these NKeys in several ways for authentication. The simplest is for the server to be configured\nwith a list of known public keys and for the clients to respond to the challenge by signing it with its private NKey\nconfigured in the `nkey_file` or `nkey` field.\n\nhttps://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[More details^].\n\n=== User credentials\n\nNATS server supports decentralized authentication based on JSON Web Tokens (JWT). Clients need an https://docs.nats.io/nats-server/configuration/securing_nats/jwt#json-web-tokens[user JWT^]\nand a corresponding https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[NKey secret^] when connecting to a server\nwhich is configured to use this authentication scheme.\n\nThe `user_credentials_file` field should point to a file containing both the private key and the JWT and can be\ngenerated with the https://docs.nats.io/nats-tools/nsc[nsc tool^].\n\nAlternatively, the `user_jwt` field can contain a plain text JWT and the `user_nkey_seed`can contain\nthe plain text NKey Seed.\n\nhttps://docs.nats.io/using-nats/developer/connecting/creds[More details^].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["nats://127.0.0.1:4222"],["nats://username:password@127.0.0.1:4222"]]},{"name":"bucket","type":"string","kind":"scalar","description":"The name of the KV bucket.","examples":["my_kv_bucket"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auth","type":"object","kind":"scalar","description":"Optional configuration of NATS authentication parameters.","is_advanced":true,"children":[{"name":"nkey_file","type":"string","kind":"scalar","description":"An optional file containing a NKey seed.","is_advanced":true,"is_optional":true,"examples":["./seed.nk"]},{"name":"nkey","type":"string","kind":"scalar","description":"The NKey seed.","is_advanced":true,"is_optional":true,"is_secret":true,"examples":["UDXU4RCSJNZOIQHZNWXHXORDPRTGNJAHAHFRGZNEEJCPQTT2M7NLCNF4"],"version":"4.38.0","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_credentials_file","type":"string","kind":"scalar","description":"An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.","is_advanced":true,"is_optional":true,"examples":["./user.creds"]},{"name":"user_jwt","type":"string","kind":"scalar","description":"An optional plain text user JWT (given along with the corresponding user NKey Seed).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_nkey_seed","type":"string","kind":"scalar","description":"An optional plain text user NKey Seed (given along with the corresponding user JWT).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},"version":"4.27.0"},{"name":"noop","type":"cache","status":"stable","plugin":true,"summary":"Noop is a cache that stores nothing, all gets returns not found. Why? Sometimes doing nothing is the braver option.","categories":null,"config":{"name":"","type":"object","kind":"scalar","default":{}},"version":"4.27.0"},{"name":"redis","type":"cache","status":"stable","plugin":true,"summary":"Use a Redis instance as a cache. The expiration can be set to zero or an empty string in order to set no expiration.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target Redis server. Database is optional and is supplied as the URL path.","examples":["redis://:6379","redis://localhost:6379","redis://foousername:foopassword@redisplace:6379","redis://:foopassword@redisplace:6379","redis://localhost:6379/1","redis://localhost:6379/1,redis://localhost:6380/1"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"kind","type":"string","kind":"scalar","description":"Specifies a simple, cluster-aware, or failover-aware redis client.","is_advanced":true,"default":"simple","options":["simple","cluster","failover"],"linter":"\nlet options = {\n \"simple\": true,\n \"cluster\": true,\n \"failover\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"master","type":"string","kind":"scalar","description":"Name of the redis master when `kind` is `failover`","is_advanced":true,"default":"","examples":["mymaster"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"prefix","type":"string","kind":"scalar","description":"An optional string to prefix item keys with in order to prevent collisions with similar services.","is_optional":true},{"name":"default_ttl","type":"string","kind":"scalar","description":"An optional default TTL to set for items, calculated from the moment the item is cached.","is_advanced":true,"is_optional":true},{"name":"retries","type":"object","kind":"scalar","description":"Determine time intervals and cut offs for retry attempts.","is_advanced":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"500ms","examples":["50ms","1s"]},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts","is_advanced":true,"default":"1s","examples":["5s","1m"]},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum overall period of time to spend on retry attempts before the request is aborted.","is_advanced":true,"default":"5s","examples":["1m","1h"]}]}]}},{"name":"ristretto","type":"cache","status":"stable","plugin":true,"summary":"Stores key/value pairs in a map held in the memory-bound https://github.com/dgraph-io/ristretto[Ristretto cache^].","description":"This cache is more efficient and appropriate for high-volume use cases than the standard memory cache. However, the add command is non-atomic, and therefore this cache is not suitable for deduplication.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"default_ttl","type":"string","kind":"scalar","description":"A default TTL to set for items, calculated from the moment the item is cached. Set to an empty string or zero duration to disable TTLs.","default":"","examples":["5m","60s"]},{"name":"get_retries","type":"object","kind":"scalar","description":"Determines how and whether get attempts should be retried if the key is not found. Ristretto is a concurrent cache that does not immediately reflect writes, and so it can sometimes be useful to enable retries at the cost of speed in cases where the key is expected to exist.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether retries should be enabled.","is_advanced":true,"default":false},{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"1s","examples":["50ms","1s"]},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts","is_advanced":true,"default":"5s","examples":["5s","1m"]},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum overall period of time to spend on retry attempts before the request is aborted.","is_advanced":true,"default":"30s","examples":["1m","1h"]}]}]}},{"name":"sql","type":"cache","status":"experimental","plugin":true,"summary":"Uses an SQL database table as a destination for storing cache key/value items.","description":"\nEach cache key/value pair will exist as a row within the specified table. Currently only the key and value columns are set, and therefore any other columns present within the target table must allow NULL values if this cache is going to be used for set and add operations.\n\nCache operations are translated into SQL statements as follows:\n\n== Get\n\nAll `get` operations are performed with a traditional `select` statement.\n\n== Delete\n\nAll `delete` operations are performed with a traditional `delete` statement.\n\n== Set\n\nThe `set` operation is performed with a traditional `insert` statement.\n\nThis will behave as an `add` operation by default, and so ideally needs to be adapted in order to provide updates instead of failing on collision\ts. Since different SQL engines implement upserts differently it is necessary to specify a `set_suffix` that modifies an `insert` statement in order to perform updates on conflict.\n\n== Add\n\nThe `add` operation is performed with a traditional `insert` statement.\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"driver","type":"string","kind":"scalar","description":"A database \u003c\u003cdrivers, driver\u003e\u003e to use.","options":["mysql","postgres","clickhouse","mssql","sqlite","oracle","snowflake","trino","gocosmos","spanner"],"linter":"\nlet options = {\n \"mysql\": true,\n \"postgres\": true,\n \"clickhouse\": true,\n \"mssql\": true,\n \"sqlite\": true,\n \"oracle\": true,\n \"snowflake\": true,\n \"trino\": true,\n \"gocosmos\": true,\n \"spanner\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"dsn","type":"string","kind":"scalar","description":"A Data Source Name to identify the target database.\n\n==== Drivers\n\n:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community, spanner=community\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n|===\n| Driver | Data Source Name Format\n\n| `clickhouse` \n| https://github.com/ClickHouse/clickhouse-go#dsn[`clickhouse://[username[:password\\]@\\][netloc\\][:port\\]/dbname[?param1=value1\u0026...\u0026paramN=valueN\\]`^] \n\n| `mysql` \n| `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` \n\n| `postgres` \n| `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` \n\n| `mssql` \n| `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` \n\n| `sqlite` \n| `file:/path/to/filename.db[?param\u0026=value1\u0026...]` \n\n| `oracle` \n| `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` \n\n| `snowflake` \n| `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` \n\n| `trino` \n| https://github.com/trinodb/trino-go-client#dsn-data-source-name[`http[s\\]://user[:pass\\]@host[:port\\][?parameters\\]`^] \n\n| `gocosmos` \n| https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage[`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e\\][;Version=\u003ccosmosdb-api-version\u003e\\][;DefaultDb/Db=\u003cdb-name\u003e\\][;AutoId=\u003ctrue/false\u003e\\][;InsecureSkipVerify=\u003ctrue/false\u003e\\]`^] \n\n| `spanner` \n| projects/[PROJECT]/instances/[INSTANCE]/databases/[DATABASE] \n|===\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String[the docs^] for more details. For https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication[key pair authentication^], the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe https://pkg.go.dev/github.com/microsoft/gocosmos[`gocosmos`^] driver is still experimental, but it has support for https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys[hierarchical partition keys^] as well as https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query[cross-partition queries^]. Please refer to the https://github.com/microsoft/gocosmos/blob/main/SQL.md[SQL notes^] for details.","examples":["clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60","foouser:foopassword@tcp(localhost:3306)/foodb","postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable","oracle://foouser:foopass@localhost:1521/service_name"]},{"name":"table","type":"string","kind":"scalar","description":"The table to insert/read/delete cache items.","examples":["foo"]},{"name":"key_column","type":"string","kind":"scalar","description":"The name of a column to be used for storing cache item keys. This column should support strings of arbitrary size.","examples":["foo"]},{"name":"value_column","type":"string","kind":"scalar","description":"The name of a column to be used for storing cache item values. This column should support strings of arbitrary size.","examples":["bar"]},{"name":"set_suffix","type":"string","kind":"scalar","description":"An optional suffix to append to each insert query for a cache `set` operation. This should modify an insert statement into an upsert appropriate for the given SQL engine.","is_optional":true,"examples":["ON DUPLICATE KEY UPDATE bar=VALUES(bar)","ON CONFLICT (foo) DO UPDATE SET bar=excluded.bar","ON CONFLICT (foo) DO NOTHING"]},{"name":"init_files","type":"string","kind":"array","description":"\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":[["./init/*.sql"],["./foo.sql","./bar.sql"]],"version":"4.10.0"},{"name":"init_statement","type":"string","kind":"scalar","description":"\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":["\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n"],"version":"4.10.0"},{"name":"conn_max_idle_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time.","is_advanced":true,"is_optional":true},{"name":"conn_max_life_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age.","is_advanced":true,"is_optional":true},{"name":"conn_max_idle","type":"int","kind":"scalar","description":"An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release.","is_advanced":true,"is_optional":true,"default":2},{"name":"conn_max_open","type":"int","kind":"scalar","description":"An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited).","is_advanced":true,"is_optional":true}]},"version":"4.26.0"},{"name":"ttlru","type":"cache","status":"stable","plugin":true,"summary":"Stores key/value pairs in a ttlru in-memory cache. This cache is therefore reset every time the service restarts.","description":"The cache ttlru provides a simple, goroutine safe, cache with a fixed number of entries. Each entry has a per-cache defined TTL.\n\nThis TTL is reset on both modification and access of the value. As a result, if the cache is full, and no items have expired, when adding a new item, the item with the soonest expiration will be evicted.\n\nIt uses the package https://github.com/hashicorp/golang-lru/v2/expirable[`expirable`^]\n\nThe field init_values can be used to pre-populate the memory cache with any number of key/value pairs:\n\n```yaml\ncache_resources:\n - label: foocache\n ttlru:\n default_ttl: '5m'\n cap: 1024\n init_values:\n foo: bar\n```\n\nThese values can be overridden during execution.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"cap","type":"int","kind":"scalar","description":"The cache maximum capacity (number of entries)","default":1024},{"name":"default_ttl","type":"string","kind":"scalar","description":"The cache ttl of each element","default":"5m0s","version":"4.21.0"},{"name":"ttl","type":"string","kind":"scalar","description":"Deprecated. Please use `default_ttl` field","is_advanced":true,"is_optional":true},{"name":"init_values","type":"string","kind":"map","description":"A table of key/value pairs that should be present in the cache on initialization. This can be used to create static lookup tables.","default":{},"examples":[{"Nickelback":"1995","Spice Girls":"1994","The Human League":"1977"}]},{"name":"optimistic","type":"bool","kind":"scalar","description":"If true, we do not lock on read/write events. The ttlru package is thread-safe, however the ADD operation is not atomic.","is_advanced":true,"default":false}]}}],"inputs":[{"name":"amqp_0_9","type":"input","status":"stable","plugin":true,"summary":"Connects to an AMQP (0.91) queue. AMQP is a messaging protocol used by various message brokers, including RabbitMQ.","description":"\nTLS is automatic when connecting to an `amqps` URL, but custom settings can be enabled in the `tls` section.\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n- amqp_content_type\n- amqp_content_encoding\n- amqp_delivery_mode\n- amqp_priority\n- amqp_correlation_id\n- amqp_reply_to\n- amqp_expiration\n- amqp_message_id\n- amqp_timestamp\n- amqp_type\n- amqp_user_id\n- amqp_app_id\n- amqp_consumer_tag\n- amqp_delivery_tag\n- amqp_redelivered\n- amqp_exchange\n- amqp_routing_key\n- All existing message headers, including nested headers prefixed with the key of their respective parent.\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolations].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. The first URL to successfully establish a connection will be used until the connection is closed. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["amqp://guest:guest@127.0.0.1:5672/"],["amqp://127.0.0.1:5672/,amqp://127.0.0.2:5672/"],["amqp://127.0.0.1:5672/","amqp://127.0.0.2:5672/"]],"version":"3.58.0","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"queue","type":"string","kind":"scalar","description":"An AMQP queue to consume from."},{"name":"queue_declare","type":"object","kind":"scalar","description":"Allows you to passively declare the target queue. If the queue already exists then the declaration passively verifies that they match the target fields.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to enable queue declaration.","is_advanced":true,"default":false},{"name":"durable","type":"bool","kind":"scalar","description":"Whether the declared queue is durable.","is_advanced":true,"default":true},{"name":"auto_delete","type":"bool","kind":"scalar","description":"Whether the declared queue will auto-delete.","is_advanced":true,"default":false},{"name":"arguments","type":"string","kind":"map","description":"\nOptional arguments specific to the server's implementation of the queue that can be sent for queue types which require extra parameters.\n\n== Arguments\n\n- x-queue-type\n\nIs used to declare quorum and stream queues. Accepted values are: 'classic' (default), 'quorum', 'stream', 'drop-head', 'reject-publish' and 'reject-publish-dlx'.\n\n- x-max-length\n\nMaximum number of messages, is a non-negative integer value.\n\n- x-max-length-bytes\n\nMaximum number of messages, is a non-negative integer value.\n\n- x-overflow\n\nSets overflow behaviour. Possible values are: 'drop-head' (default), 'reject-publish', 'reject-publish-dlx'.\n\n- x-message-ttl\n\nTTL period in milliseconds. Must be a string representation of the number.\n\n- x-expires\n\nExpiration policy, describes the expiration period in milliseconds. Must be a positive integer.\n\n- x-max-age\n\nControls the retention of a stream. Must be a strin, valid units: (Y, M, D, h, m, s) e.g. '7D' for a week.\n\n- x-stream-max-segment-size-bytes\n\nControls the size of the segment files on disk (default 500000000). Must be a positive integer.\n\n- x-queue-version\n\ndeclares the Classic Queue version to use. Expects an integer, either 1 or 2.\n\n- x-consumer-timeout\n\nInteger specified in milliseconds.\n\n- x-single-active-consumer\n\nEnables Single Active Consumer, Expects a Boolean.\n\nSee https://github.com/rabbitmq/amqp091-go/blob/b3d409fe92c34bea04d8123a136384c85e8dc431/types.go#L282-L362 for more information on available arguments.","is_advanced":true,"is_optional":true,"examples":[{"x-max-length":1000,"x-max-length-bytes":4096,"x-queue-type":"quorum"}]}]},{"name":"bindings_declare","type":"object","kind":"array","description":"Allows you to passively declare bindings for the target queue.","is_advanced":true,"is_optional":true,"examples":[[{"exchange":"foo","key":"bar"}]],"children":[{"name":"exchange","type":"string","kind":"scalar","description":"The exchange of the declared binding.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"The key of the declared binding.","is_advanced":true,"default":""}]},{"name":"consumer_tag","type":"string","kind":"scalar","description":"A consumer tag.","default":""},{"name":"auto_ack","type":"bool","kind":"scalar","description":"Acknowledge messages automatically as they are consumed rather than waiting for acknowledgments from downstream. This can improve throughput and prevent the pipeline from blocking but at the cost of eliminating delivery guarantees.","is_advanced":true,"default":false},{"name":"nack_reject_patterns","type":"string","kind":"array","description":"A list of regular expression patterns whereby if a message that has failed to be delivered by Redpanda Connect has an error that matches it will be dropped (or delivered to a dead-letter queue if one exists). By default failed messages are nacked with requeue enabled.","is_advanced":true,"default":[],"examples":[["^reject me please:.+$"]],"version":"3.64.0"},{"name":"prefetch_count","type":"int","kind":"scalar","description":"The maximum number of pending messages to have consumed at a time.","default":10},{"name":"prefetch_size","type":"int","kind":"scalar","description":"The maximum amount of pending messages measured in bytes to have consumed at a time.","is_advanced":true,"default":0},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]}]}},{"name":"amqp_1","type":"input","status":"stable","plugin":true,"summary":"Reads messages from an AMQP (1.0) server.","description":"== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- amqp_content_type\n- amqp_content_encoding\n- amqp_creation_time\n- All string typed message annotations\n```\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].\n\nBy setting `read_header` to `true`, additional message header properties will be added to each message:\n\n```text\n- amqp_durable\n- amqp_priority\n- amqp_ttl\n- amqp_first_acquirer\n- amqp_delivery_count\n```\n\n== Performance\n\nThis input benefits from receiving multiple messages in flight in parallel for improved performance.\nYou can tune the max number of in flight messages with the field `credit`.\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"A URL to connect to.","is_deprecated":true,"is_optional":true,"examples":["amqp://localhost:5672/","amqps://guest:guest@localhost:5672/"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. The first URL to successfully establish a connection will be used until the connection is closed. If an item of the list contains commas it will be expanded into multiple URLs.","is_optional":true,"examples":[["amqp://guest:guest@127.0.0.1:5672/"],["amqp://127.0.0.1:5672/,amqp://127.0.0.2:5672/"],["amqp://127.0.0.1:5672/","amqp://127.0.0.2:5672/"]],"version":"4.23.0","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"source_address","type":"string","kind":"scalar","description":"The source address to consume from.","examples":["/foo","queue:/bar","topic:/baz"]},{"name":"azure_renew_lock","type":"bool","kind":"scalar","description":"Experimental: Azure service bus specific option to renew lock if processing takes more then configured lock time","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"read_header","type":"bool","kind":"scalar","description":"Read additional message header fields into `amqp_*` metadata properties.","is_advanced":true,"default":false,"version":"4.25.0"},{"name":"credit","type":"int","kind":"scalar","description":"Specifies the maximum number of unacknowledged messages the sender can transmit. Once this limit is reached, no more messages will arrive until messages are acknowledged and settled.","is_advanced":true,"default":64,"version":"4.26.0","linter":"root = if this \u003c 1 { [ \"credit must be at least 1\" ] }"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"sasl","type":"object","kind":"scalar","description":"Enables SASL authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"mechanism","type":"string","kind":"scalar","description":"The SASL authentication mechanism to use.","is_advanced":true,"default":"none","annotated_options":[["anonymous","Anonymous SASL authentication."],["none","No SASL based authentication."],["plain","Plain text SASL authentication."]],"linter":"\nlet options = {\n \"anonymous\": true,\n \"none\": true,\n \"plain\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"user","type":"string","kind":"scalar","description":"A SASL plain text username. It is recommended that you use environment variables to populate this field.","is_advanced":true,"default":"","examples":["${USER}"]},{"name":"password","type":"string","kind":"scalar","description":"A SASL plain text password. It is recommended that you use environment variables to populate this field.","is_advanced":true,"is_secret":true,"default":"","examples":["${PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}],"linter":"\nroot = if this.url.or(\"\") == \"\" \u0026\u0026 this.urls.or([]).length() == 0 {\n \"field 'urls' must be set\"\n}\n"}},{"name":"aws_kinesis","type":"input","status":"stable","plugin":true,"summary":"Receive messages from one or more Kinesis streams.","description":"\nConsumes messages from one or more Kinesis streams either by automatically balancing shards across other instances of this input, or by consuming shards listed explicitly. The latest message sequence consumed by this input is stored within a \u003c\u003ctable-schema,DynamoDB table\u003e\u003e, which allows it to resume at the correct sequence of the shard during restarts. This table is also used for coordination across distributed inputs when shard balancing.\n\nRedpanda Connect will not store a consumed sequence unless it is acknowledged at the output level, which ensures at-least-once delivery guarantees.\n\n== Ordering\n\nBy default messages of a shard can be processed in parallel, up to a limit determined by the field `checkpoint_limit`. However, if strict ordered processing is required then this value must be set to 1 in order to process shard messages in lock-step. When doing so it is recommended that you perform batching at this component for performance as it will not be possible to batch lock-stepped messages at the output level.\n\n== Table schema\n\nIt's possible to configure Redpanda Connect to create the DynamoDB table required for coordination if it does not already exist. However, if you wish to create this yourself (recommended) then create a table with a string HASH key `StreamID` and a string RANGE key `ShardID`.\n\n== Batching\n\nUse the `batching` fields to configure an optional xref:configuration:batching.adoc#batch-policy[batching policy]. Each stream shard will be batched separately in order to ensure that acknowledgements aren't contaminated.\n","categories":["Services","AWS"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"streams","type":"string","kind":"array","description":"One or more Kinesis data streams to consume from. Streams can either be specified by their name or full ARN. Shards of a stream are automatically balanced across consumers by coordinating through the provided DynamoDB table. Multiple comma separated streams can be listed in a single element. Shards are automatically distributed across consumers of a stream by coordinating through the provided DynamoDB table. Alternatively, it's possible to specify an explicit shard to consume from with a colon after the stream name, e.g. `foo:0` would consume the shard `0` of the stream `foo`.","examples":[["foo","arn:aws:kinesis:*:111122223333:stream/my-stream"]]},{"name":"dynamodb","type":"object","kind":"scalar","description":"Determines the table used for storing and accessing the latest consumed sequence for shards, and for coordinating balanced consumers of streams.","children":[{"name":"table","type":"string","kind":"scalar","description":"The name of the table to access.","default":""},{"name":"create","type":"bool","kind":"scalar","description":"Whether, if the table does not exist, it should be created.","default":false},{"name":"billing_mode","type":"string","kind":"scalar","description":"When creating the table determines the billing mode.","is_advanced":true,"default":"PAY_PER_REQUEST","options":["PROVISIONED","PAY_PER_REQUEST"],"linter":"\nlet options = {\n \"provisioned\": true,\n \"pay_per_request\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"read_capacity_units","type":"int","kind":"scalar","description":"Set the provisioned read capacity when creating the table with a `billing_mode` of `PROVISIONED`.","is_advanced":true,"default":0},{"name":"write_capacity_units","type":"int","kind":"scalar","description":"Set the provisioned write capacity when creating the table with a `billing_mode` of `PROVISIONED`.","is_advanced":true,"default":0},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]},{"name":"checkpoint_limit","type":"int","kind":"scalar","description":"The maximum gap between the in flight sequence versus the latest acknowledged sequence at a given time. Increasing this limit enables parallel processing and batching at the output level to work on individual shards. Any given sequence will not be committed unless all messages under that offset are delivered in order to preserve at least once delivery guarantees.","default":1024},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"commit_period","type":"string","kind":"scalar","description":"The period of time between each update to the checkpoint table.","default":"5s"},{"name":"rebalance_period","type":"string","kind":"scalar","description":"The period of time between each attempt to rebalance shards across clients.","is_advanced":true,"default":"30s"},{"name":"lease_period","type":"string","kind":"scalar","description":"The period of time after which a client that has failed to update a shard checkpoint is assumed to be inactive.","is_advanced":true,"default":"30s"},{"name":"start_from_oldest","type":"bool","kind":"scalar","description":"Whether to consume from the oldest message when a sequence does not yet exist for the stream.","default":true},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]},"version":"3.36.0"},{"name":"aws_s3","type":"input","status":"stable","plugin":true,"summary":"Downloads objects within an Amazon S3 bucket, optionally filtered by a prefix, either by walking the items in the bucket or by streaming upload notifications in realtime.","description":"\n== Stream objects on upload with SQS\n\nA common pattern for consuming S3 objects is to emit upload notification events from the bucket either directly to an SQS queue, or to an SNS topic that is consumed by an SQS queue, and then have your consumer listen for events which prompt it to download the newly uploaded objects. More information about this pattern and how to set it up can be found at in the https://docs.aws.amazon.com/AmazonS3/latest/dev/ways-to-add-notification-config-to-bucket.html[Amazon S3 docs].\n\nRedpanda Connect is able to follow this pattern when you configure an `sqs.url`, where it consumes events from SQS and only downloads object keys received within those events. In order for this to work Redpanda Connect needs to know where within the event the key and bucket names can be found, specified as xref:configuration:field_paths.adoc[dot paths] with the fields `sqs.key_path` and `sqs.bucket_path`. The default values for these fields should already be correct when following the guide above.\n\nIf your notification events are being routed to SQS via an SNS topic then the events will be enveloped by SNS, in which case you also need to specify the field `sqs.envelope_path`, which in the case of SNS to SQS will usually be `Message`.\n\nWhen using SQS please make sure you have sensible values for `sqs.max_messages` and also the visibility timeout of the queue itself. When Redpanda Connect consumes an S3 object the SQS message that triggered it is not deleted until the S3 object has been sent onwards. This ensures at-least-once crash resiliency, but also means that if the S3 object takes longer to process than the visibility timeout of your queue then the same objects might be processed multiple times.\n\n== Download large files\n\nWhen downloading large files it's often necessary to process it in streamed parts in order to avoid loading the entire file in memory at a given time. In order to do this a \u003c\u003cscanner, `scanner`\u003e\u003e can be specified that determines how to break the input into smaller individual messages.\n\n== Credentials\n\nBy default Redpanda Connect will use a shared credentials file when connecting to AWS services. It's also possible to set them explicitly at the component level, allowing you to transfer data across accounts. You can find out more in xref:guides:cloud/aws.adoc[].\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n- s3_key\n- s3_bucket\n- s3_last_modified_unix\n- s3_last_modified (RFC3339)\n- s3_content_type\n- s3_content_encoding\n- s3_version_id\n- All user defined metadata\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation]. Note that user defined metadata is case insensitive within AWS, and it is likely that the keys will be received in a capitalized form, if you wish to make them consistent you can map all metadata keys to lower or uppercase using a Bloblang mapping such as `meta = meta().map_each_key(key -\u003e key.lowercase())`.","categories":["Services","AWS"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"bucket","type":"string","kind":"scalar","description":"The bucket to consume from. If the field `sqs.url` is specified this field is optional.","default":""},{"name":"prefix","type":"string","kind":"scalar","description":"An optional path prefix, if set only objects with the prefix are consumed when walking a bucket.","default":""},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]},{"name":"force_path_style_urls","type":"bool","kind":"scalar","description":"Forces the client API to use path style URLs for downloading keys, which is often required when connecting to custom endpoints.","is_advanced":true,"default":false},{"name":"delete_objects","type":"bool","kind":"scalar","description":"Whether to delete downloaded objects from the bucket once they are processed.","is_advanced":true,"default":false},{"name":"codec","type":"string","kind":"scalar","description":"The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.","is_deprecated":true,"is_optional":true,"examples":["lines","delim:\t","delim:foobar","gzip/csv"],"annotated_options":[["auto","EXPERIMENTAL: Attempts to derive a codec for each file based on information such as the extension. For example, a .tar.gz file would be consumed with the `gzip/tar` codec. Defaults to all-bytes."],["all-bytes","Consume the entire file as a single binary message."],["avro-ocf:marshaler=x","EXPERIMENTAL: Consume a stream of Avro OCF datum. The `marshaler` parameter is optional and has the options: `goavro` (default), `json`. Use `goavro` if OCF contains logical types."],["chunker:x","Consume the file in chunks of a given number of bytes."],["csv","Consume structured rows as comma separated values, the first row must be a header row."],["csv:x","Consume structured rows as values separated by a custom delimiter, the first row must be a header row. The custom delimiter must be a single character, e.g. the codec `\"csv:\\t\"` would consume a tab delimited file."],["csv-safe","Consume structured rows like `csv`, but sends messages with empty maps on failure to parse. Includes row number and parsing errors (if any) in the message's metadata."],["csv-safe:x","Consume structured rows like `csv:x` as values separated by a custom delimiter, but sends messages with empty maps on failure to parse. The custom delimiter must be a single character, e.g. the codec `\"csv-safe:\\t\"` would consume a tab delimited file. Includes row number and parsing errors (if any) in the message's metadata."],["delim:x","Consume the file in segments divided by a custom delimiter."],["gzip","Decompress a gzip file, this codec should precede another codec, e.g. `gzip/all-bytes`, `gzip/tar`, `gzip/csv`, etc."],["pgzip","Decompress a gzip file in parallel, this codec should precede another codec, e.g. `pgzip/all-bytes`, `pgzip/tar`, `pgzip/csv`, etc."],["lines","Consume the file in segments divided by linebreaks."],["multipart","Consumes the output of another codec and batches messages together. A batch ends when an empty message is consumed. For example, the codec `lines/multipart` could be used to consume multipart messages where an empty line indicates the end of each batch."],["regex:(?m)^\\d\\d:\\d\\d:\\d\\d","Consume the file in segments divided by regular expression."],["skipbom","Skip one or more byte order marks for each opened reader, this codec should precede another codec, e.g. `skipbom/csv`, etc."],["tar","Parse the file as a tar archive, and consume each file of the archive as a message."]]},{"name":"max_buffer","type":"int","kind":"scalar","is_deprecated":true,"default":1000000},{"name":"scanner","type":"scanner","kind":"scalar","description":"The xref:components:scanners/about.adoc[scanner] by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once.","is_optional":true,"default":{"to_the_end":{}},"version":"4.25.0"},{"name":"sqs","type":"object","kind":"scalar","description":"Consume SQS messages in order to trigger key downloads.","is_optional":true,"children":[{"name":"url","type":"string","kind":"scalar","description":"An optional SQS URL to connect to. When specified this queue will control which objects are downloaded.","default":""},{"name":"endpoint","type":"string","kind":"scalar","description":"A custom endpoint to use when connecting to SQS.","is_advanced":true,"default":""},{"name":"key_path","type":"string","kind":"scalar","description":"A xref:configuration:field_paths.adoc[dot path] whereby object keys are found in SQS messages.","default":"Records.*.s3.object.key"},{"name":"bucket_path","type":"string","kind":"scalar","description":"A xref:configuration:field_paths.adoc[dot path] whereby the bucket name can be found in SQS messages.","default":"Records.*.s3.bucket.name"},{"name":"envelope_path","type":"string","kind":"scalar","description":"A xref:configuration:field_paths.adoc[dot path] of a field to extract an enveloped JSON payload for further extracting the key and bucket from SQS messages. This is specifically useful when subscribing an SQS queue to an SNS topic that receives bucket events.","default":"","examples":["Message"]},{"name":"delay_period","type":"string","kind":"scalar","description":"An optional period of time to wait from when a notification was originally sent to when the target key download is attempted.","is_advanced":true,"default":"","examples":["10s","5m"]},{"name":"max_messages","type":"int","kind":"scalar","description":"The maximum number of SQS messages to consume from each request.","is_advanced":true,"default":10},{"name":"wait_time_seconds","type":"int","kind":"scalar","description":"Whether to set the wait time. Enabling this activates long-polling. Valid values: 0 to 20.","is_advanced":true,"default":0}]}]}},{"name":"aws_sqs","type":"input","status":"stable","plugin":true,"summary":"Consume messages from an AWS SQS URL.","description":"\n== Credentials\n\nBy default Redpanda Connect will use a shared credentials file when connecting to AWS\nservices. It's also possible to set them explicitly at the component level,\nallowing you to transfer data across accounts. You can find out more in\nxref:guides:cloud/aws.adoc[].\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n- sqs_message_id\n- sqs_receipt_handle\n- sqs_approximate_receive_count\n- All message attributes\n\nYou can access these metadata fields using\nxref:configuration:interpolation.adoc#bloblang-queries[function interpolation].","categories":["Services","AWS"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The SQS URL to consume from.","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"delete_message","type":"bool","kind":"scalar","description":"Whether to delete the consumed message once it is acked. Disabling allows you to handle the deletion using a different mechanism.","is_advanced":true,"default":true},{"name":"reset_visibility","type":"bool","kind":"scalar","description":"Whether to set the visibility timeout of the consumed message to zero once it is nacked. Disabling honors the preset visibility timeout specified for the queue.","is_advanced":true,"default":true,"version":"3.58.0"},{"name":"max_number_of_messages","type":"int","kind":"scalar","description":"The maximum number of messages to return on one poll. Valid values: 1 to 10.","is_advanced":true,"default":10},{"name":"max_outstanding_messages","type":"int","kind":"scalar","description":"The maximum number of outstanding pending messages to be consumed at a given time.","default":1000},{"name":"wait_time_seconds","type":"int","kind":"scalar","description":"Whether to set the wait time. Enabling this activates long-polling. Valid values: 0 to 20.","is_advanced":true,"default":0},{"name":"message_timeout","type":"string","kind":"scalar","description":"The time to process messages before needing to refresh the receipt handle. Messages will be eligible for refresh when half of the timeout has elapsed. This sets MessageVisibility for each received message.","is_advanced":true,"default":"30s"},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]}},{"name":"azure_blob_storage","type":"input","status":"beta","plugin":true,"summary":"Downloads objects within an Azure Blob Storage container, optionally filtered by a prefix.","description":"\nSupports multiple authentication methods but only one of the following is required:\n\n- `storage_connection_string`\n- `storage_account` and `storage_access_key`\n- `storage_account` and `storage_sas_token`\n- `storage_account` to access via https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity#DefaultAzureCredential[DefaultAzureCredential^]\n\nIf multiple are set then the `storage_connection_string` is given priority.\n\nIf the `storage_connection_string` does not contain the `AccountName` parameter, please specify it in the\n`storage_account` field.\n\n== Download large files\n\nWhen downloading large files it's often necessary to process it in streamed parts in order to avoid loading the entire file in memory at a given time. In order to do this a \u003c\u003cscanner, `scanner`\u003e\u003e can be specified that determines how to break the input into smaller individual messages.\n\n== Stream new files\n\nBy default this input will consume all files found within the target container and will then gracefully terminate. This is referred to as a \"batch\" mode of operation. However, it's possible to instead configure a container as https://learn.microsoft.com/en-gb/azure/event-grid/event-schema-blob-storage[an Event Grid source^] and then use this as a \u003c\u003ctargetsinput, `targets_input`\u003e\u003e, in which case new files are consumed as they're uploaded and Redpanda Connect will continue listening for and downloading files as they arrive. This is referred to as a \"streamed\" mode of operation.\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n- blob_storage_key\n- blob_storage_container\n- blob_storage_last_modified\n- blob_storage_last_modified_unix\n- blob_storage_content_type\n- blob_storage_content_encoding\n- All user defined metadata\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].","categories":["Services","Azure"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"storage_account","type":"string","kind":"scalar","description":"The storage account to access. This field is ignored if `storage_connection_string` is set.","default":""},{"name":"storage_access_key","type":"string","kind":"scalar","description":"The storage account access key. This field is ignored if `storage_connection_string` is set.","default":""},{"name":"storage_connection_string","type":"string","kind":"scalar","description":"A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set.","default":""},{"name":"storage_sas_token","type":"string","kind":"scalar","description":"The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set.","default":""},{"name":"container","type":"string","kind":"scalar","description":"The name of the container from which to download blobs.","interpolated":true},{"name":"prefix","type":"string","kind":"scalar","description":"An optional path prefix, if set only objects with the prefix are consumed.","default":""},{"name":"codec","type":"string","kind":"scalar","description":"The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.","is_deprecated":true,"is_optional":true,"examples":["lines","delim:\t","delim:foobar","gzip/csv"],"annotated_options":[["auto","EXPERIMENTAL: Attempts to derive a codec for each file based on information such as the extension. For example, a .tar.gz file would be consumed with the `gzip/tar` codec. Defaults to all-bytes."],["all-bytes","Consume the entire file as a single binary message."],["avro-ocf:marshaler=x","EXPERIMENTAL: Consume a stream of Avro OCF datum. The `marshaler` parameter is optional and has the options: `goavro` (default), `json`. Use `goavro` if OCF contains logical types."],["chunker:x","Consume the file in chunks of a given number of bytes."],["csv","Consume structured rows as comma separated values, the first row must be a header row."],["csv:x","Consume structured rows as values separated by a custom delimiter, the first row must be a header row. The custom delimiter must be a single character, e.g. the codec `\"csv:\\t\"` would consume a tab delimited file."],["csv-safe","Consume structured rows like `csv`, but sends messages with empty maps on failure to parse. Includes row number and parsing errors (if any) in the message's metadata."],["csv-safe:x","Consume structured rows like `csv:x` as values separated by a custom delimiter, but sends messages with empty maps on failure to parse. The custom delimiter must be a single character, e.g. the codec `\"csv-safe:\\t\"` would consume a tab delimited file. Includes row number and parsing errors (if any) in the message's metadata."],["delim:x","Consume the file in segments divided by a custom delimiter."],["gzip","Decompress a gzip file, this codec should precede another codec, e.g. `gzip/all-bytes`, `gzip/tar`, `gzip/csv`, etc."],["pgzip","Decompress a gzip file in parallel, this codec should precede another codec, e.g. `pgzip/all-bytes`, `pgzip/tar`, `pgzip/csv`, etc."],["lines","Consume the file in segments divided by linebreaks."],["multipart","Consumes the output of another codec and batches messages together. A batch ends when an empty message is consumed. For example, the codec `lines/multipart` could be used to consume multipart messages where an empty line indicates the end of each batch."],["regex:(?m)^\\d\\d:\\d\\d:\\d\\d","Consume the file in segments divided by regular expression."],["skipbom","Skip one or more byte order marks for each opened reader, this codec should precede another codec, e.g. `skipbom/csv`, etc."],["tar","Parse the file as a tar archive, and consume each file of the archive as a message."]]},{"name":"max_buffer","type":"int","kind":"scalar","is_deprecated":true,"default":1000000},{"name":"scanner","type":"scanner","kind":"scalar","description":"The xref:components:scanners/about.adoc[scanner] by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once.","is_optional":true,"default":{"to_the_end":{}},"version":"4.25.0"},{"name":"delete_objects","type":"bool","kind":"scalar","description":"Whether to delete downloaded objects from the blob once they are processed.","is_advanced":true,"default":false},{"name":"targets_input","type":"input","kind":"scalar","description":"EXPERIMENTAL: An optional source of download targets, configured as a xref:components:inputs/about.adoc[regular Redpanda Connect input]. Each message yielded by this input should be a single structured object containing a field `name`, which represents the blob to be downloaded.","is_optional":true,"examples":[{"mqtt":{"topics":["some-topic"],"urls":["example.westeurope-1.ts.eventgrid.azure.net:8883"]},"processors":[{"unarchive":{"format":"json_array"}},{"mapping":"if this.eventType == \"Microsoft.Storage.BlobCreated\" {\n root.name = this.data.url.parse_url().path.trim_prefix(\"/foocontainer/\")\n} else {\n root = deleted()\n}"}]}],"version":"4.27.0"}],"linter":"root = if this.storage_connection_string != \"\" \u0026\u0026 !this.storage_connection_string.contains(\"AccountName=\") \u0026\u0026 !this.storage_connection_string.contains(\"UseDevelopmentStorage=true;\") \u0026\u0026 this.storage_account == \"\" { [ \"storage_account must be set if storage_connection_string does not contain the \\\"AccountName\\\" parameter\" ] }"},"version":"3.36.0"},{"name":"azure_cosmosdb","type":"input","status":"experimental","plugin":true,"summary":"Executes a SQL query against https://learn.microsoft.com/en-us/azure/cosmos-db/introduction[Azure CosmosDB^] and creates a batch of messages from each page of items.","description":"\n== Cross-partition queries\n\nCross-partition queries are currently not supported by the underlying driver. For every query, the PartitionKey values must be known in advance and specified in the config. https://github.com/Azure/azure-sdk-for-go/issues/18578#issuecomment-1222510989[See details^].\n\n\n== Credentials\n\nYou can use one of the following authentication mechanisms:\n\n- Set the `endpoint` field and the `account_key` field\n- Set only the `endpoint` field to use https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity#DefaultAzureCredential[DefaultAzureCredential^]\n- Set the `connection_string` field\n\n\n== Metadata\n\nThis component adds the following metadata fields to each message:\n```\n- activity_id\n- request_charge\n```\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].\n","categories":["Azure"],"footnotes":"\n\n== CosmosDB emulator\n\nIf you wish to run the CosmosDB emulator that is referenced in the documentation https://learn.microsoft.com/en-us/azure/cosmos-db/linux-emulator[here^], the following Docker command should do the trick:\n\n```bash\n\u003e docker run --rm -it -p 8081:8081 --name=cosmosdb -e AZURE_COSMOS_EMULATOR_PARTITION_COUNT=10 -e AZURE_COSMOS_EMULATOR_ENABLE_DATA_PERSISTENCE=false mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator\n```\n\nNote: `AZURE_COSMOS_EMULATOR_PARTITION_COUNT` controls the number of partitions that will be supported by the emulator. The bigger the value, the longer it takes for the container to start up.\n\nAdditionally, instead of installing the container self-signed certificate which is exposed via `https://localhost:8081/_explorer/emulator.pem`, you can run https://mitmproxy.org/[mitmproxy^] like so:\n\n```bash\n\u003e mitmproxy -k --mode \"reverse:https://localhost:8081\"\n```\n\nThen you can access the CosmosDB UI via `http://localhost:8080/_explorer/index.html` and use `http://localhost:8080` as the CosmosDB endpoint.\n","examples":[{"title":"Query container","summary":"Execute a parametrized SQL query to select documents from a container.","config":"\ninput:\n azure_cosmosdb:\n endpoint: http://localhost:8080\n account_key: C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==\n database: blobbase\n container: blobfish\n partition_keys_map: root = \"AbyssalPlain\"\n query: SELECT * FROM blobfish AS b WHERE b.species = @species\n args_mapping: |\n root = [\n { \"Name\": \"@species\", \"Value\": \"smooth-head\" },\n ]\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"endpoint","type":"string","kind":"scalar","description":"CosmosDB endpoint.","is_optional":true,"examples":["https://localhost:8081"]},{"name":"account_key","type":"string","kind":"scalar","description":"Account key.","is_optional":true,"is_secret":true,"examples":["C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw=="],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"connection_string","type":"string","kind":"scalar","description":"Connection string.","is_optional":true,"is_secret":true,"examples":["AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==;"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"database","type":"string","kind":"scalar","description":"Database.","examples":["testdb"]},{"name":"container","type":"string","kind":"scalar","description":"Container.","examples":["testcontainer"]},{"name":"partition_keys_map","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to a single partition key value or an array of partition key values of type string, integer or boolean. Currently, hierarchical partition keys are not supported so only one value may be provided.","bloblang":true,"examples":["root = \"blobfish\"","root = 41","root = true","root = null","root = now().ts_format(\"2006-01-02\")"]},{"name":"query","type":"string","kind":"scalar","description":"The query to execute","examples":["SELECT c.foo FROM testcontainer AS c WHERE c.bar = \"baz\" AND c.timestamp \u003c @timestamp"]},{"name":"args_mapping","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] that, for each message, creates a list of arguments to use with the query.","is_optional":true,"bloblang":true,"examples":["root = [\n { \"Name\": \"@name\", \"Value\": \"benthos\" },\n]"]},{"name":"batch_count","type":"int","kind":"scalar","description":"The maximum number of messages that should be accumulated into each batch. Use '-1' specify dynamic page size.","is_advanced":true,"default":-1,"linter":"root = if this \u003c -1 || this == 0 || this \u003e 2147483647 { [ \"batch_count must be must be \u003e 0 and smaller than 2147483647 or -1.\" ] }"},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}],"linter":"root = []\nlet hasEndpoint = this.endpoint.or(\"\") != \"\"\nlet hasConnectionString = this.connection_string.or(\"\") != \"\"\n\nroot.\"-\" = if !$hasEndpoint \u0026\u0026 !$hasConnectionString {\n \"Either `endpoint` or `connection_string` must be set.\"\n}\n"},"version":"v4.25.0"},{"name":"azure_queue_storage","type":"input","status":"beta","plugin":true,"summary":"Dequeue objects from an Azure Storage Queue.","description":"\nThis input adds the following metadata fields to each message:\n\n```\n- queue_storage_insertion_time\n- queue_storage_queue_name\n- queue_storage_message_lag (if 'track_properties' set to true)\n- All user defined queue metadata\n```\n\nOnly one authentication method is required, `storage_connection_string` or `storage_account` and `storage_access_key`. If both are set then the `storage_connection_string` is given priority.","categories":["Services","Azure"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"storage_account","type":"string","kind":"scalar","description":"The storage account to access. This field is ignored if `storage_connection_string` is set.","default":""},{"name":"storage_access_key","type":"string","kind":"scalar","description":"The storage account access key. This field is ignored if `storage_connection_string` is set.","default":""},{"name":"storage_connection_string","type":"string","kind":"scalar","description":"A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set.","default":""},{"name":"storage_sas_token","type":"string","kind":"scalar","is_deprecated":true,"default":""},{"name":"queue_name","type":"string","kind":"scalar","description":"The name of the source storage queue.","interpolated":true,"examples":["foo_queue","${! env(\"MESSAGE_TYPE\").lowercase() }"]},{"name":"dequeue_visibility_timeout","type":"string","kind":"scalar","description":"The timeout duration until a dequeued message gets visible again, 30s by default","is_advanced":true,"default":"30s","version":"3.45.0"},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of unprocessed messages to fetch at a given time.","is_advanced":true,"default":10},{"name":"track_properties","type":"bool","kind":"scalar","description":"If set to `true` the queue is polled on each read request for information such as the queue message lag. These properties are added to consumed messages as metadata, but will also have a negative performance impact.","is_advanced":true,"default":false}],"linter":"root = if this.storage_connection_string != \"\" \u0026\u0026 !this.storage_connection_string.contains(\"AccountName=\") \u0026\u0026 !this.storage_connection_string.contains(\"UseDevelopmentStorage=true;\") \u0026\u0026 this.storage_account == \"\" { [ \"storage_account must be set if storage_connection_string does not contain the \\\"AccountName\\\" parameter\" ] }"},"version":"3.42.0"},{"name":"azure_table_storage","type":"input","status":"beta","plugin":true,"summary":"Queries an Azure Storage Account Table, optionally with multiple filters.","description":"\nQueries an Azure Storage Account Table, optionally with multiple filters.\n== Metadata\nThis input adds the following metadata fields to each message:\n\n- table_storage_name\n- row_num\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].","categories":["Services","Azure"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"storage_account","type":"string","kind":"scalar","description":"The storage account to access. This field is ignored if `storage_connection_string` is set.","default":""},{"name":"storage_access_key","type":"string","kind":"scalar","description":"The storage account access key. This field is ignored if `storage_connection_string` is set.","default":""},{"name":"storage_connection_string","type":"string","kind":"scalar","description":"A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set.","default":""},{"name":"storage_sas_token","type":"string","kind":"scalar","description":"The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set.","default":""},{"name":"table_name","type":"string","kind":"scalar","description":"The table to read messages from.","examples":["Foo"]},{"name":"filter","type":"string","kind":"scalar","description":"OData filter expression. Is not set all rows are returned. Valid operators are `eq, ne, gt, lt, ge and le`","is_advanced":true,"default":"","examples":["PartitionKey eq 'foo' and RowKey gt '1000'"]},{"name":"select","type":"string","kind":"scalar","description":"Select expression using OData notation. Limits the columns on each record to just those requested.","is_advanced":true,"default":"","examples":["PartitionKey,RowKey,Foo,Bar,Timestamp"]},{"name":"page_size","type":"int","kind":"scalar","description":"Maximum number of records to return on each page.","is_advanced":true,"default":1000}],"linter":"root = if this.storage_connection_string != \"\" \u0026\u0026 !this.storage_connection_string.contains(\"AccountName=\") \u0026\u0026 !this.storage_connection_string.contains(\"UseDevelopmentStorage=true;\") \u0026\u0026 this.storage_account == \"\" { [ \"storage_account must be set if storage_connection_string does not contain the \\\"AccountName\\\" parameter\" ] }"},"version":"4.10.0"},{"name":"batched","type":"input","status":"stable","plugin":true,"summary":"Consumes data from a child input and applies a batching policy to the stream.","description":"Batching at the input level is sometimes useful for processing across micro-batches, and can also sometimes be a useful performance trick. However, most inputs are fine without it so unless you have a specific plan for batching this component is not worth using.","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"child","type":"input","kind":"scalar","description":"The child input."},{"name":"policy","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]},"version":"4.11.0"},{"name":"beanstalkd","type":"input","status":"experimental","plugin":true,"summary":"Reads messages from a Beanstalkd queue.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"address","type":"string","kind":"scalar","description":"An address to connect to.","examples":["127.0.0.1:11300"]}]},"version":"4.7.0"},{"name":"broker","type":"input","status":"stable","plugin":true,"summary":"Allows you to combine multiple inputs into a single stream of data, where each input will be read in parallel.","description":"\nA broker type is configured with its own list of input configurations and a field to specify how many copies of the list of inputs should be created.\n\nAdding more input types allows you to combine streams from multiple sources into one. For example, reading from both RabbitMQ and Kafka:\n\n```yaml\ninput:\n broker:\n copies: 1\n inputs:\n - amqp_0_9:\n urls:\n - amqp://guest:guest@localhost:5672/\n consumer_tag: benthos-consumer\n queue: benthos-queue\n\n # Optional list of input specific processing steps\n processors:\n - mapping: |\n root.message = this\n root.meta.link_count = this.links.length()\n root.user.age = this.user.age.number()\n\n - kafka:\n addresses:\n - localhost:9092\n client_id: benthos_kafka_input\n consumer_group: benthos_consumer_group\n topics: [ benthos_stream:0 ]\n```\n\nIf the number of copies is greater than zero the list will be copied that number of times. For example, if your inputs were of type foo and bar, with 'copies' set to '2', you would end up with two 'foo' inputs and two 'bar' inputs.\n\n== Batching\n\nIt's possible to configure a xref:configuration:batching.adoc#batch-policy[batch policy] with a broker using the `batching` fields. When doing this the feeds from all child inputs are combined. Some inputs do not support broker based batching and specify this in their documentation.\n\n== Processors\n\nIt is possible to configure xref:components:processors/about.adoc[processors] at the broker level, where they will be applied to _all_ child inputs, as well as on the individual child inputs. If you have processors at both the broker level _and_ on child inputs then the broker processors will be applied _after_ the child nodes processors.","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"copies","type":"int","kind":"scalar","description":"Whatever is specified within `inputs` will be created this many times.","is_advanced":true,"default":1},{"name":"inputs","type":"input","kind":"array","description":"A list of inputs to create."},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]}},{"name":"cassandra","type":"input","status":"experimental","plugin":true,"summary":"Executes a find query and creates a message for each row received.","categories":["Services"],"examples":[{"title":"Minimal Select (Cassandra/Scylla)","summary":"\nLet's presume that we have 3 Cassandra nodes, like in this tutorial by Sebastian Sigl from freeCodeCamp:\n\nhttps://www.freecodecamp.org/news/the-apache-cassandra-beginner-tutorial/\n\nThen if we want to select everything from the table users_by_country, we should use the configuration below.\nIf we specify the stdin output, the result will look like:\n\n```json\n{\"age\":23,\"country\":\"UK\",\"first_name\":\"Bob\",\"last_name\":\"Sandler\",\"user_email\":\"bob@email.com\"}\n```\n\nThis configuration also works for Scylla.\n","config":"\ninput:\n cassandra:\n addresses:\n - 172.17.0.2\n query:\n 'SELECT * FROM learn_cassandra.users_by_country'\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"addresses","type":"string","kind":"array","description":"A list of Cassandra nodes to connect to. Multiple comma separated addresses can be specified on a single line.","examples":[["localhost:9042"],["foo:9042","bar:9042"],["foo:9042,bar:9042"]]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"password_authenticator","type":"object","kind":"scalar","description":"Optional configuration of Cassandra authentication parameters.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use password authentication","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"The username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"The password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"disable_initial_host_lookup","type":"bool","kind":"scalar","description":"If enabled the driver will not attempt to get host info from the system.peers table. This can speed up queries but will mean that data_centre, rack and token information will not be available.","is_advanced":true,"default":false},{"name":"max_retries","type":"int","kind":"scalar","description":"The maximum number of retries before giving up on a request.","is_advanced":true,"default":3},{"name":"backoff","type":"object","kind":"scalar","description":"Control time intervals between retry attempts.","is_advanced":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"1s"},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts.","is_advanced":true,"default":"5s"}]},{"name":"timeout","type":"string","kind":"scalar","description":"The client connection timeout.","default":"600ms"},{"name":"query","type":"string","kind":"scalar","description":"A query to execute."},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]}},{"name":"cockroachdb_changefeed","type":"input","status":"experimental","plugin":true,"summary":"Listens to a https://www.cockroachlabs.com/docs/stable/changefeed-examples[CockroachDB Core Changefeed^] and creates a message for each row received. Each message is a json object looking like: \n```json\n{\n\t\"primary_key\": \"[\\\"1a7ff641-3e3b-47ee-94fe-a0cadb56cd8f\\\", 2]\", // stringifed JSON array\n\t\"row\": \"{\\\"after\\\": {\\\"k\\\": \\\"1a7ff641-3e3b-47ee-94fe-a0cadb56cd8f\\\", \\\"v\\\": 2}, \\\"updated\\\": \\\"1637953249519902405.0000000000\\\"}\", // stringified JSON object\n\t\"table\": \"strm_2\"\n}\n```","description":"This input will continue to listen to the changefeed until shutdown. A backfill of the full current state of the table will be delivered upon each run unless a cache is configured for storing cursor timestamps, as this is how Redpanda Connect keeps track as to which changes have been successfully delivered.\n\nNote: You must have `SET CLUSTER SETTING kv.rangefeed.enabled = true;` on your CRDB cluster, for more information refer to https://www.cockroachlabs.com/docs/stable/changefeed-examples?filters=core[the official CockroachDB documentation^].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"dsn","type":"string","kind":"scalar","description":"A Data Source Name to identify the target database.","examples":["postgres://user:password@example.com:26257/defaultdb?sslmode=require"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"tables","type":"string","kind":"array","description":"CSV of tables to be included in the changefeed","examples":[["table1","table2"]]},{"name":"cursor_cache","type":"string","kind":"scalar","description":"A https://docs.redpanda.com/redpanda-connect/components/caches/about[cache resource^] to use for storing the current latest cursor that has been successfully delivered, this allows Redpanda Connect to continue from that cursor upon restart, rather than consume the entire state of the table.","is_optional":true},{"name":"options","type":"string","kind":"array","description":"A list of options to be included in the changefeed (WITH X, Y...).\n\nNOTE: Both the CURSOR option and UPDATED will be ignored from these options when a `cursor_cache` is specified, as they are set explicitly by Redpanda Connect in this case.","is_advanced":true,"is_optional":true,"examples":[["virtual_columns=\"omitted\""]]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]}},{"name":"csv","type":"input","status":"stable","plugin":true,"summary":"Reads one or more CSV files as structured records following the format described in RFC 4180.","description":"\nThis input offers more control over CSV parsing than the xref:components:inputs/file.adoc[`file` input].\n\nWhen parsing with a header row each line of the file will be consumed as a structured object, where the key names are determined from the header now. For example, the following CSV file:\n\n```csv\nfoo,bar,baz\nfirst foo,first bar,first baz\nsecond foo,second bar,second baz\n```\n\nWould produce the following messages:\n\n```json\n{\"foo\":\"first foo\",\"bar\":\"first bar\",\"baz\":\"first baz\"}\n{\"foo\":\"second foo\",\"bar\":\"second bar\",\"baz\":\"second baz\"}\n```\n\nIf, however, the field `parse_header_row` is set to `false` then arrays are produced instead, like follows:\n\n```json\n[\"first foo\",\"first bar\",\"first baz\"]\n[\"second foo\",\"second bar\",\"second baz\"]\n```\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- header\n- path\n- mod_time_unix\n- mod_time (RFC3339)\n```\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].\n\nNote: The `header` field is only set when `parse_header_row` is `true`.\n\n=== Output CSV column order\n\nWhen xref:guides:bloblang/advanced.adoc#creating-csv[creating CSV] from Redpanda Connect messages, the columns must be sorted lexicographically to make the output deterministic. Alternatively, when using the `csv` input, one can leverage the `header` metadata field to retrieve the column order:\n\n```yaml\ninput:\n csv:\n paths:\n - ./foo.csv\n - ./bar.csv\n parse_header_row: true\n\n processors:\n - mapping: |\n map escape_csv {\n root = if this.re_match(\"[\\\"\\n,]+\") {\n \"\\\"\" + this.replace_all(\"\\\"\", \"\\\"\\\"\") + \"\\\"\"\n } else {\n this\n }\n }\n\n let header = if count(@path) == 1 {\n @header.map_each(c -\u003e c.apply(\"escape_csv\")).join(\",\") + \"\\n\"\n } else { \"\" }\n\n root = $header + @header.map_each(c -\u003e this.get(c).string().apply(\"escape_csv\")).join(\",\")\n\noutput:\n file:\n path: ./output/${! @path.filepath_split().index(-1) }\n```\n","categories":["Local"],"footnotes":"This input is particularly useful when consuming CSV from files too large to parse entirely within memory. However, in cases where CSV is consumed from other input types it's also possible to parse them using the xref:guides:bloblang/methods.adoc#parse_csv[Bloblang `parse_csv` method].","config":{"name":"","type":"object","kind":"scalar","children":[{"name":"paths","type":"string","kind":"array","description":"A list of file paths to read from. Each file will be read sequentially until the list is exhausted, at which point the input will close. Glob patterns are supported, including super globs (double star).","examples":[["/tmp/foo.csv","/tmp/bar/*.csv","/tmp/data/**/*.csv"]]},{"name":"parse_header_row","type":"bool","kind":"scalar","description":"Whether to reference the first row as a header row. If set to true the output structure for messages will be an object where field keys are determined by the header row. Otherwise, each message will consist of an array of values from the corresponding CSV row.","default":true},{"name":"delimiter","type":"string","kind":"scalar","description":"The delimiter to use for splitting values in each record. It must be a single character.","default":","},{"name":"lazy_quotes","type":"bool","kind":"scalar","description":"If set to `true`, a quote may appear in an unquoted field and a non-doubled quote may appear in a quoted field.","default":false,"version":"4.1.0"},{"name":"delete_on_finish","type":"bool","kind":"scalar","description":"Whether to delete input files from the disk once they are fully consumed.","is_advanced":true,"default":false},{"name":"batch_count","type":"int","kind":"scalar","description":"Optionally process records in batches. This can help to speed up the consumption of exceptionally large CSV files. When the end of the file is reached the remaining records are processed as a (potentially smaller) batch.","is_advanced":true,"default":1},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]}},{"name":"discord","type":"input","status":"experimental","plugin":true,"summary":"Consumes messages posted in a Discord channel.","description":"This input works by authenticating as a bot using token based authentication. The ID of the newest message consumed and acked is stored in a cache in order to perform a backfill of unread messages each time the input is initialised. Ideally this cache should be persisted across restarts.","categories":["Services","Social"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"channel_id","type":"string","kind":"scalar","description":"A discord channel ID to consume messages from."},{"name":"bot_token","type":"string","kind":"scalar","description":"A bot token used for authentication."},{"name":"cache","type":"string","kind":"scalar","description":"A cache resource to use for performing unread message backfills, the ID of the last message received will be stored in this cache and used for subsequent requests."},{"name":"cache_key","type":"string","kind":"scalar","description":"The key identifier used when storing the ID of the last message received.","is_advanced":true,"default":"last_message_id"},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"poll_period","type":"string","kind":"scalar","description":"The length of time (as a duration string) to wait between each poll for backlogged messages. This field can be set empty, in which case requests are made at the limit set by the rate limit. This field also supports cron expressions.","is_deprecated":true,"default":"1m"},{"name":"limit","type":"int","kind":"scalar","description":"The maximum number of messages to receive in a single request.","is_deprecated":true,"default":100},{"name":"rate_limit","type":"string","kind":"scalar","is_deprecated":true,"default":"An optional rate limit resource to restrict API requests with."}]}},{"name":"dynamic","type":"input","status":"stable","plugin":true,"summary":"A special broker type where the inputs are identified by unique labels and can be created, changed and removed during runtime via a REST HTTP interface.","categories":["Utility"],"footnotes":"\n== Endpoints\n\n=== GET `/inputs`\n\nReturns a JSON object detailing all dynamic inputs, providing information such as their current uptime and configuration.\n\n=== GET `/inputs/\\{id}`\n\nReturns the configuration of an input.\n\n=== POST `/inputs/\\{id}`\n\nCreates or updates an input with a configuration provided in the request body (in YAML or JSON format).\n\n=== DELETE `/inputs/\\{id}`\n\nStops and removes an input.\n\n=== GET `/inputs/\\{id}/uptime`\n\nReturns the uptime of an input as a duration string (of the form \"72h3m0.5s\"), or \"stopped\" in the case where the input has gracefully terminated.","config":{"name":"","type":"object","kind":"scalar","children":[{"name":"inputs","type":"input","kind":"map","description":"A map of inputs to statically create.","default":{}},{"name":"prefix","type":"string","kind":"scalar","description":"A path prefix for HTTP endpoints that are registered.","default":""}]}},{"name":"file","type":"input","status":"stable","plugin":true,"summary":"Consumes data from files on disk, emitting messages according to a chosen codec.","description":"\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- path\n- mod_time_unix\n- mod_time (RFC3339)\n```\n\nYou can access these metadata fields using\nxref:configuration:interpolation.adoc#bloblang-queries[function interpolation].","categories":["Local"],"examples":[{"title":"Read a Bunch of CSVs","summary":"If we wished to consume a directory of CSV files as structured documents we can use a glob pattern and the `csv` scanner:","config":"\ninput:\n file:\n paths: [ ./data/*.csv ]\n scanner:\n csv: {}\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"paths","type":"string","kind":"array","description":"A list of paths to consume sequentially. Glob patterns are supported, including super globs (double star)."},{"name":"codec","type":"string","kind":"scalar","description":"The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.","is_deprecated":true,"is_optional":true,"examples":["lines","delim:\t","delim:foobar","gzip/csv"],"annotated_options":[["auto","EXPERIMENTAL: Attempts to derive a codec for each file based on information such as the extension. For example, a .tar.gz file would be consumed with the `gzip/tar` codec. Defaults to all-bytes."],["all-bytes","Consume the entire file as a single binary message."],["avro-ocf:marshaler=x","EXPERIMENTAL: Consume a stream of Avro OCF datum. The `marshaler` parameter is optional and has the options: `goavro` (default), `json`. Use `goavro` if OCF contains logical types."],["chunker:x","Consume the file in chunks of a given number of bytes."],["csv","Consume structured rows as comma separated values, the first row must be a header row."],["csv:x","Consume structured rows as values separated by a custom delimiter, the first row must be a header row. The custom delimiter must be a single character, e.g. the codec `\"csv:\\t\"` would consume a tab delimited file."],["csv-safe","Consume structured rows like `csv`, but sends messages with empty maps on failure to parse. Includes row number and parsing errors (if any) in the message's metadata."],["csv-safe:x","Consume structured rows like `csv:x` as values separated by a custom delimiter, but sends messages with empty maps on failure to parse. The custom delimiter must be a single character, e.g. the codec `\"csv-safe:\\t\"` would consume a tab delimited file. Includes row number and parsing errors (if any) in the message's metadata."],["delim:x","Consume the file in segments divided by a custom delimiter."],["gzip","Decompress a gzip file, this codec should precede another codec, e.g. `gzip/all-bytes`, `gzip/tar`, `gzip/csv`, etc."],["pgzip","Decompress a gzip file in parallel, this codec should precede another codec, e.g. `pgzip/all-bytes`, `pgzip/tar`, `pgzip/csv`, etc."],["lines","Consume the file in segments divided by linebreaks."],["multipart","Consumes the output of another codec and batches messages together. A batch ends when an empty message is consumed. For example, the codec `lines/multipart` could be used to consume multipart messages where an empty line indicates the end of each batch."],["regex:(?m)^\\d\\d:\\d\\d:\\d\\d","Consume the file in segments divided by regular expression."],["skipbom","Skip one or more byte order marks for each opened reader, this codec should precede another codec, e.g. `skipbom/csv`, etc."],["tar","Parse the file as a tar archive, and consume each file of the archive as a message."]]},{"name":"max_buffer","type":"int","kind":"scalar","is_deprecated":true,"default":1000000},{"name":"scanner","type":"scanner","kind":"scalar","description":"The xref:components:scanners/about.adoc[scanner] by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once.","is_optional":true,"default":{"lines":{}},"version":"4.25.0"},{"name":"delete_on_finish","type":"bool","kind":"scalar","description":"Whether to delete input files from the disk once they are fully consumed.","is_advanced":true,"default":false},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]}},{"name":"gateway","type":"input","status":"stable","plugin":true,"summary":"Receive messages delivered over HTTP.","description":"\nThe field `rate_limit` allows you to specify an optional xref:components:rate_limits/about.adoc[`rate_limit` resource], which will be applied to each HTTP request made and each websocket payload received.\n\nWhen the rate limit is breached HTTP requests will have a 429 response returned with a Retry-After header.\n\n== Responses\n\nIt's possible to return a response for each message received using xref:guides:sync_responses.adoc[synchronous responses]. When doing so you can customize headers with the `sync_response` field `headers`, which can also use xref:configuration:interpolation.adoc#bloblang-queries[function interpolation] in the value based on the response message contents.\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- http_server_user_agent\n- http_server_request_path\n- http_server_verb\n- http_server_remote_ip\n- All headers (only first values are taken)\n- All query parameters\n- All path parameters\n- All cookies\n```\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].","categories":["Network"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"path","type":"string","kind":"scalar","description":"The endpoint path to listen for data delivery requests.","default":"/"},{"name":"rate_limit","type":"string","kind":"scalar","description":"An optional xref:components:rate_limits/about.adoc[rate limit] to throttle requests by.","default":""},{"name":"sync_response","type":"object","kind":"scalar","description":"Customize messages returned via xref:guides:sync_responses.adoc[synchronous responses].","is_advanced":true,"children":[{"name":"status","type":"string","kind":"scalar","description":"Specify the status code to return with synchronous responses. This is a string value, which allows you to customize it based on resulting payloads and their metadata.","is_advanced":true,"default":"200","interpolated":true,"examples":["${! json(\"status\") }","${! meta(\"status\") }"]},{"name":"headers","type":"string","kind":"map","description":"Specify headers to return with synchronous responses.","is_advanced":true,"default":{"Content-Type":"application/octet-stream"},"interpolated":true},{"name":"metadata_headers","type":"object","kind":"scalar","description":"Specify criteria for which metadata values are added to the response as headers.","is_advanced":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","is_advanced":true,"default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","is_advanced":true,"default":[],"examples":[[".*"],["_timestamp_unix$"]]}]}]}]}},{"name":"gcp_bigquery_select","type":"input","status":"beta","plugin":true,"summary":"Executes a `SELECT` query against BigQuery and creates a message for each row received.","description":"Once the rows from the query are exhausted, this input shuts down, allowing the pipeline to gracefully terminate (or the next input in a xref:components:inputs/sequence.adoc[sequence] to execute).","categories":["Services","GCP"],"examples":[{"title":"Word counts","summary":"\nHere we query the public corpus of Shakespeare's works to generate a stream of the top 10 words that are 3 or more characters long:","config":"\ninput:\n gcp_bigquery_select:\n project: sample-project\n table: bigquery-public-data.samples.shakespeare\n columns:\n - word\n - sum(word_count) as total_count\n where: length(word) \u003e= ?\n suffix: |\n GROUP BY word\n ORDER BY total_count DESC\n LIMIT 10\n args_mapping: |\n root = [ 3 ]\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"project","type":"string","kind":"scalar","description":"GCP project where the query job will execute."},{"name":"credentials_json","type":"string","kind":"scalar","description":"An optional field to set Google Service Account Credentials json.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"table","type":"string","kind":"scalar","description":"Fully-qualified BigQuery table name to query.","examples":["bigquery-public-data.samples.shakespeare"]},{"name":"columns","type":"string","kind":"array","description":"A list of columns to query."},{"name":"where","type":"string","kind":"scalar","description":"An optional where clause to add. Placeholder arguments are populated with the `args_mapping` field. Placeholders should always be question marks (`?`).","is_optional":true,"examples":["type = ? and created_at \u003e ?","user_id = ?"]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"job_labels","type":"string","kind":"map","description":"A list of labels to add to the query job.","default":{}},{"name":"priority","type":"string","kind":"scalar","description":"The priority with which to schedule the query.","default":""},{"name":"args_mapping","type":"string","kind":"scalar","description":"An optional xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `where`.","is_optional":true,"bloblang":true,"examples":["root = [ \"article\", now().ts_format(\"2006-01-02\") ]"]},{"name":"prefix","type":"string","kind":"scalar","description":"An optional prefix to prepend to the select query (before SELECT).","is_optional":true},{"name":"suffix","type":"string","kind":"scalar","description":"An optional suffix to append to the select query.","is_optional":true}]},"version":"3.63.0"},{"name":"gcp_cloud_storage","type":"input","status":"beta","plugin":true,"summary":"Downloads objects within a Google Cloud Storage bucket, optionally filtered by a prefix.","description":"\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```\n- gcs_key\n- gcs_bucket\n- gcs_last_modified\n- gcs_last_modified_unix\n- gcs_content_type\n- gcs_content_encoding\n- All user defined metadata\n```\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].\n\n=== Credentials\n\nBy default Redpanda Connect will use a shared credentials file when connecting to GCP services. You can find out more in xref:guides:cloud/gcp.adoc[].","categories":["Services","GCP"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"bucket","type":"string","kind":"scalar","description":"The name of the bucket from which to download objects."},{"name":"prefix","type":"string","kind":"scalar","description":"An optional path prefix, if set only objects with the prefix are consumed.","default":""},{"name":"credentials_json","type":"string","kind":"scalar","description":"An optional field to set Google Service Account Credentials json.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"codec","type":"string","kind":"scalar","description":"The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.","is_deprecated":true,"is_optional":true,"examples":["lines","delim:\t","delim:foobar","gzip/csv"],"annotated_options":[["auto","EXPERIMENTAL: Attempts to derive a codec for each file based on information such as the extension. For example, a .tar.gz file would be consumed with the `gzip/tar` codec. Defaults to all-bytes."],["all-bytes","Consume the entire file as a single binary message."],["avro-ocf:marshaler=x","EXPERIMENTAL: Consume a stream of Avro OCF datum. The `marshaler` parameter is optional and has the options: `goavro` (default), `json`. Use `goavro` if OCF contains logical types."],["chunker:x","Consume the file in chunks of a given number of bytes."],["csv","Consume structured rows as comma separated values, the first row must be a header row."],["csv:x","Consume structured rows as values separated by a custom delimiter, the first row must be a header row. The custom delimiter must be a single character, e.g. the codec `\"csv:\\t\"` would consume a tab delimited file."],["csv-safe","Consume structured rows like `csv`, but sends messages with empty maps on failure to parse. Includes row number and parsing errors (if any) in the message's metadata."],["csv-safe:x","Consume structured rows like `csv:x` as values separated by a custom delimiter, but sends messages with empty maps on failure to parse. The custom delimiter must be a single character, e.g. the codec `\"csv-safe:\\t\"` would consume a tab delimited file. Includes row number and parsing errors (if any) in the message's metadata."],["delim:x","Consume the file in segments divided by a custom delimiter."],["gzip","Decompress a gzip file, this codec should precede another codec, e.g. `gzip/all-bytes`, `gzip/tar`, `gzip/csv`, etc."],["pgzip","Decompress a gzip file in parallel, this codec should precede another codec, e.g. `pgzip/all-bytes`, `pgzip/tar`, `pgzip/csv`, etc."],["lines","Consume the file in segments divided by linebreaks."],["multipart","Consumes the output of another codec and batches messages together. A batch ends when an empty message is consumed. For example, the codec `lines/multipart` could be used to consume multipart messages where an empty line indicates the end of each batch."],["regex:(?m)^\\d\\d:\\d\\d:\\d\\d","Consume the file in segments divided by regular expression."],["skipbom","Skip one or more byte order marks for each opened reader, this codec should precede another codec, e.g. `skipbom/csv`, etc."],["tar","Parse the file as a tar archive, and consume each file of the archive as a message."]]},{"name":"max_buffer","type":"int","kind":"scalar","is_deprecated":true,"default":1000000},{"name":"scanner","type":"scanner","kind":"scalar","description":"The xref:components:scanners/about.adoc[scanner] by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once.","is_optional":true,"default":{"to_the_end":{}},"version":"4.25.0"},{"name":"delete_objects","type":"bool","kind":"scalar","description":"Whether to delete downloaded objects from the bucket once they are processed.","is_advanced":true,"default":false}]},"version":"3.43.0"},{"name":"gcp_pubsub","type":"input","status":"stable","plugin":true,"summary":"Consumes messages from a GCP Cloud Pub/Sub subscription.","description":"\nFor information on how to set up credentials see https://cloud.google.com/docs/authentication/production[this guide^].\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n- gcp_pubsub_publish_time_unix - The time at which the message was published to the topic.\n- gcp_pubsub_delivery_attempt - When dead lettering is enabled, this is set to the number of times PubSub has attempted to deliver a message.\n- All message attributes\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].\n","categories":["Services","GCP"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"project","type":"string","kind":"scalar","description":"The project ID of the target subscription."},{"name":"credentials_json","type":"string","kind":"scalar","description":"An optional field to set Google Service Account Credentials json.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"subscription","type":"string","kind":"scalar","description":"The target subscription ID."},{"name":"endpoint","type":"string","kind":"scalar","description":"An optional endpoint to override the default of `pubsub.googleapis.com:443`. This can be used to connect to a region specific pubsub endpoint. For a list of valid values, see https://cloud.google.com/pubsub/docs/reference/service_apis_overview#list_of_regional_endpoints[this document^].","default":"","examples":["us-central1-pubsub.googleapis.com:443","us-west3-pubsub.googleapis.com:443"]},{"name":"sync","type":"bool","kind":"scalar","description":"Enable synchronous pull mode.","default":false},{"name":"max_outstanding_messages","type":"int","kind":"scalar","description":"The maximum number of outstanding pending messages to be consumed at a given time.","default":1000},{"name":"max_outstanding_bytes","type":"int","kind":"scalar","description":"The maximum number of outstanding pending messages to be consumed measured in bytes.","default":1000000000},{"name":"create_subscription","type":"object","kind":"scalar","description":"Allows you to configure the input subscription and creates if it doesn't exist.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to configure subscription or not.","is_advanced":true,"default":false},{"name":"topic","type":"string","kind":"scalar","description":"Defines the topic that the subscription should be vinculated to.","is_advanced":true,"default":""}]}]}},{"name":"generate","type":"input","status":"stable","plugin":true,"summary":"Generates messages at a given interval using a xref:guides:bloblang/about.adoc[Bloblang] mapping executed without a context. This allows you to generate messages for testing your pipeline configs.","categories":["Utility"],"examples":[{"title":"Cron Scheduled Processing","summary":"A common use case for the generate input is to trigger processors on a schedule so that the processors themselves can behave similarly to an input. The following configuration reads rows from a PostgreSQL table every 5 minutes.","config":"\ninput:\n generate:\n interval: '@every 5m'\n mapping: 'root = {}'\n processors:\n - sql_select:\n driver: postgres\n dsn: postgres://foouser:foopass@localhost:5432/testdb?sslmode=disable\n table: foo\n columns: [ \"*\" ]\n"},{"title":"Generate 100 Rows","summary":"The generate input can be used as a convenient way to generate test data. The following example generates 100 rows of structured data by setting an explicit count. The interval field is set to empty, which means data is generated as fast as the downstream components can consume it.","config":"\ninput:\n generate:\n count: 100\n interval: \"\"\n mapping: |\n root = if random_int() % 2 == 0 {\n {\n \"type\": \"foo\",\n \"foo\": \"is yummy\"\n }\n } else {\n {\n \"type\": \"bar\",\n \"bar\": \"is gross\"\n }\n }\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"mapping","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang] mapping to use for generating messages.","bloblang":true,"examples":["root = \"hello world\"","root = {\"test\":\"message\",\"id\":uuid_v4()}"]},{"name":"interval","type":"string","kind":"scalar","description":"The time interval at which messages should be generated, expressed either as a duration string or as a cron expression. If set to an empty string messages will be generated as fast as downstream services can process them. Cron expressions can specify a timezone by prefixing the expression with `TZ=\u003clocation name\u003e`, where the location name corresponds to a file within the IANA Time Zone database.","default":"1s","examples":["5s","1m","1h","@every 1s","0,30 */2 * * * *","TZ=Europe/London 30 3-6,20-23 * * *"]},{"name":"count","type":"int","kind":"scalar","description":"An optional number of messages to generate, if set above 0 the specified number of messages is generated and then the input will shut down.","default":0},{"name":"batch_size","type":"int","kind":"scalar","description":"The number of generated messages that should be accumulated into each batch flushed at the specified interval.","default":1},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]},"version":"3.40.0"},{"name":"git","type":"input","status":"beta","plugin":true,"summary":"A Git input that clones (or pulls) a repository and reads the repository contents.","description":"\nThe git input clones the specified repository (or pulls updates if already cloned) and reads \nthe content of the specified file. It periodically polls the repository for new commits and emits \na message when changes are detected.\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n- git_file_path\n- git_file_size\n- git_file_mode\n- git_file_modified\n- git_commit\n- git_mime_type\n- git_is_binary\n- git_encoding (present if the file was base64 encoded)\n- git_deleted (only present if the file was deleted)\n\nYou can access these metadata fields using function interpolation.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"repository_url","type":"string","kind":"scalar","description":"The URL of the Git repository to clone.","examples":["https://github.com/username/repo.git"]},{"name":"branch","type":"string","kind":"scalar","description":"The branch to check out.","default":"main"},{"name":"poll_interval","type":"string","kind":"scalar","description":"Duration between polling attempts","default":"10s","examples":["10s"]},{"name":"include_patterns","type":"string","kind":"array","description":"A list of file patterns to include (for example, '**/*.md', 'configs/*.yaml'). If empty, all files will be included. Supports glob patterns: *, /**/, ?, and character ranges [a-z]. Any character with a special meaning can be escaped with a backslash.","is_optional":true,"default":[]},{"name":"exclude_patterns","type":"string","kind":"array","description":"A list of file patterns to exclude (for example, '.git/**', '**/*.png'). These patterns take precedence over include_patterns. Supports glob patterns: *, /**/, ?, and character ranges [a-z]. Any character with a special meaning can be escaped with a backslash.","is_optional":true,"default":[]},{"name":"max_file_size","type":"int","kind":"scalar","description":"The maximum size of files to include in bytes. Files larger than this will be skipped. Set to 0 for no limit.","default":10485760},{"name":"checkpoint_cache","type":"string","kind":"scalar","description":"A cache resource to store the last processed commit hash, allowing the input to resume from where it left off after a restart.","is_optional":true},{"name":"checkpoint_key","type":"string","kind":"scalar","description":"The key to use when storing the last processed commit hash in the cache.","is_optional":true,"default":"git_last_commit"},{"name":"auth","type":"object","kind":"scalar","description":"Authentication options for the Git repository","is_optional":true,"children":[{"name":"basic","type":"object","kind":"scalar","description":"Basic authentication credentials","is_optional":true,"children":[{"name":"username","type":"string","kind":"scalar","description":"Username for basic authentication","is_optional":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"Password for basic authentication","is_optional":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"ssh_key","type":"object","kind":"scalar","description":"SSH key authentication","is_optional":true,"children":[{"name":"private_key_path","type":"string","kind":"scalar","description":"Path to SSH private key file","is_optional":true,"default":""},{"name":"private_key","type":"string","kind":"scalar","description":"SSH private key content","is_optional":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"passphrase","type":"string","kind":"scalar","description":"Passphrase for the SSH private key","is_optional":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"token","type":"object","kind":"scalar","description":"Token-based authentication","is_optional":true,"children":[{"name":"value","type":"string","kind":"scalar","description":"Token value for token-based authentication","is_optional":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]},"version":"4.51.0"},{"name":"hdfs","type":"input","status":"stable","plugin":true,"summary":"Reads files from a HDFS directory, where each discrete file will be consumed as a single message payload.","description":"\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n- hdfs_name\n- hdfs_path\n\nYou can access these metadata fields using\nxref:configuration:interpolation.adoc#bloblang-queries[function interpolation].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"hosts","type":"string","kind":"array","description":"A list of target host addresses to connect to.","examples":["localhost:9000"]},{"name":"user","type":"string","kind":"scalar","description":"A user ID to connect as.","default":""},{"name":"directory","type":"string","kind":"scalar","description":"The directory to consume from."}]}},{"name":"http_client","type":"input","status":"stable","plugin":true,"summary":"Connects to a server and continuously performs requests for a single message.","description":"\nThe URL and header values of this type can be dynamically set using function interpolations described xref:configuration:interpolation.adoc#bloblang-queries[here].\n\n== Streaming\n\nIf you enable streaming then Redpanda Connect will consume the body of the response as a continuous stream of data, breaking messages out following a chosen scanner. This allows you to consume APIs that provide long lived streamed data feeds (such as Twitter).\n\n== Pagination\n\nThis input supports interpolation functions in the `url` and `headers` fields where data from the previous successfully consumed message (if there was one) can be referenced. This can be used in order to support basic levels of pagination. However, in cases where pagination depends on logic it is recommended that you use an xref:components:processors/http.adoc[`http` processor] instead, often combined with a xref:components:inputs/generate.adoc[`generate` input] in order to schedule the processor.","categories":["Network"],"examples":[{"title":"Basic Pagination","summary":"Interpolation functions within the `url` and `headers` fields can be used to reference the previously consumed message, which allows simple pagination.","config":"\ninput:\n http_client:\n url: \u003e-\n https://api.example.com/search?query=allmyfoos\u0026start_time=${! (\n (timestamp_unix()-300).ts_format(\"2006-01-02T15:04:05Z\",\"UTC\").escape_url_query()\n ) }${! (\"\u0026next_token=\"+this.meta.next_token.not_null()) | \"\" }\n verb: GET\n rate_limit: foo_searches\n oauth2:\n enabled: true\n token_url: https://api.example.com/oauth2/token\n client_key: \"${EXAMPLE_KEY}\"\n client_secret: \"${EXAMPLE_SECRET}\"\n\nrate_limit_resources:\n - label: foo_searches\n local:\n count: 1\n interval: 30s\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL to connect to.","interpolated":true},{"name":"verb","type":"string","kind":"scalar","description":"A verb to connect with","default":"GET","examples":["POST","GET","DELETE"]},{"name":"headers","type":"string","kind":"map","description":"A map of headers to add to the request.","default":{},"interpolated":true,"examples":[{"Content-Type":"application/octet-stream","traceparent":"${! tracing_span().traceparent }"}]},{"name":"metadata","type":"object","kind":"scalar","description":"Specify optional matching rules to determine which metadata keys should be added to the HTTP request as headers.","is_advanced":true,"is_optional":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","is_advanced":true,"default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","is_advanced":true,"default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"dump_request_log_level","type":"string","kind":"scalar","description":"EXPERIMENTAL: Optionally set a level at which the request and response payload of each request made will be logged.","is_advanced":true,"default":"","options":["TRACE","DEBUG","INFO","WARN","ERROR","FATAL",""],"version":"4.12.0","linter":"\nlet options = {\n \"trace\": true,\n \"debug\": true,\n \"info\": true,\n \"warn\": true,\n \"error\": true,\n \"fatal\": true,\n \"\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"oauth","type":"object","kind":"scalar","description":"Allows you to specify open authentication via OAuth version 1.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use OAuth version 1 in requests.","is_advanced":true,"default":false},{"name":"consumer_key","type":"string","kind":"scalar","description":"A value used to identify the client to the service provider.","is_advanced":true,"default":""},{"name":"consumer_secret","type":"string","kind":"scalar","description":"A secret used to establish ownership of the consumer key.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"access_token","type":"string","kind":"scalar","description":"A value used to gain access to the protected resources on behalf of the user.","is_advanced":true,"default":""},{"name":"access_token_secret","type":"string","kind":"scalar","description":"A secret provided in order to establish ownership of a given access token.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"oauth2","type":"object","kind":"scalar","description":"Allows you to specify open authentication via OAuth version 2 using the client credentials token flow.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use OAuth version 2 in requests.","is_advanced":true,"default":false},{"name":"client_key","type":"string","kind":"scalar","description":"A value used to identify the client to the token provider.","is_advanced":true,"default":""},{"name":"client_secret","type":"string","kind":"scalar","description":"A secret used to establish ownership of the client key.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token_url","type":"string","kind":"scalar","description":"The URL of the token provider.","is_advanced":true,"default":"","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"scopes","type":"string","kind":"array","description":"A list of optional requested permissions.","is_advanced":true,"default":[],"version":"3.45.0"},{"name":"endpoint_params","type":"unknown","kind":"map","description":"A list of optional endpoint parameters, values should be arrays of strings.","is_advanced":true,"is_optional":true,"default":{},"examples":[{"bar":["woof"],"foo":["meow","quack"]}],"version":"4.21.0","linter":"\nroot = if this.type() == \"object\" {\n this.values().map_each(ele -\u003e if ele.type() != \"array\" {\n \"field must be an object containing arrays of strings, got %s (%v)\".format(ele.format_json(no_indent: true), ele.type())\n } else {\n ele.map_each(str -\u003e if str.type() != \"string\" {\n \"field values must be strings, got %s (%v)\".format(str.format_json(no_indent: true), str.type())\n } else { deleted() })\n }).\n flatten()\n}\n"}]},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"jwt","type":"object","kind":"scalar","description":"BETA: Allows you to specify JWT authentication.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use JWT authentication in requests.","is_advanced":true,"default":false},{"name":"private_key_file","type":"string","kind":"scalar","description":"A file with the PEM encoded via PKCS1 or PKCS8 as private key.","is_advanced":true,"default":""},{"name":"signing_method","type":"string","kind":"scalar","description":"A method used to sign the token such as RS256, RS384, RS512 or EdDSA.","is_advanced":true,"default":""},{"name":"claims","type":"unknown","kind":"map","description":"A value used to identify the claims that issued the JWT.","is_advanced":true,"default":{}},{"name":"headers","type":"unknown","kind":"map","description":"Add optional key/value headers to the JWT.","is_advanced":true,"default":{}}]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"extract_headers","type":"object","kind":"scalar","description":"Specify which response headers should be added to resulting messages as metadata. Header keys are lowercased before matching, so ensure that your patterns target lowercased versions of the header keys that you expect.","is_advanced":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","is_advanced":true,"default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","is_advanced":true,"default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"rate_limit","type":"string","kind":"scalar","description":"An optional xref:components:rate_limits/about.adoc[rate limit] to throttle requests by.","is_optional":true},{"name":"timeout","type":"string","kind":"scalar","description":"A static timeout to apply to requests.","default":"5s"},{"name":"retry_period","type":"string","kind":"scalar","description":"The base period to wait between failed requests.","is_advanced":true,"default":"1s"},{"name":"max_retry_backoff","type":"string","kind":"scalar","description":"The maximum period to wait between failed requests.","is_advanced":true,"default":"300s"},{"name":"retries","type":"int","kind":"scalar","description":"The maximum number of retry attempts to make.","is_advanced":true,"default":3},{"name":"follow_redirects","type":"bool","kind":"scalar","description":"Whether or not to transparently follow redirects, i.e. responses with 300-399 status codes. If disabled, the response message will contain the body, status, and headers from the redirect response and the processor will not make a request to the URL set in the Location header of the response.","is_advanced":true,"default":true},{"name":"backoff_on","type":"int","kind":"array","description":"A list of status codes whereby the request should be considered to have failed and retries should be attempted, but the period between them should be increased gradually.","is_advanced":true,"default":[429]},{"name":"drop_on","type":"int","kind":"array","description":"A list of status codes whereby the request should be considered to have failed but retries should not be attempted. This is useful for preventing wasted retries for requests that will never succeed. Note that with these status codes the _request_ is dropped, but _message_ that caused the request will not be dropped.","is_advanced":true,"default":[]},{"name":"successful_on","type":"int","kind":"array","description":"A list of status codes whereby the attempt should be considered successful, this is useful for dropping requests that return non-2XX codes indicating that the message has been dealt with, such as a 303 See Other or a 409 Conflict. All 2XX codes are considered successful unless they are present within `backoff_on` or `drop_on`, regardless of this field.","is_advanced":true,"default":[]},{"name":"proxy_url","type":"string","kind":"scalar","description":"An optional HTTP proxy URL.","is_advanced":true,"is_optional":true},{"name":"disable_http2","type":"bool","kind":"scalar","description":"Whether or not to disable disable HTTP/2","is_advanced":true,"default":false,"version":"4.44.0"},{"name":"payload","type":"string","kind":"scalar","description":"An optional payload to deliver for each request.","is_optional":true,"interpolated":true},{"name":"drop_empty_bodies","type":"bool","kind":"scalar","description":"Whether empty payloads received from the target server should be dropped.","is_advanced":true,"default":true},{"name":"stream","type":"object","kind":"scalar","description":"Allows you to set streaming mode, where requests are kept open and messages are processed line-by-line.","is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Enables streaming mode.","default":false},{"name":"reconnect","type":"bool","kind":"scalar","description":"Sets whether to re-establish the connection once it is lost.","default":true},{"name":"codec","type":"string","kind":"scalar","description":"The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.","is_deprecated":true,"is_optional":true,"examples":["lines","delim:\t","delim:foobar","gzip/csv"],"annotated_options":[["auto","EXPERIMENTAL: Attempts to derive a codec for each file based on information such as the extension. For example, a .tar.gz file would be consumed with the `gzip/tar` codec. Defaults to all-bytes."],["all-bytes","Consume the entire file as a single binary message."],["avro-ocf:marshaler=x","EXPERIMENTAL: Consume a stream of Avro OCF datum. The `marshaler` parameter is optional and has the options: `goavro` (default), `json`. Use `goavro` if OCF contains logical types."],["chunker:x","Consume the file in chunks of a given number of bytes."],["csv","Consume structured rows as comma separated values, the first row must be a header row."],["csv:x","Consume structured rows as values separated by a custom delimiter, the first row must be a header row. The custom delimiter must be a single character, e.g. the codec `\"csv:\\t\"` would consume a tab delimited file."],["csv-safe","Consume structured rows like `csv`, but sends messages with empty maps on failure to parse. Includes row number and parsing errors (if any) in the message's metadata."],["csv-safe:x","Consume structured rows like `csv:x` as values separated by a custom delimiter, but sends messages with empty maps on failure to parse. The custom delimiter must be a single character, e.g. the codec `\"csv-safe:\\t\"` would consume a tab delimited file. Includes row number and parsing errors (if any) in the message's metadata."],["delim:x","Consume the file in segments divided by a custom delimiter."],["gzip","Decompress a gzip file, this codec should precede another codec, e.g. `gzip/all-bytes`, `gzip/tar`, `gzip/csv`, etc."],["pgzip","Decompress a gzip file in parallel, this codec should precede another codec, e.g. `pgzip/all-bytes`, `pgzip/tar`, `pgzip/csv`, etc."],["lines","Consume the file in segments divided by linebreaks."],["multipart","Consumes the output of another codec and batches messages together. A batch ends when an empty message is consumed. For example, the codec `lines/multipart` could be used to consume multipart messages where an empty line indicates the end of each batch."],["regex:(?m)^\\d\\d:\\d\\d:\\d\\d","Consume the file in segments divided by regular expression."],["skipbom","Skip one or more byte order marks for each opened reader, this codec should precede another codec, e.g. `skipbom/csv`, etc."],["tar","Parse the file as a tar archive, and consume each file of the archive as a message."]]},{"name":"max_buffer","type":"int","kind":"scalar","is_deprecated":true,"default":1000000},{"name":"scanner","type":"scanner","kind":"scalar","description":"The xref:components:scanners/about.adoc[scanner] by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once.","is_optional":true,"default":{"lines":{}},"version":"4.25.0"}]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]}},{"name":"http_server","type":"input","status":"stable","plugin":true,"summary":"Receive messages POSTed over HTTP(S). HTTP 2.0 is supported when using TLS, which is enabled when key and cert files are specified.","description":"\nIf the `address` config field is left blank the xref:components:http/about.adoc[service-wide HTTP server] will be used.\n\nThe field `rate_limit` allows you to specify an optional xref:components:rate_limits/about.adoc[`rate_limit` resource], which will be applied to each HTTP request made and each websocket payload received.\n\nWhen the rate limit is breached HTTP requests will have a 429 response returned with a Retry-After header. Websocket payloads will be dropped and an optional response payload will be sent as per `ws_rate_limit_message`.\n\n== Responses\n\nIt's possible to return a response for each message received using xref:guides:sync_responses.adoc[synchronous responses]. When doing so you can customize headers with the `sync_response` field `headers`, which can also use xref:configuration:interpolation.adoc#bloblang-queries[function interpolation] in the value based on the response message contents.\n\n== Endpoints\n\nThe following fields specify endpoints that are registered for sending messages, and support path parameters of the form `/\\{foo}`, which are added to ingested messages as metadata. A path ending in `/` will match against all extensions of that path:\n\n=== `path` (defaults to `/post`)\n\nThis endpoint expects POST requests where the entire request body is consumed as a single message.\n\nIf the request contains a multipart `content-type` header as per https://www.w3.org/Protocols/rfc1341/7_2_Multipart.html[RFC1341^] then the multiple parts are consumed as a batch of messages, where each body part is a message of the batch.\n\n=== `ws_path` (defaults to `/post/ws`)\n\nCreates a websocket connection, where payloads received on the socket are passed through the pipeline as a batch of one message.\n\n\n[CAUTION]\n.Endpoint caveats\n====\nComponents within a Redpanda Connect config will register their respective endpoints in a non-deterministic order. This means that establishing precedence of endpoints that are registered via multiple `http_server` inputs or outputs (either within brokers or from cohabiting streams) is not possible in a predictable way.\n\nThis ambiguity makes it difficult to ensure that paths which are both a subset of a path registered by a separate component, and end in a slash (`/`) and will therefore match against all extensions of that path, do not prevent the more specific path from matching against requests.\n\nIt is therefore recommended that you ensure paths of separate components do not collide unless they are explicitly non-competing.\n\nFor example, if you were to deploy two separate `http_server` inputs, one with a path `/foo/` and the other with a path `/foo/bar`, it would not be possible to ensure that the path `/foo/` does not swallow requests made to `/foo/bar`.\n====\n\nYou may specify an optional `ws_welcome_message`, which is a static payload to be sent to all clients once a websocket connection is first established.\n\nIt's also possible to specify a `ws_rate_limit_message`, which is a static payload to be sent to clients that have triggered the servers rate limit.\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- http_server_user_agent\n- http_server_request_path\n- http_server_verb\n- http_server_remote_ip\n- All headers (only first values are taken)\n- All query parameters\n- All path parameters\n- All cookies\n```\n\nIf HTTPS is enabled, the following fields are added as well:\n```text\n- http_server_tls_version\n- http_server_tls_subject\n- http_server_tls_cipher_suite\n```\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].","categories":["Network"],"examples":[{"title":"Path Switching","summary":"This example shows an `http_server` input that captures all requests and processes them by switching on that path:","config":"\ninput:\n http_server:\n path: /\n allowed_verbs: [ GET, POST ]\n sync_response:\n headers:\n Content-Type: application/json\n\n processors:\n - switch:\n - check: '@http_server_request_path == \"/foo\"'\n processors:\n - mapping: |\n root.title = \"You Got Fooed!\"\n root.result = content().string().uppercase()\n\n - check: '@http_server_request_path == \"/bar\"'\n processors:\n - mapping: 'root.title = \"Bar Is Slow\"'\n - sleep: # Simulate a slow endpoint\n duration: 1s\n"},{"title":"Mock OAuth 2.0 Server","summary":"This example shows an `http_server` input that mocks an OAuth 2.0 Client Credentials flow server at the endpoint `/oauth2_test`:","config":"\ninput:\n http_server:\n path: /oauth2_test\n allowed_verbs: [ GET, POST ]\n sync_response:\n headers:\n Content-Type: application/json\n\n processors:\n - log:\n message: \"Received request\"\n level: INFO\n fields_mapping: |\n root = @\n root.body = content().string()\n\n - mapping: |\n root.access_token = \"MTQ0NjJkZmQ5OTM2NDE1ZTZjNGZmZjI3\"\n root.token_type = \"Bearer\"\n root.expires_in = 3600\n\n - sync_response: {}\n - mapping: 'root = deleted()'\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"address","type":"string","kind":"scalar","description":"An alternative address to host from. If left empty the service wide address is used.","default":""},{"name":"path","type":"string","kind":"scalar","description":"The endpoint path to listen for POST requests.","default":"/post"},{"name":"ws_path","type":"string","kind":"scalar","description":"The endpoint path to create websocket connections from.","default":"/post/ws"},{"name":"ws_welcome_message","type":"string","kind":"scalar","description":"An optional message to deliver to fresh websocket connections.","is_advanced":true,"default":""},{"name":"ws_rate_limit_message","type":"string","kind":"scalar","description":"An optional message to delivery to websocket connections that are rate limited.","is_advanced":true,"default":""},{"name":"allowed_verbs","type":"string","kind":"array","description":"An array of verbs that are allowed for the `path` endpoint.","default":["POST"],"version":"3.33.0"},{"name":"timeout","type":"string","kind":"scalar","description":"Timeout for requests. If a consumed messages takes longer than this to be delivered the connection is closed, but the message may still be delivered.","default":"5s"},{"name":"rate_limit","type":"string","kind":"scalar","description":"An optional xref:components:rate_limits/about.adoc[rate limit] to throttle requests by.","default":""},{"name":"cert_file","type":"string","kind":"scalar","description":"Enable TLS by specifying a certificate and key file. Only valid with a custom `address`.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"Enable TLS by specifying a certificate and key file. Only valid with a custom `address`.","is_advanced":true,"default":""},{"name":"cors","type":"object","kind":"scalar","description":"Adds Cross-Origin Resource Sharing headers. Only valid with a custom `address`.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to allow CORS requests.","is_advanced":true,"default":false},{"name":"allowed_origins","type":"string","kind":"array","description":"An explicit list of origins that are allowed for CORS requests.","is_advanced":true,"default":[]}],"version":"3.63.0"},{"name":"sync_response","type":"object","kind":"scalar","description":"Customize messages returned via xref:guides:sync_responses.adoc[synchronous responses].","is_advanced":true,"children":[{"name":"status","type":"string","kind":"scalar","description":"Specify the status code to return with synchronous responses. This is a string value, which allows you to customize it based on resulting payloads and their metadata.","is_advanced":true,"default":"200","interpolated":true,"examples":["${! json(\"status\") }","${! meta(\"status\") }"]},{"name":"headers","type":"string","kind":"map","description":"Specify headers to return with synchronous responses.","is_advanced":true,"default":{"Content-Type":"application/octet-stream"},"interpolated":true},{"name":"metadata_headers","type":"object","kind":"scalar","description":"Specify criteria for which metadata values are added to the response as headers.","is_advanced":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","is_advanced":true,"default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","is_advanced":true,"default":[],"examples":[[".*"],["_timestamp_unix$"]]}]}]}]}},{"name":"inproc","type":"input","status":"stable","plugin":true,"description":"\nDirectly connect to an output within a Redpanda Connect process by referencing it by a chosen ID. This allows you to hook up isolated streams whilst running Redpanda Connect in xref:guides:streams_mode/about.adoc[streams mode], it is NOT recommended that you connect the inputs of a stream with an output of the same stream, as feedback loops can lead to deadlocks in your message flow.\n\nIt is possible to connect multiple inputs to the same inproc ID, resulting in messages dispatching in a round-robin fashion to connected inputs. However, only one output can assume an inproc ID, and will replace existing outputs if a collision occurs.","categories":["Utility"],"config":{"name":"","type":"string","kind":"scalar","default":""}},{"name":"kafka","type":"input","status":"stable","plugin":true,"summary":"Connects to Kafka brokers and consumes one or more topics.","description":"\nOffsets are managed within Kafka under the specified consumer group, and partitions for each topic are automatically balanced across members of the consumer group.\n\nThe Kafka input allows parallel processing of messages from different topic partitions, and messages of the same topic partition are processed with a maximum parallelism determined by the field \u003c\u003ccheckpoint_limit,`checkpoint_limit`\u003e\u003e.\n\nIn order to enforce ordered processing of partition messages set the \u003ccheckpoint_limit,`checkpoint_limit`\u003e\u003e to `1` and this will force partitions to be processed in lock-step, where a message will only be processed once the prior message is delivered.\n\nBatching messages before processing can be enabled using the \u003c\u003cbatching,`batching`\u003e\u003e field, and this batching is performed per-partition such that messages of a batch will always originate from the same partition. This batching mechanism is capable of creating batches of greater size than the \u003c\u003ccheckpoint_limit,`checkpoint_limit`\u003e\u003e, in which case the next batch will only be created upon delivery of the current one.\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n- kafka_key\n- kafka_topic\n- kafka_partition\n- kafka_offset\n- kafka_lag\n- kafka_timestamp_ms\n- kafka_timestamp_unix\n- kafka_tombstone_message\n- All existing message headers (version 0.11+)\n\nThe field `kafka_lag` is the calculated difference between the high water mark offset of the partition at the time of ingestion and the current message offset.\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].\n\n== Ordering\n\nBy default messages of a topic partition can be processed in parallel, up to a limit determined by the field `checkpoint_limit`. However, if strict ordered processing is required then this value must be set to 1 in order to process shard messages in lock-step. When doing so it is recommended that you perform batching at this component for performance as it will not be possible to batch lock-stepped messages at the output level.\n\n== Troubleshooting\n\nIf you're seeing issues writing to or reading from Kafka with this component then it's worth trying out the newer xref:components:inputs/kafka_franz.adoc[`kafka_franz` input].\n\n- I'm seeing logs that report `Failed to connect to kafka: kafka: client has run out of available brokers to talk to (Is your cluster reachable?)`, but the brokers are definitely reachable.\n\nUnfortunately this error message will appear for a wide range of connection problems even when the broker endpoint can be reached. Double check your authentication configuration and also ensure that you have \u003c\u003ctlsenabled, enabled TLS\u003e\u003e if applicable.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"addresses","type":"string","kind":"array","description":"A list of broker addresses to connect to. If an item of the list contains commas it will be expanded into multiple addresses.","examples":[["localhost:9092"],["localhost:9041,localhost:9042"],["localhost:9041","localhost:9042"]]},{"name":"topics","type":"string","kind":"array","description":"A list of topics to consume from. Multiple comma separated topics can be listed in a single element. Partitions are automatically distributed across consumers of a topic. Alternatively, it's possible to specify explicit partitions to consume from with a colon after the topic name, e.g. `foo:0` would consume the partition 0 of the topic foo. This syntax supports ranges, e.g. `foo:0-10` would consume partitions 0 through to 10 inclusive.","examples":[["foo","bar"],["foo,bar"],["foo:0","bar:1","bar:3"],["foo:0,bar:1,bar:3"],["foo:0-5"]],"version":"3.33.0"},{"name":"target_version","type":"string","kind":"scalar","description":"The version of the Kafka protocol to use. This limits the capabilities used by the client and should ideally match the version of your brokers. Defaults to the oldest supported stable version.","is_optional":true,"examples":["2.1.0","3.1.0"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"sasl","type":"object","kind":"scalar","description":"Enables SASL authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"mechanism","type":"string","kind":"scalar","description":"The SASL authentication mechanism, if left empty SASL authentication is not used.","is_advanced":true,"default":"none","annotated_options":[["OAUTHBEARER","OAuth Bearer based authentication."],["PLAIN","Plain text authentication. NOTE: When using plain text auth it is extremely likely that you'll also need to \u003c\u003ctls-enabled, enable TLS\u003e\u003e."],["SCRAM-SHA-256","Authentication using the SCRAM-SHA-256 mechanism."],["SCRAM-SHA-512","Authentication using the SCRAM-SHA-512 mechanism."],["none","Default, no SASL authentication."]],"linter":"\nlet options = {\n \"oauthbearer\": true,\n \"plain\": true,\n \"scram-sha-256\": true,\n \"scram-sha-512\": true,\n \"none\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"user","type":"string","kind":"scalar","description":"A PLAIN username. It is recommended that you use environment variables to populate this field.","is_advanced":true,"default":"","examples":["${USER}"]},{"name":"password","type":"string","kind":"scalar","description":"A PLAIN password. It is recommended that you use environment variables to populate this field.","is_advanced":true,"is_secret":true,"default":"","examples":["${PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"access_token","type":"string","kind":"scalar","description":"A static OAUTHBEARER access token","is_advanced":true,"default":""},{"name":"token_cache","type":"string","kind":"scalar","description":"Instead of using a static `access_token` allows you to query a xref:components:caches/about.adoc[`cache`] resource to fetch OAUTHBEARER tokens from","is_advanced":true,"default":""},{"name":"token_key","type":"string","kind":"scalar","description":"Required when using a `token_cache`, the key to query the cache with for tokens.","is_advanced":true,"default":""}]},{"name":"consumer_group","type":"string","kind":"scalar","description":"An identifier for the consumer group of the connection. This field can be explicitly made empty in order to disable stored offsets for the consumed topic partitions.","default":""},{"name":"client_id","type":"string","kind":"scalar","description":"An identifier for the client connection.","is_advanced":true,"default":"benthos"},{"name":"instance_id","type":"string","kind":"scalar","description":"When using consumer groups, an identifier for this specific input so that it can be identified over restarts of this process. This should be unique per input.","is_advanced":true,"is_optional":true},{"name":"rack_id","type":"string","kind":"scalar","description":"A rack identifier for this client.","is_advanced":true,"default":""},{"name":"start_from_oldest","type":"bool","kind":"scalar","description":"Determines whether to consume from the oldest available offset, otherwise messages are consumed from the latest offset. The setting is applied when creating a new consumer group or the saved offset no longer exists.","is_advanced":true,"default":true},{"name":"checkpoint_limit","type":"int","kind":"scalar","description":"The maximum number of messages of the same topic and partition that can be processed at a given time. Increasing this limit enables parallel processing and batching at the output level to work on individual partitions. Any given offset will not be committed unless all messages under that offset are delivered in order to preserve at least once delivery guarantees.","default":1024,"version":"3.33.0"},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"commit_period","type":"string","kind":"scalar","description":"The period of time between each commit of the current partition offsets. Offsets are always committed during shutdown.","is_advanced":true,"default":"1s"},{"name":"max_processing_period","type":"string","kind":"scalar","description":"A maximum estimate for the time taken to process a message, this is used for tuning consumer group synchronization.","is_advanced":true,"default":"100ms"},{"name":"extract_tracing_map","type":"string","kind":"scalar","description":"EXPERIMENTAL: A xref:guides:bloblang/about.adoc[Bloblang mapping] that attempts to extract an object containing tracing propagation information, which will then be used as the root tracing span for the message. The specification of the extracted fields must match the format used by the service wide tracer.","is_advanced":true,"is_optional":true,"bloblang":true,"examples":["root = @","root = this.meta.span"],"version":"3.45.0"},{"name":"group","type":"object","kind":"scalar","description":"Tuning parameters for consumer group synchronization.","is_advanced":true,"children":[{"name":"session_timeout","type":"string","kind":"scalar","description":"A period after which a consumer of the group is kicked after no heartbeats.","is_advanced":true,"default":"10s"},{"name":"heartbeat_interval","type":"string","kind":"scalar","description":"A period in which heartbeats should be sent out.","is_advanced":true,"default":"3s"},{"name":"rebalance_timeout","type":"string","kind":"scalar","description":"A period after which rebalancing is abandoned if unresolved.","is_advanced":true,"default":"60s"}]},{"name":"fetch_buffer_cap","type":"int","kind":"scalar","description":"The maximum number of unprocessed messages to fetch at a given time.","is_advanced":true,"default":256},{"name":"multi_header","type":"bool","kind":"scalar","description":"Decode headers into lists to allow handling of multiple values with the same key","is_advanced":true,"default":false},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","is_advanced":true,"examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","is_advanced":true,"default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","is_advanced":true,"default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","is_advanced":true,"default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","is_advanced":true,"default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]}},{"name":"kafka_franz","type":"input","status":"beta","plugin":true,"summary":"A Kafka input using the https://github.com/twmb/franz-go[Franz Kafka client library^].","description":"\nWhen a consumer group is specified this input consumes one or more topics where partitions will automatically balance across any other connected clients with the same consumer group. When a consumer group is not specified topics can either be consumed in their entirety or with explicit partitions.\n\nThis input often out-performs the traditional `kafka` input as well as providing more useful logs and error messages.\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- kafka_key\n- kafka_topic\n- kafka_partition\n- kafka_offset\n- kafka_timestamp_ms\n- kafka_timestamp_unix\n- kafka_tombstone_message\n- All record headers\n```\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"seed_brokers","type":"string","kind":"array","description":"A list of broker addresses to connect to in order to establish connections. If an item of the list contains commas it will be expanded into multiple addresses.","examples":[["localhost:9092"],["foo:9092","bar:9092"],["foo:9092,bar:9092"]]},{"name":"client_id","type":"string","kind":"scalar","description":"An identifier for the client connection.","is_advanced":true,"default":"benthos"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"sasl","type":"object","kind":"array","description":"Specify one or more methods of SASL authentication. SASL is tried in order; if the broker supports the first mechanism, all connections will use that mechanism. If the first mechanism fails, the client will pick the first supported mechanism. If the broker does not support any client mechanisms, connections will fail.","is_advanced":true,"is_optional":true,"examples":[[{"mechanism":"SCRAM-SHA-512","password":"bar","username":"foo"}]],"children":[{"name":"mechanism","type":"string","kind":"scalar","description":"The SASL mechanism to use.","is_advanced":true,"annotated_options":[["AWS_MSK_IAM","AWS IAM based authentication as specified by the 'aws-msk-iam-auth' java library."],["OAUTHBEARER","OAuth Bearer based authentication."],["PLAIN","Plain text authentication."],["SCRAM-SHA-256","SCRAM based authentication as specified in RFC5802."],["SCRAM-SHA-512","SCRAM based authentication as specified in RFC5802."],["none","Disable sasl authentication"]],"linter":"\nlet options = {\n \"aws_msk_iam\": true,\n \"oauthbearer\": true,\n \"plain\": true,\n \"scram-sha-256\": true,\n \"scram-sha-512\": true,\n \"none\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"username","type":"string","kind":"scalar","description":"A username to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token to use for a single session's OAUTHBEARER authentication.","is_advanced":true,"default":""},{"name":"extensions","type":"string","kind":"map","description":"Key/value pairs to add to OAUTHBEARER authentication requests.","is_advanced":true,"is_optional":true},{"name":"aws","type":"object","kind":"scalar","description":"Contains AWS specific fields for when the `mechanism` is set to `AWS_MSK_IAM`.","is_advanced":true,"is_optional":true,"children":[{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]}]},{"name":"metadata_max_age","type":"string","kind":"scalar","description":"The maximum age of metadata before it is refreshed.","is_advanced":true,"default":"5m"},{"name":"request_timeout_overhead","type":"string","kind":"scalar","description":"The request time overhead. Uses the given time as overhead while deadlining requests. Roughly equivalent to request.timeout.ms, but grants additional time to requests that have timeout fields.","is_advanced":true,"default":"10s"},{"name":"conn_idle_timeout","type":"string","kind":"scalar","description":"The rough amount of time to allow connections to idle before they are closed.","is_advanced":true,"default":"20s"},{"name":"topics","type":"string","kind":"array","description":"\nA list of topics to consume from. Multiple comma separated topics can be listed in a single element. When a `consumer_group` is specified partitions are automatically distributed across consumers of a topic, otherwise all partitions are consumed.\n\nAlternatively, it's possible to specify explicit partitions to consume from with a colon after the topic name, e.g. `foo:0` would consume the partition 0 of the topic foo. This syntax supports ranges, e.g. `foo:0-10` would consume partitions 0 through to 10 inclusive.\n\nFinally, it's also possible to specify an explicit offset to consume from by adding another colon after the partition, e.g. `foo:0:10` would consume the partition 0 of the topic foo starting from the offset 10. If the offset is not present (or remains unspecified) then the field `start_from_oldest` determines which offset to start from.","examples":[["foo","bar"],["things.*"],["foo,bar"],["foo:0","bar:1","bar:3"],["foo:0,bar:1,bar:3"],["foo:0-5"]]},{"name":"regexp_topics","type":"bool","kind":"scalar","description":"Whether listed topics should be interpreted as regular expression patterns for matching multiple topics. When topics are specified with explicit partitions this field must remain set to `false`.","default":false},{"name":"rack_id","type":"string","kind":"scalar","description":"A rack specifies where the client is physically located and changes fetch requests to consume from the closest replica as opposed to the leader replica.","is_advanced":true,"default":""},{"name":"instance_id","type":"string","kind":"scalar","description":"When using a consumer group, an instance ID specifies the groups static membership, which can prevent rebalances during reconnects. When using a instance ID the client does NOT leave the group when closing. To actually leave the group one must use an external admin command to leave the group on behalf of this instance ID. This ID must be unique per consumer within the group.","is_advanced":true,"default":""},{"name":"rebalance_timeout","type":"string","kind":"scalar","description":"When using a consumer group, `rebalance_timeout` sets how long group members are allowed to take when a rebalance has begun. This timeout is how long all members are allowed to complete work and commit offsets, minus the time it took to detect the rebalance (from a heartbeat).","is_advanced":true,"default":"45s"},{"name":"session_timeout","type":"string","kind":"scalar","description":"When using a consumer group, `session_timeout` sets how long a member in hte group can go between heartbeats. If a member does not heartbeat in this timeout, the broker will remove the member from the group and initiate a rebalance.","is_advanced":true,"default":"1m"},{"name":"heartbeat_interval","type":"string","kind":"scalar","description":"When using a consumer group, `heartbeat_interval` sets how long a group member goes between heartbeats to Kafka. Kafka uses heartbeats to ensure that a group member's sesion stays active. This value should be no higher than 1/3rd of the `session_timeout`. This is equivalent to the Java heartbeat.interval.ms setting.","is_advanced":true,"default":"3s"},{"name":"start_from_oldest","type":"bool","kind":"scalar","description":"Determines whether to consume from the oldest available offset, otherwise messages are consumed from the latest offset. The setting is applied when creating a new consumer group or the saved offset no longer exists.","is_advanced":true,"is_deprecated":true,"default":true},{"name":"start_offset","type":"string","kind":"scalar","description":"Sets the offset to start consuming from, or if OffsetOutOfRange is seen while fetching, to restart consuming from.","is_advanced":true,"default":"earliest","annotated_options":[["committed","Prevents consuming a partition in a group if the partition has no prior commits. Corresponds to Kafka's `auto.offset.reset=none` option"],["earliest","Start from the earliest offset. Corresponds to Kafka's `auto.offset.reset=earliest` option."],["latest","Start from the latest offset. Corresponds to Kafka's `auto.offset.reset=latest` option."]],"linter":"\nlet options = {\n \"committed\": true,\n \"earliest\": true,\n \"latest\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"fetch_max_bytes","type":"string","kind":"scalar","description":"Sets the maximum amount of bytes a broker will try to send during a fetch. Note that brokers may not obey this limit if it has records larger than this limit. This is the equivalent to the Java fetch.max.bytes setting.","is_advanced":true,"default":"50MiB"},{"name":"fetch_max_wait","type":"string","kind":"scalar","description":"Sets the maximum amount of time a broker will wait for a fetch response to hit the minimum number of required bytes. This is the equivalent to the Java fetch.max.wait.ms setting.","is_advanced":true,"default":"5s"},{"name":"fetch_min_bytes","type":"string","kind":"scalar","description":"Sets the minimum amount of bytes a broker will try to send during a fetch. This is the equivalent to the Java fetch.min.bytes setting.","is_advanced":true,"default":"1B"},{"name":"fetch_max_partition_bytes","type":"string","kind":"scalar","description":"Sets the maximum amount of bytes that will be consumed for a single partition in a fetch request. Note that if a single batch is larger than this number, that batch will still be returned so the client can make progress. This is the equivalent to the Java fetch.max.partition.bytes setting.","is_advanced":true,"default":"1MiB"},{"name":"transaction_isolation_level","type":"string","kind":"scalar","description":"The transaction isolation level","default":"read_uncommitted","annotated_options":[["read_committed","If set, only committed transactional records are processed."],["read_uncommitted","If set, then uncommitted records are processed."]],"linter":"\nlet options = {\n \"read_committed\": true,\n \"read_uncommitted\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"consumer_group","type":"string","kind":"scalar","description":"An optional consumer group to consume as. When specified the partitions of specified topics are automatically distributed across consumers sharing a consumer group, and partition offsets are automatically committed and resumed under this name. Consumer groups are not supported when specifying explicit partitions to consume from in the `topics` field.","is_optional":true},{"name":"checkpoint_limit","type":"int","kind":"scalar","description":"Determines how many messages of the same partition can be processed in parallel before applying back pressure. When a message of a given offset is delivered to the output the offset is only allowed to be committed when all messages of prior offsets have also been delivered, this ensures at-least-once delivery guarantees. However, this mechanism also increases the likelihood of duplicates in the event of crashes or server faults, reducing the checkpoint limit will mitigate this.","is_advanced":true,"default":1024},{"name":"commit_period","type":"string","kind":"scalar","description":"The period of time between each commit of the current partition offsets. Offsets are always committed during shutdown.","is_advanced":true,"default":"5s"},{"name":"multi_header","type":"bool","kind":"scalar","description":"Decode headers into lists to allow handling of multiple values with the same key","is_advanced":true,"default":false},{"name":"batching","type":"object","kind":"","description":"Allows you to configure a xref:configuration:batching.adoc[batching policy] that applies to individual topic partitions in order to batch messages together before flushing them for processing. Batching can be beneficial for performance as well as useful for windowed processing, and doing so this way preserves the ordering of topic partitions.","is_advanced":true,"examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","is_advanced":true,"default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","is_advanced":true,"default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","is_advanced":true,"default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","is_advanced":true,"default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}],"linter":"\nlet has_topic_partitions = this.topics.any(t -\u003e t.contains(\":\"))\n\nroot = [\n if $has_topic_partitions {\n if this.consumer_group.or(\"\") != \"\" {\n \"this input does not support both a consumer group and explicit topic partitions\"\n } else if this.regexp_topics {\n \"this input does not support both regular expression topics and explicit topic partitions\"\n }\n } else {\n if this.consumer_group.or(\"\") == \"\" {\n \"a consumer group is mandatory when not using explicit topic partitions\"\n }\n },\n # We don't have any way to distinguish between start_from_oldest set explicitly to true and not set at all, so we\n # assume users will be OK if start_offset overwrites it silently\n if this.start_from_oldest == false \u0026\u0026 this.start_offset == \"earliest\" {\n \"start_from_oldest cannot be set to false when start_offset is set to earliest\"\n }\n]\n"},"version":"3.61.0"},{"name":"mongodb","type":"input","status":"experimental","plugin":true,"summary":"Executes a query and creates a message for each document received.","description":"Once the documents from the query are exhausted, this input shuts down, allowing the pipeline to gracefully terminate (or the next input in a xref:components:inputs/sequence.adoc[sequence] to execute).","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target MongoDB server.","examples":["mongodb://localhost:27017"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"database","type":"string","kind":"scalar","description":"The name of the target MongoDB database."},{"name":"username","type":"string","kind":"scalar","description":"The username to connect to the database.","default":""},{"name":"password","type":"string","kind":"scalar","description":"The password to connect to the database.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"app_name","type":"string","kind":"scalar","description":"The client application name.","is_advanced":true,"default":"benthos","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"collection","type":"string","kind":"scalar","description":"The collection to select from."},{"name":"operation","type":"string","kind":"scalar","description":"The mongodb operation to perform.","is_advanced":true,"default":"find","options":["find","aggregate"],"version":"4.2.0","linter":"\nlet options = {\n \"find\": true,\n \"aggregate\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"json_marshal_mode","type":"string","kind":"scalar","description":"The json_marshal_mode setting is optional and controls the format of the output message.","is_advanced":true,"default":"canonical","annotated_options":[["canonical","A string format that emphasizes type preservation at the expense of readability and interoperability. That is, conversion from canonical to BSON will generally preserve type information except in certain specific cases. "],["relaxed","A string format that emphasizes readability and interoperability at the expense of type preservation.That is, conversion from relaxed format to BSON can lose type information."]],"version":"4.7.0","linter":"\nlet options = {\n \"canonical\": true,\n \"relaxed\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"query","type":"string","kind":"scalar","description":"Bloblang expression describing MongoDB query.","bloblang":true,"examples":["\n root.from = {\"$lte\": timestamp_unix()}\n root.to = {\"$gte\": timestamp_unix()}\n"]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"batch_size","type":"int","kind":"scalar","description":"A explicit number of documents to batch up before flushing them for processing. Must be greater than `0`. Operations: `find`, `aggregate`","is_optional":true,"examples":[1000],"version":"4.26.0"},{"name":"sort","type":"int","kind":"map","description":"An object specifying fields to sort by, and the respective sort order (`1` ascending, `-1` descending). Note: The driver currently appears to support only one sorting key. Operations: `find`","is_optional":true,"examples":[{"name":1},{"age":-1}],"version":"4.26.0"},{"name":"limit","type":"int","kind":"scalar","description":"An explicit maximum number of documents to return. Operations: `find`","is_optional":true,"version":"4.26.0"}]},"version":"3.64.0"},{"name":"mongodb_cdc","type":"input","status":"experimental","plugin":true,"summary":"Streams changes from a MongoDB replica set.","description":"Read from a MongoDB replica set using https://www.mongodb.com/docs/manual/changeStreams/[^Change Streams]. It's only possible to watch for changes when using a sharded MongoDB or a MongoDB cluster running as a replica set.\n\nBy default MongoDB does not propagate changes in all cases. In order to capture all changes (including deletes) in a MongoDB cluster one needs to enable pre and post image saving and the collection needs to also enable saving these pre and post images. For more information see https://www.mongodb.com/docs/manual/changeStreams/#change-streams-with-document-pre--and-post-images[^MongoDB documentation].\n\n== Metadata\n\nEach message omitted by this plugin has the following metadata:\n\n- operation: either \"create\", \"replace\", \"delete\" or \"update\" for changes streamed. Documents from the initial snapshot have the operation set to \"read\".\n- collection: the collection the document was written to.\n- operation_time: the oplog time for when this operation occurred.\n ","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target MongoDB server.","examples":["mongodb://localhost:27017"]},{"name":"database","type":"string","kind":"scalar","description":"The name of the target MongoDB database."},{"name":"username","type":"string","kind":"scalar","description":"The username to connect to the database.","default":""},{"name":"password","type":"string","kind":"scalar","description":"The password to connect to the database.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"collections","type":"string","kind":"array","description":"The collections to stream changes from."},{"name":"checkpoint_key","type":"string","kind":"scalar","description":"Checkpoint cache key name.","default":"mongodb_cdc_checkpoint"},{"name":"checkpoint_cache","type":"string","kind":"scalar","description":"Checkpoint cache name."},{"name":"checkpoint_interval","type":"string","kind":"scalar","description":"The interval between writing checkpoints to the cache.","default":"5s"},{"name":"checkpoint_limit","type":"int","kind":"scalar","default":1000},{"name":"read_batch_size","type":"int","kind":"scalar","description":"The batch size of documents for MongoDB to return.","default":1000},{"name":"read_max_wait","type":"string","kind":"scalar","description":"The maximum time MongoDB waits to fulfill `read_batch_size` on the change stream before returning documents.","default":"1s"},{"name":"stream_snapshot","type":"bool","kind":"scalar","description":"If to read initial snapshot before streaming changes.","default":false},{"name":"snapshot_parallelism","type":"int","kind":"scalar","description":"Parallelism for snapshot phase.","default":1,"linter":"match {\n this \u003c 1 =\u003e [\"field snapshot_parallelism must be greater or equal to 1.\"],\n}"},{"name":"snapshot_auto_bucket_sharding","type":"bool","kind":"scalar","description":"If true, determine parallel snapshot chunks using `$bucketAuto` instead of the `splitVector` command. This allows parallel collection reading in environments where privledged access to the MongoDB cluster is not allowed such as MongoDB Atlas.","is_advanced":true,"default":false},{"name":"document_mode","type":"string","kind":"scalar","description":"The mode in which to emit documents, specifically updates and deletes.","is_advanced":true,"default":"update_lookup","annotated_options":[["partial_update","In this mode update operations only have a description of the update operation, which follows the following schema:\n {\n \"_id\": \u003cdocument_id\u003e,\n \"operations\": [\n # type == set means that the value was updated like so:\n # root.foo.\"bar.baz\" = \"world\"\n {\"path\": [\"foo\", \"bar.baz\"], \"type\": \"set\", \"value\":\"world\"},\n # type == unset means that the value was deleted like so:\n # root.qux = deleted()\n {\"path\": [\"qux\"], \"type\": \"unset\", \"value\": null},\n # type == truncatedArray means that the array at that path was truncated to value number of elements\n # root.array = this.array.slice(2)\n {\"path\": [\"array\"], \"type\": \"truncatedArray\", \"value\": 2}\n ]\n }\n "],["pre_and_post_images","Uses pre and post image collection to emit the full documents for update and delete operations. To use and configure this mode see the setup steps in the https://www.mongodb.com/docs/manual/changeStreams/#change-streams-with-document-pre--and-post-images[^MongoDB documentation]."],["update_lookup","In this mode insert, replace and update operations have the full document emitted and deletes only have the _id field populated. Documents updates lookup the full document. This corresponds to the updateLookup option, see the https://www.mongodb.com/docs/manual/changeStreams/#std-label-change-streams-updateLookup[^MongoDB documentation] for more information."]],"linter":"\nlet options = {\n \"partial_update\": true,\n \"pre_and_post_images\": true,\n \"update_lookup\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"json_marshal_mode","type":"string","kind":"scalar","description":"The json_marshal_mode setting is optional and controls the format of the output message.","is_advanced":true,"default":"canonical","annotated_options":[["canonical","A string format that emphasizes type preservation at the expense of readability and interoperability. That is, conversion from canonical to BSON will generally preserve type information except in certain specific cases. "],["relaxed","A string format that emphasizes readability and interoperability at the expense of type preservation.That is, conversion from relaxed format to BSON can lose type information."]],"linter":"\nlet options = {\n \"canonical\": true,\n \"relaxed\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"app_name","type":"string","kind":"scalar","description":"The client application name.","is_advanced":true,"default":"benthos"},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]}},{"name":"mqtt","type":"input","status":"stable","plugin":true,"summary":"Subscribe to topics on MQTT brokers.","description":"\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n- mqtt_duplicate\n- mqtt_qos\n- mqtt_retained\n- mqtt_topic\n- mqtt_message_id\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. The format should be `scheme://host:port` where `scheme` is one of `tcp`, `ssl`, or `ws`, `host` is the ip-address (or hostname) and `port` is the port on which the broker is accepting connections. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["tcp://localhost:1883"]],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"client_id","type":"string","kind":"scalar","description":"An identifier for the client connection.","default":""},{"name":"dynamic_client_id_suffix","type":"string","kind":"scalar","description":"Append a dynamically generated suffix to the specified `client_id` on each run of the pipeline. This can be useful when clustering Redpanda Connect producers.","is_advanced":true,"is_optional":true,"annotated_options":[["nanoid","append a nanoid of length 21 characters"]],"linter":"root = []"},{"name":"connect_timeout","type":"string","kind":"scalar","description":"The maximum amount of time to wait in order to establish a connection before the attempt is abandoned.","default":"30s","examples":["1s","500ms"],"version":"3.58.0"},{"name":"will","type":"object","kind":"scalar","description":"Set last will message in case of Redpanda Connect failure","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to enable last will messages.","is_advanced":true,"default":false},{"name":"qos","type":"int","kind":"scalar","description":"Set QoS for last will message. Valid values are: 0, 1, 2.","is_advanced":true,"default":0},{"name":"retained","type":"bool","kind":"scalar","description":"Set retained for last will message.","is_advanced":true,"default":false},{"name":"topic","type":"string","kind":"scalar","description":"Set topic for last will message.","is_advanced":true,"default":""},{"name":"payload","type":"string","kind":"scalar","description":"Set payload for last will message.","is_advanced":true,"default":""}]},{"name":"user","type":"string","kind":"scalar","description":"A username to connect with.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to connect with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"keepalive","type":"int","kind":"scalar","description":"Max seconds of inactivity before a keepalive message is sent.","is_advanced":true,"default":30},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"topics","type":"string","kind":"array","description":"A list of topics to consume from."},{"name":"qos","type":"int","kind":"scalar","description":"The level of delivery guarantee to enforce. Has options 0, 1, 2.","is_advanced":true,"default":1},{"name":"clean_session","type":"bool","kind":"scalar","description":"Set whether the connection is non-persistent.","is_advanced":true,"default":true},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]}},{"name":"mysql_cdc","type":"input","status":"beta","plugin":true,"summary":"Enables MySQL streaming for RedPanda Connect.","description":"\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n- operation\n- table\n- binlog_position\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"flavor","type":"string","kind":"scalar","description":"The type of MySQL database to connect to.","default":"mysql","annotated_options":[["mariadb","MariaDB flavored databases."],["mysql","MySQL flavored databases."]],"linter":"\nlet options = {\n \"mariadb\": true,\n \"mysql\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"dsn","type":"string","kind":"scalar","description":"The DSN of the MySQL database to connect to.","examples":["user:password@tcp(localhost:3306)/database"]},{"name":"tables","type":"string","kind":"array","description":"A list of tables to stream from the database.","examples":[["table1","table2"]]},{"name":"checkpoint_cache","type":"string","kind":"scalar","description":"A https://www.docs.redpanda.com/redpanda-connect/components/caches/about[cache resource^] to use for storing the current latest BinLog Position that has been successfully delivered, this allows Redpanda Connect to continue from that BinLog Position upon restart, rather than consume the entire state of the table."},{"name":"checkpoint_key","type":"string","kind":"scalar","description":"The key to use to store the snapshot position in `checkpoint_cache`. An alternative key can be provided if multiple CDC inputs share the same cache.","default":"mysql_binlog_position"},{"name":"snapshot_max_batch_size","type":"int","kind":"scalar","description":"The maximum number of rows to be streamed in a single batch when taking a snapshot.","default":1000},{"name":"stream_snapshot","type":"bool","kind":"scalar","description":"If set to true, the connector will query all the existing data as a part of snapshot process. Otherwise, it will start from the current binlog position."},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"checkpoint_limit","type":"int","kind":"scalar","description":"The maximum number of messages that can be processed at a given time. Increasing this limit enables parallel processing and batching at the output level. Any given BinLog Position will not be acknowledged unless all messages under that offset are delivered in order to preserve at least once delivery guarantees.","default":1024},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]},"version":"4.45.0"},{"name":"nanomsg","type":"input","status":"stable","plugin":true,"summary":"Consumes messages via Nanomsg sockets (scalability protocols).","description":"Currently only PULL and SUB sockets are supported.","categories":["Network"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to (or as). If an item of the list contains commas it will be expanded into multiple URLs.","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"bind","type":"bool","kind":"scalar","description":"Whether the URLs provided should be connected to, or bound as.","default":true},{"name":"socket_type","type":"string","kind":"scalar","description":"The socket type to use.","default":"PULL","options":["PULL","SUB"],"linter":"\nlet options = {\n \"pull\": true,\n \"sub\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"sub_filters","type":"string","kind":"array","description":"A list of subscription topic filters to use when consuming from a SUB socket. Specifying a single sub_filter of `''` will subscribe to everything.","default":[]},{"name":"poll_timeout","type":"string","kind":"scalar","description":"The period to wait until a poll is abandoned and reattempted.","is_advanced":true,"default":"5s"}]}},{"name":"nats","type":"input","status":"stable","plugin":true,"summary":"Subscribe to a NATS subject.","description":"\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- nats_subject\n- nats_reply_subject\n- All message headers (when supported by the connection)\n```\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].\n\n== Connection name\n\nWhen monitoring and managing a production NATS system, it is often useful to\nknow which connection a message was send/received from. This can be achieved by\nsetting the connection name option when creating a NATS connection.\n\nRedpanda Connect will automatically set the connection name based off the label of the given\nNATS component, so that monitoring tools between NATS and Redpanda Connect can stay in sync.\n\n\n== Authentication\n\nThere are several components within Redpanda Connect which uses NATS services. You will find that each of these components\nsupport optional advanced authentication parameters for https://docs.nats.io/nats-server/configuration/securing_nats/auth_intro/nkey_auth[NKeys^]\nand https://docs.nats.io/using-nats/developer/connecting/creds[User Credentials^].\n\nSee an https://docs.nats.io/running-a-nats-service/nats_admin/security/jwt[in-depth tutorial^].\n\n=== NKey file\n\nThe NATS server can use these NKeys in several ways for authentication. The simplest is for the server to be configured\nwith a list of known public keys and for the clients to respond to the challenge by signing it with its private NKey\nconfigured in the `nkey_file` or `nkey` field.\n\nhttps://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[More details^].\n\n=== User credentials\n\nNATS server supports decentralized authentication based on JSON Web Tokens (JWT). Clients need an https://docs.nats.io/nats-server/configuration/securing_nats/jwt#json-web-tokens[user JWT^]\nand a corresponding https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[NKey secret^] when connecting to a server\nwhich is configured to use this authentication scheme.\n\nThe `user_credentials_file` field should point to a file containing both the private key and the JWT and can be\ngenerated with the https://docs.nats.io/nats-tools/nsc[nsc tool^].\n\nAlternatively, the `user_jwt` field can contain a plain text JWT and the `user_nkey_seed`can contain\nthe plain text NKey Seed.\n\nhttps://docs.nats.io/using-nats/developer/connecting/creds[More details^].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["nats://127.0.0.1:4222"],["nats://username:password@127.0.0.1:4222"]]},{"name":"subject","type":"string","kind":"scalar","description":"A subject to consume from. Supports wildcards for consuming multiple subjects. Either a subject or stream must be specified.","examples":["foo.bar.baz","foo.*.baz","foo.bar.*","foo.\u003e"]},{"name":"queue","type":"string","kind":"scalar","description":"An optional queue group to consume as.","is_optional":true},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"send_ack","type":"bool","kind":"scalar","description":"Control whether ACKS are sent as a reply to each message. When enabled, these replies are sent only once the data has been delivered to all outputs.","default":true},{"name":"nak_delay","type":"string","kind":"scalar","description":"An optional delay duration on redelivering a message when negatively acknowledged.","is_advanced":true,"is_optional":true,"examples":["1m"]},{"name":"prefetch_count","type":"int","kind":"scalar","description":"The maximum number of messages to pull at a time.","is_advanced":true,"default":524288,"linter":"root = if this \u003c 0 { [\"prefetch count must be greater than or equal to zero\"] }"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auth","type":"object","kind":"scalar","description":"Optional configuration of NATS authentication parameters.","is_advanced":true,"children":[{"name":"nkey_file","type":"string","kind":"scalar","description":"An optional file containing a NKey seed.","is_advanced":true,"is_optional":true,"examples":["./seed.nk"]},{"name":"nkey","type":"string","kind":"scalar","description":"The NKey seed.","is_advanced":true,"is_optional":true,"is_secret":true,"examples":["UDXU4RCSJNZOIQHZNWXHXORDPRTGNJAHAHFRGZNEEJCPQTT2M7NLCNF4"],"version":"4.38.0","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_credentials_file","type":"string","kind":"scalar","description":"An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.","is_advanced":true,"is_optional":true,"examples":["./user.creds"]},{"name":"user_jwt","type":"string","kind":"scalar","description":"An optional plain text user JWT (given along with the corresponding user NKey Seed).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_nkey_seed","type":"string","kind":"scalar","description":"An optional plain text user NKey Seed (given along with the corresponding user JWT).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"extract_tracing_map","type":"string","kind":"scalar","description":"EXPERIMENTAL: A xref:guides:bloblang/about.adoc[Bloblang mapping] that attempts to extract an object containing tracing propagation information, which will then be used as the root tracing span for the message. The specification of the extracted fields must match the format used by the service wide tracer.","is_advanced":true,"is_optional":true,"bloblang":true,"examples":["root = @","root = this.meta.span"],"version":"4.23.0"}]}},{"name":"nats_jetstream","type":"input","status":"stable","plugin":true,"summary":"Reads messages from NATS JetStream subjects.","description":"\n== Consume mirrored streams\n\nIn the case where a stream being consumed is mirrored from a different JetStream domain the stream cannot be resolved from the subject name alone, and so the stream name as well as the subject (if applicable) must both be specified.\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- nats_subject\n- nats_sequence_stream\n- nats_sequence_consumer\n- nats_num_delivered\n- nats_num_pending\n- nats_domain\n- nats_timestamp_unix_nano\n```\n\nYou can access these metadata fields using\nxref:configuration:interpolation.adoc#bloblang-queries[function interpolation].\n\n== Connection name\n\nWhen monitoring and managing a production NATS system, it is often useful to\nknow which connection a message was send/received from. This can be achieved by\nsetting the connection name option when creating a NATS connection.\n\nRedpanda Connect will automatically set the connection name based off the label of the given\nNATS component, so that monitoring tools between NATS and Redpanda Connect can stay in sync.\n\n\n== Authentication\n\nThere are several components within Redpanda Connect which uses NATS services. You will find that each of these components\nsupport optional advanced authentication parameters for https://docs.nats.io/nats-server/configuration/securing_nats/auth_intro/nkey_auth[NKeys^]\nand https://docs.nats.io/using-nats/developer/connecting/creds[User Credentials^].\n\nSee an https://docs.nats.io/running-a-nats-service/nats_admin/security/jwt[in-depth tutorial^].\n\n=== NKey file\n\nThe NATS server can use these NKeys in several ways for authentication. The simplest is for the server to be configured\nwith a list of known public keys and for the clients to respond to the challenge by signing it with its private NKey\nconfigured in the `nkey_file` or `nkey` field.\n\nhttps://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[More details^].\n\n=== User credentials\n\nNATS server supports decentralized authentication based on JSON Web Tokens (JWT). Clients need an https://docs.nats.io/nats-server/configuration/securing_nats/jwt#json-web-tokens[user JWT^]\nand a corresponding https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[NKey secret^] when connecting to a server\nwhich is configured to use this authentication scheme.\n\nThe `user_credentials_file` field should point to a file containing both the private key and the JWT and can be\ngenerated with the https://docs.nats.io/nats-tools/nsc[nsc tool^].\n\nAlternatively, the `user_jwt` field can contain a plain text JWT and the `user_nkey_seed`can contain\nthe plain text NKey Seed.\n\nhttps://docs.nats.io/using-nats/developer/connecting/creds[More details^].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["nats://127.0.0.1:4222"],["nats://username:password@127.0.0.1:4222"]]},{"name":"queue","type":"string","kind":"scalar","description":"An optional queue group to consume as. Used to configure a push consumer.","is_optional":true},{"name":"subject","type":"string","kind":"scalar","description":"A subject to consume from. Supports wildcards for consuming multiple subjects. Either a subject or stream must be specified.","is_optional":true,"examples":["foo.bar.baz","foo.*.baz","foo.bar.*","foo.\u003e"]},{"name":"durable","type":"string","kind":"scalar","description":"Preserve the state of your consumer under a durable name. Used to configure a pull consumer.","is_optional":true},{"name":"stream","type":"string","kind":"scalar","description":"A stream to consume from. Either a subject or stream must be specified.","is_optional":true},{"name":"bind","type":"bool","kind":"scalar","description":"Indicates that the subscription should use an existing consumer.","is_optional":true},{"name":"deliver","type":"string","kind":"scalar","description":"Determines which messages to deliver when consuming without a durable subscriber.","default":"all","annotated_options":[["all","Deliver all available messages."],["last","Deliver starting with the last published messages."],["last_per_subject","Deliver starting with the last published message per subject."],["new","Deliver starting from now, not taking into account any previous messages."]],"linter":"\nlet options = {\n \"all\": true,\n \"last\": true,\n \"last_per_subject\": true,\n \"new\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"ack_wait","type":"string","kind":"scalar","description":"The maximum amount of time NATS server should wait for an ack from consumer.","is_advanced":true,"default":"30s","examples":["100ms","5m"]},{"name":"max_ack_pending","type":"int","kind":"scalar","description":"The maximum number of outstanding acks to be allowed before consuming is halted.","is_advanced":true,"default":1024},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auth","type":"object","kind":"scalar","description":"Optional configuration of NATS authentication parameters.","is_advanced":true,"children":[{"name":"nkey_file","type":"string","kind":"scalar","description":"An optional file containing a NKey seed.","is_advanced":true,"is_optional":true,"examples":["./seed.nk"]},{"name":"nkey","type":"string","kind":"scalar","description":"The NKey seed.","is_advanced":true,"is_optional":true,"is_secret":true,"examples":["UDXU4RCSJNZOIQHZNWXHXORDPRTGNJAHAHFRGZNEEJCPQTT2M7NLCNF4"],"version":"4.38.0","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_credentials_file","type":"string","kind":"scalar","description":"An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.","is_advanced":true,"is_optional":true,"examples":["./user.creds"]},{"name":"user_jwt","type":"string","kind":"scalar","description":"An optional plain text user JWT (given along with the corresponding user NKey Seed).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_nkey_seed","type":"string","kind":"scalar","description":"An optional plain text user NKey Seed (given along with the corresponding user JWT).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"extract_tracing_map","type":"string","kind":"scalar","description":"EXPERIMENTAL: A xref:guides:bloblang/about.adoc[Bloblang mapping] that attempts to extract an object containing tracing propagation information, which will then be used as the root tracing span for the message. The specification of the extracted fields must match the format used by the service wide tracer.","is_advanced":true,"is_optional":true,"bloblang":true,"examples":["root = @","root = this.meta.span"],"version":"4.23.0"}],"linter":"root = match {\n\t\t\tthis.exists(\"queue\") \u0026\u0026 this.queue != \"\" \u0026\u0026 this.exists(\"durable\") \u0026\u0026 this.durable != \"\" =\u003e [ \"both 'queue' and 'durable' can't be set simultaneously\" ],\n\t\t\t}"},"version":"3.46.0"},{"name":"nats_kv","type":"input","status":"beta","plugin":true,"summary":"Watches for updates in a NATS key-value bucket.","description":"\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n``` text\n- nats_kv_key\n- nats_kv_bucket\n- nats_kv_revision\n- nats_kv_delta\n- nats_kv_operation\n- nats_kv_created\n```\n\n== Connection name\n\nWhen monitoring and managing a production NATS system, it is often useful to\nknow which connection a message was send/received from. This can be achieved by\nsetting the connection name option when creating a NATS connection.\n\nRedpanda Connect will automatically set the connection name based off the label of the given\nNATS component, so that monitoring tools between NATS and Redpanda Connect can stay in sync.\n\n\n== Authentication\n\nThere are several components within Redpanda Connect which uses NATS services. You will find that each of these components\nsupport optional advanced authentication parameters for https://docs.nats.io/nats-server/configuration/securing_nats/auth_intro/nkey_auth[NKeys^]\nand https://docs.nats.io/using-nats/developer/connecting/creds[User Credentials^].\n\nSee an https://docs.nats.io/running-a-nats-service/nats_admin/security/jwt[in-depth tutorial^].\n\n=== NKey file\n\nThe NATS server can use these NKeys in several ways for authentication. The simplest is for the server to be configured\nwith a list of known public keys and for the clients to respond to the challenge by signing it with its private NKey\nconfigured in the `nkey_file` or `nkey` field.\n\nhttps://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[More details^].\n\n=== User credentials\n\nNATS server supports decentralized authentication based on JSON Web Tokens (JWT). Clients need an https://docs.nats.io/nats-server/configuration/securing_nats/jwt#json-web-tokens[user JWT^]\nand a corresponding https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[NKey secret^] when connecting to a server\nwhich is configured to use this authentication scheme.\n\nThe `user_credentials_file` field should point to a file containing both the private key and the JWT and can be\ngenerated with the https://docs.nats.io/nats-tools/nsc[nsc tool^].\n\nAlternatively, the `user_jwt` field can contain a plain text JWT and the `user_nkey_seed`can contain\nthe plain text NKey Seed.\n\nhttps://docs.nats.io/using-nats/developer/connecting/creds[More details^].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["nats://127.0.0.1:4222"],["nats://username:password@127.0.0.1:4222"]]},{"name":"bucket","type":"string","kind":"scalar","description":"The name of the KV bucket.","examples":["my_kv_bucket"]},{"name":"key","type":"string","kind":"scalar","description":"Key to watch for updates, can include wildcards.","default":"\u003e","examples":["foo.bar.baz","foo.*.baz","foo.bar.*","foo.\u003e"]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"ignore_deletes","type":"bool","kind":"scalar","description":"Do not send delete markers as messages.","is_advanced":true,"default":false},{"name":"include_history","type":"bool","kind":"scalar","description":"Include all the history per key, not just the last one.","is_advanced":true,"default":false},{"name":"meta_only","type":"bool","kind":"scalar","description":"Retrieve only the metadata of the entry","is_advanced":true,"default":false},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auth","type":"object","kind":"scalar","description":"Optional configuration of NATS authentication parameters.","is_advanced":true,"children":[{"name":"nkey_file","type":"string","kind":"scalar","description":"An optional file containing a NKey seed.","is_advanced":true,"is_optional":true,"examples":["./seed.nk"]},{"name":"nkey","type":"string","kind":"scalar","description":"The NKey seed.","is_advanced":true,"is_optional":true,"is_secret":true,"examples":["UDXU4RCSJNZOIQHZNWXHXORDPRTGNJAHAHFRGZNEEJCPQTT2M7NLCNF4"],"version":"4.38.0","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_credentials_file","type":"string","kind":"scalar","description":"An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.","is_advanced":true,"is_optional":true,"examples":["./user.creds"]},{"name":"user_jwt","type":"string","kind":"scalar","description":"An optional plain text user JWT (given along with the corresponding user NKey Seed).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_nkey_seed","type":"string","kind":"scalar","description":"An optional plain text user NKey Seed (given along with the corresponding user JWT).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},"version":"4.12.0"},{"name":"nats_stream","type":"input","status":"stable","plugin":true,"summary":"Subscribe to a NATS Stream subject. Joining a queue is optional and allows multiple clients of a subject to consume using queue semantics.","description":"\n[CAUTION]\n.Deprecation notice\n====\nThe NATS Streaming Server is being deprecated. Critical bug fixes and security fixes will be applied until June of 2023. NATS-enabled applications requiring persistence should use https://docs.nats.io/nats-concepts/jetstream[JetStream^].\n====\n\nTracking and persisting offsets through a durable name is also optional and works with or without a queue. If a durable name is not provided then subjects are consumed from the most recently published message.\n\nWhen a consumer closes its connection it unsubscribes, when all consumers of a durable queue do this the offsets are deleted. In order to avoid this you can stop the consumers from unsubscribing by setting the field `unsubscribe_on_close` to `false`.\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n- nats_stream_subject\n- nats_stream_sequence\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].\n\n\n\n== Authentication\n\nThere are several components within Redpanda Connect which uses NATS services. You will find that each of these components\nsupport optional advanced authentication parameters for https://docs.nats.io/nats-server/configuration/securing_nats/auth_intro/nkey_auth[NKeys^]\nand https://docs.nats.io/using-nats/developer/connecting/creds[User Credentials^].\n\nSee an https://docs.nats.io/running-a-nats-service/nats_admin/security/jwt[in-depth tutorial^].\n\n=== NKey file\n\nThe NATS server can use these NKeys in several ways for authentication. The simplest is for the server to be configured\nwith a list of known public keys and for the clients to respond to the challenge by signing it with its private NKey\nconfigured in the `nkey_file` or `nkey` field.\n\nhttps://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[More details^].\n\n=== User credentials\n\nNATS server supports decentralized authentication based on JSON Web Tokens (JWT). Clients need an https://docs.nats.io/nats-server/configuration/securing_nats/jwt#json-web-tokens[user JWT^]\nand a corresponding https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[NKey secret^] when connecting to a server\nwhich is configured to use this authentication scheme.\n\nThe `user_credentials_file` field should point to a file containing both the private key and the JWT and can be\ngenerated with the https://docs.nats.io/nats-tools/nsc[nsc tool^].\n\nAlternatively, the `user_jwt` field can contain a plain text JWT and the `user_nkey_seed`can contain\nthe plain text NKey Seed.\n\nhttps://docs.nats.io/using-nats/developer/connecting/creds[More details^].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["nats://127.0.0.1:4222"],["nats://username:password@127.0.0.1:4222"]]},{"name":"cluster_id","type":"string","kind":"scalar","description":"The ID of the cluster to consume from."},{"name":"client_id","type":"string","kind":"scalar","description":"A client ID to connect as.","default":""},{"name":"queue","type":"string","kind":"scalar","description":"The queue to consume from.","default":""},{"name":"subject","type":"string","kind":"scalar","description":"A subject to consume from.","default":""},{"name":"durable_name","type":"string","kind":"scalar","description":"Preserve the state of your consumer under a durable name.","default":""},{"name":"unsubscribe_on_close","type":"bool","kind":"scalar","description":"Whether the subscription should be destroyed when this client disconnects.","default":false},{"name":"start_from_oldest","type":"bool","kind":"scalar","description":"If a position is not found for a queue, determines whether to consume from the oldest available message, otherwise messages are consumed from the latest.","is_advanced":true,"default":true},{"name":"max_inflight","type":"int","kind":"scalar","description":"The maximum number of unprocessed messages to fetch at a given time.","is_advanced":true,"default":1024},{"name":"ack_wait","type":"string","kind":"scalar","description":"An optional duration to specify at which a message that is yet to be acked will be automatically retried.","is_advanced":true,"default":"30s"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auth","type":"object","kind":"scalar","description":"Optional configuration of NATS authentication parameters.","is_advanced":true,"children":[{"name":"nkey_file","type":"string","kind":"scalar","description":"An optional file containing a NKey seed.","is_advanced":true,"is_optional":true,"examples":["./seed.nk"]},{"name":"nkey","type":"string","kind":"scalar","description":"The NKey seed.","is_advanced":true,"is_optional":true,"is_secret":true,"examples":["UDXU4RCSJNZOIQHZNWXHXORDPRTGNJAHAHFRGZNEEJCPQTT2M7NLCNF4"],"version":"4.38.0","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_credentials_file","type":"string","kind":"scalar","description":"An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.","is_advanced":true,"is_optional":true,"examples":["./user.creds"]},{"name":"user_jwt","type":"string","kind":"scalar","description":"An optional plain text user JWT (given along with the corresponding user NKey Seed).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_nkey_seed","type":"string","kind":"scalar","description":"An optional plain text user NKey Seed (given along with the corresponding user JWT).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"extract_tracing_map","type":"string","kind":"scalar","description":"EXPERIMENTAL: A xref:guides:bloblang/about.adoc[Bloblang mapping] that attempts to extract an object containing tracing propagation information, which will then be used as the root tracing span for the message. The specification of the extracted fields must match the format used by the service wide tracer.","is_advanced":true,"is_optional":true,"bloblang":true,"examples":["root = @","root = this.meta.span"],"version":"4.23.0"}]}},{"name":"nsq","type":"input","status":"stable","plugin":true,"summary":"Subscribe to an NSQ instance topic and channel.","description":"\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n- nsq_attempts\n- nsq_id\n- nsq_nsqd_address\n- nsq_timestamp\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"nsqd_tcp_addresses","type":"string","kind":"array","description":"A list of nsqd addresses to connect to."},{"name":"lookupd_http_addresses","type":"string","kind":"array","description":"A list of nsqlookupd addresses to connect to."},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"topic","type":"string","kind":"scalar","description":"The topic to consume from."},{"name":"channel","type":"string","kind":"scalar","description":"The channel to consume from."},{"name":"user_agent","type":"string","kind":"scalar","description":"A user agent to assume when connecting.","is_optional":true},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of pending messages to consume at any given time.","default":100},{"name":"max_attempts","type":"int","kind":"scalar","description":"The maximum number of attempts to successfully consume a messages.","default":5}]}},{"name":"ockam_kafka","type":"input","status":"experimental","plugin":true,"summary":"Ockam","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"kafka","type":"object","kind":"scalar","children":[{"name":"seed_brokers","type":"string","kind":"array","description":"A list of broker addresses to connect to in order to establish connections. If an item of the list contains commas it will be expanded into multiple addresses.","is_optional":true,"examples":[["localhost:9092"],["foo:9092","bar:9092"],["foo:9092,bar:9092"]]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"topics","type":"string","kind":"array","description":"\nA list of topics to consume from. Multiple comma separated topics can be listed in a single element. When a `consumer_group` is specified partitions are automatically distributed across consumers of a topic, otherwise all partitions are consumed.\n\nAlternatively, it's possible to specify explicit partitions to consume from with a colon after the topic name, e.g. `foo:0` would consume the partition 0 of the topic foo. This syntax supports ranges, e.g. `foo:0-10` would consume partitions 0 through to 10 inclusive.\n\nFinally, it's also possible to specify an explicit offset to consume from by adding another colon after the partition, e.g. `foo:0:10` would consume the partition 0 of the topic foo starting from the offset 10. If the offset is not present (or remains unspecified) then the field `start_from_oldest` determines which offset to start from.","examples":[["foo","bar"],["things.*"],["foo,bar"],["foo:0","bar:1","bar:3"],["foo:0,bar:1,bar:3"],["foo:0-5"]]},{"name":"regexp_topics","type":"bool","kind":"scalar","description":"Whether listed topics should be interpreted as regular expression patterns for matching multiple topics. When topics are specified with explicit partitions this field must remain set to `false`.","default":false},{"name":"rack_id","type":"string","kind":"scalar","description":"A rack specifies where the client is physically located and changes fetch requests to consume from the closest replica as opposed to the leader replica.","is_advanced":true,"default":""},{"name":"instance_id","type":"string","kind":"scalar","description":"When using a consumer group, an instance ID specifies the groups static membership, which can prevent rebalances during reconnects. When using a instance ID the client does NOT leave the group when closing. To actually leave the group one must use an external admin command to leave the group on behalf of this instance ID. This ID must be unique per consumer within the group.","is_advanced":true,"default":""},{"name":"rebalance_timeout","type":"string","kind":"scalar","description":"When using a consumer group, `rebalance_timeout` sets how long group members are allowed to take when a rebalance has begun. This timeout is how long all members are allowed to complete work and commit offsets, minus the time it took to detect the rebalance (from a heartbeat).","is_advanced":true,"default":"45s"},{"name":"session_timeout","type":"string","kind":"scalar","description":"When using a consumer group, `session_timeout` sets how long a member in hte group can go between heartbeats. If a member does not heartbeat in this timeout, the broker will remove the member from the group and initiate a rebalance.","is_advanced":true,"default":"1m"},{"name":"heartbeat_interval","type":"string","kind":"scalar","description":"When using a consumer group, `heartbeat_interval` sets how long a group member goes between heartbeats to Kafka. Kafka uses heartbeats to ensure that a group member's sesion stays active. This value should be no higher than 1/3rd of the `session_timeout`. This is equivalent to the Java heartbeat.interval.ms setting.","is_advanced":true,"default":"3s"},{"name":"start_from_oldest","type":"bool","kind":"scalar","description":"Determines whether to consume from the oldest available offset, otherwise messages are consumed from the latest offset. The setting is applied when creating a new consumer group or the saved offset no longer exists.","is_advanced":true,"is_deprecated":true,"default":true},{"name":"start_offset","type":"string","kind":"scalar","description":"Sets the offset to start consuming from, or if OffsetOutOfRange is seen while fetching, to restart consuming from.","is_advanced":true,"default":"earliest","annotated_options":[["committed","Prevents consuming a partition in a group if the partition has no prior commits. Corresponds to Kafka's `auto.offset.reset=none` option"],["earliest","Start from the earliest offset. Corresponds to Kafka's `auto.offset.reset=earliest` option."],["latest","Start from the latest offset. Corresponds to Kafka's `auto.offset.reset=latest` option."]],"linter":"\nlet options = {\n \"committed\": true,\n \"earliest\": true,\n \"latest\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"fetch_max_bytes","type":"string","kind":"scalar","description":"Sets the maximum amount of bytes a broker will try to send during a fetch. Note that brokers may not obey this limit if it has records larger than this limit. This is the equivalent to the Java fetch.max.bytes setting.","is_advanced":true,"default":"50MiB"},{"name":"fetch_max_wait","type":"string","kind":"scalar","description":"Sets the maximum amount of time a broker will wait for a fetch response to hit the minimum number of required bytes. This is the equivalent to the Java fetch.max.wait.ms setting.","is_advanced":true,"default":"5s"},{"name":"fetch_min_bytes","type":"string","kind":"scalar","description":"Sets the minimum amount of bytes a broker will try to send during a fetch. This is the equivalent to the Java fetch.min.bytes setting.","is_advanced":true,"default":"1B"},{"name":"fetch_max_partition_bytes","type":"string","kind":"scalar","description":"Sets the maximum amount of bytes that will be consumed for a single partition in a fetch request. Note that if a single batch is larger than this number, that batch will still be returned so the client can make progress. This is the equivalent to the Java fetch.max.partition.bytes setting.","is_advanced":true,"default":"1MiB"},{"name":"transaction_isolation_level","type":"string","kind":"scalar","description":"The transaction isolation level","default":"read_uncommitted","annotated_options":[["read_committed","If set, only committed transactional records are processed."],["read_uncommitted","If set, then uncommitted records are processed."]],"linter":"\nlet options = {\n \"read_committed\": true,\n \"read_uncommitted\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"consumer_group","type":"string","kind":"scalar","description":"An optional consumer group to consume as. When specified the partitions of specified topics are automatically distributed across consumers sharing a consumer group, and partition offsets are automatically committed and resumed under this name. Consumer groups are not supported when specifying explicit partitions to consume from in the `topics` field.","is_optional":true},{"name":"checkpoint_limit","type":"int","kind":"scalar","description":"Determines how many messages of the same partition can be processed in parallel before applying back pressure. When a message of a given offset is delivered to the output the offset is only allowed to be committed when all messages of prior offsets have also been delivered, this ensures at-least-once delivery guarantees. However, this mechanism also increases the likelihood of duplicates in the event of crashes or server faults, reducing the checkpoint limit will mitigate this.","is_advanced":true,"default":1024},{"name":"commit_period","type":"string","kind":"scalar","description":"The period of time between each commit of the current partition offsets. Offsets are always committed during shutdown.","is_advanced":true,"default":"5s"},{"name":"multi_header","type":"bool","kind":"scalar","description":"Decode headers into lists to allow handling of multiple values with the same key","is_advanced":true,"default":false},{"name":"batching","type":"object","kind":"","description":"Allows you to configure a xref:configuration:batching.adoc[batching policy] that applies to individual topic partitions in order to batch messages together before flushing them for processing. Batching can be beneficial for performance as well as useful for windowed processing, and doing so this way preserves the ordering of topic partitions.","is_advanced":true,"examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","is_advanced":true,"default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","is_advanced":true,"default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","is_advanced":true,"default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","is_advanced":true,"default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}],"linter":"\nlet has_topic_partitions = this.topics.any(t -\u003e t.contains(\":\"))\n\nroot = [\n if $has_topic_partitions {\n if this.consumer_group.or(\"\") != \"\" {\n \"this input does not support both a consumer group and explicit topic partitions\"\n } else if this.regexp_topics {\n \"this input does not support both regular expression topics and explicit topic partitions\"\n }\n } else {\n if this.consumer_group.or(\"\") == \"\" {\n \"a consumer group is mandatory when not using explicit topic partitions\"\n }\n },\n # We don't have any way to distinguish between start_from_oldest set explicitly to true and not set at all, so we\n # assume users will be OK if start_offset overwrites it silently\n if this.start_from_oldest == false \u0026\u0026 this.start_offset == \"earliest\" {\n \"start_from_oldest cannot be set to false when start_offset is set to earliest\"\n }\n]\n"},{"name":"disable_content_encryption","type":"bool","kind":"scalar","default":false},{"name":"enrollment_ticket","type":"string","kind":"scalar","is_optional":true},{"name":"identity_name","type":"string","kind":"scalar","is_optional":true},{"name":"allow","type":"string","kind":"scalar","default":"self"},{"name":"route_to_kafka_outlet","type":"string","kind":"scalar","default":"self"},{"name":"allow_producer","type":"string","kind":"scalar","default":"self"},{"name":"relay","type":"string","kind":"scalar","is_optional":true},{"name":"node_address","type":"string","kind":"scalar","default":"127.0.0.1:6262"},{"name":"encrypted_fields","type":"string","kind":"array","description":"The fields to encrypt in the kafka messages, assuming the record is a valid JSON map. By default, the whole record is encrypted.","default":[]}]}},{"name":"parquet","type":"input","status":"experimental","plugin":true,"summary":"Reads and decodes https://parquet.apache.org/docs/[Parquet files^] into a stream of structured messages.","description":"\nThis input uses https://github.com/parquet-go/parquet-go[https://github.com/parquet-go/parquet-go^], which is itself experimental. Therefore changes could be made into how this processor functions outside of major version releases.\n\nBy default any BYTE_ARRAY or FIXED_LEN_BYTE_ARRAY value will be extracted as a byte slice (`[]byte`) unless the logical type is UTF8, in which case they are extracted as a string (`string`).\n\nWhen a value extracted as a byte slice exists within a document which is later JSON serialized by default it will be base 64 encoded into strings, which is the default for arbitrary data fields. It is possible to convert these binary values to strings (or other data types) using Bloblang transformations such as `root.foo = this.foo.string()` or `root.foo = this.foo.encode(\"hex\")`, etc.","categories":["Local"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"paths","type":"string","kind":"array","description":"A list of file paths to read from. Each file will be read sequentially until the list is exhausted, at which point the input will close. Glob patterns are supported, including super globs (double star).","examples":["/tmp/foo.parquet","/tmp/bar/*.parquet","/tmp/data/**/*.parquet"]},{"name":"batch_count","type":"int","kind":"scalar","description":"Optionally process records in batches. This can help to speed up the consumption of exceptionally large files. When the end of the file is reached the remaining records are processed as a (potentially smaller) batch.","is_advanced":true,"default":1},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]},"version":"4.8.0"},{"name":"pg_stream","type":"input","status":"deprecated","plugin":true,"summary":"Streams changes from a PostgreSQL database using logical replication.","description":"Streams changes from a PostgreSQL database for Change Data Capture (CDC).\nAdditionally, if `stream_snapshot` is set to true, then the existing data in the database is also streamed too.\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n- table (Name of the table that the message originated from)\n- operation (Type of operation that generated the message: \"read\", \"insert\", \"update\", or \"delete\". \"read\" is from messages that are read in the initial snapshot phase. This will also be \"begin\" and \"commit\" if `include_transaction_markers` is enabled)\n- lsn (the log sequence number in postgres)\n\t\t","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"dsn","type":"string","kind":"scalar","description":"The Data Source Name for the PostgreSQL database in the form of `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]`. Please note that Postgres enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.","examples":["postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable"]},{"name":"include_transaction_markers","type":"bool","kind":"scalar","description":"When set to true, empty messages with operation types BEGIN and COMMIT are generated for the beginning and end of each transaction. Messages with operation metadata set to \"begin\" or \"commit\" will have null message payloads.","default":false},{"name":"stream_snapshot","type":"bool","kind":"scalar","description":"When set to true, the plugin will first stream a snapshot of all existing data in the database before streaming changes. In order to use this the tables that are being snapshot MUST have a primary key set so that reading from the table can be parallelized.","default":false,"examples":[true]},{"name":"snapshot_memory_safety_factor","type":"float","kind":"scalar","description":"Determines the fraction of available memory that can be used for streaming the snapshot. Values between 0 and 1 represent the percentage of memory to use. Lower values make initial streaming slower but help prevent out-of-memory errors.","is_deprecated":true,"default":1,"examples":[0.2]},{"name":"snapshot_batch_size","type":"int","kind":"scalar","description":"The number of rows to fetch in each batch when querying the snapshot.","default":1000,"examples":[10000]},{"name":"schema","type":"string","kind":"scalar","description":"The PostgreSQL schema from which to replicate data.","examples":["public","\"MyCaseSensitiveSchemaNeedingQuotes\""]},{"name":"tables","type":"string","kind":"array","description":"A list of table names to include in the logical replication. Each table should be specified as a separate item.","examples":[["my_table_1","\"MyCaseSensitiveTableNeedingQuotes\""]]},{"name":"checkpoint_limit","type":"int","kind":"scalar","description":"The maximum number of messages that can be processed at a given time. Increasing this limit enables parallel processing and batching at the output level. Any given LSN will not be acknowledged unless all messages under that offset are delivered in order to preserve at least once delivery guarantees.","default":1024},{"name":"temporary_slot","type":"bool","kind":"scalar","description":"If set to true, creates a temporary replication slot that is automatically dropped when the connection is closed.","default":false},{"name":"slot_name","type":"string","kind":"scalar","description":"The name of the PostgreSQL logical replication slot to use. If not provided, a random name will be generated. You can create this slot manually before starting replication if desired.","examples":["my_test_slot"]},{"name":"pg_standby_timeout","type":"string","kind":"scalar","description":"Specify the standby timeout before refreshing an idle connection.","default":"10s","examples":["30s"]},{"name":"pg_wal_monitor_interval","type":"string","kind":"scalar","description":"How often to report changes to the replication lag.","default":"3s","examples":["6s"]},{"name":"max_parallel_snapshot_tables","type":"int","kind":"scalar","description":"Int specifies a number of tables that will be processed in parallel during the snapshot processing stage","default":1},{"name":"unchanged_toast_value","type":"unknown","kind":"scalar","description":"The value to emit when there are unchanged TOAST values in the stream. This occurs for updates and deletes where REPLICA IDENTITY is not FULL.","is_advanced":true,"default":null,"examples":["__redpanda_connect_unchanged_toast_value__"]},{"name":"heartbeat_interval","type":"string","kind":"scalar","description":"The interval at which to write heartbeat messages. Heartbeat messages are needed in scenarios when the subscribed tables are low frequency, but there are other high frequency tables writing. Due to the checkpointing mechanism for replication slots, not having new messages to acknowledge will prevent postgres from reclaiming the write ahead log, which can exhaust the local disk. Having heartbeats allows Redpanda Connect to safely acknowledge data periodically and move forward the committed point in the log so it can be reclaimed. Setting the duration to 0s will disable heartbeats entirely. Heartbeats are created by periodically writing logical messages to the write ahead log using `pg_logical_emit_message`.","is_advanced":true,"default":"1h","examples":["0s","24h"]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]},"version":"4.39.0"},{"name":"postgres_cdc","type":"input","status":"beta","plugin":true,"summary":"Streams changes from a PostgreSQL database using logical replication.","description":"Streams changes from a PostgreSQL database for Change Data Capture (CDC).\nAdditionally, if `stream_snapshot` is set to true, then the existing data in the database is also streamed too.\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n- table (Name of the table that the message originated from)\n- operation (Type of operation that generated the message: \"read\", \"insert\", \"update\", or \"delete\". \"read\" is from messages that are read in the initial snapshot phase. This will also be \"begin\" and \"commit\" if `include_transaction_markers` is enabled)\n- lsn (the log sequence number in postgres)\n\t\t","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"dsn","type":"string","kind":"scalar","description":"The Data Source Name for the PostgreSQL database in the form of `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]`. Please note that Postgres enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.","examples":["postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable"]},{"name":"include_transaction_markers","type":"bool","kind":"scalar","description":"When set to true, empty messages with operation types BEGIN and COMMIT are generated for the beginning and end of each transaction. Messages with operation metadata set to \"begin\" or \"commit\" will have null message payloads.","default":false},{"name":"stream_snapshot","type":"bool","kind":"scalar","description":"When set to true, the plugin will first stream a snapshot of all existing data in the database before streaming changes. In order to use this the tables that are being snapshot MUST have a primary key set so that reading from the table can be parallelized.","default":false,"examples":[true]},{"name":"snapshot_memory_safety_factor","type":"float","kind":"scalar","description":"Determines the fraction of available memory that can be used for streaming the snapshot. Values between 0 and 1 represent the percentage of memory to use. Lower values make initial streaming slower but help prevent out-of-memory errors.","is_deprecated":true,"default":1,"examples":[0.2]},{"name":"snapshot_batch_size","type":"int","kind":"scalar","description":"The number of rows to fetch in each batch when querying the snapshot.","default":1000,"examples":[10000]},{"name":"schema","type":"string","kind":"scalar","description":"The PostgreSQL schema from which to replicate data.","examples":["public","\"MyCaseSensitiveSchemaNeedingQuotes\""]},{"name":"tables","type":"string","kind":"array","description":"A list of table names to include in the logical replication. Each table should be specified as a separate item.","examples":[["my_table_1","\"MyCaseSensitiveTableNeedingQuotes\""]]},{"name":"checkpoint_limit","type":"int","kind":"scalar","description":"The maximum number of messages that can be processed at a given time. Increasing this limit enables parallel processing and batching at the output level. Any given LSN will not be acknowledged unless all messages under that offset are delivered in order to preserve at least once delivery guarantees.","default":1024},{"name":"temporary_slot","type":"bool","kind":"scalar","description":"If set to true, creates a temporary replication slot that is automatically dropped when the connection is closed.","default":false},{"name":"slot_name","type":"string","kind":"scalar","description":"The name of the PostgreSQL logical replication slot to use. If not provided, a random name will be generated. You can create this slot manually before starting replication if desired.","examples":["my_test_slot"]},{"name":"pg_standby_timeout","type":"string","kind":"scalar","description":"Specify the standby timeout before refreshing an idle connection.","default":"10s","examples":["30s"]},{"name":"pg_wal_monitor_interval","type":"string","kind":"scalar","description":"How often to report changes to the replication lag.","default":"3s","examples":["6s"]},{"name":"max_parallel_snapshot_tables","type":"int","kind":"scalar","description":"Int specifies a number of tables that will be processed in parallel during the snapshot processing stage","default":1},{"name":"unchanged_toast_value","type":"unknown","kind":"scalar","description":"The value to emit when there are unchanged TOAST values in the stream. This occurs for updates and deletes where REPLICA IDENTITY is not FULL.","is_advanced":true,"default":null,"examples":["__redpanda_connect_unchanged_toast_value__"]},{"name":"heartbeat_interval","type":"string","kind":"scalar","description":"The interval at which to write heartbeat messages. Heartbeat messages are needed in scenarios when the subscribed tables are low frequency, but there are other high frequency tables writing. Due to the checkpointing mechanism for replication slots, not having new messages to acknowledge will prevent postgres from reclaiming the write ahead log, which can exhaust the local disk. Having heartbeats allows Redpanda Connect to safely acknowledge data periodically and move forward the committed point in the log so it can be reclaimed. Setting the duration to 0s will disable heartbeats entirely. Heartbeats are created by periodically writing logical messages to the write ahead log using `pg_logical_emit_message`.","is_advanced":true,"default":"1h","examples":["0s","24h"]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]},"version":"4.39.0"},{"name":"pulsar","type":"input","status":"experimental","plugin":true,"summary":"Reads messages from an Apache Pulsar server.","description":"\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- pulsar_message_id\n- pulsar_key\n- pulsar_ordering_key\n- pulsar_event_time_unix\n- pulsar_publish_time_unix\n- pulsar_topic\n- pulsar_producer_name\n- pulsar_redelivery_count\n- All properties of the message\n```\n\nYou can access these metadata fields using\nxref:configuration:interpolation.adoc#bloblang-queries[function interpolation].\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"A URL to connect to.","examples":["pulsar://localhost:6650","pulsar://pulsar.us-west.example.com:6650","pulsar+ssl://pulsar.us-west.example.com:6651"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"topics","type":"string","kind":"array","description":"A list of topics to subscribe to. This or topics_pattern must be set.","is_optional":true},{"name":"topics_pattern","type":"string","kind":"scalar","description":"A regular expression matching the topics to subscribe to. This or topics must be set.","is_optional":true},{"name":"subscription_name","type":"string","kind":"scalar","description":"Specify the subscription name for this consumer."},{"name":"subscription_type","type":"string","kind":"scalar","description":"Specify the subscription type for this consumer.\n\n\u003e NOTE: Using a `key_shared` subscription type will __allow out-of-order delivery__ since nack-ing messages sets non-zero nack delivery delay - this can potentially cause consumers to stall. See https://pulsar.apache.org/docs/en/2.8.1/concepts-messaging/#negative-acknowledgement[Pulsar documentation^] and https://github.com/apache/pulsar/issues/12208[this Github issue^] for more details.","default":"shared","options":["shared","key_shared","failover","exclusive"],"linter":"\nlet options = {\n \"shared\": true,\n \"key_shared\": true,\n \"failover\": true,\n \"exclusive\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"subscription_initial_position","type":"string","kind":"scalar","description":"Specify the subscription initial position for this consumer.","default":"latest","options":["latest","earliest"],"linter":"\nlet options = {\n \"latest\": true,\n \"earliest\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"tls","type":"object","kind":"scalar","description":"Specify the path to a custom CA certificate to trust broker TLS service.","children":[{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","default":"","examples":["./root_cas.pem"]}]},{"name":"auth","type":"object","kind":"scalar","description":"Optional configuration of Pulsar authentication methods.","is_advanced":true,"is_optional":true,"children":[{"name":"oauth2","type":"object","kind":"scalar","description":"Parameters for Pulsar OAuth2 authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether OAuth2 is enabled.","is_advanced":true,"default":false},{"name":"audience","type":"string","kind":"scalar","description":"OAuth2 audience.","is_advanced":true,"default":""},{"name":"issuer_url","type":"string","kind":"scalar","description":"OAuth2 issuer URL.","is_advanced":true,"default":"","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"scope","type":"string","kind":"scalar","description":"OAuth2 scope to request.","is_advanced":true,"default":"","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"private_key_file","type":"string","kind":"scalar","description":"The path to a file containing a private key.","is_advanced":true,"default":""}]},{"name":"token","type":"object","kind":"scalar","description":"Parameters for Pulsar Token authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether Token Auth is enabled.","is_advanced":true,"default":false},{"name":"token","type":"string","kind":"scalar","description":"Actual base64 encoded token.","is_advanced":true,"default":""}]}],"version":"3.60.0"}]},"version":"3.43.0"},{"name":"read_until","type":"input","status":"stable","plugin":true,"summary":"Reads messages from a child input until a consumed message passes a xref:guides:bloblang/about.adoc[Bloblang query], at which point the input closes. It is also possible to configure a timeout after which the input is closed if no new messages arrive in that period.","description":"\nMessages are read continuously while the query check returns false, when the query returns true the message that triggered the check is sent out and the input is closed. Use this to define inputs where the stream should end once a certain message appears.\n\nIf the idle timeout is configured, the input will be closed if no new messages arrive after that period of time. Use this field if you want to empty out and close an input that doesn't have a logical end.\n\nSometimes inputs close themselves. For example, when the `file` input type reaches the end of a file it will shut down. By default this type will also shut down. If you wish for the input type to be restarted every time it shuts down until the query check is met then set `restart_input` to `true`.\n\n== Metadata\n\nA metadata key `benthos_read_until` containing the value `final` is added to the first part of the message that triggers the input to stop.","categories":["Utility"],"examples":[{"title":"Consume N Messages","summary":"A common reason to use this input is to consume only N messages from an input and then stop. This can easily be done with the xref:guides:bloblang/functions.adoc#count[`count` function]:","config":"\n# Only read 100 messages, and then exit.\ninput:\n read_until:\n check: count(\"messages\") \u003e= 100\n input:\n kafka:\n addresses: [ TODO ]\n topics: [ foo, bar ]\n consumer_group: foogroup\n"},{"title":"Read from a kafka and close when empty","summary":"A common reason to use this input is a job that consumes all messages and exits once its empty:","config":"\n# Consumes all messages and exit when the last message was consumed 5s ago.\ninput:\n read_until:\n idle_timeout: 5s\n input:\n kafka:\n addresses: [ TODO ]\n topics: [ foo, bar ]\n consumer_group: foogroup\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"input","type":"input","kind":"scalar","description":"The child input to consume from."},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether the input should now be closed.","is_optional":true,"bloblang":true,"examples":["this.type == \"foo\"","count(\"messages\") \u003e= 100"]},{"name":"idle_timeout","type":"string","kind":"scalar","description":"The maximum amount of time without receiving new messages after which the input is closed.","is_optional":true,"examples":["5s"]},{"name":"restart_input","type":"bool","kind":"scalar","description":"Whether the input should be reopened if it closes itself before the condition has resolved to true.","default":false}]}},{"name":"redis_list","type":"input","status":"stable","plugin":true,"summary":"Pops messages from the beginning of a Redis list using the BLPop command.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target Redis server. Database is optional and is supplied as the URL path.","examples":["redis://:6379","redis://localhost:6379","redis://foousername:foopassword@redisplace:6379","redis://:foopassword@redisplace:6379","redis://localhost:6379/1","redis://localhost:6379/1,redis://localhost:6380/1"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"kind","type":"string","kind":"scalar","description":"Specifies a simple, cluster-aware, or failover-aware redis client.","is_advanced":true,"default":"simple","options":["simple","cluster","failover"],"linter":"\nlet options = {\n \"simple\": true,\n \"cluster\": true,\n \"failover\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"master","type":"string","kind":"scalar","description":"Name of the redis master when `kind` is `failover`","is_advanced":true,"default":"","examples":["mymaster"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"key","type":"string","kind":"scalar","description":"The key of a list to read from."},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"max_in_flight","type":"int","kind":"scalar","description":"Optionally sets a limit on the number of messages that can be flowing through a Redpanda Connect stream pending acknowledgment from the input at any given time. Once a message has been either acknowledged or rejected (nacked) it is no longer considered pending. If the input produces logical batches then each batch is considered a single count against the maximum. **WARNING**: Batching policies at the output level will stall if this field limits the number of messages below the batching threshold. Zero (default) or lower implies no limit.","is_advanced":true,"default":0,"version":"4.9.0"},{"name":"timeout","type":"string","kind":"scalar","description":"The length of time to poll for new messages before reattempting.","is_advanced":true,"default":"5s"},{"name":"command","type":"string","kind":"scalar","description":"The command used to pop elements from the Redis list","is_advanced":true,"default":"blpop","options":["blpop","brpop"],"version":"4.22.0","linter":"\nlet options = {\n \"blpop\": true,\n \"brpop\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"}]}},{"name":"redis_pubsub","type":"input","status":"stable","plugin":true,"summary":"Consume from a Redis publish/subscribe channel using either the SUBSCRIBE or PSUBSCRIBE commands.","description":"\nIn order to subscribe to channels using the `PSUBSCRIBE` command set the field `use_patterns` to `true`, then you can include glob-style patterns in your channel names. For example:\n\n- `h?llo` subscribes to hello, hallo and hxllo\n- `h*llo` subscribes to hllo and heeeello\n- `h[ae]llo` subscribes to hello and hallo, but not hillo\n\nUse `\\` to escape special characters if you want to match them verbatim.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target Redis server. Database is optional and is supplied as the URL path.","examples":["redis://:6379","redis://localhost:6379","redis://foousername:foopassword@redisplace:6379","redis://:foopassword@redisplace:6379","redis://localhost:6379/1","redis://localhost:6379/1,redis://localhost:6380/1"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"kind","type":"string","kind":"scalar","description":"Specifies a simple, cluster-aware, or failover-aware redis client.","is_advanced":true,"default":"simple","options":["simple","cluster","failover"],"linter":"\nlet options = {\n \"simple\": true,\n \"cluster\": true,\n \"failover\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"master","type":"string","kind":"scalar","description":"Name of the redis master when `kind` is `failover`","is_advanced":true,"default":"","examples":["mymaster"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"channels","type":"string","kind":"array","description":"A list of channels to consume from."},{"name":"use_patterns","type":"bool","kind":"scalar","description":"Whether to use the PSUBSCRIBE command, allowing for glob-style patterns within target channel names.","default":false},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]}},{"name":"redis_scan","type":"input","status":"experimental","plugin":true,"summary":"Scans the set of keys in the current selected database and gets their values, using the Scan and Get commands.","description":"Optionally, iterates only elements matching a blob-style pattern. For example:\n\n- `*foo*` iterates only keys which contain `foo` in it.\n- `foo*` iterates only keys starting with `foo`.\n\nThis input generates a message for each key value pair in the following format:\n\n```json\n{\"key\":\"foo\",\"value\":\"bar\"}\n```\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target Redis server. Database is optional and is supplied as the URL path.","examples":["redis://:6379","redis://localhost:6379","redis://foousername:foopassword@redisplace:6379","redis://:foopassword@redisplace:6379","redis://localhost:6379/1","redis://localhost:6379/1,redis://localhost:6380/1"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"kind","type":"string","kind":"scalar","description":"Specifies a simple, cluster-aware, or failover-aware redis client.","is_advanced":true,"default":"simple","options":["simple","cluster","failover"],"linter":"\nlet options = {\n \"simple\": true,\n \"cluster\": true,\n \"failover\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"master","type":"string","kind":"scalar","description":"Name of the redis master when `kind` is `failover`","is_advanced":true,"default":"","examples":["mymaster"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"match","type":"string","kind":"scalar","description":"Iterates only elements matching the optional glob-style pattern. By default, it matches all elements.","default":"","examples":["*","1*","foo*","foo","*4*"]}]},"version":"4.27.0"},{"name":"redis_streams","type":"input","status":"stable","plugin":true,"summary":"Pulls messages from Redis (v5.0+) streams with the XREADGROUP command. The `client_id` should be unique for each consumer of a group.","description":"Redis stream entries are key/value pairs, as such it is necessary to specify the key that contains the body of the message. All other keys/value pairs are saved as metadata fields.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target Redis server. Database is optional and is supplied as the URL path.","examples":["redis://:6379","redis://localhost:6379","redis://foousername:foopassword@redisplace:6379","redis://:foopassword@redisplace:6379","redis://localhost:6379/1","redis://localhost:6379/1,redis://localhost:6380/1"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"kind","type":"string","kind":"scalar","description":"Specifies a simple, cluster-aware, or failover-aware redis client.","is_advanced":true,"default":"simple","options":["simple","cluster","failover"],"linter":"\nlet options = {\n \"simple\": true,\n \"cluster\": true,\n \"failover\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"master","type":"string","kind":"scalar","description":"Name of the redis master when `kind` is `failover`","is_advanced":true,"default":"","examples":["mymaster"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"body_key","type":"string","kind":"scalar","description":"The field key to extract the raw message from. All other keys will be stored in the message as metadata.","default":"body"},{"name":"streams","type":"string","kind":"array","description":"A list of streams to consume from."},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"limit","type":"int","kind":"scalar","description":"The maximum number of messages to consume from a single request.","default":10},{"name":"client_id","type":"string","kind":"scalar","description":"An identifier for the client connection.","default":""},{"name":"consumer_group","type":"string","kind":"scalar","description":"An identifier for the consumer group of the stream.","default":""},{"name":"create_streams","type":"bool","kind":"scalar","description":"Create subscribed streams if they do not exist (MKSTREAM option).","is_advanced":true,"default":true},{"name":"start_from_oldest","type":"bool","kind":"scalar","description":"If an offset is not found for a stream, determines whether to consume from the oldest available offset, otherwise messages are consumed from the latest offset.","is_advanced":true,"default":true},{"name":"commit_period","type":"string","kind":"scalar","description":"The period of time between each commit of the current offset. Offsets are always committed during shutdown.","is_advanced":true,"default":"1s"},{"name":"timeout","type":"string","kind":"scalar","description":"The length of time to poll for new messages before reattempting.","is_advanced":true,"default":"1s"}]}},{"name":"redpanda","type":"input","status":"beta","plugin":true,"summary":"A Kafka input using the https://github.com/twmb/franz-go[Franz Kafka client library^].","description":"\nWhen a consumer group is specified this input consumes one or more topics where partitions will automatically balance across any other connected clients with the same consumer group. When a consumer group is not specified topics can either be consumed in their entirety or with explicit partitions.\n\n== Delivery Guarantees\n\nWhen using consumer groups the offsets of \"delivered\" records will be committed automatically and continuously, and in the event of restarts these committed offsets will be used in order to resume from where the input left off. Redpanda Connect guarantees at least once delivery by ensuring that records are only considerd to be delivered when all configured outputs that the record is routed to have confirmed delivery.\n\n== Ordering\n\nIn order to preserve ordering of topic partitions, records consumed from each partition are processed and delivered in the order that they are received, and only one batch of records of a given partition will ever be processed at a time. This means that parallel processing can only occur when multiple topic partitions are being consumed, but ensures that data is processed in a sequential order as determined from the source partition.\n\nHowever, one way in which the order of records can be mixed is when delivery errors occur and error handling mechanisms kick in. Redpanda Connect always leans towards at least once delivery unless instructed otherwise, and this includes reattempting delivery of data when the ordering of that data can no longer be guaranteed.\n\nFor example, a batch of records may have been sent to an output broker and only a subset of records were delivered, in this case Redpanda Connect by default will reattempt to deliver the records that failed, even though these failed records may have come before records that were previously delivered successfully.\n\nIn order to avoid this scenario you must specify in your configuration an alternative way to handle delivery errors in the form of a xref:components:outputs/fallback.adoc[`fallback`] output. It is good practice to also disable the field `auto_retry_nacks` by setting it to `false` when you've added an explicit fallback output as this will improve the throughput of your pipeline. For example, the following config avoids ordering issues by specifying a fallback output into a DLQ topic, which is also retried indefinitely as a way to apply back pressure during connectivity issues:\n\n```yaml\noutput:\n fallback:\n - redpanda:\n seed_brokers: [ localhost:9092 ]\n topic: foo\n - retry:\n output:\n redpanda:\n seed_brokers: [ localhost:9092 ]\n topic: foo_dlq\n```\n\n== Batching\n\nRecords are processed and delivered from each partition in batches as received from brokers. These batch sizes are therefore dynamically sized in order to optimise throughput, but can be tuned with the config fields `fetch_max_partition_bytes` and `fetch_max_bytes`. Batches can be further broken down using the xref:components:processors/split.adoc[`split`] processor.\n\n== Metrics\n\nEmits a `redpanda_lag` metric with `topic` and `partition` labels for each consumed topic.\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- kafka_key\n- kafka_topic\n- kafka_partition\n- kafka_offset\n- kafka_lag\n- kafka_timestamp_ms\n- kafka_timestamp_unix\n- kafka_tombstone_message\n- All record headers\n```\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"seed_brokers","type":"string","kind":"array","description":"A list of broker addresses to connect to in order to establish connections. If an item of the list contains commas it will be expanded into multiple addresses.","examples":[["localhost:9092"],["foo:9092","bar:9092"],["foo:9092,bar:9092"]]},{"name":"client_id","type":"string","kind":"scalar","description":"An identifier for the client connection.","is_advanced":true,"default":"benthos"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"sasl","type":"object","kind":"array","description":"Specify one or more methods of SASL authentication. SASL is tried in order; if the broker supports the first mechanism, all connections will use that mechanism. If the first mechanism fails, the client will pick the first supported mechanism. If the broker does not support any client mechanisms, connections will fail.","is_advanced":true,"is_optional":true,"examples":[[{"mechanism":"SCRAM-SHA-512","password":"bar","username":"foo"}]],"children":[{"name":"mechanism","type":"string","kind":"scalar","description":"The SASL mechanism to use.","is_advanced":true,"annotated_options":[["AWS_MSK_IAM","AWS IAM based authentication as specified by the 'aws-msk-iam-auth' java library."],["OAUTHBEARER","OAuth Bearer based authentication."],["PLAIN","Plain text authentication."],["SCRAM-SHA-256","SCRAM based authentication as specified in RFC5802."],["SCRAM-SHA-512","SCRAM based authentication as specified in RFC5802."],["none","Disable sasl authentication"]],"linter":"\nlet options = {\n \"aws_msk_iam\": true,\n \"oauthbearer\": true,\n \"plain\": true,\n \"scram-sha-256\": true,\n \"scram-sha-512\": true,\n \"none\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"username","type":"string","kind":"scalar","description":"A username to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token to use for a single session's OAUTHBEARER authentication.","is_advanced":true,"default":""},{"name":"extensions","type":"string","kind":"map","description":"Key/value pairs to add to OAUTHBEARER authentication requests.","is_advanced":true,"is_optional":true},{"name":"aws","type":"object","kind":"scalar","description":"Contains AWS specific fields for when the `mechanism` is set to `AWS_MSK_IAM`.","is_advanced":true,"is_optional":true,"children":[{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]}]},{"name":"metadata_max_age","type":"string","kind":"scalar","description":"The maximum age of metadata before it is refreshed.","is_advanced":true,"default":"5m"},{"name":"request_timeout_overhead","type":"string","kind":"scalar","description":"The request time overhead. Uses the given time as overhead while deadlining requests. Roughly equivalent to request.timeout.ms, but grants additional time to requests that have timeout fields.","is_advanced":true,"default":"10s"},{"name":"conn_idle_timeout","type":"string","kind":"scalar","description":"The rough amount of time to allow connections to idle before they are closed.","is_advanced":true,"default":"20s"},{"name":"topics","type":"string","kind":"array","description":"\nA list of topics to consume from. Multiple comma separated topics can be listed in a single element. When a `consumer_group` is specified partitions are automatically distributed across consumers of a topic, otherwise all partitions are consumed.\n\nAlternatively, it's possible to specify explicit partitions to consume from with a colon after the topic name, e.g. `foo:0` would consume the partition 0 of the topic foo. This syntax supports ranges, e.g. `foo:0-10` would consume partitions 0 through to 10 inclusive.\n\nFinally, it's also possible to specify an explicit offset to consume from by adding another colon after the partition, e.g. `foo:0:10` would consume the partition 0 of the topic foo starting from the offset 10. If the offset is not present (or remains unspecified) then the field `start_from_oldest` determines which offset to start from.","examples":[["foo","bar"],["things.*"],["foo,bar"],["foo:0","bar:1","bar:3"],["foo:0,bar:1,bar:3"],["foo:0-5"]]},{"name":"regexp_topics","type":"bool","kind":"scalar","description":"Whether listed topics should be interpreted as regular expression patterns for matching multiple topics. When topics are specified with explicit partitions this field must remain set to `false`.","default":false},{"name":"rack_id","type":"string","kind":"scalar","description":"A rack specifies where the client is physically located and changes fetch requests to consume from the closest replica as opposed to the leader replica.","is_advanced":true,"default":""},{"name":"instance_id","type":"string","kind":"scalar","description":"When using a consumer group, an instance ID specifies the groups static membership, which can prevent rebalances during reconnects. When using a instance ID the client does NOT leave the group when closing. To actually leave the group one must use an external admin command to leave the group on behalf of this instance ID. This ID must be unique per consumer within the group.","is_advanced":true,"default":""},{"name":"rebalance_timeout","type":"string","kind":"scalar","description":"When using a consumer group, `rebalance_timeout` sets how long group members are allowed to take when a rebalance has begun. This timeout is how long all members are allowed to complete work and commit offsets, minus the time it took to detect the rebalance (from a heartbeat).","is_advanced":true,"default":"45s"},{"name":"session_timeout","type":"string","kind":"scalar","description":"When using a consumer group, `session_timeout` sets how long a member in hte group can go between heartbeats. If a member does not heartbeat in this timeout, the broker will remove the member from the group and initiate a rebalance.","is_advanced":true,"default":"1m"},{"name":"heartbeat_interval","type":"string","kind":"scalar","description":"When using a consumer group, `heartbeat_interval` sets how long a group member goes between heartbeats to Kafka. Kafka uses heartbeats to ensure that a group member's sesion stays active. This value should be no higher than 1/3rd of the `session_timeout`. This is equivalent to the Java heartbeat.interval.ms setting.","is_advanced":true,"default":"3s"},{"name":"start_from_oldest","type":"bool","kind":"scalar","description":"Determines whether to consume from the oldest available offset, otherwise messages are consumed from the latest offset. The setting is applied when creating a new consumer group or the saved offset no longer exists.","is_advanced":true,"is_deprecated":true,"default":true},{"name":"start_offset","type":"string","kind":"scalar","description":"Sets the offset to start consuming from, or if OffsetOutOfRange is seen while fetching, to restart consuming from.","is_advanced":true,"default":"earliest","annotated_options":[["committed","Prevents consuming a partition in a group if the partition has no prior commits. Corresponds to Kafka's `auto.offset.reset=none` option"],["earliest","Start from the earliest offset. Corresponds to Kafka's `auto.offset.reset=earliest` option."],["latest","Start from the latest offset. Corresponds to Kafka's `auto.offset.reset=latest` option."]],"linter":"\nlet options = {\n \"committed\": true,\n \"earliest\": true,\n \"latest\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"fetch_max_bytes","type":"string","kind":"scalar","description":"Sets the maximum amount of bytes a broker will try to send during a fetch. Note that brokers may not obey this limit if it has records larger than this limit. This is the equivalent to the Java fetch.max.bytes setting.","is_advanced":true,"default":"50MiB"},{"name":"fetch_max_wait","type":"string","kind":"scalar","description":"Sets the maximum amount of time a broker will wait for a fetch response to hit the minimum number of required bytes. This is the equivalent to the Java fetch.max.wait.ms setting.","is_advanced":true,"default":"5s"},{"name":"fetch_min_bytes","type":"string","kind":"scalar","description":"Sets the minimum amount of bytes a broker will try to send during a fetch. This is the equivalent to the Java fetch.min.bytes setting.","is_advanced":true,"default":"1B"},{"name":"fetch_max_partition_bytes","type":"string","kind":"scalar","description":"Sets the maximum amount of bytes that will be consumed for a single partition in a fetch request. Note that if a single batch is larger than this number, that batch will still be returned so the client can make progress. This is the equivalent to the Java fetch.max.partition.bytes setting.","is_advanced":true,"default":"1MiB"},{"name":"transaction_isolation_level","type":"string","kind":"scalar","description":"The transaction isolation level","default":"read_uncommitted","annotated_options":[["read_committed","If set, only committed transactional records are processed."],["read_uncommitted","If set, then uncommitted records are processed."]],"linter":"\nlet options = {\n \"read_committed\": true,\n \"read_uncommitted\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"consumer_group","type":"string","kind":"scalar","description":"An optional consumer group to consume as. When specified the partitions of specified topics are automatically distributed across consumers sharing a consumer group, and partition offsets are automatically committed and resumed under this name. Consumer groups are not supported when specifying explicit partitions to consume from in the `topics` field.","is_optional":true},{"name":"commit_period","type":"string","kind":"scalar","description":"The period of time between each commit of the current partition offsets. Offsets are always committed during shutdown.","is_advanced":true,"default":"5s"},{"name":"partition_buffer_bytes","type":"string","kind":"scalar","description":"A buffer size (in bytes) for each consumed partition, allowing records to be queued internally before flushing. Increasing this may improve throughput at the cost of higher memory utilisation. Note that each buffer can grow slightly beyond this value.","is_advanced":true,"default":"1MB"},{"name":"topic_lag_refresh_period","type":"string","kind":"scalar","description":"The period of time between each topic lag refresh cycle.","is_advanced":true,"default":"5s"},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}],"linter":"\nlet has_topic_partitions = this.topics.any(t -\u003e t.contains(\":\"))\n\nroot = [\n if $has_topic_partitions {\n if this.consumer_group.or(\"\") != \"\" {\n \"this input does not support both a consumer group and explicit topic partitions\"\n } else if this.regexp_topics {\n \"this input does not support both regular expression topics and explicit topic partitions\"\n }\n } else {\n if this.consumer_group.or(\"\") == \"\" {\n \"a consumer group is mandatory when not using explicit topic partitions\"\n }\n },\n # We don't have any way to distinguish between start_from_oldest set explicitly to true and not set at all, so we\n # assume users will be OK if start_offset overwrites it silently\n if this.start_from_oldest == false \u0026\u0026 this.start_offset == \"earliest\" {\n \"start_from_oldest cannot be set to false when start_offset is set to earliest\"\n }\n]\n"}},{"name":"redpanda_common","type":"input","status":"beta","plugin":true,"summary":"Consumes data from a Redpanda (Kafka) broker, using credentials defined in a common top-level `redpanda` config block.","description":"\nWhen a consumer group is specified this input consumes one or more topics where partitions will automatically balance across any other connected clients with the same consumer group. When a consumer group is not specified topics can either be consumed in their entirety or with explicit partitions.\n\n== Delivery Guarantees\n\nWhen using consumer groups the offsets of \"delivered\" records will be committed automatically and continuously, and in the event of restarts these committed offsets will be used in order to resume from where the input left off. Redpanda Connect guarantees at least once delivery by ensuring that records are only considerd to be delivered when all configured outputs that the record is routed to have confirmed delivery.\n\n== Ordering\n\nIn order to preserve ordering of topic partitions, records consumed from each partition are processed and delivered in the order that they are received, and only one batch of records of a given partition will ever be processed at a time. This means that parallel processing can only occur when multiple topic partitions are being consumed, but ensures that data is processed in a sequential order as determined from the source partition.\n\nHowever, one way in which the order of records can be mixed is when delivery errors occur and error handling mechanisms kick in. Redpanda Connect always leans towards at least once delivery unless instructed otherwise, and this includes reattempting delivery of data when the ordering of that data can no longer be guaranteed.\n\nFor example, a batch of records may have been sent to an output broker and only a subset of records were delivered, in this case Redpanda Connect by default will reattempt to deliver the records that failed, even though these failed records may have come before records that were previously delivered successfully.\n\nIn order to avoid this scenario you must specify in your configuration an alternative way to handle delivery errors in the form of a xref:components:outputs/fallback.adoc[`fallback`] output. It is good practice to also disable the field `auto_retry_nacks` by setting it to `false` when you've added an explicit fallback output as this will improve the throughput of your pipeline. For example, the following config avoids ordering issues by specifying a fallback output into a DLQ topic, which is also retried indefinitely as a way to apply back pressure during connectivity issues:\n\n```yaml\noutput:\n fallback:\n - redpanda_common:\n topic: foo\n - retry:\n output:\n redpanda_common:\n topic: foo_dlq\n```\n\n== Batching\n\nRecords are processed and delivered from each partition in batches as received from brokers. These batch sizes are therefore dynamically sized in order to optimise throughput, but can be tuned with the config fields `fetch_max_partition_bytes` and `fetch_max_bytes`. Batches can be further broken down using the xref:components:processors/split.adoc[`split`] processor.\n\n== Metrics\n\nEmits a `redpanda_lag` metric with `topic` and `partition` labels for each consumed topic.\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- kafka_key\n- kafka_topic\n- kafka_partition\n- kafka_offset\n- kafka_lag\n- kafka_timestamp_ms\n- kafka_timestamp_unix\n- kafka_tombstone_message\n- All record headers\n```\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"topics","type":"string","kind":"array","description":"\nA list of topics to consume from. Multiple comma separated topics can be listed in a single element. When a `consumer_group` is specified partitions are automatically distributed across consumers of a topic, otherwise all partitions are consumed.\n\nAlternatively, it's possible to specify explicit partitions to consume from with a colon after the topic name, e.g. `foo:0` would consume the partition 0 of the topic foo. This syntax supports ranges, e.g. `foo:0-10` would consume partitions 0 through to 10 inclusive.\n\nFinally, it's also possible to specify an explicit offset to consume from by adding another colon after the partition, e.g. `foo:0:10` would consume the partition 0 of the topic foo starting from the offset 10. If the offset is not present (or remains unspecified) then the field `start_from_oldest` determines which offset to start from.","examples":[["foo","bar"],["things.*"],["foo,bar"],["foo:0","bar:1","bar:3"],["foo:0,bar:1,bar:3"],["foo:0-5"]]},{"name":"regexp_topics","type":"bool","kind":"scalar","description":"Whether listed topics should be interpreted as regular expression patterns for matching multiple topics. When topics are specified with explicit partitions this field must remain set to `false`.","default":false},{"name":"rack_id","type":"string","kind":"scalar","description":"A rack specifies where the client is physically located and changes fetch requests to consume from the closest replica as opposed to the leader replica.","is_advanced":true,"default":""},{"name":"instance_id","type":"string","kind":"scalar","description":"When using a consumer group, an instance ID specifies the groups static membership, which can prevent rebalances during reconnects. When using a instance ID the client does NOT leave the group when closing. To actually leave the group one must use an external admin command to leave the group on behalf of this instance ID. This ID must be unique per consumer within the group.","is_advanced":true,"default":""},{"name":"rebalance_timeout","type":"string","kind":"scalar","description":"When using a consumer group, `rebalance_timeout` sets how long group members are allowed to take when a rebalance has begun. This timeout is how long all members are allowed to complete work and commit offsets, minus the time it took to detect the rebalance (from a heartbeat).","is_advanced":true,"default":"45s"},{"name":"session_timeout","type":"string","kind":"scalar","description":"When using a consumer group, `session_timeout` sets how long a member in hte group can go between heartbeats. If a member does not heartbeat in this timeout, the broker will remove the member from the group and initiate a rebalance.","is_advanced":true,"default":"1m"},{"name":"heartbeat_interval","type":"string","kind":"scalar","description":"When using a consumer group, `heartbeat_interval` sets how long a group member goes between heartbeats to Kafka. Kafka uses heartbeats to ensure that a group member's sesion stays active. This value should be no higher than 1/3rd of the `session_timeout`. This is equivalent to the Java heartbeat.interval.ms setting.","is_advanced":true,"default":"3s"},{"name":"start_from_oldest","type":"bool","kind":"scalar","description":"Determines whether to consume from the oldest available offset, otherwise messages are consumed from the latest offset. The setting is applied when creating a new consumer group or the saved offset no longer exists.","is_advanced":true,"is_deprecated":true,"default":true},{"name":"start_offset","type":"string","kind":"scalar","description":"Sets the offset to start consuming from, or if OffsetOutOfRange is seen while fetching, to restart consuming from.","is_advanced":true,"default":"earliest","annotated_options":[["committed","Prevents consuming a partition in a group if the partition has no prior commits. Corresponds to Kafka's `auto.offset.reset=none` option"],["earliest","Start from the earliest offset. Corresponds to Kafka's `auto.offset.reset=earliest` option."],["latest","Start from the latest offset. Corresponds to Kafka's `auto.offset.reset=latest` option."]],"linter":"\nlet options = {\n \"committed\": true,\n \"earliest\": true,\n \"latest\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"fetch_max_bytes","type":"string","kind":"scalar","description":"Sets the maximum amount of bytes a broker will try to send during a fetch. Note that brokers may not obey this limit if it has records larger than this limit. This is the equivalent to the Java fetch.max.bytes setting.","is_advanced":true,"default":"50MiB"},{"name":"fetch_max_wait","type":"string","kind":"scalar","description":"Sets the maximum amount of time a broker will wait for a fetch response to hit the minimum number of required bytes. This is the equivalent to the Java fetch.max.wait.ms setting.","is_advanced":true,"default":"5s"},{"name":"fetch_min_bytes","type":"string","kind":"scalar","description":"Sets the minimum amount of bytes a broker will try to send during a fetch. This is the equivalent to the Java fetch.min.bytes setting.","is_advanced":true,"default":"1B"},{"name":"fetch_max_partition_bytes","type":"string","kind":"scalar","description":"Sets the maximum amount of bytes that will be consumed for a single partition in a fetch request. Note that if a single batch is larger than this number, that batch will still be returned so the client can make progress. This is the equivalent to the Java fetch.max.partition.bytes setting.","is_advanced":true,"default":"1MiB"},{"name":"transaction_isolation_level","type":"string","kind":"scalar","description":"The transaction isolation level","default":"read_uncommitted","annotated_options":[["read_committed","If set, only committed transactional records are processed."],["read_uncommitted","If set, then uncommitted records are processed."]],"linter":"\nlet options = {\n \"read_committed\": true,\n \"read_uncommitted\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"consumer_group","type":"string","kind":"scalar","description":"An optional consumer group to consume as. When specified the partitions of specified topics are automatically distributed across consumers sharing a consumer group, and partition offsets are automatically committed and resumed under this name. Consumer groups are not supported when specifying explicit partitions to consume from in the `topics` field.","is_optional":true},{"name":"commit_period","type":"string","kind":"scalar","description":"The period of time between each commit of the current partition offsets. Offsets are always committed during shutdown.","is_advanced":true,"default":"5s"},{"name":"partition_buffer_bytes","type":"string","kind":"scalar","description":"A buffer size (in bytes) for each consumed partition, allowing records to be queued internally before flushing. Increasing this may improve throughput at the cost of higher memory utilisation. Note that each buffer can grow slightly beyond this value.","is_advanced":true,"default":"1MB"},{"name":"topic_lag_refresh_period","type":"string","kind":"scalar","description":"The period of time between each topic lag refresh cycle.","is_advanced":true,"default":"5s"},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}],"linter":"\nlet has_topic_partitions = this.topics.any(t -\u003e t.contains(\":\"))\n\nroot = [\n if $has_topic_partitions {\n if this.consumer_group.or(\"\") != \"\" {\n \"this input does not support both a consumer group and explicit topic partitions\"\n } else if this.regexp_topics {\n \"this input does not support both regular expression topics and explicit topic partitions\"\n }\n } else {\n if this.consumer_group.or(\"\") == \"\" {\n \"a consumer group is mandatory when not using explicit topic partitions\"\n }\n },\n # We don't have any way to distinguish between start_from_oldest set explicitly to true and not set at all, so we\n # assume users will be OK if start_offset overwrites it silently\n if this.start_from_oldest == false \u0026\u0026 this.start_offset == \"earliest\" {\n \"start_from_oldest cannot be set to false when start_offset is set to earliest\"\n }\n]\n"}},{"name":"redpanda_migrator","type":"input","status":"beta","plugin":true,"summary":"A Redpanda Migrator input using the https://github.com/twmb/franz-go[Franz Kafka client library^].","description":"\nReads a batch of messages from a Kafka broker and waits for the output to acknowledge the writes before updating the Kafka consumer group offset.\n\nThis input should be used in combination with a `redpanda_migrator` output.\n\nWhen a consumer group is specified this input consumes one or more topics where partitions will automatically balance across any other connected clients with the same consumer group. When a consumer group is not specified topics can either be consumed in their entirety or with explicit partitions.\n\nIt provides the same delivery guarantees and ordering semantics as the `redpanda` input.\n\n== Metrics\n\nEmits a `input_redpanda_migrator_lag` metric with `topic` and `partition` labels for each consumed topic.\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- kafka_key\n- kafka_topic\n- kafka_partition\n- kafka_offset\n- kafka_lag\n- kafka_timestamp_ms\n- kafka_timestamp_unix\n- kafka_tombstone_message\n- All record headers\n```\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"seed_brokers","type":"string","kind":"array","description":"A list of broker addresses to connect to in order to establish connections. If an item of the list contains commas it will be expanded into multiple addresses.","examples":[["localhost:9092"],["foo:9092","bar:9092"],["foo:9092,bar:9092"]]},{"name":"client_id","type":"string","kind":"scalar","description":"An identifier for the client connection.","is_advanced":true,"default":"benthos"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"sasl","type":"object","kind":"array","description":"Specify one or more methods of SASL authentication. SASL is tried in order; if the broker supports the first mechanism, all connections will use that mechanism. If the first mechanism fails, the client will pick the first supported mechanism. If the broker does not support any client mechanisms, connections will fail.","is_advanced":true,"is_optional":true,"examples":[[{"mechanism":"SCRAM-SHA-512","password":"bar","username":"foo"}]],"children":[{"name":"mechanism","type":"string","kind":"scalar","description":"The SASL mechanism to use.","is_advanced":true,"annotated_options":[["AWS_MSK_IAM","AWS IAM based authentication as specified by the 'aws-msk-iam-auth' java library."],["OAUTHBEARER","OAuth Bearer based authentication."],["PLAIN","Plain text authentication."],["SCRAM-SHA-256","SCRAM based authentication as specified in RFC5802."],["SCRAM-SHA-512","SCRAM based authentication as specified in RFC5802."],["none","Disable sasl authentication"]],"linter":"\nlet options = {\n \"aws_msk_iam\": true,\n \"oauthbearer\": true,\n \"plain\": true,\n \"scram-sha-256\": true,\n \"scram-sha-512\": true,\n \"none\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"username","type":"string","kind":"scalar","description":"A username to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token to use for a single session's OAUTHBEARER authentication.","is_advanced":true,"default":""},{"name":"extensions","type":"string","kind":"map","description":"Key/value pairs to add to OAUTHBEARER authentication requests.","is_advanced":true,"is_optional":true},{"name":"aws","type":"object","kind":"scalar","description":"Contains AWS specific fields for when the `mechanism` is set to `AWS_MSK_IAM`.","is_advanced":true,"is_optional":true,"children":[{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]}]},{"name":"metadata_max_age","type":"string","kind":"scalar","description":"The maximum age of metadata before it is refreshed.","is_advanced":true,"default":"5m"},{"name":"request_timeout_overhead","type":"string","kind":"scalar","description":"The request time overhead. Uses the given time as overhead while deadlining requests. Roughly equivalent to request.timeout.ms, but grants additional time to requests that have timeout fields.","is_advanced":true,"default":"10s"},{"name":"conn_idle_timeout","type":"string","kind":"scalar","description":"The rough amount of time to allow connections to idle before they are closed.","is_advanced":true,"default":"20s"},{"name":"topics","type":"string","kind":"array","description":"\nA list of topics to consume from. Multiple comma separated topics can be listed in a single element. When a `consumer_group` is specified partitions are automatically distributed across consumers of a topic, otherwise all partitions are consumed.\n\nAlternatively, it's possible to specify explicit partitions to consume from with a colon after the topic name, e.g. `foo:0` would consume the partition 0 of the topic foo. This syntax supports ranges, e.g. `foo:0-10` would consume partitions 0 through to 10 inclusive.\n\nFinally, it's also possible to specify an explicit offset to consume from by adding another colon after the partition, e.g. `foo:0:10` would consume the partition 0 of the topic foo starting from the offset 10. If the offset is not present (or remains unspecified) then the field `start_from_oldest` determines which offset to start from.","examples":[["foo","bar"],["things.*"],["foo,bar"],["foo:0","bar:1","bar:3"],["foo:0,bar:1,bar:3"],["foo:0-5"]]},{"name":"regexp_topics","type":"bool","kind":"scalar","description":"Whether listed topics should be interpreted as regular expression patterns for matching multiple topics. When topics are specified with explicit partitions this field must remain set to `false`.","default":false},{"name":"rack_id","type":"string","kind":"scalar","description":"A rack specifies where the client is physically located and changes fetch requests to consume from the closest replica as opposed to the leader replica.","is_advanced":true,"default":""},{"name":"instance_id","type":"string","kind":"scalar","description":"When using a consumer group, an instance ID specifies the groups static membership, which can prevent rebalances during reconnects. When using a instance ID the client does NOT leave the group when closing. To actually leave the group one must use an external admin command to leave the group on behalf of this instance ID. This ID must be unique per consumer within the group.","is_advanced":true,"default":""},{"name":"rebalance_timeout","type":"string","kind":"scalar","description":"When using a consumer group, `rebalance_timeout` sets how long group members are allowed to take when a rebalance has begun. This timeout is how long all members are allowed to complete work and commit offsets, minus the time it took to detect the rebalance (from a heartbeat).","is_advanced":true,"default":"45s"},{"name":"session_timeout","type":"string","kind":"scalar","description":"When using a consumer group, `session_timeout` sets how long a member in hte group can go between heartbeats. If a member does not heartbeat in this timeout, the broker will remove the member from the group and initiate a rebalance.","is_advanced":true,"default":"1m"},{"name":"heartbeat_interval","type":"string","kind":"scalar","description":"When using a consumer group, `heartbeat_interval` sets how long a group member goes between heartbeats to Kafka. Kafka uses heartbeats to ensure that a group member's sesion stays active. This value should be no higher than 1/3rd of the `session_timeout`. This is equivalent to the Java heartbeat.interval.ms setting.","is_advanced":true,"default":"3s"},{"name":"start_from_oldest","type":"bool","kind":"scalar","description":"Determines whether to consume from the oldest available offset, otherwise messages are consumed from the latest offset. The setting is applied when creating a new consumer group or the saved offset no longer exists.","is_advanced":true,"is_deprecated":true,"default":true},{"name":"start_offset","type":"string","kind":"scalar","description":"Sets the offset to start consuming from, or if OffsetOutOfRange is seen while fetching, to restart consuming from.","is_advanced":true,"default":"earliest","annotated_options":[["committed","Prevents consuming a partition in a group if the partition has no prior commits. Corresponds to Kafka's `auto.offset.reset=none` option"],["earliest","Start from the earliest offset. Corresponds to Kafka's `auto.offset.reset=earliest` option."],["latest","Start from the latest offset. Corresponds to Kafka's `auto.offset.reset=latest` option."]],"linter":"\nlet options = {\n \"committed\": true,\n \"earliest\": true,\n \"latest\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"fetch_max_bytes","type":"string","kind":"scalar","description":"Sets the maximum amount of bytes a broker will try to send during a fetch. Note that brokers may not obey this limit if it has records larger than this limit. This is the equivalent to the Java fetch.max.bytes setting.","is_advanced":true,"default":"50MiB"},{"name":"fetch_max_wait","type":"string","kind":"scalar","description":"Sets the maximum amount of time a broker will wait for a fetch response to hit the minimum number of required bytes. This is the equivalent to the Java fetch.max.wait.ms setting.","is_advanced":true,"default":"5s"},{"name":"fetch_min_bytes","type":"string","kind":"scalar","description":"Sets the minimum amount of bytes a broker will try to send during a fetch. This is the equivalent to the Java fetch.min.bytes setting.","is_advanced":true,"default":"1B"},{"name":"fetch_max_partition_bytes","type":"string","kind":"scalar","description":"Sets the maximum amount of bytes that will be consumed for a single partition in a fetch request. Note that if a single batch is larger than this number, that batch will still be returned so the client can make progress. This is the equivalent to the Java fetch.max.partition.bytes setting.","is_advanced":true,"default":"1MiB"},{"name":"transaction_isolation_level","type":"string","kind":"scalar","description":"The transaction isolation level","default":"read_uncommitted","annotated_options":[["read_committed","If set, only committed transactional records are processed."],["read_uncommitted","If set, then uncommitted records are processed."]],"linter":"\nlet options = {\n \"read_committed\": true,\n \"read_uncommitted\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"consumer_group","type":"string","kind":"scalar","description":"An optional consumer group to consume as. When specified the partitions of specified topics are automatically distributed across consumers sharing a consumer group, and partition offsets are automatically committed and resumed under this name. Consumer groups are not supported when specifying explicit partitions to consume from in the `topics` field.","is_optional":true},{"name":"commit_period","type":"string","kind":"scalar","description":"The period of time between each commit of the current partition offsets. Offsets are always committed during shutdown.","is_advanced":true,"default":"5s"},{"name":"partition_buffer_bytes","type":"string","kind":"scalar","description":"A buffer size (in bytes) for each consumed partition, allowing records to be queued internally before flushing. Increasing this may improve throughput at the cost of higher memory utilisation. Note that each buffer can grow slightly beyond this value.","is_advanced":true,"default":"1MB"},{"name":"topic_lag_refresh_period","type":"string","kind":"scalar","description":"The period of time between each topic lag refresh cycle.","is_advanced":true,"default":"5s"},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"output_resource","type":"string","kind":"scalar","description":"The label of the redpanda_migrator output in which the currently selected topics need to be created before attempting to read messages.","is_advanced":true,"is_deprecated":true,"default":"redpanda_migrator_output"},{"name":"replication_factor_override","type":"bool","kind":"scalar","description":"Use the specified replication factor when creating topics.","is_advanced":true,"is_deprecated":true,"default":true},{"name":"replication_factor","type":"int","kind":"scalar","description":"Replication factor for created topics. This is only used when `replication_factor_override` is set to `true`.","is_advanced":true,"is_deprecated":true,"default":3},{"name":"multi_header","type":"bool","kind":"scalar","description":"Decode headers into lists to allow handling of multiple values with the same key","is_advanced":true,"is_deprecated":true,"default":false},{"name":"batch_size","type":"int","kind":"scalar","description":"The maximum number of messages that should be accumulated into each batch.","is_advanced":true,"is_deprecated":true,"default":1024}],"linter":"\nlet has_topic_partitions = this.topics.any(t -\u003e t.contains(\":\"))\n\nroot = [\n if $has_topic_partitions {\n if this.consumer_group.or(\"\") != \"\" {\n \"this input does not support both a consumer group and explicit topic partitions\"\n } else if this.regexp_topics {\n \"this input does not support both regular expression topics and explicit topic partitions\"\n }\n } else {\n if this.consumer_group.or(\"\") == \"\" {\n \"a consumer group is mandatory when not using explicit topic partitions\"\n }\n },\n # We don't have any way to distinguish between start_from_oldest set explicitly to true and not set at all, so we\n # assume users will be OK if start_offset overwrites it silently\n if this.start_from_oldest == false \u0026\u0026 this.start_offset == \"earliest\" {\n \"start_from_oldest cannot be set to false when start_offset is set to earliest\"\n }\n]\n"},"version":"4.37.0"},{"name":"redpanda_migrator_bundle","type":"input","status":"experimental","plugin":true,"summary":"Redpanda Migrator bundle input","description":"All-in-one input which reads messages and schemas from a Kafka or Redpanda cluster. This input is meant to be used\ntogether with the `redpanda_migrator_bundle` output.\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"redpanda_migrator","type":"unknown","kind":"map","description":"The `redpanda_migrator` input configuration.\n"},{"name":"schema_registry","type":"unknown","kind":"map","description":"The `schema_registry` input configuration.\n"},{"name":"migrate_schemas_before_data","type":"bool","kind":"scalar","description":"Migrate all schemas first before starting to migrate data.\n","default":true}]}},{"name":"redpanda_migrator_offsets","type":"input","status":"beta","plugin":true,"summary":"Redpanda Migrator consumer group offsets input using the https://github.com/twmb/franz-go[Franz Kafka client library^].","description":"\nTODO: Description\n\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- kafka_key\n- kafka_topic\n- kafka_partition\n- kafka_offset\n- kafka_timestamp_unix\n- kafka_timestamp_ms\n- kafka_tombstone_message\n- kafka_offset_topic\n- kafka_offset_group\n- kafka_offset_partition\n- kafka_offset_commit_timestamp\n- kafka_offset_metadata\n- kafka_is_high_watermark\n```\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"seed_brokers","type":"string","kind":"array","description":"A list of broker addresses to connect to in order to establish connections. If an item of the list contains commas it will be expanded into multiple addresses.","examples":[["localhost:9092"],["foo:9092","bar:9092"],["foo:9092,bar:9092"]]},{"name":"client_id","type":"string","kind":"scalar","description":"An identifier for the client connection.","is_advanced":true,"default":"benthos"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"sasl","type":"object","kind":"array","description":"Specify one or more methods of SASL authentication. SASL is tried in order; if the broker supports the first mechanism, all connections will use that mechanism. If the first mechanism fails, the client will pick the first supported mechanism. If the broker does not support any client mechanisms, connections will fail.","is_advanced":true,"is_optional":true,"examples":[[{"mechanism":"SCRAM-SHA-512","password":"bar","username":"foo"}]],"children":[{"name":"mechanism","type":"string","kind":"scalar","description":"The SASL mechanism to use.","is_advanced":true,"annotated_options":[["AWS_MSK_IAM","AWS IAM based authentication as specified by the 'aws-msk-iam-auth' java library."],["OAUTHBEARER","OAuth Bearer based authentication."],["PLAIN","Plain text authentication."],["SCRAM-SHA-256","SCRAM based authentication as specified in RFC5802."],["SCRAM-SHA-512","SCRAM based authentication as specified in RFC5802."],["none","Disable sasl authentication"]],"linter":"\nlet options = {\n \"aws_msk_iam\": true,\n \"oauthbearer\": true,\n \"plain\": true,\n \"scram-sha-256\": true,\n \"scram-sha-512\": true,\n \"none\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"username","type":"string","kind":"scalar","description":"A username to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token to use for a single session's OAUTHBEARER authentication.","is_advanced":true,"default":""},{"name":"extensions","type":"string","kind":"map","description":"Key/value pairs to add to OAUTHBEARER authentication requests.","is_advanced":true,"is_optional":true},{"name":"aws","type":"object","kind":"scalar","description":"Contains AWS specific fields for when the `mechanism` is set to `AWS_MSK_IAM`.","is_advanced":true,"is_optional":true,"children":[{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]}]},{"name":"metadata_max_age","type":"string","kind":"scalar","description":"The maximum age of metadata before it is refreshed.","is_advanced":true,"default":"5m"},{"name":"request_timeout_overhead","type":"string","kind":"scalar","description":"The request time overhead. Uses the given time as overhead while deadlining requests. Roughly equivalent to request.timeout.ms, but grants additional time to requests that have timeout fields.","is_advanced":true,"default":"10s"},{"name":"conn_idle_timeout","type":"string","kind":"scalar","description":"The rough amount of time to allow connections to idle before they are closed.","is_advanced":true,"default":"20s"},{"name":"topics","type":"string","kind":"array","description":"\nA list of topics to consume from. Multiple comma separated topics can be listed in a single element. When a `consumer_group` is specified partitions are automatically distributed across consumers of a topic, otherwise all partitions are consumed.","examples":[["foo","bar"],["things.*"],["foo,bar"]],"linter":"if this.length() == 0 { [\"at least one topic must be specified\"] }"},{"name":"regexp_topics","type":"bool","kind":"scalar","description":"Whether listed topics should be interpreted as regular expression patterns for matching multiple topics.","default":false},{"name":"rack_id","type":"string","kind":"scalar","description":"A rack specifies where the client is physically located and changes fetch requests to consume from the closest replica as opposed to the leader replica.","is_advanced":true,"default":""},{"name":"consumer_group","type":"string","kind":"scalar","description":"An optional consumer group to consume as. When specified the partitions of specified topics are automatically distributed across consumers sharing a consumer group, and partition offsets are automatically committed and resumed under this name. Consumer groups are not supported when specifying explicit partitions to consume from in the `topics` field.","is_optional":true},{"name":"commit_period","type":"string","kind":"scalar","description":"The period of time between each commit of the current partition offsets. Offsets are always committed during shutdown.","is_advanced":true,"default":"5s"},{"name":"partition_buffer_bytes","type":"string","kind":"scalar","description":"A buffer size (in bytes) for each consumed partition, allowing records to be queued internally before flushing. Increasing this may improve throughput at the cost of higher memory utilisation. Note that each buffer can grow slightly beyond this value.","is_advanced":true,"default":"1MB"},{"name":"topic_lag_refresh_period","type":"string","kind":"scalar","description":"The period of time between each topic lag refresh cycle.","is_advanced":true,"default":"5s"},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]},"version":"4.45.0"},{"name":"resource","type":"input","status":"stable","plugin":true,"summary":"Resource is an input type that channels messages from a resource input, identified by its name.","description":"Resources allow you to tidy up deeply nested configs. For example, the config:\n\n```yaml\ninput:\n broker:\n inputs:\n - kafka:\n addresses: [ TODO ]\n topics: [ foo ]\n consumer_group: foogroup\n - gcp_pubsub:\n project: bar\n subscription: baz\n```\n\nCould also be expressed as:\n\n```yaml\ninput:\n broker:\n inputs:\n - resource: foo\n - resource: bar\n\ninput_resources:\n - label: foo\n kafka:\n addresses: [ TODO ]\n topics: [ foo ]\n consumer_group: foogroup\n\n - label: bar\n gcp_pubsub:\n project: bar\n subscription: baz\n```\n\nResources also allow you to reference a single input in multiple places, such as multiple streams mode configs, or multiple entries in a broker input. However, when a resource is referenced more than once the messages it produces are distributed across those references, so each message will only be directed to a single reference, not all of them.\n\nYou can find out more about resources in xref:configuration:resources.adoc[].","categories":["Utility"],"config":{"name":"","type":"string","kind":"scalar","default":""}},{"name":"schema_registry","type":"input","status":"beta","plugin":true,"summary":"Reads schemas from SchemaRegistry.","description":"\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- schema_registry_subject\n- schema_registry_version\n```\n\nYou can access these metadata fields using\nxref:configuration:interpolation.adoc#bloblang-queries[function interpolation].\n\n","categories":["Integration"],"examples":[{"title":"Read schemas","summary":"Read all schemas (including deleted) from a Schema Registry instance which are associated with subjects matching the `^foo.*` filter.","config":"\ninput:\n schema_registry:\n url: http://localhost:8081\n include_deleted: true\n subject_filter: ^foo.*\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The base URL of the schema registry service."},{"name":"include_deleted","type":"bool","kind":"scalar","description":"Include deleted entities.","is_advanced":true,"default":false},{"name":"subject_filter","type":"string","kind":"scalar","description":"Include only subjects which match the regular expression filter. All subjects are selected when not set.","is_advanced":true,"default":""},{"name":"fetch_in_order","type":"bool","kind":"scalar","description":"Fetch all schemas on connect and sort them by ID. Should be set to `true` when schema references are used.","is_advanced":true,"default":true,"version":"4.37.0"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"oauth","type":"object","kind":"scalar","description":"Allows you to specify open authentication via OAuth version 1.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use OAuth version 1 in requests.","is_advanced":true,"default":false},{"name":"consumer_key","type":"string","kind":"scalar","description":"A value used to identify the client to the service provider.","is_advanced":true,"default":""},{"name":"consumer_secret","type":"string","kind":"scalar","description":"A secret used to establish ownership of the consumer key.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"access_token","type":"string","kind":"scalar","description":"A value used to gain access to the protected resources on behalf of the user.","is_advanced":true,"default":""},{"name":"access_token_secret","type":"string","kind":"scalar","description":"A secret provided in order to establish ownership of a given access token.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"jwt","type":"object","kind":"scalar","description":"BETA: Allows you to specify JWT authentication.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use JWT authentication in requests.","is_advanced":true,"default":false},{"name":"private_key_file","type":"string","kind":"scalar","description":"A file with the PEM encoded via PKCS1 or PKCS8 as private key.","is_advanced":true,"default":""},{"name":"signing_method","type":"string","kind":"scalar","description":"A method used to sign the token such as RS256, RS384, RS512 or EdDSA.","is_advanced":true,"default":""},{"name":"claims","type":"unknown","kind":"map","description":"A value used to identify the claims that issued the JWT.","is_advanced":true,"default":{}},{"name":"headers","type":"unknown","kind":"map","description":"Add optional key/value headers to the JWT.","is_advanced":true,"default":{}}]}]},"version":"4.32.2"},{"name":"sequence","type":"input","status":"stable","plugin":true,"summary":"Reads messages from a sequence of child inputs, starting with the first and once that input gracefully terminates starts consuming from the next, and so on.","description":"This input is useful for consuming from inputs that have an explicit end but must not be consumed in parallel.","categories":["Utility"],"examples":[{"title":"End of Stream Message","summary":"A common use case for sequence might be to generate a message at the end of our main input. With the following config once the records within `./dataset.csv` are exhausted our final payload `{\"status\":\"finished\"}` will be routed through the pipeline.","config":"\ninput:\n sequence:\n inputs:\n - file:\n paths: [ ./dataset.csv ]\n scanner:\n csv: {}\n - generate:\n count: 1\n mapping: 'root = {\"status\":\"finished\"}'\n"},{"title":"Joining Data (Simple)","summary":"Redpanda Connect can be used to join unordered data from fragmented datasets in memory by specifying a common identifier field and a number of sharded iterations. For example, given two CSV files, the first called \"main.csv\", which contains rows of user data:\n\n```csv\nuuid,name,age\nAAA,Melanie,34\nBBB,Emma,28\nCCC,Geri,45\n```\n\nAnd the second called \"hobbies.csv\" that, for each user, contains zero or more rows of hobbies:\n\n```csv\nuuid,hobby\nCCC,pokemon go\nAAA,rowing\nAAA,golf\n```\n\nWe can parse and join this data into a single dataset:\n\n```json\n{\"uuid\":\"AAA\",\"name\":\"Melanie\",\"age\":34,\"hobbies\":[\"rowing\",\"golf\"]}\n{\"uuid\":\"BBB\",\"name\":\"Emma\",\"age\":28}\n{\"uuid\":\"CCC\",\"name\":\"Geri\",\"age\":45,\"hobbies\":[\"pokemon go\"]}\n```\n\nWith the following config:","config":"\ninput:\n sequence:\n sharded_join:\n type: full-outer\n id_path: uuid\n merge_strategy: array\n inputs:\n - file:\n paths:\n - ./hobbies.csv\n - ./main.csv\n scanner:\n csv: {}\n"},{"title":"Joining Data (Advanced)","summary":"In this example we are able to join unordered and fragmented data from a combination of CSV files and newline-delimited JSON documents by specifying multiple sequence inputs with their own processors for extracting the structured data.\n\nThe first file \"main.csv\" contains straight forward CSV data:\n\n```csv\nuuid,name,age\nAAA,Melanie,34\nBBB,Emma,28\nCCC,Geri,45\n```\n\nAnd the second file called \"hobbies.ndjson\" contains JSON documents, one per line, that associate an identifier with an array of hobbies. However, these data objects are in a nested format:\n\n```json\n{\"document\":{\"uuid\":\"CCC\",\"hobbies\":[{\"type\":\"pokemon go\"}]}}\n{\"document\":{\"uuid\":\"AAA\",\"hobbies\":[{\"type\":\"rowing\"},{\"type\":\"golf\"}]}}\n```\n\nAnd so we will want to map these into a flattened structure before the join, and then we will end up with a single dataset that looks like this:\n\n```json\n{\"uuid\":\"AAA\",\"name\":\"Melanie\",\"age\":34,\"hobbies\":[\"rowing\",\"golf\"]}\n{\"uuid\":\"BBB\",\"name\":\"Emma\",\"age\":28}\n{\"uuid\":\"CCC\",\"name\":\"Geri\",\"age\":45,\"hobbies\":[\"pokemon go\"]}\n```\n\nWith the following config:","config":"\ninput:\n sequence:\n sharded_join:\n type: full-outer\n id_path: uuid\n iterations: 10\n merge_strategy: array\n inputs:\n - file:\n paths: [ ./main.csv ]\n scanner:\n csv: {}\n - file:\n paths: [ ./hobbies.ndjson ]\n scanner:\n lines: {}\n processors:\n - mapping: |\n root.uuid = this.document.uuid\n root.hobbies = this.document.hobbies.map_each(this.type)\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"sharded_join","type":"object","kind":"scalar","description":"EXPERIMENTAL: Provides a way to perform outer joins of arbitrarily structured and unordered data resulting from the input sequence, even when the overall size of the data surpasses the memory available on the machine.\n\nWhen configured the sequence of inputs will be consumed one or more times according to the number of iterations, and when more than one iteration is specified each iteration will process an entirely different set of messages by sharding them by the ID field. Increasing the number of iterations reduces the memory consumption at the cost of needing to fully parse the data each time.\n\nEach message must be structured (JSON or otherwise processed into a structured form) and the fields will be aggregated with those of other messages sharing the ID. At the end of each iteration the joined messages are flushed downstream before the next iteration begins, hence keeping memory usage limited.","is_advanced":true,"children":[{"name":"type","type":"string","kind":"scalar","description":"The type of join to perform. A `full-outer` ensures that all identifiers seen in any of the input sequences are sent, and is performed by consuming all input sequences before flushing the joined results. An `outer` join consumes all input sequences but only writes data joined from the last input in the sequence, similar to a left or right outer join. With an `outer` join if an identifier appears multiple times within the final sequence input it will be flushed each time it appears. `full-outter` and `outter` have been deprecated in favour of `full-outer` and `outer`.","is_advanced":true,"default":"none","options":["none","full-outer","outer","full-outter","outter"],"linter":"\nlet options = {\n \"none\": true,\n \"full-outer\": true,\n \"outer\": true,\n \"full-outter\": true,\n \"outter\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"id_path","type":"string","kind":"scalar","description":"A xref:configuration:field_paths.adoc[dot path] that points to a common field within messages of each fragmented data set and can be used to join them. Messages that are not structured or are missing this field will be dropped. This field must be set in order to enable joins.","is_advanced":true,"default":""},{"name":"iterations","type":"int","kind":"scalar","description":"The total number of iterations (shards), increasing this number will increase the overall time taken to process the data, but reduces the memory used in the process. The real memory usage required is significantly higher than the real size of the data and therefore the number of iterations should be at least an order of magnitude higher than the available memory divided by the overall size of the dataset.","is_advanced":true,"default":1},{"name":"merge_strategy","type":"string","kind":"scalar","description":"The chosen strategy to use when a data join would otherwise result in a collision of field values. The strategy `array` means non-array colliding values are placed into an array and colliding arrays are merged. The strategy `replace` replaces old values with new values. The strategy `keep` keeps the old value.","is_advanced":true,"default":"array","options":["array","replace","keep"],"linter":"\nlet options = {\n \"array\": true,\n \"replace\": true,\n \"keep\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"}],"version":"3.40.0"},{"name":"inputs","type":"input","kind":"array","description":"An array of inputs to read from sequentially."}]}},{"name":"sftp","type":"input","status":"beta","plugin":true,"summary":"Consumes files from an SFTP server.","description":"\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n- sftp_path\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].","categories":["Network"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"address","type":"string","kind":"scalar","description":"The address of the server to connect to."},{"name":"credentials","type":"object","kind":"scalar","description":"The credentials to use to log into the target server.","children":[{"name":"username","type":"string","kind":"scalar","description":"The username to connect to the SFTP server.","default":""},{"name":"password","type":"string","kind":"scalar","description":"The password for the username to connect to the SFTP server.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"private_key_file","type":"string","kind":"scalar","description":"The private key for the username to connect to the SFTP server.","default":""},{"name":"private_key","type":"string","kind":"scalar","description":"The private key file for the username to connect to the SFTP server.","is_secret":true,"default":"","linter":"root = match { this.exists(\"private_key\") \u0026\u0026 this.exists(\"private_key_file\") =\u003e [ \"both private_key and private_key_file can't be set simultaneously\" ], }","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"private_key_pass","type":"string","kind":"scalar","description":"Optional passphrase for private key.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"paths","type":"string","kind":"array","description":"A list of paths to consume sequentially. Glob patterns are supported."},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"codec","type":"string","kind":"scalar","description":"The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.","is_deprecated":true,"is_optional":true,"examples":["lines","delim:\t","delim:foobar","gzip/csv"],"annotated_options":[["auto","EXPERIMENTAL: Attempts to derive a codec for each file based on information such as the extension. For example, a .tar.gz file would be consumed with the `gzip/tar` codec. Defaults to all-bytes."],["all-bytes","Consume the entire file as a single binary message."],["avro-ocf:marshaler=x","EXPERIMENTAL: Consume a stream of Avro OCF datum. The `marshaler` parameter is optional and has the options: `goavro` (default), `json`. Use `goavro` if OCF contains logical types."],["chunker:x","Consume the file in chunks of a given number of bytes."],["csv","Consume structured rows as comma separated values, the first row must be a header row."],["csv:x","Consume structured rows as values separated by a custom delimiter, the first row must be a header row. The custom delimiter must be a single character, e.g. the codec `\"csv:\\t\"` would consume a tab delimited file."],["csv-safe","Consume structured rows like `csv`, but sends messages with empty maps on failure to parse. Includes row number and parsing errors (if any) in the message's metadata."],["csv-safe:x","Consume structured rows like `csv:x` as values separated by a custom delimiter, but sends messages with empty maps on failure to parse. The custom delimiter must be a single character, e.g. the codec `\"csv-safe:\\t\"` would consume a tab delimited file. Includes row number and parsing errors (if any) in the message's metadata."],["delim:x","Consume the file in segments divided by a custom delimiter."],["gzip","Decompress a gzip file, this codec should precede another codec, e.g. `gzip/all-bytes`, `gzip/tar`, `gzip/csv`, etc."],["pgzip","Decompress a gzip file in parallel, this codec should precede another codec, e.g. `pgzip/all-bytes`, `pgzip/tar`, `pgzip/csv`, etc."],["lines","Consume the file in segments divided by linebreaks."],["multipart","Consumes the output of another codec and batches messages together. A batch ends when an empty message is consumed. For example, the codec `lines/multipart` could be used to consume multipart messages where an empty line indicates the end of each batch."],["regex:(?m)^\\d\\d:\\d\\d:\\d\\d","Consume the file in segments divided by regular expression."],["skipbom","Skip one or more byte order marks for each opened reader, this codec should precede another codec, e.g. `skipbom/csv`, etc."],["tar","Parse the file as a tar archive, and consume each file of the archive as a message."]]},{"name":"max_buffer","type":"int","kind":"scalar","is_deprecated":true,"default":1000000},{"name":"scanner","type":"scanner","kind":"scalar","description":"The xref:components:scanners/about.adoc[scanner] by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once.","is_optional":true,"default":{"to_the_end":{}},"version":"4.25.0"},{"name":"delete_on_finish","type":"bool","kind":"scalar","description":"Whether to delete files from the server once they are processed.","is_advanced":true,"default":false},{"name":"watcher","type":"object","kind":"scalar","description":"An experimental mode whereby the input will periodically scan the target paths for new files and consume them, when all files are consumed the input will continue polling for new files.","children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether file watching is enabled.","default":false},{"name":"minimum_age","type":"string","kind":"scalar","description":"The minimum period of time since a file was last updated before attempting to consume it. Increasing this period decreases the likelihood that a file will be consumed whilst it is still being written to.","default":"1s","examples":["10s","1m","10m"]},{"name":"poll_interval","type":"string","kind":"scalar","description":"The interval between each attempt to scan the target paths for new files.","default":"1s","examples":["100ms","1s"]},{"name":"cache","type":"string","kind":"scalar","description":"A xref:components:caches/about.adoc[cache resource] for storing the paths of files already consumed.","default":""}],"version":"3.42.0"}]},"version":"3.39.0"},{"name":"slack","type":"input","status":"experimental","plugin":true,"description":"Connects to Slack using https://api.slack.com/apis/socket-mode[^Socket Mode]. This allows for receiving events, interactions and slash commands. Each message emitted from this input has a @type metadata of the event type \"events_api\", \"interactions\" or \"slash_commands\".","categories":null,"examples":[{"title":"Echo Slackbot","summary":"A slackbot that echo messages from other users","config":"\ninput:\n slack:\n app_token: \"${APP_TOKEN:xapp-demo}\"\n bot_token: \"${BOT_TOKEN:xoxb-demo}\"\npipeline:\n processors:\n - mutation: |\n # ignore hidden or non message events\n if this.event.type != \"message\" || (this.event.hidden | false) {\n root = deleted()\n }\n # Don't respond to our own messages\n if this.authorizations.any(auth -\u003e auth.user_id == this.event.user) {\n root = deleted()\n }\noutput:\n slack_post:\n bot_token: \"${BOT_TOKEN:xoxb-demo}\"\n channel_id: \"${!this.event.channel}\"\n thread_ts: \"${!this.event.ts}\"\n text: \"ECHO: ${!this.event.text}\"\n "}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"app_token","type":"string","kind":"scalar","description":"The Slack App token to use.","linter":"\n root = if !this.has_prefix(\"xapp-\") { [ \"field must start with xapp-\" ] }\n "},{"name":"bot_token","type":"string","kind":"scalar","description":"The Slack Bot User OAuth token to use.","linter":"\n root = if !this.has_prefix(\"xoxb-\") { [ \"field must start with xoxb-\" ] }\n "},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]}},{"name":"slack_users","type":"input","status":"experimental","plugin":true,"description":"Reads all users in a slack organization (optionally filtered by a team ID).","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"bot_token","type":"string","kind":"scalar","description":"The Slack Bot User OAuth token to use.","linter":"\n root = if !this.has_prefix(\"xoxb-\") { [ \"field must start with xoxb-\" ] }\n "},{"name":"team_id","type":"string","kind":"scalar","description":"The team ID to filter by","default":""},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]}},{"name":"socket","type":"input","status":"stable","plugin":true,"summary":"Connects to a tcp or unix socket and consumes a continuous stream of messages.","categories":["Network"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"network","type":"string","kind":"scalar","description":"A network type to assume (unix|tcp).","options":["unix","tcp"],"linter":"\nlet options = {\n \"unix\": true,\n \"tcp\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"address","type":"string","kind":"scalar","description":"The address to connect to.","examples":["/tmp/benthos.sock","127.0.0.1:6000"]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"codec","type":"string","kind":"scalar","description":"The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.","is_deprecated":true,"is_optional":true,"examples":["lines","delim:\t","delim:foobar","gzip/csv"],"annotated_options":[["auto","EXPERIMENTAL: Attempts to derive a codec for each file based on information such as the extension. For example, a .tar.gz file would be consumed with the `gzip/tar` codec. Defaults to all-bytes."],["all-bytes","Consume the entire file as a single binary message."],["avro-ocf:marshaler=x","EXPERIMENTAL: Consume a stream of Avro OCF datum. The `marshaler` parameter is optional and has the options: `goavro` (default), `json`. Use `goavro` if OCF contains logical types."],["chunker:x","Consume the file in chunks of a given number of bytes."],["csv","Consume structured rows as comma separated values, the first row must be a header row."],["csv:x","Consume structured rows as values separated by a custom delimiter, the first row must be a header row. The custom delimiter must be a single character, e.g. the codec `\"csv:\\t\"` would consume a tab delimited file."],["csv-safe","Consume structured rows like `csv`, but sends messages with empty maps on failure to parse. Includes row number and parsing errors (if any) in the message's metadata."],["csv-safe:x","Consume structured rows like `csv:x` as values separated by a custom delimiter, but sends messages with empty maps on failure to parse. The custom delimiter must be a single character, e.g. the codec `\"csv-safe:\\t\"` would consume a tab delimited file. Includes row number and parsing errors (if any) in the message's metadata."],["delim:x","Consume the file in segments divided by a custom delimiter."],["gzip","Decompress a gzip file, this codec should precede another codec, e.g. `gzip/all-bytes`, `gzip/tar`, `gzip/csv`, etc."],["pgzip","Decompress a gzip file in parallel, this codec should precede another codec, e.g. `pgzip/all-bytes`, `pgzip/tar`, `pgzip/csv`, etc."],["lines","Consume the file in segments divided by linebreaks."],["multipart","Consumes the output of another codec and batches messages together. A batch ends when an empty message is consumed. For example, the codec `lines/multipart` could be used to consume multipart messages where an empty line indicates the end of each batch."],["regex:(?m)^\\d\\d:\\d\\d:\\d\\d","Consume the file in segments divided by regular expression."],["skipbom","Skip one or more byte order marks for each opened reader, this codec should precede another codec, e.g. `skipbom/csv`, etc."],["tar","Parse the file as a tar archive, and consume each file of the archive as a message."]]},{"name":"max_buffer","type":"int","kind":"scalar","is_deprecated":true,"default":1000000},{"name":"scanner","type":"scanner","kind":"scalar","description":"The xref:components:scanners/about.adoc[scanner] by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once.","is_optional":true,"default":{"lines":{}},"version":"4.25.0"}]}},{"name":"socket_server","type":"input","status":"stable","plugin":true,"summary":"Creates a server that receives a stream of messages over a TCP, UDP or Unix socket.","categories":["Network"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"network","type":"string","kind":"scalar","description":"A network type to accept.","options":["unix","tcp","udp","tls"],"linter":"\nlet options = {\n \"unix\": true,\n \"tcp\": true,\n \"udp\": true,\n \"tls\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"address","type":"string","kind":"scalar","description":"The address to listen from.","examples":["/tmp/benthos.sock","0.0.0.0:6000"]},{"name":"address_cache","type":"string","kind":"scalar","description":"An optional xref:components:caches/about.adoc[`cache`] within which this input should write it's bound address once known. The key of the cache item containing the address will be the label of the component suffixed with `_address` (e.g. `foo_address`), or `socket_server_address` when a label has not been provided. This is useful in situations where the address is dynamically allocated by the server (`127.0.0.1:0`) and you want to store the allocated address somewhere for reference by other systems and components.","is_optional":true,"version":"4.25.0"},{"name":"tls","type":"object","kind":"scalar","description":"TLS specific configuration, valid when the `network` is set to `tls`.","is_optional":true,"children":[{"name":"cert_file","type":"string","kind":"scalar","description":"PEM encoded certificate for use with TLS.","is_optional":true},{"name":"key_file","type":"string","kind":"scalar","description":"PEM encoded private key for use with TLS.","is_optional":true},{"name":"self_signed","type":"bool","kind":"scalar","description":"Whether to generate self signed certificates.","default":false}]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"codec","type":"string","kind":"scalar","description":"The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.","is_deprecated":true,"is_optional":true,"examples":["lines","delim:\t","delim:foobar","gzip/csv"],"annotated_options":[["auto","EXPERIMENTAL: Attempts to derive a codec for each file based on information such as the extension. For example, a .tar.gz file would be consumed with the `gzip/tar` codec. Defaults to all-bytes."],["all-bytes","Consume the entire file as a single binary message."],["avro-ocf:marshaler=x","EXPERIMENTAL: Consume a stream of Avro OCF datum. The `marshaler` parameter is optional and has the options: `goavro` (default), `json`. Use `goavro` if OCF contains logical types."],["chunker:x","Consume the file in chunks of a given number of bytes."],["csv","Consume structured rows as comma separated values, the first row must be a header row."],["csv:x","Consume structured rows as values separated by a custom delimiter, the first row must be a header row. The custom delimiter must be a single character, e.g. the codec `\"csv:\\t\"` would consume a tab delimited file."],["csv-safe","Consume structured rows like `csv`, but sends messages with empty maps on failure to parse. Includes row number and parsing errors (if any) in the message's metadata."],["csv-safe:x","Consume structured rows like `csv:x` as values separated by a custom delimiter, but sends messages with empty maps on failure to parse. The custom delimiter must be a single character, e.g. the codec `\"csv-safe:\\t\"` would consume a tab delimited file. Includes row number and parsing errors (if any) in the message's metadata."],["delim:x","Consume the file in segments divided by a custom delimiter."],["gzip","Decompress a gzip file, this codec should precede another codec, e.g. `gzip/all-bytes`, `gzip/tar`, `gzip/csv`, etc."],["pgzip","Decompress a gzip file in parallel, this codec should precede another codec, e.g. `pgzip/all-bytes`, `pgzip/tar`, `pgzip/csv`, etc."],["lines","Consume the file in segments divided by linebreaks."],["multipart","Consumes the output of another codec and batches messages together. A batch ends when an empty message is consumed. For example, the codec `lines/multipart` could be used to consume multipart messages where an empty line indicates the end of each batch."],["regex:(?m)^\\d\\d:\\d\\d:\\d\\d","Consume the file in segments divided by regular expression."],["skipbom","Skip one or more byte order marks for each opened reader, this codec should precede another codec, e.g. `skipbom/csv`, etc."],["tar","Parse the file as a tar archive, and consume each file of the archive as a message."]]},{"name":"max_buffer","type":"int","kind":"scalar","is_deprecated":true,"default":1000000},{"name":"scanner","type":"scanner","kind":"scalar","description":"The xref:components:scanners/about.adoc[scanner] by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once.","is_optional":true,"default":{"lines":{}},"version":"4.25.0"}]}},{"name":"spicedb_watch","type":"input","status":"stable","plugin":true,"summary":"Consume messages from the Watch API from SpiceDB.","description":"\nThe SpiceDB input allows you to consume messages from the Watch API of a SpiceDB instance.\nThis input is useful for applications that need to react to changes in the data managed by SpiceDB in real-time.\n\n== Credentials\n\nYou need to provide the endpoint of your SpiceDB instance and a Bearer token for authentication.\n\n== Cache\n\nThe zed token of the newest update consumed and acked is stored in a cache in order to start reading from it each time the input is initialised.\nIdeally this cache should be persisted across restarts.\n","categories":["Services","SpiceDB"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"endpoint","type":"string","kind":"scalar","description":"The SpiceDB endpoint.","examples":["grpc.authzed.com:443"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"bearer_token","type":"string","kind":"scalar","description":"The SpiceDB Bearer token used to authenticate against the SpiceDB instance.","is_secret":true,"default":"","examples":["t_your_token_here_1234567deadbeef"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"max_receive_message_bytes","type":"string","kind":"scalar","description":"Maximum message size in bytes the SpiceDB client can receive.","is_advanced":true,"default":"4MB","examples":["100MB","50mib"]},{"name":"cache","type":"string","kind":"scalar","description":"A cache resource to use for performing unread message backfills, the ID of the last message received will be stored in this cache and used for subsequent requests."},{"name":"cache_key","type":"string","kind":"scalar","description":"The key identifier used when storing the ID of the last message received.","is_advanced":true,"default":"authzed.com/spicedb/watch/last_zed_token"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]}]}},{"name":"splunk","type":"input","status":"beta","plugin":true,"summary":"Consumes messages from Splunk.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"Full HTTP Search API endpoint URL.","examples":["https://foobar.splunkcloud.com/services/search/v2/jobs/export"]},{"name":"user","type":"string","kind":"scalar","description":"Splunk account user."},{"name":"password","type":"string","kind":"scalar","description":"Splunk account password.","is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"query","type":"string","kind":"scalar","description":"Splunk search query."},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]},"version":"4.30.0"},{"name":"sql_raw","type":"input","status":"beta","plugin":true,"summary":"Executes a select query and creates a message for each row received.","description":"Once the rows from the query are exhausted this input shuts down, allowing the pipeline to gracefully terminate (or the next input in a xref:components:inputs/sequence.adoc[sequence] to execute).","categories":["Services"],"examples":[{"title":"Consumes an SQL table using a query as an input.","summary":"\nHere we preform an aggregate over a list of names in a table that are less than 3600 seconds old.","config":"\ninput:\n sql_raw:\n driver: postgres\n dsn: postgres://foouser:foopass@localhost:5432/testdb?sslmode=disable\n query: \"SELECT name, count(*) FROM person WHERE last_updated \u003c $1 GROUP BY name;\"\n args_mapping: |\n root = [\n now().ts_unix() - 3600\n ]\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"driver","type":"string","kind":"scalar","description":"A database \u003c\u003cdrivers, driver\u003e\u003e to use.","options":["mysql","postgres","clickhouse","mssql","sqlite","oracle","snowflake","trino","gocosmos","spanner"],"linter":"\nlet options = {\n \"mysql\": true,\n \"postgres\": true,\n \"clickhouse\": true,\n \"mssql\": true,\n \"sqlite\": true,\n \"oracle\": true,\n \"snowflake\": true,\n \"trino\": true,\n \"gocosmos\": true,\n \"spanner\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"dsn","type":"string","kind":"scalar","description":"A Data Source Name to identify the target database.\n\n==== Drivers\n\n:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community, spanner=community\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n|===\n| Driver | Data Source Name Format\n\n| `clickhouse` \n| https://github.com/ClickHouse/clickhouse-go#dsn[`clickhouse://[username[:password\\]@\\][netloc\\][:port\\]/dbname[?param1=value1\u0026...\u0026paramN=valueN\\]`^] \n\n| `mysql` \n| `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` \n\n| `postgres` \n| `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` \n\n| `mssql` \n| `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` \n\n| `sqlite` \n| `file:/path/to/filename.db[?param\u0026=value1\u0026...]` \n\n| `oracle` \n| `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` \n\n| `snowflake` \n| `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` \n\n| `trino` \n| https://github.com/trinodb/trino-go-client#dsn-data-source-name[`http[s\\]://user[:pass\\]@host[:port\\][?parameters\\]`^] \n\n| `gocosmos` \n| https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage[`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e\\][;Version=\u003ccosmosdb-api-version\u003e\\][;DefaultDb/Db=\u003cdb-name\u003e\\][;AutoId=\u003ctrue/false\u003e\\][;InsecureSkipVerify=\u003ctrue/false\u003e\\]`^] \n\n| `spanner` \n| projects/[PROJECT]/instances/[INSTANCE]/databases/[DATABASE] \n|===\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String[the docs^] for more details. For https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication[key pair authentication^], the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe https://pkg.go.dev/github.com/microsoft/gocosmos[`gocosmos`^] driver is still experimental, but it has support for https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys[hierarchical partition keys^] as well as https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query[cross-partition queries^]. Please refer to the https://github.com/microsoft/gocosmos/blob/main/SQL.md[SQL notes^] for details.","examples":["clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60","foouser:foopassword@tcp(localhost:3306)/foodb","postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable","oracle://foouser:foopass@localhost:1521/service_name"]},{"name":"query","type":"string","kind":"scalar","description":"The query to execute. The style of placeholder to use depends on the driver, some drivers require question marks (`?`) whereas others expect incrementing dollar signs (`$1`, `$2`, and so on) or colons (`:1`, `:2` and so on). The style to use is outlined in this table:\n\n| Driver | Placeholder Style |\n|---|---|\n| `clickhouse` | Dollar sign |\n| `mysql` | Question mark |\n| `postgres` | Dollar sign |\n| `mssql` | Question mark |\n| `sqlite` | Question mark |\n| `oracle` | Colon |\n| `snowflake` | Question mark |\n| `trino` | Question mark |\n| `gocosmos` | Colon |\n","examples":["SELECT * FROM footable WHERE user_id = $1;"]},{"name":"args_mapping","type":"string","kind":"scalar","description":"An optional xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `query`.","is_optional":true,"bloblang":true,"examples":["root = [ this.cat.meow, this.doc.woofs[0] ]","root = [ meta(\"user.id\") ]"]},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"init_files","type":"string","kind":"array","description":"\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":[["./init/*.sql"],["./foo.sql","./bar.sql"]],"version":"4.10.0"},{"name":"init_statement","type":"string","kind":"scalar","description":"\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":["\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n"],"version":"4.10.0"},{"name":"conn_max_idle_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time.","is_advanced":true,"is_optional":true},{"name":"conn_max_life_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age.","is_advanced":true,"is_optional":true},{"name":"conn_max_idle","type":"int","kind":"scalar","description":"An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release.","is_advanced":true,"is_optional":true,"default":2},{"name":"conn_max_open","type":"int","kind":"scalar","description":"An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited).","is_advanced":true,"is_optional":true}]},"version":"4.10.0"},{"name":"sql_select","type":"input","status":"beta","plugin":true,"summary":"Executes a select query and creates a message for each row received.","description":"Once the rows from the query are exhausted this input shuts down, allowing the pipeline to gracefully terminate (or the next input in a xref:components:inputs/sequence.adoc[sequence] to execute).","categories":["Services"],"examples":[{"title":"Consume a Table (PostgreSQL)","summary":"\nHere we define a pipeline that will consume all rows from a table created within the last hour by comparing the unix timestamp stored in the row column \"created_at\":","config":"\ninput:\n sql_select:\n driver: postgres\n dsn: postgres://foouser:foopass@localhost:5432/testdb?sslmode=disable\n table: footable\n columns: [ '*' ]\n where: created_at \u003e= ?\n args_mapping: |\n root = [\n now().ts_unix() - 3600\n ]\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"driver","type":"string","kind":"scalar","description":"A database \u003c\u003cdrivers, driver\u003e\u003e to use.","options":["mysql","postgres","clickhouse","mssql","sqlite","oracle","snowflake","trino","gocosmos","spanner"],"linter":"\nlet options = {\n \"mysql\": true,\n \"postgres\": true,\n \"clickhouse\": true,\n \"mssql\": true,\n \"sqlite\": true,\n \"oracle\": true,\n \"snowflake\": true,\n \"trino\": true,\n \"gocosmos\": true,\n \"spanner\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"dsn","type":"string","kind":"scalar","description":"A Data Source Name to identify the target database.\n\n==== Drivers\n\n:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community, spanner=community\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n|===\n| Driver | Data Source Name Format\n\n| `clickhouse` \n| https://github.com/ClickHouse/clickhouse-go#dsn[`clickhouse://[username[:password\\]@\\][netloc\\][:port\\]/dbname[?param1=value1\u0026...\u0026paramN=valueN\\]`^] \n\n| `mysql` \n| `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` \n\n| `postgres` \n| `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` \n\n| `mssql` \n| `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` \n\n| `sqlite` \n| `file:/path/to/filename.db[?param\u0026=value1\u0026...]` \n\n| `oracle` \n| `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` \n\n| `snowflake` \n| `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` \n\n| `trino` \n| https://github.com/trinodb/trino-go-client#dsn-data-source-name[`http[s\\]://user[:pass\\]@host[:port\\][?parameters\\]`^] \n\n| `gocosmos` \n| https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage[`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e\\][;Version=\u003ccosmosdb-api-version\u003e\\][;DefaultDb/Db=\u003cdb-name\u003e\\][;AutoId=\u003ctrue/false\u003e\\][;InsecureSkipVerify=\u003ctrue/false\u003e\\]`^] \n\n| `spanner` \n| projects/[PROJECT]/instances/[INSTANCE]/databases/[DATABASE] \n|===\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String[the docs^] for more details. For https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication[key pair authentication^], the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe https://pkg.go.dev/github.com/microsoft/gocosmos[`gocosmos`^] driver is still experimental, but it has support for https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys[hierarchical partition keys^] as well as https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query[cross-partition queries^]. Please refer to the https://github.com/microsoft/gocosmos/blob/main/SQL.md[SQL notes^] for details.","examples":["clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60","foouser:foopassword@tcp(localhost:3306)/foodb","postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable","oracle://foouser:foopass@localhost:1521/service_name"]},{"name":"table","type":"string","kind":"scalar","description":"The table to select from.","examples":["foo"]},{"name":"columns","type":"string","kind":"array","description":"A list of columns to select.","examples":[["*"],["foo","bar","baz"]]},{"name":"where","type":"string","kind":"scalar","description":"An optional where clause to add. Placeholder arguments are populated with the `args_mapping` field. Placeholders should always be question marks, and will automatically be converted to dollar syntax when the postgres or clickhouse drivers are used.","is_optional":true,"examples":["type = ? and created_at \u003e ?","user_id = ?"]},{"name":"args_mapping","type":"string","kind":"scalar","description":"An optional xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `where`.","is_optional":true,"bloblang":true,"examples":["root = [ \"article\", now().ts_format(\"2006-01-02\") ]"]},{"name":"prefix","type":"string","kind":"scalar","description":"An optional prefix to prepend to the select query (before SELECT).","is_advanced":true,"is_optional":true},{"name":"suffix","type":"string","kind":"scalar","description":"An optional suffix to append to the select query.","is_advanced":true,"is_optional":true},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"init_files","type":"string","kind":"array","description":"\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":[["./init/*.sql"],["./foo.sql","./bar.sql"]],"version":"4.10.0"},{"name":"init_statement","type":"string","kind":"scalar","description":"\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":["\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n"],"version":"4.10.0"},{"name":"conn_max_idle_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time.","is_advanced":true,"is_optional":true},{"name":"conn_max_life_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age.","is_advanced":true,"is_optional":true},{"name":"conn_max_idle","type":"int","kind":"scalar","description":"An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release.","is_advanced":true,"is_optional":true,"default":2},{"name":"conn_max_open","type":"int","kind":"scalar","description":"An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited).","is_advanced":true,"is_optional":true}]},"version":"3.59.0"},{"name":"stdin","type":"input","status":"stable","plugin":true,"summary":"Consumes data piped to stdin, chopping it into individual messages according to the specified scanner.","categories":["Local"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"codec","type":"string","kind":"scalar","description":"The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.","is_deprecated":true,"is_optional":true,"examples":["lines","delim:\t","delim:foobar","gzip/csv"],"annotated_options":[["auto","EXPERIMENTAL: Attempts to derive a codec for each file based on information such as the extension. For example, a .tar.gz file would be consumed with the `gzip/tar` codec. Defaults to all-bytes."],["all-bytes","Consume the entire file as a single binary message."],["avro-ocf:marshaler=x","EXPERIMENTAL: Consume a stream of Avro OCF datum. The `marshaler` parameter is optional and has the options: `goavro` (default), `json`. Use `goavro` if OCF contains logical types."],["chunker:x","Consume the file in chunks of a given number of bytes."],["csv","Consume structured rows as comma separated values, the first row must be a header row."],["csv:x","Consume structured rows as values separated by a custom delimiter, the first row must be a header row. The custom delimiter must be a single character, e.g. the codec `\"csv:\\t\"` would consume a tab delimited file."],["csv-safe","Consume structured rows like `csv`, but sends messages with empty maps on failure to parse. Includes row number and parsing errors (if any) in the message's metadata."],["csv-safe:x","Consume structured rows like `csv:x` as values separated by a custom delimiter, but sends messages with empty maps on failure to parse. The custom delimiter must be a single character, e.g. the codec `\"csv-safe:\\t\"` would consume a tab delimited file. Includes row number and parsing errors (if any) in the message's metadata."],["delim:x","Consume the file in segments divided by a custom delimiter."],["gzip","Decompress a gzip file, this codec should precede another codec, e.g. `gzip/all-bytes`, `gzip/tar`, `gzip/csv`, etc."],["pgzip","Decompress a gzip file in parallel, this codec should precede another codec, e.g. `pgzip/all-bytes`, `pgzip/tar`, `pgzip/csv`, etc."],["lines","Consume the file in segments divided by linebreaks."],["multipart","Consumes the output of another codec and batches messages together. A batch ends when an empty message is consumed. For example, the codec `lines/multipart` could be used to consume multipart messages where an empty line indicates the end of each batch."],["regex:(?m)^\\d\\d:\\d\\d:\\d\\d","Consume the file in segments divided by regular expression."],["skipbom","Skip one or more byte order marks for each opened reader, this codec should precede another codec, e.g. `skipbom/csv`, etc."],["tar","Parse the file as a tar archive, and consume each file of the archive as a message."]]},{"name":"max_buffer","type":"int","kind":"scalar","is_deprecated":true,"default":1000000},{"name":"scanner","type":"scanner","kind":"scalar","description":"The xref:components:scanners/about.adoc[scanner] by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once.","is_optional":true,"default":{"lines":{}},"version":"4.25.0"},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true}]}},{"name":"subprocess","type":"input","status":"beta","plugin":true,"summary":"Executes a command, runs it as a subprocess, and consumes messages from it over stdout.","description":"\nMessages are consumed according to a specified codec. The command is executed once and if it terminates the input also closes down gracefully. Alternatively, the field `restart_on_close` can be set to `true` in order to have Redpanda Connect re-execute the command each time it stops.\n\nThe field `max_buffer` defines the maximum message size able to be read from the subprocess. This value should be set significantly above the real expected maximum message size.\n\nThe execution environment of the subprocess is the same as the Redpanda Connect instance, including environment variables and the current working directory.","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"name","type":"string","kind":"scalar","description":"The command to execute as a subprocess.","examples":["cat","sed","awk"]},{"name":"args","type":"string","kind":"array","description":"A list of arguments to provide the command.","default":[]},{"name":"codec","type":"string","kind":"scalar","description":"The way in which messages should be consumed from the subprocess.","default":"lines","options":["lines"],"linter":"\nlet options = {\n \"lines\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"restart_on_exit","type":"bool","kind":"scalar","description":"Whether the command should be re-executed each time the subprocess ends.","default":false},{"name":"max_buffer","type":"int","kind":"scalar","description":"The maximum expected size of an individual message.","is_advanced":true,"default":65536}]}},{"name":"timeplus","type":"input","status":"experimental","plugin":true,"summary":"Executes a query on Timeplus Enterprise and creates a message from each row received","description":"\nThis input can execute a query on Timeplus Enterprise Cloud, Timeplus Enterprise (self-hosted) or Timeplusd. A structured message will be created\nfrom each row received.\n\nIf it is a streaming query, this input will keep running until the query is terminated. If it is a table query, this input will shut down once the rows from the query are exhausted.","categories":["Services"],"examples":[{"title":"From Timeplus Enterprise Cloud via HTTP","summary":"You will need to create API Key on Timeplus Enterprise Cloud Web console first and then set the `apikey` field.","config":"\ninput:\n timeplus:\n url: https://us-west-2.timeplus.cloud\n workspace: my_workspace_id\n query: select * from iot\n apikey: \u003cYour API Key\u003e"},{"title":"From Timeplus Enterprise (self-hosted) via HTTP","summary":"For self-housted Timeplus Enterprise, you will need to specify the username and password as well as the URL of the App server","config":"\ninput:\n timeplus:\n url: http://localhost:8000\n workspace: my_workspace_id\n query: select * from iot\n username: username\n password: pw"},{"title":"From Timeplus Enterprise (self-hosted) via TCP","summary":"Make sure the the schema of url is tcp","config":"\ninput:\n timeplus:\n url: tcp://localhost:8463\n query: select * from iot\n username: timeplus\n password: timeplus"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"query","type":"string","kind":"scalar","description":"The query to run","examples":["select * from iot","select count(*) from table(iot)"]},{"name":"url","type":"string","kind":"scalar","description":"The url should always include schema and host.","default":"tcp://localhost:8463","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"workspace","type":"string","kind":"scalar","description":"ID of the workspace. Required when reads from Timeplus Enterprise.","is_optional":true},{"name":"apikey","type":"string","kind":"scalar","description":"The API key. Required when reads from Timeplus Enterprise Cloud","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"username","type":"string","kind":"scalar","description":"The username. Required when reads from Timeplus Enterprise (self-hosted) or Timeplusd","is_optional":true},{"name":"password","type":"string","kind":"scalar","description":"The password. Required when reads from Timeplus Enterprise (self-hosted) or Timeplusd","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}},{"name":"twitter_search","type":"input","status":"experimental","plugin":true,"summary":"Consumes tweets matching a given search using the Twitter recent search V2 API.","description":"Continuously polls the https://developer.twitter.com/en/docs/twitter-api/tweets/search/api-reference/get-tweets-search-recent[Twitter recent search V2 API^] for tweets that match a given search query.\n\nEach tweet received is emitted as a JSON object message, with a field `id` and `text` by default. Extra fields https://developer.twitter.com/en/docs/twitter-api/fields[can be obtained from the search API^] when listed with the `tweet_fields` field.\n\nIn order to paginate requests that are made the ID of the latest received tweet is stored in a xref:components:caches/about.adoc[cache resource], which is then used by subsequent requests to ensure only tweets after it are consumed. It is recommended that the cache you use is persistent so that Redpanda Connect can resume searches at the correct place on a restart.\n\nAuthentication is done using OAuth 2.0 credentials which can be generated within the https://developer.twitter.com[Twitter developer portal^].\n","categories":["Services","Social"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"query","type":"string","kind":"scalar","description":"A search expression to use."},{"name":"tweet_fields","type":"string","kind":"array","description":"An optional list of additional fields to obtain for each tweet, by default only the fields `id` and `text` are returned. For more info refer to the https://developer.twitter.com/en/docs/twitter-api/fields[twitter API docs^].","default":[]},{"name":"poll_period","type":"string","kind":"scalar","description":"The length of time (as a duration string) to wait between each search request. This field can be set empty, in which case requests are made at the limit set by the rate limit. This field also supports cron expressions.","default":"1m"},{"name":"backfill_period","type":"string","kind":"scalar","description":"A duration string indicating the maximum age of tweets to acquire when starting a search.","default":"5m"},{"name":"cache","type":"string","kind":"scalar","description":"A cache resource to use for request pagination."},{"name":"cache_key","type":"string","kind":"scalar","description":"The key identifier used when storing the ID of the last tweet received.","is_advanced":true,"default":"last_tweet_id"},{"name":"rate_limit","type":"string","kind":"scalar","description":"An optional rate limit resource to restrict API requests with.","is_advanced":true,"default":""},{"name":"api_key","type":"string","kind":"scalar","description":"An API key for OAuth 2.0 authentication. It is recommended that you populate this field using xref:configuration:interpolation.adoc[environment variables]."},{"name":"api_secret","type":"string","kind":"scalar","description":"An API secret for OAuth 2.0 authentication. It is recommended that you populate this field using xref:configuration:interpolation.adoc[environment variables]."}]}},{"name":"websocket","type":"input","status":"stable","plugin":true,"summary":"Connects to a websocket server and continuously receives messages.","description":"It is possible to configure an `open_message`, which when set to a non-empty string will be sent to the websocket server each time a connection is first established.","categories":["Network"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL to connect to.","examples":["ws://localhost:4195/get/ws"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"proxy_url","type":"string","kind":"scalar","description":"An optional HTTP proxy URL.","is_advanced":true,"is_optional":true,"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"open_message","type":"string","kind":"scalar","description":"An optional message to send to the server upon connection.","is_advanced":true,"is_optional":true},{"name":"open_message_type","type":"string","kind":"scalar","description":"An optional flag to indicate the data type of open_message.","is_advanced":true,"default":"binary","annotated_options":[["binary","Binary data open_message."],["text","Text data open_message. The text message payload is interpreted as UTF-8 encoded text data."]],"linter":"\nlet options = {\n \"binary\": true,\n \"text\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"auto_replay_nacks","type":"bool","kind":"scalar","description":"Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.","default":true},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"connection","type":"object","kind":"scalar","description":"Customise how websocket connection attempts are made.","is_advanced":true,"is_optional":true,"children":[{"name":"max_retries","type":"int","kind":"scalar","description":"An optional limit to the number of consecutive retry attempts that will be made before abandoning the connection altogether and gracefully terminating the input. When all inputs terminate in this way the service (or stream) will shut down. If set to zero connections will never be reattempted upon a failure. If set below zero this field is ignored (effectively unset).","is_advanced":true,"is_optional":true,"examples":[-1,10]}]},{"name":"oauth","type":"object","kind":"scalar","description":"Allows you to specify open authentication via OAuth version 1.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use OAuth version 1 in requests.","is_advanced":true,"default":false},{"name":"consumer_key","type":"string","kind":"scalar","description":"A value used to identify the client to the service provider.","is_advanced":true,"default":""},{"name":"consumer_secret","type":"string","kind":"scalar","description":"A secret used to establish ownership of the consumer key.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"access_token","type":"string","kind":"scalar","description":"A value used to gain access to the protected resources on behalf of the user.","is_advanced":true,"default":""},{"name":"access_token_secret","type":"string","kind":"scalar","description":"A secret provided in order to establish ownership of a given access token.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"jwt","type":"object","kind":"scalar","description":"BETA: Allows you to specify JWT authentication.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use JWT authentication in requests.","is_advanced":true,"default":false},{"name":"private_key_file","type":"string","kind":"scalar","description":"A file with the PEM encoded via PKCS1 or PKCS8 as private key.","is_advanced":true,"default":""},{"name":"signing_method","type":"string","kind":"scalar","description":"A method used to sign the token such as RS256, RS384, RS512 or EdDSA.","is_advanced":true,"default":""},{"name":"claims","type":"unknown","kind":"map","description":"A value used to identify the claims that issued the JWT.","is_advanced":true,"default":{}},{"name":"headers","type":"unknown","kind":"map","description":"Add optional key/value headers to the JWT.","is_advanced":true,"default":{}}]}]}}],"outputs":[{"name":"amqp_0_9","type":"output","status":"stable","plugin":true,"summary":"Sends messages to an AMQP (0.91) exchange. AMQP is a messaging protocol used by various message brokers, including RabbitMQ.Connects to an AMQP (0.91) queue. AMQP is a messaging protocol used by various message brokers, including RabbitMQ.","description":"The metadata from each message are delivered as headers.\n\nIt's possible for this output type to create the target exchange by setting `exchange_declare.enabled` to `true`, if the exchange already exists then the declaration passively verifies that the settings match.\n\nTLS is automatic when connecting to an `amqps` URL, but custom settings can be enabled in the `tls` section.\n\nThe fields 'key', 'exchange' and 'type' can be dynamically set using xref:configuration:interpolation.adoc#bloblang-queries[function interpolations].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. The first URL to successfully establish a connection will be used until the connection is closed. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["amqp://guest:guest@127.0.0.1:5672/"],["amqp://127.0.0.1:5672/,amqp://127.0.0.2:5672/"],["amqp://127.0.0.1:5672/","amqp://127.0.0.2:5672/"]],"version":"3.58.0","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"exchange","type":"string","kind":"scalar","description":"An AMQP exchange to publish to.","interpolated":true},{"name":"exchange_declare","type":"object","kind":"scalar","description":"Optionally declare the target exchange (passive).","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to declare the exchange.","is_advanced":true,"default":false},{"name":"type","type":"string","kind":"scalar","description":"The type of the exchange.","is_advanced":true,"default":"direct","options":["direct","fanout","topic","headers","x-custom"],"linter":"\nlet options = {\n \"direct\": true,\n \"fanout\": true,\n \"topic\": true,\n \"headers\": true,\n \"x-custom\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"durable","type":"bool","kind":"scalar","description":"Whether the exchange should be durable.","is_advanced":true,"default":true},{"name":"arguments","type":"string","kind":"map","description":"Optional arguments specific to the server's implementation of the exchange that can be sent for exchange types which require extra parameters.","is_advanced":true,"is_optional":true,"examples":[{"alternate-exchange":"my-ae"}]}]},{"name":"key","type":"string","kind":"scalar","description":"The binding key to set for each message.","default":"","interpolated":true},{"name":"type","type":"string","kind":"scalar","description":"The type property to set for each message.","default":"","interpolated":true},{"name":"content_type","type":"string","kind":"scalar","description":"The content type attribute to set for each message.","is_advanced":true,"default":"application/octet-stream","interpolated":true},{"name":"content_encoding","type":"string","kind":"scalar","description":"The content encoding attribute to set for each message.","is_advanced":true,"default":"","interpolated":true},{"name":"correlation_id","type":"string","kind":"scalar","description":"Set the correlation ID of each message with a dynamic interpolated expression.","is_advanced":true,"default":"","interpolated":true},{"name":"reply_to","type":"string","kind":"scalar","description":"Carries response queue name - set with a dynamic interpolated expression.","is_advanced":true,"default":"","interpolated":true},{"name":"expiration","type":"string","kind":"scalar","description":"Set the per-message TTL","is_advanced":true,"default":"","interpolated":true},{"name":"message_id","type":"string","kind":"scalar","description":"Set the message ID of each message with a dynamic interpolated expression.","is_advanced":true,"default":"","interpolated":true},{"name":"user_id","type":"string","kind":"scalar","description":"Set the user ID to the name of the publisher. If this property is set by a publisher, its value must be equal to the name of the user used to open the connection.","is_advanced":true,"default":"","interpolated":true},{"name":"app_id","type":"string","kind":"scalar","description":"Set the application ID of each message with a dynamic interpolated expression.","is_advanced":true,"default":"","interpolated":true},{"name":"metadata","type":"object","kind":"scalar","description":"Specify criteria for which metadata values are attached to messages as headers.","children":[{"name":"exclude_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages.","default":[]}]},{"name":"priority","type":"string","kind":"scalar","description":"Set the priority of each message with a dynamic interpolated expression.","is_advanced":true,"default":"","interpolated":true,"examples":["0","${! meta(\"amqp_priority\") }","${! json(\"doc.priority\") }"]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"persistent","type":"bool","kind":"scalar","description":"Whether message delivery should be persistent (transient by default).","is_advanced":true,"default":false},{"name":"mandatory","type":"bool","kind":"scalar","description":"Whether to set the mandatory flag on published messages. When set if a published message is routed to zero queues it is returned.","is_advanced":true,"default":false},{"name":"immediate","type":"bool","kind":"scalar","description":"Whether to set the immediate flag on published messages. When set if there are no ready consumers of a queue then the message is dropped instead of waiting.","is_advanced":true,"default":false},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period to wait before abandoning it and reattempting. If not set, wait indefinitely.","is_advanced":true,"default":""},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]}]}},{"name":"amqp_1","type":"output","status":"stable","plugin":true,"summary":"Sends messages to an AMQP (1.0) server.","description":"\n== Metadata\n\nMessage metadata is added to each AMQP message as string annotations. In order to control which metadata keys are added use the `metadata` config field.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"A URL to connect to.","is_deprecated":true,"is_optional":true,"examples":["amqp://localhost:5672/","amqps://guest:guest@localhost:5672/"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. The first URL to successfully establish a connection will be used until the connection is closed. If an item of the list contains commas it will be expanded into multiple URLs.","is_optional":true,"examples":[["amqp://guest:guest@127.0.0.1:5672/"],["amqp://127.0.0.1:5672/,amqp://127.0.0.2:5672/"],["amqp://127.0.0.1:5672/","amqp://127.0.0.2:5672/"]],"version":"4.23.0","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"target_address","type":"string","kind":"scalar","description":"The target address to write to.","examples":["/foo","queue:/bar","topic:/baz"]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"application_properties_map","type":"string","kind":"scalar","description":"An optional Bloblang mapping that can be defined in order to set the `application-properties` on output messages.","is_advanced":true,"is_optional":true,"bloblang":true},{"name":"sasl","type":"object","kind":"scalar","description":"Enables SASL authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"mechanism","type":"string","kind":"scalar","description":"The SASL authentication mechanism to use.","is_advanced":true,"default":"none","annotated_options":[["anonymous","Anonymous SASL authentication."],["none","No SASL based authentication."],["plain","Plain text SASL authentication."]],"linter":"\nlet options = {\n \"anonymous\": true,\n \"none\": true,\n \"plain\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"user","type":"string","kind":"scalar","description":"A SASL plain text username. It is recommended that you use environment variables to populate this field.","is_advanced":true,"default":"","examples":["${USER}"]},{"name":"password","type":"string","kind":"scalar","description":"A SASL plain text password. It is recommended that you use environment variables to populate this field.","is_advanced":true,"is_secret":true,"default":"","examples":["${PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"metadata","type":"object","kind":"scalar","description":"Specify criteria for which metadata values are attached to messages as headers.","children":[{"name":"exclude_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages.","default":[]}]},{"name":"content_type","type":"string","kind":"scalar","description":"Specify the message body content type. The option `string` will transfer the message as an AMQP value of type string. Consider choosing the option `string` if your intention is to transfer UTF-8 string messages (like JSON messages) to the destination.","is_advanced":true,"default":"opaque_binary","options":["opaque_binary","string"],"linter":"\nlet options = {\n \"opaque_binary\": true,\n \"string\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"}],"linter":"\nroot = if this.url.or(\"\") == \"\" \u0026\u0026 this.urls.or([]).length() == 0 {\n \"field 'urls' must be set\"\n}\n"}},{"name":"aws_dynamodb","type":"output","status":"stable","plugin":true,"summary":"Inserts items into a DynamoDB table.","description":"\nThe field `string_columns` is a map of column names to string values, where the values are xref:configuration:interpolation.adoc#bloblang-queries[function interpolated] per message of a batch. This allows you to populate string columns of an item by extracting fields within the document payload or metadata like follows:\n\n```yml\nstring_columns:\n id: ${!json(\"id\")}\n title: ${!json(\"body.title\")}\n topic: ${!meta(\"kafka_topic\")}\n full_content: ${!content()}\n```\n\nThe field `json_map_columns` is a map of column names to json paths, where the xref:configuration:field_paths.adoc[dot path] is extracted from each document and converted into a map value. Both an empty path and the path `.` are interpreted as the root of the document. This allows you to populate map columns of an item like follows:\n\n```yml\njson_map_columns:\n user: path.to.user\n whole_document: .\n```\n\nA column name can be empty:\n\n```yml\njson_map_columns:\n \"\": .\n```\n\nIn which case the top level document fields will be written at the root of the item, potentially overwriting previously defined column values. If a path is not found within a document the column will not be populated.\n\n== Credentials\n\nBy default Redpanda Connect will use a shared credentials file when connecting to AWS services. It's also possible to set them explicitly at the component level, allowing you to transfer data across accounts. You can find out more in xref:guides:cloud/aws.adoc[].\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].\n","categories":["Services","AWS"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"table","type":"string","kind":"scalar","description":"The table to store messages in."},{"name":"string_columns","type":"string","kind":"map","description":"A map of column keys to string values to store.","default":{},"interpolated":true,"examples":[{"full_content":"${!content()}","id":"${!json(\"id\")}","title":"${!json(\"body.title\")}","topic":"${!meta(\"kafka_topic\")}"}]},{"name":"json_map_columns","type":"string","kind":"map","description":"A map of column keys to xref:configuration:field_paths.adoc[field paths] pointing to value data within messages.","default":{},"examples":[{"user":"path.to.user","whole_document":"."},{"":"."}]},{"name":"ttl","type":"string","kind":"scalar","description":"An optional TTL to set for items, calculated from the moment the message is sent.","is_advanced":true,"default":""},{"name":"ttl_key","type":"string","kind":"scalar","description":"The column key to place the TTL value within.","is_advanced":true,"default":""},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]},{"name":"max_retries","type":"int","kind":"scalar","description":"The maximum number of retries before giving up on the request. If set to zero there is no discrete limit.","is_advanced":true,"default":3},{"name":"backoff","type":"object","kind":"scalar","description":"Control time intervals between retry attempts.","is_advanced":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"1s"},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts.","is_advanced":true,"default":"5s"},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum period to wait before retry attempts are abandoned. If zero then no limit is used.","is_advanced":true,"default":"30s"}]}]},"version":"3.36.0"},{"name":"aws_kinesis","type":"output","status":"stable","plugin":true,"summary":"Sends messages to a Kinesis stream.","description":"\nBoth the `partition_key`(required) and `hash_key` (optional) fields can be dynamically set using function interpolations described xref:configuration:interpolation.adoc#bloblang-queries[here]. When sending batched messages the interpolations are performed per message part.\n\n== Credentials\n\nBy default Redpanda Connect will use a shared credentials file when connecting to AWS services. It's also possible to set them explicitly at the component level, allowing you to transfer data across accounts. You can find out more in xref:guides:cloud/aws.adoc[].\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services","AWS"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"stream","type":"string","kind":"scalar","description":"The stream to publish messages to. Streams can either be specified by their name or full ARN.","examples":["foo","arn:aws:kinesis:*:111122223333:stream/my-stream"]},{"name":"partition_key","type":"string","kind":"scalar","description":"A required key for partitioning messages.","interpolated":true},{"name":"hash_key","type":"string","kind":"scalar","description":"A optional hash key for partitioning messages.","is_advanced":true,"is_optional":true,"interpolated":true},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of parallel message batches to have in flight at any given time.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]},{"name":"max_retries","type":"int","kind":"scalar","description":"The maximum number of retries before giving up on the request. If set to zero there is no discrete limit.","is_advanced":true,"default":0},{"name":"backoff","type":"object","kind":"scalar","description":"Control time intervals between retry attempts.","is_advanced":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"1s"},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts.","is_advanced":true,"default":"5s"},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum period to wait before retry attempts are abandoned. If zero then no limit is used.","is_advanced":true,"default":"30s"}]}]},"version":"3.36.0"},{"name":"aws_kinesis_firehose","type":"output","status":"stable","plugin":true,"summary":"Sends messages to a Kinesis Firehose delivery stream.","description":"\n== Credentials\n\nBy default Redpanda Connect will use a shared credentials file when connecting to AWS services. It's also possible to set them explicitly at the component level, allowing you to transfer data across accounts. You can find out more in xref:guides:cloud/aws.adoc[].\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].\n","categories":["Services","AWS"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"stream","type":"string","kind":"scalar","description":"The stream to publish messages to."},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]},{"name":"max_retries","type":"int","kind":"scalar","description":"The maximum number of retries before giving up on the request. If set to zero there is no discrete limit.","is_advanced":true,"default":0},{"name":"backoff","type":"object","kind":"scalar","description":"Control time intervals between retry attempts.","is_advanced":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"1s"},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts.","is_advanced":true,"default":"5s"},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum period to wait before retry attempts are abandoned. If zero then no limit is used.","is_advanced":true,"default":"30s"}]}]},"version":"3.36.0"},{"name":"aws_s3","type":"output","status":"stable","plugin":true,"summary":"Sends message parts as objects to an Amazon S3 bucket. Each object is uploaded with the path specified with the `path` field.","description":"\nIn order to have a different path for each object you should use function interpolations described in xref:configuration:interpolation.adoc#bloblang-queries[Bloblang queries], which are calculated per message of a batch.\n\n== Metadata\n\nMetadata fields on messages will be sent as headers, in order to mutate these values (or remove them) check out the xref:configuration:metadata.adoc[metadata docs].\n\n== Tags\n\nThe tags field allows you to specify key/value pairs to attach to objects as tags, where the values support xref:configuration:interpolation.adoc#bloblang-queries[interpolation functions]:\n\n```yaml\noutput:\n aws_s3:\n bucket: TODO\n path: ${!counter()}-${!timestamp_unix_nano()}.tar.gz\n tags:\n Key1: Value1\n Timestamp: ${!meta(\"Timestamp\")}\n```\n\n=== Credentials\n\nBy default Redpanda Connect will use a shared credentials file when connecting to AWS services. It's also possible to set them explicitly at the component level, allowing you to transfer data across accounts. You can find out more in xref:guides:cloud/aws.adoc[].\n\n== Batching\n\nIt's common to want to upload messages to S3 as batched archives, the easiest way to do this is to batch your messages at the output level and join the batch of messages with an xref:components:processors/archive.adoc[`archive`] and/or xref:components:processors/compress.adoc[`compress`] processor.\n\nFor example, if we wished to upload messages as a .tar.gz archive of documents we could achieve that with the following config:\n\n```yaml\noutput:\n aws_s3:\n bucket: TODO\n path: ${!counter()}-${!timestamp_unix_nano()}.tar.gz\n batching:\n count: 100\n period: 10s\n processors:\n - archive:\n format: tar\n - compress:\n algorithm: gzip\n```\n\nAlternatively, if we wished to upload JSON documents as a single large document containing an array of objects we can do that with:\n\n```yaml\noutput:\n aws_s3:\n bucket: TODO\n path: ${!counter()}-${!timestamp_unix_nano()}.json\n batching:\n count: 100\n processors:\n - archive:\n format: json_array\n```\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.","categories":["Services","AWS"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"bucket","type":"string","kind":"scalar","description":"The bucket to upload messages to."},{"name":"path","type":"string","kind":"scalar","description":"The path of each message to upload.","default":"${!counter()}-${!timestamp_unix_nano()}.txt","interpolated":true,"examples":["${!counter()}-${!timestamp_unix_nano()}.txt","${!meta(\"kafka_key\")}.json","${!json(\"doc.namespace\")}/${!json(\"doc.id\")}.json"]},{"name":"tags","type":"string","kind":"map","description":"Key/value pairs to store with the object as tags.","default":{},"interpolated":true,"examples":[{"Key1":"Value1","Timestamp":"${!meta(\"Timestamp\")}"}]},{"name":"content_type","type":"string","kind":"scalar","description":"The content type to set for each object.","default":"application/octet-stream","interpolated":true},{"name":"content_encoding","type":"string","kind":"scalar","description":"An optional content encoding to set for each object.","is_advanced":true,"default":"","interpolated":true},{"name":"cache_control","type":"string","kind":"scalar","description":"The cache control to set for each object.","is_advanced":true,"default":"","interpolated":true},{"name":"content_disposition","type":"string","kind":"scalar","description":"The content disposition to set for each object.","is_advanced":true,"default":"","interpolated":true},{"name":"content_language","type":"string","kind":"scalar","description":"The content language to set for each object.","is_advanced":true,"default":"","interpolated":true},{"name":"content_md5","type":"string","kind":"scalar","description":"The content MD5 to set for each object.","is_advanced":true,"default":"","interpolated":true},{"name":"website_redirect_location","type":"string","kind":"scalar","description":"The website redirect location to set for each object.","is_advanced":true,"default":"","interpolated":true},{"name":"metadata","type":"object","kind":"scalar","description":"Specify criteria for which metadata values are attached to objects as headers.","children":[{"name":"exclude_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages.","default":[]}]},{"name":"storage_class","type":"string","kind":"scalar","description":"The storage class to set for each object.","is_advanced":true,"default":"STANDARD","interpolated":true,"options":["STANDARD","REDUCED_REDUNDANCY","GLACIER","STANDARD_IA","ONEZONE_IA","INTELLIGENT_TIERING","DEEP_ARCHIVE"],"linter":"\nlet options = {\n \"standard\": true,\n \"reduced_redundancy\": true,\n \"glacier\": true,\n \"standard_ia\": true,\n \"onezone_ia\": true,\n \"intelligent_tiering\": true,\n \"deep_archive\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"kms_key_id","type":"string","kind":"scalar","description":"An optional server side encryption key.","is_advanced":true,"default":""},{"name":"checksum_algorithm","type":"string","kind":"scalar","description":"The algorithm used to create the checksum for each object.","is_advanced":true,"default":"","options":["CRC32","CRC32C","SHA1","SHA256"],"linter":"\nlet options = {\n \"crc32\": true,\n \"crc32c\": true,\n \"sha1\": true,\n \"sha256\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"server_side_encryption","type":"string","kind":"scalar","description":"An optional server side encryption algorithm.","is_advanced":true,"default":"","version":"3.63.0"},{"name":"force_path_style_urls","type":"bool","kind":"scalar","description":"Forces the client API to use path style URLs, which helps when connecting to custom endpoints.","is_advanced":true,"default":false},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period to wait on an upload before abandoning it and reattempting.","is_advanced":true,"default":"5s"},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]},"version":"3.36.0"},{"name":"aws_sns","type":"output","status":"stable","plugin":true,"summary":"Sends messages to an AWS SNS topic.","description":"\n== Credentials\n\nBy default Redpanda Connect will use a shared credentials file when connecting to AWS services. It's also possible to set them explicitly at the component level, allowing you to transfer data across accounts. You can find out more in xref:guides:cloud/aws.adoc[].\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.","categories":["Services","AWS"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"topic_arn","type":"string","kind":"scalar","description":"The topic to publish to."},{"name":"message_group_id","type":"string","kind":"scalar","description":"An optional group ID to set for messages.","is_optional":true,"interpolated":true,"version":"3.60.0"},{"name":"message_deduplication_id","type":"string","kind":"scalar","description":"An optional deduplication ID to set for messages.","is_optional":true,"interpolated":true,"version":"3.60.0"},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"metadata","type":"object","kind":"scalar","description":"Specify criteria for which metadata values are sent as headers.","children":[{"name":"exclude_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages.","default":[]}],"version":"3.60.0"},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period to wait on an upload before abandoning it and reattempting.","is_advanced":true,"default":"5s"},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]},"version":"3.36.0"},{"name":"aws_sqs","type":"output","status":"stable","plugin":true,"summary":"Sends messages to an SQS queue.","description":"\nMetadata values are sent along with the payload as attributes with the data type String. If the number of metadata values in a message exceeds the message attribute limit (10) then the top ten keys ordered alphabetically will be selected.\n\nThe fields `message_group_id`, `message_deduplication_id` and `delay_seconds` can be set dynamically using xref:configuration:interpolation.adoc#bloblang-queries[function interpolations], which are resolved individually for each message of a batch.\n\n== Credentials\n\nBy default Redpanda Connect will use a shared credentials file when connecting to AWS services. It's also possible to set them explicitly at the component level, allowing you to transfer data across accounts. You can find out more in xref:guides:cloud/aws.adoc[].\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services","AWS"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target SQS queue.","interpolated":true},{"name":"message_group_id","type":"string","kind":"scalar","description":"An optional group ID to set for messages.","is_optional":true,"interpolated":true},{"name":"message_deduplication_id","type":"string","kind":"scalar","description":"An optional deduplication ID to set for messages.","is_optional":true,"interpolated":true},{"name":"delay_seconds","type":"string","kind":"scalar","description":"An optional delay time in seconds for message. Value between 0 and 900","is_optional":true,"interpolated":true},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of parallel message batches to have in flight at any given time.","default":64},{"name":"metadata","type":"object","kind":"scalar","description":"Specify criteria for which metadata values are sent as headers.","children":[{"name":"exclude_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages.","default":[]}]},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"max_records_per_request","type":"int","kind":"scalar","description":"Customize the maximum number of records delivered in a single SQS request. This value must be greater than 0 but no greater than 10.","is_advanced":true,"default":10,"linter":"if this \u003c= 0 || this \u003e 10 { \"this field must be \u003e0 and \u003c=10\" } "},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]},{"name":"max_retries","type":"int","kind":"scalar","description":"The maximum number of retries before giving up on the request. If set to zero there is no discrete limit.","is_advanced":true,"default":0},{"name":"backoff","type":"object","kind":"scalar","description":"Control time intervals between retry attempts.","is_advanced":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"1s"},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts.","is_advanced":true,"default":"5s"},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum period to wait before retry attempts are abandoned. If zero then no limit is used.","is_advanced":true,"default":"30s"}]}]},"version":"3.36.0"},{"name":"azure_blob_storage","type":"output","status":"beta","plugin":true,"summary":"Sends message parts as objects to an Azure Blob Storage Account container. Each object is uploaded with the filename specified with the `container` field.","description":"\nIn order to have a different path for each object you should use function\ninterpolations described xref:configuration:interpolation.adoc#bloblang-queries[here], which are\ncalculated per message of a batch.\n\nSupports multiple authentication methods but only one of the following is required:\n\n- `storage_connection_string`\n- `storage_account` and `storage_access_key`\n- `storage_account` and `storage_sas_token`\n- `storage_account` to access via https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity#DefaultAzureCredential[DefaultAzureCredential^]\n\nIf multiple are set then the `storage_connection_string` is given priority.\n\nIf the `storage_connection_string` does not contain the `AccountName` parameter, please specify it in the\n`storage_account` field.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.","categories":["Services","Azure"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"storage_account","type":"string","kind":"scalar","description":"The storage account to access. This field is ignored if `storage_connection_string` is set.","default":""},{"name":"storage_access_key","type":"string","kind":"scalar","description":"The storage account access key. This field is ignored if `storage_connection_string` is set.","default":""},{"name":"storage_connection_string","type":"string","kind":"scalar","description":"A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set.","default":""},{"name":"storage_sas_token","type":"string","kind":"scalar","description":"The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set.","default":""},{"name":"container","type":"string","kind":"scalar","description":"The container for uploading the messages to.","interpolated":true,"examples":["messages-${!timestamp(\"2006\")}"]},{"name":"path","type":"string","kind":"scalar","description":"The path of each message to upload.","default":"${!counter()}-${!timestamp_unix_nano()}.txt","interpolated":true,"examples":["${!counter()}-${!timestamp_unix_nano()}.json","${!meta(\"kafka_key\")}.json","${!json(\"doc.namespace\")}/${!json(\"doc.id\")}.json"]},{"name":"blob_type","type":"string","kind":"scalar","description":"Block and Append blobs are comprized of blocks, and each blob can support up to 50,000 blocks. The default value is `+\"`BLOCK`\"+`.`","is_advanced":true,"default":"BLOCK","interpolated":true,"options":["BLOCK","APPEND"],"linter":"\nlet options = {\n \"block\": true,\n \"append\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"public_access_level","type":"string","kind":"scalar","description":"The container's public access level. The default value is `PRIVATE`.","is_advanced":true,"default":"PRIVATE","interpolated":true,"options":["PRIVATE","BLOB","CONTAINER"],"linter":"\nlet options = {\n \"private\": true,\n \"blob\": true,\n \"container\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64}],"linter":"root = if this.storage_connection_string != \"\" \u0026\u0026 !this.storage_connection_string.contains(\"AccountName=\") \u0026\u0026 !this.storage_connection_string.contains(\"UseDevelopmentStorage=true;\") \u0026\u0026 this.storage_account == \"\" { [ \"storage_account must be set if storage_connection_string does not contain the \\\"AccountName\\\" parameter\" ] }"},"version":"3.36.0"},{"name":"azure_cosmosdb","type":"output","status":"experimental","plugin":true,"summary":"Creates or updates messages as JSON documents in https://learn.microsoft.com/en-us/azure/cosmos-db/introduction[Azure CosmosDB^].","description":"\nWhen creating documents, each message must have the `id` property (case-sensitive) set (or use `auto_id: true`). It is the unique name that identifies the document, that is, no two documents share the same `id` within a logical partition. The `id` field must not exceed 255 characters. https://learn.microsoft.com/en-us/rest/api/cosmos-db/documents[See details^].\n\nThe `partition_keys` field must resolve to the same value(s) across the entire message batch.\n\n\n== Credentials\n\nYou can use one of the following authentication mechanisms:\n\n- Set the `endpoint` field and the `account_key` field\n- Set only the `endpoint` field to use https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity#DefaultAzureCredential[DefaultAzureCredential^]\n- Set the `connection_string` field\n\n\n== Batching\n\nCosmosDB limits the maximum batch size to 100 messages and the payload must not exceed 2MB (https://learn.microsoft.com/en-us/azure/cosmos-db/concepts-limits#per-request-limits[details here^]).\n\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Azure"],"footnotes":"\n\n== CosmosDB emulator\n\nIf you wish to run the CosmosDB emulator that is referenced in the documentation https://learn.microsoft.com/en-us/azure/cosmos-db/linux-emulator[here^], the following Docker command should do the trick:\n\n```bash\n\u003e docker run --rm -it -p 8081:8081 --name=cosmosdb -e AZURE_COSMOS_EMULATOR_PARTITION_COUNT=10 -e AZURE_COSMOS_EMULATOR_ENABLE_DATA_PERSISTENCE=false mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator\n```\n\nNote: `AZURE_COSMOS_EMULATOR_PARTITION_COUNT` controls the number of partitions that will be supported by the emulator. The bigger the value, the longer it takes for the container to start up.\n\nAdditionally, instead of installing the container self-signed certificate which is exposed via `https://localhost:8081/_explorer/emulator.pem`, you can run https://mitmproxy.org/[mitmproxy^] like so:\n\n```bash\n\u003e mitmproxy -k --mode \"reverse:https://localhost:8081\"\n```\n\nThen you can access the CosmosDB UI via `http://localhost:8080/_explorer/index.html` and use `http://localhost:8080` as the CosmosDB endpoint.\n","examples":[{"title":"Create documents","summary":"Create new documents in the `blobfish` container with partition key `/habitat`.","config":"\noutput:\n azure_cosmosdb:\n endpoint: http://localhost:8080\n account_key: C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==\n database: blobbase\n container: blobfish\n partition_keys_map: root = json(\"habitat\")\n operation: Create\n"},{"title":"Patch documents","summary":"Execute the Patch operation on documents from the `blobfish` container.","config":"\noutput:\n azure_cosmosdb:\n endpoint: http://localhost:8080\n account_key: C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==\n database: testdb\n container: blobfish\n partition_keys_map: root = json(\"habitat\")\n item_id: ${! json(\"id\") }\n operation: Patch\n patch_operations:\n # Add a new /diet field\n - operation: Add\n path: /diet\n value_map: root = json(\"diet\")\n # Remove the first location from the /locations array field\n - operation: Remove\n path: /locations/0\n # Add new location at the end of the /locations array field\n - operation: Add\n path: /locations/-\n value_map: root = \"Challenger Deep\"\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"endpoint","type":"string","kind":"scalar","description":"CosmosDB endpoint.","is_optional":true,"examples":["https://localhost:8081"]},{"name":"account_key","type":"string","kind":"scalar","description":"Account key.","is_optional":true,"is_secret":true,"examples":["C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw=="],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"connection_string","type":"string","kind":"scalar","description":"Connection string.","is_optional":true,"is_secret":true,"examples":["AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==;"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"database","type":"string","kind":"scalar","description":"Database.","examples":["testdb"]},{"name":"container","type":"string","kind":"scalar","description":"Container.","examples":["testcontainer"]},{"name":"partition_keys_map","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to a single partition key value or an array of partition key values of type string, integer or boolean. Currently, hierarchical partition keys are not supported so only one value may be provided.","bloblang":true,"examples":["root = \"blobfish\"","root = 41","root = true","root = null","root = json(\"blobfish\").depth"]},{"name":"operation","type":"string","kind":"scalar","description":"Operation.","default":"Create","annotated_options":[["Create","Create operation."],["Delete","Delete operation."],["Patch","Patch operation."],["Replace","Replace operation."],["Upsert","Upsert operation."]],"linter":"\nlet options = {\n \"create\": true,\n \"delete\": true,\n \"patch\": true,\n \"replace\": true,\n \"upsert\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"patch_operations","type":"object","kind":"array","description":"Patch operations to be performed when `operation: Patch` .","is_advanced":true,"is_optional":true,"children":[{"name":"operation","type":"string","kind":"scalar","description":"Operation.","is_advanced":true,"default":"Add","annotated_options":[["Add","Add patch operation."],["Increment","Increment patch operation."],["Remove","Remove patch operation."],["Replace","Replace patch operation."],["Set","Set patch operation."]],"linter":"\nlet options = {\n \"add\": true,\n \"increment\": true,\n \"remove\": true,\n \"replace\": true,\n \"set\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"path","type":"string","kind":"scalar","description":"Path.","is_advanced":true,"examples":["/foo/bar/baz"]},{"name":"value_map","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to a value of any type that is supported by CosmosDB.","is_advanced":true,"is_optional":true,"bloblang":true,"examples":["root = \"blobfish\"","root = 41","root = true","root = json(\"blobfish\").depth","root = [1, 2, 3]"]}]},{"name":"patch_condition","type":"string","kind":"scalar","description":"Patch operation condition.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["from c where not is_defined(c.blobfish)"]},{"name":"auto_id","type":"bool","kind":"scalar","description":"Automatically set the item `id` field to a random UUID v4. If the `id` field is already set, then it will not be overwritten. Setting this to `false` can improve performance, since the messages will not have to be parsed.","is_advanced":true,"default":true},{"name":"item_id","type":"string","kind":"scalar","description":"ID of item to replace or delete. Only used by the Replace and Delete operations","is_optional":true,"interpolated":true,"examples":["${! json(\"id\") }"]},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64}],"linter":"root = []\nlet hasEndpoint = this.endpoint.or(\"\") != \"\"\nlet hasConnectionString = this.connection_string.or(\"\") != \"\"\n\nroot.\"-\" = if !$hasEndpoint \u0026\u0026 !$hasConnectionString {\n \"Either `endpoint` or `connection_string` must be set.\"\n}\n\nlet hasItemID = this.item_id.or(\"\") != \"\"\nlet hasPatchOperations = this.patch_operations.length().or(0) \u003e 0\nlet hasPatchCondition = this.patch_condition.or(\"\") != \"\"\n\nroot.\"-\" = if !$hasItemID \u0026\u0026 (this.operation == \"Replace\" || this.operation == \"Delete\" || this.operation == \"Read\" || this.operation == \"Patch\") {\n \"The `item_id` field must be set for Replace, Delete, Read and Patch operations.\"\n}\n\nroot.\"-\" = if this.operation == \"Patch\" \u0026\u0026 !$hasPatchOperations {\n \"At least one `patch_operations` must be set when `operation: Patch`.\"\n}\n\nroot.\"-\" = if $hasPatchCondition \u0026\u0026 (!$hasPatchOperations || this.operation != \"Patch\") {\n \"The `patch_condition` field only applies to `Patch` operations and it requires one or more `patch_operations`.\"\n}\n\nroot.\"-\" = if this.operation == \"Patch\" \u0026\u0026 this.patch_operations.any(o -\u003e o.operation != \"Remove\" \u0026\u0026 o.value_map.or(\"\") == \"\") {\n \"The `patch_operations` `value_map` field must be set when `operation` is not `Remove`.\"\n}\n\nroot.\"-\" = if this.operation == \"Patch\" \u0026\u0026 this.patch_operations.any(o -\u003e o.operation == \"Remove\" \u0026\u0026 o.value_map.or(\"\") != \"\") {\n \"The `patch_operations` `value_map` field must not be set when `operation` is `Remove`.\"\n}\n"},"version":"v4.25.0"},{"name":"azure_data_lake_gen2","type":"output","status":"beta","plugin":true,"summary":"Sends message parts as files to an Azure Data Lake Gen2 filesystem. Each file is uploaded with the filename specified with the `path` field.","description":"\nIn order to have a different path for each file you should use function\ninterpolations described xref:configuration:interpolation.adoc#bloblang-queries[here], which are\ncalculated per message of a batch.\n\nSupports multiple authentication methods but only one of the following is required:\n\n- `storage_connection_string`\n- `storage_account` and `storage_access_key`\n- `storage_account` and `storage_sas_token`\n- `storage_account` to access via https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity#DefaultAzureCredential[DefaultAzureCredential^]\n\nIf multiple are set then the `storage_connection_string` is given priority.\n\nIf the `storage_connection_string` does not contain the `AccountName` parameter, please specify it in the\n`storage_account` field.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.","categories":["Services","Azure"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"storage_account","type":"string","kind":"scalar","description":"The storage account to access. This field is ignored if `storage_connection_string` is set.","default":""},{"name":"storage_access_key","type":"string","kind":"scalar","description":"The storage account access key. This field is ignored if `storage_connection_string` is set.","default":""},{"name":"storage_connection_string","type":"string","kind":"scalar","description":"A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set.","default":""},{"name":"storage_sas_token","type":"string","kind":"scalar","description":"The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set.","default":""},{"name":"filesystem","type":"string","kind":"scalar","description":"The data lake storage filesystem name for uploading the messages to.","interpolated":true,"examples":["messages-${!timestamp(\"2006\")}"]},{"name":"path","type":"string","kind":"scalar","description":"The path of each message to upload within the filesystem.","default":"${!counter()}-${!timestamp_unix_nano()}.txt","interpolated":true,"examples":["${!counter()}-${!timestamp_unix_nano()}.json","${!meta(\"kafka_key\")}.json","${!json(\"doc.namespace\")}/${!json(\"doc.id\")}.json"]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64}],"linter":"root = if this.storage_connection_string != \"\" \u0026\u0026 !this.storage_connection_string.contains(\"AccountName=\") \u0026\u0026 !this.storage_connection_string.contains(\"UseDevelopmentStorage=true;\") \u0026\u0026 this.storage_account == \"\" { [ \"storage_account must be set if storage_connection_string does not contain the \\\"AccountName\\\" parameter\" ] }"},"version":"4.38.0"},{"name":"azure_queue_storage","type":"output","status":"beta","plugin":true,"summary":"Sends messages to an Azure Storage Queue.","description":"\nOnly one authentication method is required, `storage_connection_string` or `storage_account` and `storage_access_key`. If both are set then the `storage_connection_string` is given priority.\n\nIn order to set the `queue_name` you can use function interpolations described xref:configuration:interpolation.adoc#bloblang-queries[here], which are calculated per message of a batch.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services","Azure"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"storage_account","type":"string","kind":"scalar","description":"The storage account to access. This field is ignored if `storage_connection_string` is set.","default":""},{"name":"storage_access_key","type":"string","kind":"scalar","description":"The storage account access key. This field is ignored if `storage_connection_string` is set.","default":""},{"name":"storage_connection_string","type":"string","kind":"scalar","description":"A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set.","default":""},{"name":"storage_sas_token","type":"string","kind":"scalar","description":"The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set.","default":""},{"name":"queue_name","type":"string","kind":"scalar","description":"The name of the target Queue Storage queue.","interpolated":true},{"name":"ttl","type":"string","kind":"scalar","description":"The TTL of each individual message as a duration string. Defaults to 0, meaning no retention period is set","is_advanced":true,"default":"","interpolated":true,"examples":["60s","5m","36h"]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of parallel message batches to have in flight at any given time.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}],"linter":"root = if this.storage_connection_string != \"\" \u0026\u0026 !this.storage_connection_string.contains(\"AccountName=\") \u0026\u0026 !this.storage_connection_string.contains(\"UseDevelopmentStorage=true;\") \u0026\u0026 this.storage_account == \"\" { [ \"storage_account must be set if storage_connection_string does not contain the \\\"AccountName\\\" parameter\" ] }"},"version":"3.36.0"},{"name":"azure_table_storage","type":"output","status":"beta","plugin":true,"summary":"Stores messages in an Azure Table Storage table.","description":"\nOnly one authentication method is required, `storage_connection_string` or `storage_account` and `storage_access_key`. If both are set then the `storage_connection_string` is given priority.\n\nIn order to set the `table_name`, `partition_key` and `row_key` you can use function interpolations described xref:configuration:interpolation.adoc#bloblang-queries[here], which are calculated per message of a batch.\n\nIf the `properties` are not set in the config, all the `json` fields are marshalled and stored in the table, which will be created if it does not exist.\n\nThe `object` and `array` fields are marshaled as strings. e.g.:\n\nThe JSON message:\n\n```json\n{\n \"foo\": 55,\n \"bar\": {\n \"baz\": \"a\",\n \"bez\": \"b\"\n },\n \"diz\": [\"a\", \"b\"]\n}\n```\n\nWill store in the table the following properties:\n\n```yml\nfoo: '55'\nbar: '{ \"baz\": \"a\", \"bez\": \"b\" }'\ndiz: '[\"a\", \"b\"]'\n```\n\nIt's also possible to use function interpolations to get or transform the properties values, e.g.:\n\n```yml\nproperties:\n device: '${! json(\"device\") }'\n timestamp: '${! json(\"timestamp\") }'\n```\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services","Azure"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"storage_account","type":"string","kind":"scalar","description":"The storage account to access. This field is ignored if `storage_connection_string` is set.","default":""},{"name":"storage_access_key","type":"string","kind":"scalar","description":"The storage account access key. This field is ignored if `storage_connection_string` is set.","default":""},{"name":"storage_connection_string","type":"string","kind":"scalar","description":"A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set.","default":""},{"name":"storage_sas_token","type":"string","kind":"scalar","description":"The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set.","default":""},{"name":"table_name","type":"string","kind":"scalar","description":"The table to store messages into.","interpolated":true,"examples":["${! meta(\"kafka_topic\") }","${! json(\"table\") }"]},{"name":"partition_key","type":"string","kind":"scalar","description":"The partition key.","default":"","interpolated":true,"examples":["${! json(\"date\") }"]},{"name":"row_key","type":"string","kind":"scalar","description":"The row key.","default":"","interpolated":true,"examples":["${! json(\"device\")}-${!uuid_v4() }"]},{"name":"properties","type":"string","kind":"map","description":"A map of properties to store into the table.","default":{},"interpolated":true},{"name":"insert_type","type":"string","kind":"scalar","description":"Type of insert operation. Valid options are `INSERT`, `INSERT_MERGE` and `INSERT_REPLACE`","is_advanced":true,"is_deprecated":true,"default":"","interpolated":true,"examples":["${! json(\"operation\") }","${! meta(\"operation\") }","INSERT"],"options":["INSERT","INSERT_MERGE","INSERT_REPLACE"],"linter":"\nlet options = {\n \"insert\": true,\n \"insert_merge\": true,\n \"insert_replace\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"transaction_type","type":"string","kind":"scalar","description":"Type of transaction operation.","is_advanced":true,"default":"INSERT","interpolated":true,"examples":["${! json(\"operation\") }","${! meta(\"operation\") }","INSERT"],"options":["INSERT","INSERT_MERGE","INSERT_REPLACE","UPDATE_MERGE","UPDATE_REPLACE","DELETE"],"linter":"\nlet options = {\n \"insert\": true,\n \"insert_merge\": true,\n \"insert_replace\": true,\n \"update_merge\": true,\n \"update_replace\": true,\n \"delete\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of parallel message batches to have in flight at any given time.","default":64},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period to wait on an upload before abandoning it and reattempting.","is_advanced":true,"default":"5s"},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}],"linter":"root = if this.storage_connection_string != \"\" \u0026\u0026 !this.storage_connection_string.contains(\"AccountName=\") \u0026\u0026 !this.storage_connection_string.contains(\"UseDevelopmentStorage=true;\") \u0026\u0026 this.storage_account == \"\" { [ \"storage_account must be set if storage_connection_string does not contain the \\\"AccountName\\\" parameter\" ] }"},"version":"3.36.0"},{"name":"beanstalkd","type":"output","status":"experimental","plugin":true,"summary":"Write messages to a Beanstalkd queue.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"address","type":"string","kind":"scalar","description":"An address to connect to.","examples":["127.0.0.1:11300"]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase to improve throughput.","default":64}]},"version":"4.7.0"},{"name":"broker","type":"output","status":"stable","plugin":true,"summary":"Allows you to route messages to multiple child outputs using a range of brokering \u003c\u003cpatterns\u003e\u003e.","description":"\nxref:components:processors/about.adoc[Processors] can be listed to apply across individual outputs or all outputs:\n\n```yaml\noutput:\n broker:\n pattern: fan_out\n outputs:\n - resource: foo\n - resource: bar\n # Processors only applied to messages sent to bar.\n processors:\n - resource: bar_processor\n\n # Processors applied to messages sent to all brokered outputs.\n processors:\n - resource: general_processor\n```","categories":["Utility"],"footnotes":"\n== Patterns\n\nThe broker pattern determines the way in which messages are allocated and can be chosen from the following:\n\n=== `fan_out`\n\nWith the fan out pattern all outputs will be sent every message that passes through Redpanda Connect in parallel.\n\nIf an output applies back pressure it will block all subsequent messages, and if an output fails to send a message it will be retried continuously until completion or service shut down. This mechanism is in place in order to prevent one bad output from causing a larger retry loop that results in a good output from receiving unbounded message duplicates.\n\nSometimes it is useful to disable the back pressure or retries of certain fan out outputs and instead drop messages that have failed or were blocked. In this case you can wrap outputs with a xref:components:outputs/drop_on.adoc[`drop_on` output].\n\n=== `fan_out_fail_fast`\n\nThe same as the `fan_out` pattern, except that output failures will not be automatically retried. This pattern should be used with caution as busy retry loops could result in unlimited duplicates being introduced into the non-failure outputs.\n\n=== `fan_out_sequential`\n\nSimilar to the fan out pattern except outputs are written to sequentially, meaning an output is only written to once the preceding output has confirmed receipt of the same message.\n\nIf an output applies back pressure it will block all subsequent messages, and if an output fails to send a message it will be retried continuously until completion or service shut down. This mechanism is in place in order to prevent one bad output from causing a larger retry loop that results in a good output from receiving unbounded message duplicates.\n\n=== `fan_out_sequential_fail_fast`\n\nThe same as the `fan_out_sequential` pattern, except that output failures will not be automatically retried. This pattern should be used with caution as busy retry loops could result in unlimited duplicates being introduced into the non-failure outputs.\n\n=== `round_robin`\n\nWith the round robin pattern each message will be assigned a single output following their order. If an output applies back pressure it will block all subsequent messages. If an output fails to send a message then the message will be re-attempted with the next input, and so on.\n\n=== `greedy`\n\nThe greedy pattern results in higher output throughput at the cost of potentially disproportionate message allocations to those outputs. Each message is sent to a single output, which is determined by allowing outputs to claim messages as soon as they are able to process them. This results in certain faster outputs potentially processing more messages at the cost of slower outputs.","config":{"name":"","type":"object","kind":"scalar","children":[{"name":"copies","type":"int","kind":"scalar","description":"The number of copies of each configured output to spawn.","is_advanced":true,"default":1},{"name":"pattern","type":"string","kind":"scalar","description":"The brokering pattern to use.","default":"fan_out","options":["fan_out","fan_out_fail_fast","fan_out_sequential","fan_out_sequential_fail_fast","round_robin","greedy"],"linter":"\nlet options = {\n \"fan_out\": true,\n \"fan_out_fail_fast\": true,\n \"fan_out_sequential\": true,\n \"fan_out_sequential_fail_fast\": true,\n \"round_robin\": true,\n \"greedy\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"outputs","type":"output","kind":"array","description":"A list of child outputs to broker."},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]}},{"name":"cache","type":"output","status":"stable","plugin":true,"summary":"Stores each message in a xref:components:caches/about.adoc[cache].","description":"Caches are configured as xref:components:caches/about.adoc[resources], where there's a wide variety to choose from.\n\n:cache-support: aws_dynamodb=certified, aws_s3=certified, file=certified, memcached=certified, memory=certified, nats_kv=certified, redis=certified, ristretto=certified, couchbase=community, mongodb=community, sql=community, multilevel=community, ttlru=community, gcp_cloud_storage=community, lru=community, noop=community\n\nThe `target` field must reference a configured cache resource label like follows:\n\n```yaml\noutput:\n cache:\n target: foo\n key: ${!json(\"document.id\")}\n\ncache_resources:\n - label: foo\n memcached:\n addresses:\n - localhost:11211\n default_ttl: 60s\n```\n\nIn order to create a unique `key` value per item you should use function interpolations described in xref:configuration:interpolation.adoc#bloblang-queries[Bloblang queries].\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"target","type":"string","kind":"scalar","description":"The target cache to store messages in."},{"name":"key","type":"string","kind":"scalar","description":"The key to store messages by, function interpolation should be used in order to derive a unique key for each message.","default":"${!count(\"items\")}-${!timestamp_unix_nano()}","interpolated":true,"examples":["${!count(\"items\")}-${!timestamp_unix_nano()}","${!json(\"doc.id\")}","${!meta(\"kafka_key\")}"]},{"name":"ttl","type":"string","kind":"scalar","description":"The TTL of each individual item as a duration string. After this period an item will be eligible for removal during the next compaction. Not all caches support per-key TTLs, and those that do not will fall back to their generally configured TTL setting.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["60s","5m","36h"],"version":"3.33.0"},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64}]}},{"name":"cassandra","type":"output","status":"beta","plugin":true,"summary":"Runs a query against a Cassandra database for each message in order to insert data.","description":"\nQuery arguments can be set using a bloblang array for the fields using the `args_mapping` field.\n\nWhen populating timestamp columns the value must either be a string in ISO 8601 format (2006-01-02T15:04:05Z07:00), or an integer representing unix time in seconds.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":null,"examples":[{"title":"Basic Inserts","summary":"If we were to create a table with some basic columns with `CREATE TABLE foo.bar (id int primary key, content text, created_at timestamp);`, and were processing JSON documents of the form `{\"id\":\"342354354\",\"content\":\"hello world\",\"timestamp\":1605219406}` using logged batches, we could populate our table with the following config:","config":"\noutput:\n cassandra:\n addresses:\n - localhost:9042\n query: 'INSERT INTO foo.bar (id, content, created_at) VALUES (?, ?, ?)'\n args_mapping: |\n root = [\n this.id,\n this.content,\n this.timestamp\n ]\n batching:\n count: 500\n period: 1s\n"},{"title":"Insert JSON Documents","summary":"The following example inserts JSON documents into the table `footable` of the keyspace `foospace` using INSERT JSON (https://cassandra.apache.org/doc/latest/cql/json.html#insert-json).","config":"\noutput:\n cassandra:\n addresses:\n - localhost:9042\n query: 'INSERT INTO foospace.footable JSON ?'\n args_mapping: 'root = [ this ]'\n batching:\n count: 500\n period: 1s\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"addresses","type":"string","kind":"array","description":"A list of Cassandra nodes to connect to. Multiple comma separated addresses can be specified on a single line.","examples":[["localhost:9042"],["foo:9042","bar:9042"],["foo:9042,bar:9042"]]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"password_authenticator","type":"object","kind":"scalar","description":"Optional configuration of Cassandra authentication parameters.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use password authentication","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"The username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"The password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"disable_initial_host_lookup","type":"bool","kind":"scalar","description":"If enabled the driver will not attempt to get host info from the system.peers table. This can speed up queries but will mean that data_centre, rack and token information will not be available.","is_advanced":true,"default":false},{"name":"max_retries","type":"int","kind":"scalar","description":"The maximum number of retries before giving up on a request.","is_advanced":true,"default":3},{"name":"backoff","type":"object","kind":"scalar","description":"Control time intervals between retry attempts.","is_advanced":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"1s"},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts.","is_advanced":true,"default":"5s"}]},{"name":"timeout","type":"string","kind":"scalar","description":"The client connection timeout.","default":"600ms"},{"name":"query","type":"string","kind":"scalar","description":"A query to execute for each message."},{"name":"args_mapping","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] that can be used to provide arguments to Cassandra queries. The result of the query must be an array containing a matching number of elements to the query arguments.","is_optional":true,"bloblang":true,"version":"3.55.0"},{"name":"consistency","type":"string","kind":"scalar","description":"The consistency level to use.","is_advanced":true,"default":"QUORUM","options":["ANY","ONE","TWO","THREE","QUORUM","ALL","LOCAL_QUORUM","EACH_QUORUM","LOCAL_ONE"],"linter":"\nlet options = {\n \"any\": true,\n \"one\": true,\n \"two\": true,\n \"three\": true,\n \"quorum\": true,\n \"all\": true,\n \"local_quorum\": true,\n \"each_quorum\": true,\n \"local_one\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"logged_batch","type":"bool","kind":"scalar","description":"If enabled the driver will perform a logged batch. Disabling this prompts unlogged batches to be used instead, which are less efficient but necessary for alternative storages that do not support logged batches.","is_advanced":true,"default":true},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]}},{"name":"couchbase","type":"output","status":"experimental","plugin":true,"summary":"Performs operations against Couchbase for each message, allowing you to store or delete data.","description":"When inserting, replacing or upserting documents, each must have the `content` property set.\n\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Integration"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"Couchbase connection string.","examples":["couchbase://localhost:11210"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"username","type":"string","kind":"scalar","description":"Username to connect to the cluster.","is_optional":true},{"name":"password","type":"string","kind":"scalar","description":"Password to connect to the cluster.","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"bucket","type":"string","kind":"scalar","description":"Couchbase bucket."},{"name":"collection","type":"string","kind":"scalar","description":"Bucket collection.","is_advanced":true,"is_optional":true,"default":"_default"},{"name":"transcoder","type":"string","kind":"scalar","description":"Couchbase transcoder to use.","is_advanced":true,"default":"legacy","annotated_options":[["json","JSONTranscoder implements the default transcoding behavior and applies JSON transcoding to all values. This will apply the following behavior to the value: binary ([]byte) -\u003e error. default -\u003e JSON value, JSON Flags."],["legacy","LegacyTranscoder implements the behavior for a backward-compatible transcoder. This transcoder implements behavior matching that of gocb v1.This will apply the following behavior to the value: binary ([]byte) -\u003e binary bytes, Binary expectedFlags. string -\u003e string bytes, String expectedFlags. default -\u003e JSON value, JSON expectedFlags."],["raw","RawBinaryTranscoder implements passthrough behavior of raw binary data. This transcoder does not apply any serialization. This will apply the following behavior to the value: binary ([]byte) -\u003e binary bytes, binary expectedFlags. default -\u003e error."],["rawjson","RawJSONTranscoder implements passthrough behavior of JSON data. This transcoder does not apply any serialization. It will forward data across the network without incurring unnecessary parsing costs. This will apply the following behavior to the value: binary ([]byte) -\u003e JSON bytes, JSON expectedFlags. string -\u003e JSON bytes, JSON expectedFlags. default -\u003e error."],["rawstring","RawStringTranscoder implements passthrough behavior of raw string data. This transcoder does not apply any serialization. This will apply the following behavior to the value: string -\u003e string bytes, string expectedFlags. default -\u003e error."]],"linter":"\nlet options = {\n \"json\": true,\n \"legacy\": true,\n \"raw\": true,\n \"rawjson\": true,\n \"rawstring\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"timeout","type":"string","kind":"scalar","description":"Operation timeout.","is_advanced":true,"default":"15s"},{"name":"id","type":"string","kind":"scalar","description":"Document id.","interpolated":true,"examples":["${! json(\"id\") }"]},{"name":"content","type":"string","kind":"scalar","description":"Document content.","is_optional":true,"bloblang":true},{"name":"operation","type":"string","kind":"scalar","description":"Couchbase operation to perform.","default":"upsert","annotated_options":[["insert","insert a new document."],["remove","delete a document."],["replace","replace the contents of a document."],["upsert","creates a new document if it does not exist, if it does exist then it updates it."]],"linter":"\nlet options = {\n \"insert\": true,\n \"remove\": true,\n \"replace\": true,\n \"upsert\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}],"linter":"root = if ((this.operation == \"insert\" || this.operation == \"replace\" || this.operation == \"upsert\") \u0026\u0026 !this.exists(\"content\")) { [ \"content must be set for insert, replace and upsert operations.\" ] }"},"version":"4.37.0"},{"name":"cypher","type":"output","status":"experimental","plugin":true,"description":"The cypher output type writes a batch of messages to any graph database that supports the Neo4j or Bolt protocols.","categories":["Services"],"examples":[{"title":"Write to Neo4j Aura","summary":"This is an example of how to write to Neo4j Aura","config":"\noutput:\n cypher:\n uri: neo4j+s://example.databases.neo4j.io\n cypher: |\n MERGE (product:Product {id: $id})\n ON CREATE SET product.name = $product,\n product.title = $title,\n product.description = $description,\n args_mapping: |\n root = {}\n root.id = this.product.id \n root.product = this.product.summary.name\n root.title = this.product.summary.displayName\n root.description = this.product.fullDescription\n basic_auth:\n enabled: true\n username: \"${NEO4J_USER}\"\n password: \"${NEO4J_PASSWORD}\"\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"uri","type":"string","kind":"scalar","description":"The connection URI to connect to.\nSee https://neo4j.com/docs/go-manual/current/connect-advanced/[Neo4j's documentation^] for more information. ","examples":["neo4j://demo.neo4jlabs.com","neo4j+s://aura.databases.neo4j.io","neo4j+ssc://self-signed.demo.neo4jlabs.com","bolt://127.0.0.1:7687","bolt+s://core.db.server:7687","bolt+ssc://10.0.0.43"]},{"name":"cypher","type":"string","kind":"scalar","description":"The cypher expression to execute against the graph database.","examples":["MERGE (p:Person {name: $name})","MATCH (o:Organization {id: $orgId})\nMATCH (p:Person {name: $name})\nMERGE (p)-[:WORKS_FOR]-\u003e(o)"]},{"name":"database_name","type":"string","kind":"scalar","description":"Set the target database for which expressions are evaluated against.","default":""},{"name":"args_mapping","type":"string","kind":"scalar","description":"The mapping from the message to the data that is passed in as parameters to the cypher expression. Must be an object. By default the entire payload is used.","is_optional":true,"bloblang":true,"examples":["root.name = this.displayName","root = {\"orgId\": this.org.id, \"name\": this.user.name}"]},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"realm","type":"string","kind":"scalar","description":"The realm for authentication challenges.","is_advanced":true,"default":""}]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64}]},"version":"4.37.0"},{"name":"discord","type":"output","status":"experimental","plugin":true,"summary":"Writes messages to a Discord channel.","description":"\nThis output POSTs messages to the `/channels/\\{channel_id}/messages` Discord API endpoint authenticated as a bot using token based authentication.\n\nIf the format of a message is a JSON object matching the https://discord.com/developers/docs/resources/channel#message-object[Discord API message type^] then it is sent directly, otherwise an object matching the API type is created with the content of the message added as a string.\n","categories":["Services","Social"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"channel_id","type":"string","kind":"scalar","description":"A discord channel ID to write messages to."},{"name":"bot_token","type":"string","kind":"scalar","description":"A bot token used for authentication."},{"name":"rate_limit","type":"string","kind":"scalar","is_deprecated":true,"default":"An optional rate limit resource to restrict API requests with."}]}},{"name":"drop","type":"output","status":"stable","plugin":true,"summary":"Drops all messages.","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","default":{}}},{"name":"drop_on","type":"output","status":"stable","plugin":true,"summary":"Attempts to write messages to a child output and if the write fails for one of a list of configurable reasons the message is dropped (acked) instead of being reattempted (or nacked).","description":"Regular Redpanda Connect outputs will apply back pressure when downstream services aren't accessible, and Redpanda Connect retries (or nacks) all messages that fail to be delivered. However, in some circumstances, or for certain output types, we instead might want to relax these mechanisms, which is when this output becomes useful.","categories":["Utility"],"examples":[{"title":"Dropping failed HTTP requests","summary":"In this example we have a fan_out broker, where we guarantee delivery to our Kafka output, but drop messages if they fail our secondary HTTP client output.","config":"\noutput:\n broker:\n pattern: fan_out\n outputs:\n - kafka:\n addresses: [ foobar:6379 ]\n topic: foo\n - drop_on:\n error: true\n output:\n http_client:\n url: http://example.com/foo/messages\n verb: POST\n"},{"title":"Dropping from outputs that cannot connect","summary":"Most outputs that attempt to establish and long-lived connection will apply back-pressure when the connection is lost. The following example has a websocket output where if it takes longer than 10 seconds to establish a connection, or recover a lost one, pending messages are dropped.","config":"\noutput:\n drop_on:\n back_pressure: 10s\n output:\n websocket:\n url: ws://example.com/foo/messages\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"error","type":"bool","kind":"scalar","description":"Whether messages should be dropped when the child output returns an error of any type. For example, this could be when an `http_client` output gets a 4XX response code. In order to instead drop only on specific error patterns use the `error_matches` field instead.","default":false},{"name":"error_patterns","type":"string","kind":"array","description":"A list of regular expressions (re2) where if the child output returns an error that matches any part of any of these patterns the message will be dropped.","is_optional":true,"examples":[["and that was really bad$"],["roughly [0-9]+ issues occurred"]],"version":"4.27.0"},{"name":"back_pressure","type":"string","kind":"scalar","description":"An optional duration string that determines the maximum length of time to wait for a given message to be accepted by the child output before the message should be dropped instead. The most common reason for an output to block is when waiting for a lost connection to be re-established. Once a message has been dropped due to back pressure all subsequent messages are dropped immediately until the output is ready to process them again. Note that if `error` is set to `false` and this field is specified then messages dropped due to back pressure will return an error response (are nacked or reattempted).","is_optional":true,"examples":["30s","1m"]},{"name":"output","type":"output","kind":"scalar","description":"A child output to wrap with this drop mechanism."}]}},{"name":"dynamic","type":"output","status":"stable","plugin":true,"summary":"A special broker type where the outputs are identified by unique labels and can be created, changed and removed during runtime via a REST API.","description":"The broker pattern used is always `fan_out`, meaning each message will be delivered to each dynamic output.","categories":["Utility"],"footnotes":"\n== Endpoints\n\n=== GET `/outputs`\n\nReturns a JSON object detailing all dynamic outputs, providing information such as their current uptime and configuration.\n\n=== GET `/outputs/\\{id}`\n\nReturns the configuration of an output.\n\n=== POST `/outputs/\\{id}`\n\nCreates or updates an output with a configuration provided in the request body (in YAML or JSON format).\n\n=== DELETE `/outputs/\\{id}`\n\nStops and removes an output.\n\n=== GET `/outputs/\\{id}/uptime`\n\nReturns the uptime of an output as a duration string (of the form \"72h3m0.5s\").","config":{"name":"","type":"object","kind":"scalar","children":[{"name":"outputs","type":"output","kind":"map","description":"A map of outputs to statically create.","default":{}},{"name":"prefix","type":"string","kind":"scalar","description":"A path prefix for HTTP endpoints that are registered.","default":""}]}},{"name":"elasticsearch","type":"output","status":"stable","plugin":true,"summary":"Publishes messages into an Elasticsearch index. If the index does not exist then it is created with a dynamic mapping.","description":"\nBoth the `id` and `index` fields can be dynamically set using function interpolations described in xref:configuration:interpolation.adoc#bloblang-queries[Bloblang queries]. When sending batched messages these interpolations are performed per message part.\n\n== AWS\n\nIt's possible to enable AWS connectivity with this output using the `aws` fields. However, you may need to set `sniff` and `healthcheck` to false for connections to succeed.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services"],"examples":[{"title":"Elastic Cloud Serverless","summary":"This is an example of writing data to https://www.elastic.co/docs/current/serverless[Elastic Cloud serverless^].","config":"\noutput:\n elasticsearch:\n urls: [\"https://${ELASTIC_CLOUD_CLUSTER_ID}.es.us-east-1.aws.elastic.cloud:443\"]\n sniff: false\n healthcheck: false\n index: \"my-elasticsearch-index\"\n id: my-document-id-${!counter()}-${!timestamp_unix()}\n api_key: \"${ELASTIC_CLOUD_API_KEY}\"\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["http://localhost:9200"]]},{"name":"index","type":"string","kind":"scalar","description":"The index to place messages.","interpolated":true},{"name":"action","type":"string","kind":"scalar","description":"The action to take on the document. This field must resolve to one of the following action types: `create`, `index`, `update`, `upsert` or `delete`.","is_advanced":true,"default":"index","interpolated":true},{"name":"pipeline","type":"string","kind":"scalar","description":"An optional pipeline id to preprocess incoming documents.","is_advanced":true,"default":"","interpolated":true},{"name":"id","type":"string","kind":"scalar","description":"The ID for indexed messages. Interpolation should be used in order to create a unique ID for each message.","default":"${!counter()}-${!timestamp_unix()}","interpolated":true},{"name":"type","type":"string","kind":"scalar","description":"The document mapping type. This field is required for versions of elasticsearch earlier than 6.0.0, but are invalid for versions 7.0.0 or later.","default":"","interpolated":true},{"name":"routing","type":"string","kind":"scalar","description":"The routing key to use for the document.","is_advanced":true,"default":"","interpolated":true},{"name":"retry_on_conflict","type":"int","kind":"scalar","description":"When using the update or upsert action, retry_on_conflict can be used to specify how many times an update should be retried in the case of a version conflict.","is_advanced":true,"default":0},{"name":"sniff","type":"bool","kind":"scalar","description":"Prompts Redpanda Connect to sniff for brokers to connect to when establishing a connection.","is_advanced":true,"default":true},{"name":"healthcheck","type":"bool","kind":"scalar","description":"Whether to enable healthchecks.","is_advanced":true,"default":true},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum time to wait before abandoning a request (and trying again).","is_advanced":true,"default":"5s"},{"name":"api_key","type":"string","kind":"scalar","description":"The key to set in the Authorization header if using API keys for authentication.","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"max_retries","type":"int","kind":"scalar","description":"The maximum number of retries before giving up on the request. If set to zero there is no discrete limit.","is_advanced":true,"default":0},{"name":"backoff","type":"object","kind":"scalar","description":"Control time intervals between retry attempts.","is_advanced":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"1s"},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts.","is_advanced":true,"default":"5s"},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum period to wait before retry attempts are abandoned. If zero then no limit is used.","is_advanced":true,"default":"30s"}]},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"aws","type":"object","kind":"scalar","description":"Enables and customises connectivity to Amazon Elastic Service.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to connect to Amazon Elastic Service.","is_advanced":true,"default":false},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]},{"name":"gzip_compression","type":"bool","kind":"scalar","description":"Enable gzip compression on the request side.","is_advanced":true,"default":false}]}},{"name":"elasticsearch_v8","type":"output","status":"stable","plugin":true,"summary":"Publishes messages into an Elasticsearch index. If the index does not exist then it is created with a dynamic mapping.","description":"\nBoth the `id` and `index` fields can be dynamically set using function interpolations described xref:configuration:interpolation.adoc#bloblang-queries[here]. When sending batched messages these interpolations are performed per message part.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services"],"examples":[{"title":"Updating Documents","summary":"When updating documents, the request body should contain a combination of a `doc`, `upsert`, and/or `script` fields at the top level, this should be done via mapping processors. `doc` updates using a partial document, `script` performs an update using a scripting language such as the built in Painless language, and `upsert` updates an existing document or inserts a new one if it doesn’t exist. For more information on the structures and behaviors of these fields, please see the https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-update.html[Elasticsearch Update API^]","config":"\n# Partial document update\noutput:\n processors:\n - mapping: |\n meta id = this.id\n # Performs a partial update ont he document.\n root.doc = this\n elasticsearch_v8:\n urls: [localhost:9200]\n index: foo\n id: ${! @id }\n action: update\n\n# Scripted update\noutput:\n processors:\n - mapping: |\n meta id = this.id\n # Increments the field \"counter\" by 1.\n root.script.source = \"ctx._source.counter += 1\"\n elasticsearch_v8:\n urls: [localhost:9200]\n index: foo\n id: ${! @id }\n action: update\n\n# Upsert\noutput:\n processors:\n - mapping: |\n meta id = this.id\n # If the product with the ID exists, its price will be updated to 100.\n # If the product does not exist, a new document with ID 1 and a price\n # of 50 will be inserted.\n root.doc.product_price = 50\n root.upsert.product_price = 100\n elasticsearch_v8:\n urls: [localhost:9200]\n index: foo\n id: ${! @id }\n action: update\n"},{"title":"Indexing documents from Redpanda","summary":"Here we read messages from a Redpanda cluster and write them to an Elasticsearch index using a field from the message as the ID for the Elasticsearch document.","config":"\ninput:\n redpanda:\n seed_brokers: [localhost:19092]\n topics: [\"things\"]\n consumer_group: \"rpcn3\"\n processors:\n - mapping: |\n meta id = this.id\n root = this\noutput:\n elasticsearch_v8:\n urls: ['http://localhost:9200']\n index: \"things\"\n action: \"index\"\n id: ${! meta(\"id\") }\n"},{"title":"Indexing documents from S3","summary":"Here we read messages from a AWS S3 bucket and write them to an Elasticsearch index using the S3 key as the ID for the Elasticsearch document.","config":"\ninput:\n aws_s3:\n bucket: \"my-cool-bucket\"\n prefix: \"bug-facts/\"\n scanner:\n to_the_end: {}\noutput:\n elasticsearch_v8:\n urls: ['http://localhost:9200']\n index: \"cool-bug-facts\"\n action: \"index\"\n id: ${! meta(\"s3_key\") }\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["http://localhost:9200"]]},{"name":"index","type":"string","kind":"scalar","description":"The index to place messages.","interpolated":true},{"name":"action","type":"string","kind":"scalar","description":"The action to take on the document. This field must resolve to one of the following action types: `index`, `update` or `delete`. See the `Updating Documents` example for more on how the `update` action works.","interpolated":true},{"name":"id","type":"string","kind":"scalar","description":"The ID for indexed messages. Interpolation should be used in order to create a unique ID for each message.","interpolated":true,"examples":["${!counter()}-${!timestamp_unix()}"]},{"name":"pipeline","type":"string","kind":"scalar","description":"An optional pipeline id to preprocess incoming documents.","is_advanced":true,"default":"","interpolated":true},{"name":"routing","type":"string","kind":"scalar","description":"The routing key to use for the document.","is_advanced":true,"default":"","interpolated":true},{"name":"retry_on_conflict","type":"int","kind":"scalar","description":"Specify how many times should an update operation be retried when a conflict occurs","is_advanced":true,"default":0},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]}},{"name":"fallback","type":"output","status":"stable","plugin":true,"summary":"Attempts to send each message to a child output, starting from the first output on the list. If an output attempt fails then the next output in the list is attempted, and so on.","description":"\nThis pattern is useful for triggering events in the case where certain output targets have broken. For example, if you had an output type `http_client` but wished to reroute messages whenever the endpoint becomes unreachable you could use this pattern:\n\n```yaml\noutput:\n fallback:\n - http_client:\n url: http://foo:4195/post/might/become/unreachable\n retries: 3\n retry_period: 1s\n - http_client:\n url: http://bar:4196/somewhere/else\n retries: 3\n retry_period: 1s\n processors:\n - mapping: 'root = \"failed to send this message to foo: \" + content()'\n - file:\n path: /usr/local/benthos/everything_failed.jsonl\n```\n\n== Metadata\n\nWhen a given output fails the message routed to the following output will have a metadata value named `fallback_error` containing a string error message outlining the cause of the failure. The content of this string will depend on the particular output and can be used to enrich the message or provide information used to broker the data to an appropriate output using something like a `switch` output.\n\n== Batching\n\nWhen an output within a fallback sequence uses batching, like so:\n\n```yaml\noutput:\n fallback:\n - aws_dynamodb:\n table: foo\n string_columns:\n id: ${!json(\"id\")}\n content: ${!content()}\n batching:\n count: 10\n period: 1s\n - file:\n path: /usr/local/benthos/failed_stuff.jsonl\n```\n\nRedpanda Connect makes a best attempt at inferring which specific messages of the batch failed, and only propagates those individual messages to the next fallback tier.\n\nHowever, depending on the output and the error returned it is sometimes not possible to determine the individual messages that failed, in which case the whole batch is passed to the next tier in order to preserve at-least-once delivery guarantees.","categories":["Utility"],"config":{"name":"","type":"output","kind":"array","default":[]},"version":"3.58.0"},{"name":"file","type":"output","status":"stable","plugin":true,"summary":"Writes messages to files on disk based on a chosen codec.","description":"Messages can be written to different files by using xref:configuration:interpolation.adoc#bloblang-queries[interpolation functions] in the path field. However, only one file is ever open at a given time, and therefore when the path changes the previously open file is closed.","categories":["Local"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"path","type":"string","kind":"scalar","description":"The file to write to, if the file does not yet exist it will be created.","interpolated":true,"examples":["/tmp/data.txt","/tmp/${! timestamp_unix() }.txt","/tmp/${! json(\"document.id\") }.json"],"version":"3.33.0"},{"name":"codec","type":"string","kind":"scalar","description":"The way in which the bytes of messages should be written out into the output data stream. It's possible to write lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter.","default":"lines","examples":["lines","delim:\t","delim:foobar"],"annotated_options":[["all-bytes","Only applicable to file based outputs. Writes each message to a file in full, if the file already exists the old content is deleted."],["append","Append each message to the output stream without any delimiter or special encoding."],["lines","Append each message to the output stream followed by a line break."],["delim:x","Append each message to the output stream followed by a custom delimiter."]],"version":"3.33.0"}]}},{"name":"gcp_bigquery","type":"output","status":"beta","plugin":true,"summary":"Sends messages as new rows to a Google Cloud BigQuery table.","description":"\n== Credentials\n\nBy default Redpanda Connect will use a shared credentials file when connecting to GCP services. You can find out more in xref:guides:cloud/gcp.adoc[].\n\n== Format\n\nThis output currently supports only CSV, NEWLINE_DELIMITED_JSON and PARQUET, formats. Learn more about how to use GCP BigQuery with them here:\n\n- https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-json[`NEWLINE_DELIMITED_JSON`^]\n- https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-csv[`CSV`^]\n- https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-parquet[`PARQUET`^]\n\nEach message may contain multiple elements separated by newlines. For example a single message containing:\n\n```json\n{\"key\": \"1\"}\n{\"key\": \"2\"}\n```\n\nIs equivalent to two separate messages:\n\n```json\n{\"key\": \"1\"}\n```\n\nAnd:\n\n```json\n{\"key\": \"2\"}\n```\n\nThe same is true for the CSV format.\n\n=== CSV\n\nFor the CSV format when the field `csv.header` is specified a header row will be inserted as the first line of each message batch. If this field is not provided then the first message of each message batch must include a header line.\n\n=== Parquet\n\nFor parquet, the data can be encoded using the `parquet_encode` processor and each message that is sent to the output must be a full parquet message.\n\n\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["GCP","Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"project","type":"string","kind":"scalar","description":"The project ID of the dataset to insert data to. If not set, it will be inferred from the credentials or read from the GOOGLE_CLOUD_PROJECT environment variable.","default":""},{"name":"job_project","type":"string","kind":"scalar","description":"The project ID in which jobs will be exectuted. If not set, project will be used.","default":""},{"name":"dataset","type":"string","kind":"scalar","description":"The BigQuery Dataset ID."},{"name":"table","type":"string","kind":"scalar","description":"The table to insert messages to."},{"name":"format","type":"string","kind":"scalar","description":"The format of each incoming message.","default":"NEWLINE_DELIMITED_JSON","options":["NEWLINE_DELIMITED_JSON","CSV","PARQUET"],"linter":"\nlet options = {\n \"newline_delimited_json\": true,\n \"csv\": true,\n \"parquet\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of message batches to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"write_disposition","type":"string","kind":"scalar","description":"Specifies how existing data in a destination table is treated.","is_advanced":true,"default":"WRITE_APPEND","options":["WRITE_APPEND","WRITE_EMPTY","WRITE_TRUNCATE"],"linter":"\nlet options = {\n \"write_append\": true,\n \"write_empty\": true,\n \"write_truncate\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"create_disposition","type":"string","kind":"scalar","description":"Specifies the circumstances under which destination table will be created. If CREATE_IF_NEEDED is used the GCP BigQuery will create the table if it does not already exist and tables are created atomically on successful completion of a job. The CREATE_NEVER option ensures the table must already exist and will not be automatically created.","is_advanced":true,"default":"CREATE_IF_NEEDED","options":["CREATE_IF_NEEDED","CREATE_NEVER"],"linter":"\nlet options = {\n \"create_if_needed\": true,\n \"create_never\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"ignore_unknown_values","type":"bool","kind":"scalar","description":"Causes values not matching the schema to be tolerated. Unknown values are ignored. For CSV this ignores extra values at the end of a line. For JSON this ignores named values that do not match any column name. If this field is set to false (the default value), records containing unknown values are treated as bad records. The max_bad_records field can be used to customize how bad records are handled.","is_advanced":true,"default":false},{"name":"max_bad_records","type":"int","kind":"scalar","description":"The maximum number of bad records that will be ignored when reading data.","is_advanced":true,"default":0},{"name":"auto_detect","type":"bool","kind":"scalar","description":"Indicates if we should automatically infer the options and schema for CSV and JSON sources. If the table doesn't exist and this field is set to `false` the output may not be able to insert data and will throw insertion error. Be careful using this field since it delegates to the GCP BigQuery service the schema detection and values like `\"no\"` may be treated as booleans for the CSV format.","is_advanced":true,"default":false},{"name":"job_labels","type":"string","kind":"map","description":"A list of labels to add to the load job.","default":{}},{"name":"credentials_json","type":"string","kind":"scalar","description":"An optional field to set Google Service Account Credentials json.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"csv","type":"object","kind":"scalar","description":"Specify how CSV data should be interpretted.","children":[{"name":"header","type":"string","kind":"array","description":"A list of values to use as header for each batch of messages. If not specified the first line of each message will be used as header.","default":[]},{"name":"field_delimiter","type":"string","kind":"scalar","description":"The separator for fields in a CSV file, used when reading or exporting data.","default":","},{"name":"allow_jagged_rows","type":"bool","kind":"scalar","description":"Causes missing trailing optional columns to be tolerated when reading CSV data. Missing values are treated as nulls.","is_advanced":true,"default":false},{"name":"allow_quoted_newlines","type":"bool","kind":"scalar","description":"Sets whether quoted data sections containing newlines are allowed when reading CSV data.","is_advanced":true,"default":false},{"name":"encoding","type":"string","kind":"scalar","description":"Encoding is the character encoding of data to be read.","is_advanced":true,"default":"UTF-8","options":["UTF-8","ISO-8859-1"],"linter":"\nlet options = {\n \"utf-8\": true,\n \"iso-8859-1\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"skip_leading_rows","type":"int","kind":"scalar","description":"The number of rows at the top of a CSV file that BigQuery will skip when reading data. The default value is 1 since Redpanda Connect will add the specified header in the first line of each batch sent to BigQuery.","is_advanced":true,"default":1}]},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]},"version":"3.55.0"},{"name":"gcp_cloud_storage","type":"output","status":"beta","plugin":true,"summary":"Sends message parts as objects to a Google Cloud Storage bucket. Each object is uploaded with the path specified with the `path` field.","description":"\nIn order to have a different path for each object you should use function interpolations described in xref:configuration:interpolation.adoc#bloblang-queries[Bloblang queries], which are calculated per message of a batch.\n\n== Metadata\n\nMetadata fields on messages will be sent as headers, in order to mutate these values (or remove them) check out the xref:configuration:metadata.adoc[metadata docs].\n\n== Credentials\n\nBy default Redpanda Connect will use a shared credentials file when connecting to GCP services. You can find out more in xref:guides:cloud/gcp.adoc[].\n\n== Batching\n\nIt's common to want to upload messages to Google Cloud Storage as batched archives, the easiest way to do this is to batch your messages at the output level and join the batch of messages with an xref:components:processors/archive.adoc[`archive`] and/or xref:components:processors/compress.adoc[`compress`] processor.\n\nFor example, if we wished to upload messages as a .tar.gz archive of documents we could achieve that with the following config:\n\n```yaml\noutput:\n gcp_cloud_storage:\n bucket: TODO\n path: ${!counter()}-${!timestamp_unix_nano()}.tar.gz\n batching:\n count: 100\n period: 10s\n processors:\n - archive:\n format: tar\n - compress:\n algorithm: gzip\n```\n\nAlternatively, if we wished to upload JSON documents as a single large document containing an array of objects we can do that with:\n\n```yaml\noutput:\n gcp_cloud_storage:\n bucket: TODO\n path: ${!counter()}-${!timestamp_unix_nano()}.json\n batching:\n count: 100\n processors:\n - archive:\n format: json_array\n```\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services","GCP"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"bucket","type":"string","kind":"scalar","description":"The bucket to upload messages to."},{"name":"path","type":"string","kind":"scalar","description":"The path of each message to upload.","default":"${!counter()}-${!timestamp_unix_nano()}.txt","interpolated":true,"examples":["${!counter()}-${!timestamp_unix_nano()}.txt","${!meta(\"kafka_key\")}.json","${!json(\"doc.namespace\")}/${!json(\"doc.id\")}.json"]},{"name":"content_type","type":"string","kind":"scalar","description":"The content type to set for each object.","default":"application/octet-stream","interpolated":true},{"name":"content_encoding","type":"string","kind":"scalar","description":"An optional content encoding to set for each object.","is_advanced":true,"default":"","interpolated":true},{"name":"collision_mode","type":"string","kind":"scalar","description":"Determines how file path collisions should be dealt with.","default":"overwrite","annotated_options":[["append","Append the message bytes to the original file."],["error-if-exists","Return an error, this is the equivalent of a nack."],["ignore","Do not modify the original file, the new data will be dropped."],["overwrite","Replace the existing file with the new one."]],"version":"3.53.0","linter":"\nlet options = {\n \"append\": true,\n \"error-if-exists\": true,\n \"ignore\": true,\n \"overwrite\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"chunk_size","type":"int","kind":"scalar","description":"An optional chunk size which controls the maximum number of bytes of the object that the Writer will attempt to send to the server in a single request. If ChunkSize is set to zero, chunking will be disabled.","is_advanced":true,"default":16777216},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period to wait on an upload before abandoning it and reattempting.","default":"3s","examples":["1s","500ms"]},{"name":"credentials_json","type":"string","kind":"scalar","description":"An optional field to set Google Service Account Credentials json.","is_secret":true,"default":"","interpolated":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of message batches to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]},"version":"3.43.0"},{"name":"gcp_pubsub","type":"output","status":"stable","plugin":true,"summary":"Sends messages to a GCP Cloud Pub/Sub topic. xref:configuration:metadata.adoc[Metadata] from messages are sent as attributes.","description":"\nFor information on how to set up credentials, see https://cloud.google.com/docs/authentication/production[this guide^].\n\n== Troubleshooting\n\nIf you're consistently seeing `Failed to send message to gcp_pubsub: context deadline exceeded` error logs without any further information it is possible that you are encountering https://github.com/benthosdev/benthos/issues/1042, which occurs when metadata values contain characters that are not valid utf-8. This can frequently occur when consuming from Kafka as the key metadata field may be populated with an arbitrary binary value, but this issue is not exclusive to Kafka.\n\nIf you are blocked by this issue then a work around is to delete either the specific problematic keys:\n\n```yaml\npipeline:\n processors:\n - mapping: |\n meta kafka_key = deleted()\n```\n\nOr delete all keys with:\n\n```yaml\npipeline:\n processors:\n - mapping: meta = deleted()\n```","categories":["Services","GCP"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"project","type":"string","kind":"scalar","description":"The project ID of the topic to publish to."},{"name":"credentials_json","type":"string","kind":"scalar","description":"An optional field to set Google Service Account Credentials json.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"topic","type":"string","kind":"scalar","description":"The topic to publish to.","interpolated":true},{"name":"endpoint","type":"string","kind":"scalar","description":"An optional endpoint to override the default of `pubsub.googleapis.com:443`. This can be used to connect to a region specific pubsub endpoint. For a list of valid values, see https://cloud.google.com/pubsub/docs/reference/service_apis_overview#list_of_regional_endpoints[this document^].","default":"","examples":["us-central1-pubsub.googleapis.com:443","us-west3-pubsub.googleapis.com:443"]},{"name":"ordering_key","type":"string","kind":"scalar","description":"The ordering key to use for publishing messages.","is_advanced":true,"is_optional":true,"interpolated":true},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increasing this may improve throughput.","default":64},{"name":"count_threshold","type":"int","kind":"scalar","description":"Publish a pubsub buffer when it has this many messages","default":100},{"name":"delay_threshold","type":"string","kind":"scalar","description":"Publish a non-empty pubsub buffer after this delay has passed.","default":"10ms"},{"name":"byte_threshold","type":"int","kind":"scalar","description":"Publish a batch when its size in bytes reaches this value.","default":1000000},{"name":"publish_timeout","type":"string","kind":"scalar","description":"The maximum length of time to wait before abandoning a publish attempt for a message.","is_advanced":true,"default":"1m0s","examples":["10s","5m","60m"]},{"name":"metadata","type":"object","kind":"scalar","description":"Specify criteria for which metadata values are sent as attributes, all are sent by default.","is_optional":true,"children":[{"name":"exclude_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages.","default":[]}]},{"name":"flow_control","type":"object","kind":"scalar","description":"For a given topic, configures the PubSub client's internal buffer for messages to be published.","is_advanced":true,"children":[{"name":"max_outstanding_bytes","type":"int","kind":"scalar","description":"Maximum size of buffered messages to be published. If less than or equal to zero, this is disabled.","is_advanced":true,"default":-1},{"name":"max_outstanding_messages","type":"int","kind":"scalar","description":"Maximum number of buffered messages to be published. If less than or equal to zero, this is disabled.","is_advanced":true,"default":1000},{"name":"limit_exceeded_behavior","type":"string","kind":"scalar","description":"Configures the behavior when trying to publish additional messages while the flow controller is full. The available options are block (default), ignore (disable), and signal_error (publish results will return an error).","is_advanced":true,"default":"block","options":["ignore","block","signal_error"],"linter":"\nlet options = {\n \"ignore\": true,\n \"block\": true,\n \"signal_error\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"}]},{"name":"batching","type":"object","kind":"","description":"Configures a batching policy on this output. While the PubSub client maintains its own internal buffering mechanism, preparing larger batches of messages can further trade-off some latency for throughput.","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]}},{"name":"hdfs","type":"output","status":"stable","plugin":true,"summary":"Sends message parts as files to a HDFS directory.","description":"Each file is written with the path specified with the 'path' field, in order to have a different path for each object you should use function interpolations described xref:configuration:interpolation.adoc#bloblang-queries[here].\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"hosts","type":"string","kind":"array","description":"A list of target host addresses to connect to.","examples":["localhost:9000"]},{"name":"user","type":"string","kind":"scalar","description":"A user ID to connect as.","default":""},{"name":"directory","type":"string","kind":"scalar","description":"A directory to store message files within. If the directory does not exist it will be created.","interpolated":true},{"name":"path","type":"string","kind":"scalar","description":"The path to upload messages as, interpolation functions should be used in order to generate unique file paths.","default":"${!counter()}-${!timestamp_unix_nano()}.txt","interpolated":true},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]}},{"name":"http_client","type":"output","status":"stable","plugin":true,"summary":"Sends messages to an HTTP server.","description":"\nWhen the number of retries expires the output will reject the message, the behavior after this will depend on the pipeline but usually this simply means the send is attempted again until successful whilst applying back pressure.\n\nThe URL and header values of this type can be dynamically set using function interpolations described in xref:configuration:interpolation.adoc#bloblang-queries[Bloblang queries].\n\nThe body of the HTTP request is the raw contents of the message payload. If the message has multiple parts (is a batch) the request will be sent according to https://www.w3.org/Protocols/rfc1341/7_2_Multipart.html[RFC1341^]. This behavior can be disabled by setting the field \u003c\u003cbatch_as_multipart, `batch_as_multipart`\u003e\u003e to `false`.\n\n== Propagate responses\n\nIt's possible to propagate the response from each HTTP request back to the input source by setting `propagate_response` to `true`. Only inputs that support xref:guides:sync_responses.adoc[synchronous responses] are able to make use of these propagated responses.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Network"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL to connect to.","interpolated":true},{"name":"verb","type":"string","kind":"scalar","description":"A verb to connect with","default":"POST","examples":["POST","GET","DELETE"]},{"name":"headers","type":"string","kind":"map","description":"A map of headers to add to the request.","default":{},"interpolated":true,"examples":[{"Content-Type":"application/octet-stream","traceparent":"${! tracing_span().traceparent }"}]},{"name":"metadata","type":"object","kind":"scalar","description":"Specify optional matching rules to determine which metadata keys should be added to the HTTP request as headers.","is_advanced":true,"is_optional":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","is_advanced":true,"default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","is_advanced":true,"default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"dump_request_log_level","type":"string","kind":"scalar","description":"EXPERIMENTAL: Optionally set a level at which the request and response payload of each request made will be logged.","is_advanced":true,"default":"","options":["TRACE","DEBUG","INFO","WARN","ERROR","FATAL",""],"version":"4.12.0","linter":"\nlet options = {\n \"trace\": true,\n \"debug\": true,\n \"info\": true,\n \"warn\": true,\n \"error\": true,\n \"fatal\": true,\n \"\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"oauth","type":"object","kind":"scalar","description":"Allows you to specify open authentication via OAuth version 1.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use OAuth version 1 in requests.","is_advanced":true,"default":false},{"name":"consumer_key","type":"string","kind":"scalar","description":"A value used to identify the client to the service provider.","is_advanced":true,"default":""},{"name":"consumer_secret","type":"string","kind":"scalar","description":"A secret used to establish ownership of the consumer key.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"access_token","type":"string","kind":"scalar","description":"A value used to gain access to the protected resources on behalf of the user.","is_advanced":true,"default":""},{"name":"access_token_secret","type":"string","kind":"scalar","description":"A secret provided in order to establish ownership of a given access token.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"oauth2","type":"object","kind":"scalar","description":"Allows you to specify open authentication via OAuth version 2 using the client credentials token flow.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use OAuth version 2 in requests.","is_advanced":true,"default":false},{"name":"client_key","type":"string","kind":"scalar","description":"A value used to identify the client to the token provider.","is_advanced":true,"default":""},{"name":"client_secret","type":"string","kind":"scalar","description":"A secret used to establish ownership of the client key.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token_url","type":"string","kind":"scalar","description":"The URL of the token provider.","is_advanced":true,"default":"","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"scopes","type":"string","kind":"array","description":"A list of optional requested permissions.","is_advanced":true,"default":[],"version":"3.45.0"},{"name":"endpoint_params","type":"unknown","kind":"map","description":"A list of optional endpoint parameters, values should be arrays of strings.","is_advanced":true,"is_optional":true,"default":{},"examples":[{"bar":["woof"],"foo":["meow","quack"]}],"version":"4.21.0","linter":"\nroot = if this.type() == \"object\" {\n this.values().map_each(ele -\u003e if ele.type() != \"array\" {\n \"field must be an object containing arrays of strings, got %s (%v)\".format(ele.format_json(no_indent: true), ele.type())\n } else {\n ele.map_each(str -\u003e if str.type() != \"string\" {\n \"field values must be strings, got %s (%v)\".format(str.format_json(no_indent: true), str.type())\n } else { deleted() })\n }).\n flatten()\n}\n"}]},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"jwt","type":"object","kind":"scalar","description":"BETA: Allows you to specify JWT authentication.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use JWT authentication in requests.","is_advanced":true,"default":false},{"name":"private_key_file","type":"string","kind":"scalar","description":"A file with the PEM encoded via PKCS1 or PKCS8 as private key.","is_advanced":true,"default":""},{"name":"signing_method","type":"string","kind":"scalar","description":"A method used to sign the token such as RS256, RS384, RS512 or EdDSA.","is_advanced":true,"default":""},{"name":"claims","type":"unknown","kind":"map","description":"A value used to identify the claims that issued the JWT.","is_advanced":true,"default":{}},{"name":"headers","type":"unknown","kind":"map","description":"Add optional key/value headers to the JWT.","is_advanced":true,"default":{}}]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"extract_headers","type":"object","kind":"scalar","description":"Specify which response headers should be added to resulting synchronous response messages as metadata. Header keys are lowercased before matching, so ensure that your patterns target lowercased versions of the header keys that you expect. This field is not applicable unless `propagate_response` is set to `true`.","is_advanced":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","is_advanced":true,"default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","is_advanced":true,"default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"rate_limit","type":"string","kind":"scalar","description":"An optional xref:components:rate_limits/about.adoc[rate limit] to throttle requests by.","is_optional":true},{"name":"timeout","type":"string","kind":"scalar","description":"A static timeout to apply to requests.","default":"5s"},{"name":"retry_period","type":"string","kind":"scalar","description":"The base period to wait between failed requests.","is_advanced":true,"default":"1s"},{"name":"max_retry_backoff","type":"string","kind":"scalar","description":"The maximum period to wait between failed requests.","is_advanced":true,"default":"300s"},{"name":"retries","type":"int","kind":"scalar","description":"The maximum number of retry attempts to make.","is_advanced":true,"default":3},{"name":"follow_redirects","type":"bool","kind":"scalar","description":"Whether or not to transparently follow redirects, i.e. responses with 300-399 status codes. If disabled, the response message will contain the body, status, and headers from the redirect response and the processor will not make a request to the URL set in the Location header of the response.","is_advanced":true,"default":true},{"name":"backoff_on","type":"int","kind":"array","description":"A list of status codes whereby the request should be considered to have failed and retries should be attempted, but the period between them should be increased gradually.","is_advanced":true,"default":[429]},{"name":"drop_on","type":"int","kind":"array","description":"A list of status codes whereby the request should be considered to have failed but retries should not be attempted. This is useful for preventing wasted retries for requests that will never succeed. Note that with these status codes the _request_ is dropped, but _message_ that caused the request will not be dropped.","is_advanced":true,"default":[]},{"name":"successful_on","type":"int","kind":"array","description":"A list of status codes whereby the attempt should be considered successful, this is useful for dropping requests that return non-2XX codes indicating that the message has been dealt with, such as a 303 See Other or a 409 Conflict. All 2XX codes are considered successful unless they are present within `backoff_on` or `drop_on`, regardless of this field.","is_advanced":true,"default":[]},{"name":"proxy_url","type":"string","kind":"scalar","description":"An optional HTTP proxy URL.","is_advanced":true,"is_optional":true},{"name":"disable_http2","type":"bool","kind":"scalar","description":"Whether or not to disable disable HTTP/2","is_advanced":true,"default":false,"version":"4.44.0"},{"name":"batch_as_multipart","type":"bool","kind":"scalar","description":"Send message batches as a single request using https://www.w3.org/Protocols/rfc1341/7_2_Multipart.html[RFC1341^]. If disabled messages in batches will be sent as individual requests.","is_advanced":true,"default":false},{"name":"propagate_response","type":"bool","kind":"scalar","description":"Whether responses from the server should be xref:guides:sync_responses.adoc[propagated back] to the input.","is_advanced":true,"default":false},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of parallel message batches to have in flight at any given time.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"multipart","type":"object","kind":"array","description":"EXPERIMENTAL: Create explicit multipart HTTP requests by specifying an array of parts to add to the request, each part specified consists of content headers and a data field that can be populated dynamically. If this field is populated it will override the default request creation behavior.","is_advanced":true,"default":[],"children":[{"name":"content_type","type":"string","kind":"scalar","description":"The content type of the individual message part.","is_advanced":true,"default":"","interpolated":true,"examples":["application/bin"]},{"name":"content_disposition","type":"string","kind":"scalar","description":"The content disposition of the individual message part.","is_advanced":true,"default":"","interpolated":true,"examples":["form-data; name=\"bin\"; filename='${! @AttachmentName }"]},{"name":"body","type":"string","kind":"scalar","description":"The body of the individual message part.","is_advanced":true,"default":"","interpolated":true,"examples":["${! this.data.part1 }"]}],"version":"3.63.0"}]}},{"name":"http_server","type":"output","status":"stable","plugin":true,"summary":"Sets up an HTTP server that will send messages over HTTP(S) GET requests. HTTP 2.0 is supported when using TLS, which is enabled when key and cert files are specified.","description":"Sets up an HTTP server that will send messages over HTTP(S) GET requests. If the `address` config field is left blank the xref:components:http/about.adoc[service-wide HTTP server] will be used.\n\nThree endpoints will be registered at the paths specified by the fields `path`, `stream_path` and `ws_path`. Which allow you to consume a single message batch, a continuous stream of line delimited messages, or a websocket of messages for each request respectively.\n\nWhen messages are batched the `path` endpoint encodes the batch according to https://www.w3.org/Protocols/rfc1341/7_2_Multipart.html[RFC1341^]. This behavior can be overridden by xref:configuration:batching.adoc#post-batch-processing[archiving your batches].\n\nPlease note, messages are considered delivered as soon as the data is written to the client. There is no concept of at least once delivery on this output.\n\n\n[CAUTION]\n.Endpoint caveats\n====\nComponents within a Redpanda Connect config will register their respective endpoints in a non-deterministic order. This means that establishing precedence of endpoints that are registered via multiple `http_server` inputs or outputs (either within brokers or from cohabiting streams) is not possible in a predictable way.\n\nThis ambiguity makes it difficult to ensure that paths which are both a subset of a path registered by a separate component, and end in a slash (`/`) and will therefore match against all extensions of that path, do not prevent the more specific path from matching against requests.\n\nIt is therefore recommended that you ensure paths of separate components do not collide unless they are explicitly non-competing.\n\nFor example, if you were to deploy two separate `http_server` inputs, one with a path `/foo/` and the other with a path `/foo/bar`, it would not be possible to ensure that the path `/foo/` does not swallow requests made to `/foo/bar`.\n====\n","categories":["Network"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"address","type":"string","kind":"scalar","description":"An alternative address to host from. If left empty the service wide address is used.","default":""},{"name":"path","type":"string","kind":"scalar","description":"The path from which discrete messages can be consumed.","default":"/get"},{"name":"stream_path","type":"string","kind":"scalar","description":"The path from which a continuous stream of messages can be consumed.","default":"/get/stream"},{"name":"ws_path","type":"string","kind":"scalar","description":"The path from which websocket connections can be established.","default":"/get/ws"},{"name":"allowed_verbs","type":"string","kind":"array","description":"An array of verbs that are allowed for the `path` and `stream_path` HTTP endpoint.","default":["GET"]},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum time to wait before a blocking, inactive connection is dropped (only applies to the `path` endpoint).","is_advanced":true,"default":"5s"},{"name":"cert_file","type":"string","kind":"scalar","description":"Enable TLS by specifying a certificate and key file. Only valid with a custom `address`.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"Enable TLS by specifying a certificate and key file. Only valid with a custom `address`.","is_advanced":true,"default":""},{"name":"cors","type":"object","kind":"scalar","description":"Adds Cross-Origin Resource Sharing headers. Only valid with a custom `address`.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to allow CORS requests.","is_advanced":true,"default":false},{"name":"allowed_origins","type":"string","kind":"array","description":"An explicit list of origins that are allowed for CORS requests.","is_advanced":true,"default":[]}],"version":"3.63.0"}]}},{"name":"inproc","type":"output","status":"stable","plugin":true,"description":"\nSends data directly to Redpanda Connect inputs by connecting to a unique ID. This allows you to hook up isolated streams whilst running Redpanda Connect in xref:guides:streams_mode/about.adoc[streams mode], it is NOT recommended that you connect the inputs of a stream with an output of the same stream, as feedback loops can lead to deadlocks in your message flow.\n\nIt is possible to connect multiple inputs to the same inproc ID, resulting in messages dispatching in a round-robin fashion to connected inputs. However, only one output can assume an inproc ID, and will replace existing outputs if a collision occurs.","categories":["Utility"],"config":{"name":"","type":"string","kind":"scalar","default":""}},{"name":"kafka","type":"output","status":"stable","plugin":true,"summary":"The kafka output type writes a batch of messages to Kafka brokers and waits for acknowledgement before propagating it back to the input.","description":"\nThe config field `ack_replicas` determines whether we wait for acknowledgement from all replicas or just a single broker.\n\nBoth the `key` and `topic` fields can be dynamically set using function interpolations described in xref:configuration:interpolation.adoc#bloblang-queries[Bloblang queries].\n\nxref:configuration:metadata.adoc[Metadata] will be added to each message sent as headers (version 0.11+), but can be restricted using the field \u003c\u003cmetadata, `metadata`\u003e\u003e.\n\n== Strict ordering and retries\n\nWhen strict ordering is required for messages written to topic partitions it is important to ensure that both the field `max_in_flight` is set to `1` and that the field `retry_as_batch` is set to `true`.\n\nYou must also ensure that failed batches are never rerouted back to the same output. This can be done by setting the field `max_retries` to `0` and `backoff.max_elapsed_time` to empty, which will apply back pressure indefinitely until the batch is sent successfully.\n\nHowever, this also means that manual intervention will eventually be required in cases where the batch cannot be sent due to configuration problems such as an incorrect `max_msg_bytes` estimate. A less strict but automated alternative would be to route failed batches to a dead letter queue using a xref:components:outputs/fallback.adoc[`fallback` broker], but this would allow subsequent batches to be delivered in the meantime whilst those failed batches are dealt with.\n\n== Troubleshooting\n\nIf you're seeing issues writing to or reading from Kafka with this component then it's worth trying out the newer xref:components:outputs/kafka_franz.adoc[`kafka_franz` output].\n\n- I'm seeing logs that report `Failed to connect to kafka: kafka: client has run out of available brokers to talk to (Is your cluster reachable?)`, but the brokers are definitely reachable.\n\nUnfortunately this error message will appear for a wide range of connection problems even when the broker endpoint can be reached. Double check your authentication configuration and also ensure that you have \u003c\u003ctlsenabled, enabled TLS\u003e\u003e if applicable.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"addresses","type":"string","kind":"array","description":"A list of broker addresses to connect to. If an item of the list contains commas it will be expanded into multiple addresses.","examples":[["localhost:9092"],["localhost:9041,localhost:9042"],["localhost:9041","localhost:9042"]]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"sasl","type":"object","kind":"scalar","description":"Enables SASL authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"mechanism","type":"string","kind":"scalar","description":"The SASL authentication mechanism, if left empty SASL authentication is not used.","is_advanced":true,"default":"none","annotated_options":[["OAUTHBEARER","OAuth Bearer based authentication."],["PLAIN","Plain text authentication. NOTE: When using plain text auth it is extremely likely that you'll also need to \u003c\u003ctls-enabled, enable TLS\u003e\u003e."],["SCRAM-SHA-256","Authentication using the SCRAM-SHA-256 mechanism."],["SCRAM-SHA-512","Authentication using the SCRAM-SHA-512 mechanism."],["none","Default, no SASL authentication."]],"linter":"\nlet options = {\n \"oauthbearer\": true,\n \"plain\": true,\n \"scram-sha-256\": true,\n \"scram-sha-512\": true,\n \"none\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"user","type":"string","kind":"scalar","description":"A PLAIN username. It is recommended that you use environment variables to populate this field.","is_advanced":true,"default":"","examples":["${USER}"]},{"name":"password","type":"string","kind":"scalar","description":"A PLAIN password. It is recommended that you use environment variables to populate this field.","is_advanced":true,"is_secret":true,"default":"","examples":["${PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"access_token","type":"string","kind":"scalar","description":"A static OAUTHBEARER access token","is_advanced":true,"default":""},{"name":"token_cache","type":"string","kind":"scalar","description":"Instead of using a static `access_token` allows you to query a xref:components:caches/about.adoc[`cache`] resource to fetch OAUTHBEARER tokens from","is_advanced":true,"default":""},{"name":"token_key","type":"string","kind":"scalar","description":"Required when using a `token_cache`, the key to query the cache with for tokens.","is_advanced":true,"default":""}]},{"name":"topic","type":"string","kind":"scalar","description":"The topic to publish messages to.","interpolated":true},{"name":"client_id","type":"string","kind":"scalar","description":"An identifier for the client connection.","is_advanced":true,"default":"benthos"},{"name":"target_version","type":"string","kind":"scalar","description":"The version of the Kafka protocol to use. This limits the capabilities used by the client and should ideally match the version of your brokers. Defaults to the oldest supported stable version.","is_optional":true,"examples":["2.1.0","3.1.0"]},{"name":"rack_id","type":"string","kind":"scalar","description":"A rack identifier for this client.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"The key to publish messages with.","default":"","interpolated":true},{"name":"partitioner","type":"string","kind":"scalar","description":"The partitioning algorithm to use.","default":"fnv1a_hash","options":["fnv1a_hash","murmur2_hash","random","round_robin","manual"],"linter":"\nlet options = {\n \"fnv1a_hash\": true,\n \"murmur2_hash\": true,\n \"random\": true,\n \"round_robin\": true,\n \"manual\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"partition","type":"string","kind":"scalar","description":"The manually-specified partition to publish messages to, relevant only when the field `partitioner` is set to `manual`. Must be able to parse as a 32-bit integer.","is_advanced":true,"default":"","interpolated":true},{"name":"custom_topic_creation","type":"object","kind":"scalar","description":"If enabled, topics will be created with the specified number of partitions and replication factor if they do not already exist.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to enable custom topic creation.","is_advanced":true,"default":false},{"name":"partitions","type":"int","kind":"scalar","description":"The number of partitions to create for new topics. Leave at -1 to use the broker configured default. Must be \u003e= 1.","is_advanced":true,"default":-1},{"name":"replication_factor","type":"int","kind":"scalar","description":"The replication factor to use for new topics. Leave at -1 to use the broker configured default. Must be an odd number, and less then or equal to the number of brokers.","is_advanced":true,"default":-1}]},{"name":"compression","type":"string","kind":"scalar","description":"The compression algorithm to use.","default":"none","options":["none","snappy","lz4","gzip","zstd"],"linter":"\nlet options = {\n \"none\": true,\n \"snappy\": true,\n \"lz4\": true,\n \"gzip\": true,\n \"zstd\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"static_headers","type":"string","kind":"map","description":"An optional map of static headers that should be added to messages in addition to metadata.","is_optional":true,"examples":[{"first-static-header":"value-1","second-static-header":"value-2"}]},{"name":"metadata","type":"object","kind":"scalar","description":"Specify criteria for which metadata values are sent with messages as headers.","children":[{"name":"exclude_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages.","default":[]}]},{"name":"inject_tracing_map","type":"string","kind":"scalar","description":"EXPERIMENTAL: A xref:guides:bloblang/about.adoc[Bloblang mapping] used to inject an object containing tracing propagation information into outbound messages. The specification of the injected fields will match the format used by the service wide tracer.","is_advanced":true,"is_optional":true,"bloblang":true,"examples":["meta = @.merge(this)","root.meta.span = this"],"version":"3.45.0"},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"idempotent_write","type":"bool","kind":"scalar","description":"Enable the idempotent write producer option. This requires the `IDEMPOTENT_WRITE` permission on `CLUSTER` and can be disabled if this permission is not available.","is_advanced":true,"default":false},{"name":"ack_replicas","type":"bool","kind":"scalar","description":"Ensure that messages have been copied across all replicas before acknowledging receipt.","is_advanced":true,"default":false},{"name":"max_msg_bytes","type":"int","kind":"scalar","description":"The maximum size in bytes of messages sent to the target topic.","is_advanced":true,"default":1000000},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period of time to wait for message sends before abandoning the request and retrying.","is_advanced":true,"default":"5s"},{"name":"retry_as_batch","type":"bool","kind":"scalar","description":"When enabled forces an entire batch of messages to be retried if any individual message fails on a send, otherwise only the individual messages that failed are retried. Disabling this helps to reduce message duplicates during intermittent errors, but also makes it impossible to guarantee strict ordering of messages.","is_advanced":true,"default":false},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"max_retries","type":"int","kind":"scalar","description":"The maximum number of retries before giving up on the request. If set to zero there is no discrete limit.","is_advanced":true,"default":0},{"name":"backoff","type":"object","kind":"scalar","description":"Control time intervals between retry attempts.","is_advanced":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"3s","examples":["50ms","1s"]},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts","is_advanced":true,"default":"10s","examples":["5s","1m"]},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum overall period of time to spend on retry attempts before the request is aborted. Setting this value to a zeroed duration (such as `0s`) will result in unbounded retries.","is_advanced":true,"default":"30s","examples":["1m","1h"]}]},{"name":"timestamp","type":"string","kind":"scalar","description":"An optional timestamp to set for each message. When left empty, the current timestamp is used.","is_advanced":true,"is_deprecated":true,"is_optional":true,"interpolated":true,"examples":["${! timestamp_unix() }","${! metadata(\"kafka_timestamp_unix\") }"]},{"name":"timestamp_ms","type":"string","kind":"scalar","description":"An optional timestamp to set for each message expressed in milliseconds. When left empty, the current timestamp is used.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["${! timestamp_unix_milli() }","${! metadata(\"kafka_timestamp_ms\") }"]}]}},{"name":"kafka_franz","type":"output","status":"beta","plugin":true,"summary":"A Kafka output using the https://github.com/twmb/franz-go[Franz Kafka client library^].","description":"\nWrites a batch of messages to Kafka brokers and waits for acknowledgement before propagating it back to the input.\n\nThis output often out-performs the traditional `kafka` output as well as providing more useful logs and error messages.\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"seed_brokers","type":"string","kind":"array","description":"A list of broker addresses to connect to in order to establish connections. If an item of the list contains commas it will be expanded into multiple addresses.","examples":[["localhost:9092"],["foo:9092","bar:9092"],["foo:9092,bar:9092"]]},{"name":"client_id","type":"string","kind":"scalar","description":"An identifier for the client connection.","is_advanced":true,"default":"benthos"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"sasl","type":"object","kind":"array","description":"Specify one or more methods of SASL authentication. SASL is tried in order; if the broker supports the first mechanism, all connections will use that mechanism. If the first mechanism fails, the client will pick the first supported mechanism. If the broker does not support any client mechanisms, connections will fail.","is_advanced":true,"is_optional":true,"examples":[[{"mechanism":"SCRAM-SHA-512","password":"bar","username":"foo"}]],"children":[{"name":"mechanism","type":"string","kind":"scalar","description":"The SASL mechanism to use.","is_advanced":true,"annotated_options":[["AWS_MSK_IAM","AWS IAM based authentication as specified by the 'aws-msk-iam-auth' java library."],["OAUTHBEARER","OAuth Bearer based authentication."],["PLAIN","Plain text authentication."],["SCRAM-SHA-256","SCRAM based authentication as specified in RFC5802."],["SCRAM-SHA-512","SCRAM based authentication as specified in RFC5802."],["none","Disable sasl authentication"]],"linter":"\nlet options = {\n \"aws_msk_iam\": true,\n \"oauthbearer\": true,\n \"plain\": true,\n \"scram-sha-256\": true,\n \"scram-sha-512\": true,\n \"none\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"username","type":"string","kind":"scalar","description":"A username to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token to use for a single session's OAUTHBEARER authentication.","is_advanced":true,"default":""},{"name":"extensions","type":"string","kind":"map","description":"Key/value pairs to add to OAUTHBEARER authentication requests.","is_advanced":true,"is_optional":true},{"name":"aws","type":"object","kind":"scalar","description":"Contains AWS specific fields for when the `mechanism` is set to `AWS_MSK_IAM`.","is_advanced":true,"is_optional":true,"children":[{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]}]},{"name":"metadata_max_age","type":"string","kind":"scalar","description":"The maximum age of metadata before it is refreshed.","is_advanced":true,"default":"5m"},{"name":"request_timeout_overhead","type":"string","kind":"scalar","description":"The request time overhead. Uses the given time as overhead while deadlining requests. Roughly equivalent to request.timeout.ms, but grants additional time to requests that have timeout fields.","is_advanced":true,"default":"10s"},{"name":"conn_idle_timeout","type":"string","kind":"scalar","description":"The rough amount of time to allow connections to idle before they are closed.","is_advanced":true,"default":"20s"},{"name":"topic","type":"string","kind":"scalar","description":"A topic to write messages to.","interpolated":true},{"name":"key","type":"string","kind":"scalar","description":"An optional key to populate for each message.","is_optional":true,"interpolated":true},{"name":"partition","type":"string","kind":"scalar","description":"An optional explicit partition to set for each message. This field is only relevant when the `partitioner` is set to `manual`. The provided interpolation string must be a valid integer.","is_optional":true,"interpolated":true,"examples":["${! meta(\"partition\") }"]},{"name":"metadata","type":"object","kind":"scalar","description":"Determine which (if any) metadata values should be added to messages as headers.","is_optional":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"timestamp","type":"string","kind":"scalar","description":"An optional timestamp to set for each message. When left empty, the current timestamp is used.","is_advanced":true,"is_deprecated":true,"is_optional":true,"interpolated":true,"examples":["${! timestamp_unix() }","${! metadata(\"kafka_timestamp_unix\") }"]},{"name":"timestamp_ms","type":"string","kind":"scalar","description":"An optional timestamp to set for each message expressed in milliseconds. When left empty, the current timestamp is used.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["${! timestamp_unix_milli() }","${! metadata(\"kafka_timestamp_ms\") }"]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of batches to be sending in parallel at any given time.","default":10},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"rack_id","type":"string","kind":"scalar","is_deprecated":true},{"name":"partitioner","type":"string","kind":"scalar","description":"Override the default murmur2 hashing partitioner.","is_advanced":true,"is_optional":true,"annotated_options":[["least_backup","Chooses the least backed up partition (the partition with the fewest amount of buffered records). Partitions are selected per batch."],["manual","Manually select a partition for each message, requires the field `partition` to be specified."],["murmur2_hash","Kafka's default hash algorithm that uses a 32-bit murmur2 hash of the key to compute which partition the record will be on."],["round_robin","Round-robin's messages through all available partitions. This algorithm has lower throughput and causes higher CPU load on brokers, but can be useful if you want to ensure an even distribution of records to partitions."]],"linter":"\nlet options = {\n \"least_backup\": true,\n \"manual\": true,\n \"murmur2_hash\": true,\n \"round_robin\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"idempotent_write","type":"bool","kind":"scalar","description":"Enable the idempotent write producer option. This requires the `IDEMPOTENT_WRITE` permission on `CLUSTER` and can be disabled if this permission is not available.","is_advanced":true,"default":true},{"name":"compression","type":"string","kind":"scalar","description":"Optionally set an explicit compression type. The default preference is to use snappy when the broker supports it, and fall back to none if not.","is_advanced":true,"is_optional":true,"options":["lz4","snappy","gzip","none","zstd"],"linter":"\nlet options = {\n \"lz4\": true,\n \"snappy\": true,\n \"gzip\": true,\n \"none\": true,\n \"zstd\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period of time to wait for message sends before abandoning the request and retrying","is_advanced":true,"default":"10s"},{"name":"max_message_bytes","type":"string","kind":"scalar","description":"The maximum space in bytes than an individual message may take, messages larger than this value will be rejected. This field corresponds to Kafka's `max.message.bytes`.","is_advanced":true,"default":"1MiB","examples":["100MB","50mib"]},{"name":"broker_write_max_bytes","type":"string","kind":"scalar","description":"The upper bound for the number of bytes written to a broker connection in a single write. This field corresponds to Kafka's `socket.request.max.bytes`.","is_advanced":true,"default":"100MiB","examples":["128MB","50mib"]}],"linter":"root = match {\n this.partitioner == \"manual\" \u0026\u0026 this.partition.or(\"\") == \"\" =\u003e \"a partition must be specified when the partitioner is set to manual\"\n this.partitioner != \"manual\" \u0026\u0026 this.partition.or(\"\") != \"\" =\u003e \"a partition cannot be specified unless the partitioner is set to manual\"\n this.timestamp.or(\"\") != \"\" \u0026\u0026 this.timestamp_ms.or(\"\") != \"\" =\u003e \"both timestamp and timestamp_ms cannot be specified simultaneously\"\n}"},"version":"3.61.0"},{"name":"mongodb","type":"output","status":"experimental","plugin":true,"summary":"Inserts items into a MongoDB collection.","description":"\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target MongoDB server.","examples":["mongodb://localhost:27017"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"database","type":"string","kind":"scalar","description":"The name of the target MongoDB database."},{"name":"username","type":"string","kind":"scalar","description":"The username to connect to the database.","default":""},{"name":"password","type":"string","kind":"scalar","description":"The password to connect to the database.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"app_name","type":"string","kind":"scalar","description":"The client application name.","is_advanced":true,"default":"benthos","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"collection","type":"string","kind":"scalar","description":"The name of the target collection.","interpolated":true},{"name":"operation","type":"string","kind":"scalar","description":"The mongodb operation to perform.","default":"update-one","options":["insert-one","delete-one","delete-many","replace-one","update-one"],"linter":"\nlet options = {\n \"insert-one\": true,\n \"delete-one\": true,\n \"delete-many\": true,\n \"replace-one\": true,\n \"update-one\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"write_concern","type":"object","kind":"scalar","description":"The write concern settings for the mongo connection.","children":[{"name":"w","type":"string","kind":"scalar","description":"W requests acknowledgement that write operations propagate to the specified number of mongodb instances. Can be the string \"majority\" to wait for a calculated majority of nodes to acknowledge the write operation, or an integer value specifying an minimum number of nodes to acknowledge the operation, or a string specifying the name of a custom write concern configured in the cluster.","default":"majority"},{"name":"j","type":"bool","kind":"scalar","description":"J requests acknowledgement from MongoDB that write operations are written to the journal.","default":false},{"name":"w_timeout","type":"string","kind":"scalar","description":"The write concern timeout.","default":""}]},{"name":"document_map","type":"string","kind":"scalar","description":"A bloblang map representing a document to store within MongoDB, expressed as https://www.mongodb.com/docs/manual/reference/mongodb-extended-json/[extended JSON in canonical form^]. The document map is required for the operations insert-one, replace-one, update-one and aggregate.","default":"","bloblang":true,"examples":["root.a = this.foo\nroot.b = this.bar"]},{"name":"filter_map","type":"string","kind":"scalar","description":"A bloblang map representing a filter for a MongoDB command, expressed as https://www.mongodb.com/docs/manual/reference/mongodb-extended-json/[extended JSON in canonical form^]. The filter map is required for all operations except insert-one. It is used to find the document(s) for the operation. For example in a delete-one case, the filter map should have the fields required to locate the document to delete.","default":"","bloblang":true,"examples":["root.a = this.foo\nroot.b = this.bar"]},{"name":"hint_map","type":"string","kind":"scalar","description":"A bloblang map representing the hint for the MongoDB command, expressed as https://www.mongodb.com/docs/manual/reference/mongodb-extended-json/[extended JSON in canonical form^]. This map is optional and is used with all operations except insert-one. It is used to improve performance of finding the documents in the mongodb.","default":"","bloblang":true,"examples":["root.a = this.foo\nroot.b = this.bar"]},{"name":"upsert","type":"bool","kind":"scalar","description":"The upsert setting is optional and only applies for update-one and replace-one operations. If the filter specified in filter_map matches, the document is updated or replaced accordingly, otherwise it is created.","default":false,"version":"3.60.0"},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"max_retries","type":"int","kind":"scalar","description":"The maximum number of retries before giving up on the request. If set to zero there is no discrete limit.","is_advanced":true,"is_deprecated":true,"default":3},{"name":"backoff","type":"object","kind":"scalar","description":"Control time intervals between retry attempts.","is_advanced":true,"is_deprecated":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"is_deprecated":true,"default":"1s"},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts.","is_advanced":true,"is_deprecated":true,"default":"5s"},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum period to wait before retry attempts are abandoned. If zero then no limit is used.","is_advanced":true,"is_deprecated":true,"default":"30s"}]}]},"version":"3.43.0"},{"name":"mqtt","type":"output","status":"stable","plugin":true,"summary":"Pushes messages to an MQTT broker.","description":"\nThe `topic` field can be dynamically set using function interpolations described xref:configuration:interpolation.adoc#bloblang-queries[here]. When sending batched messages these interpolations are performed per message part.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. The format should be `scheme://host:port` where `scheme` is one of `tcp`, `ssl`, or `ws`, `host` is the ip-address (or hostname) and `port` is the port on which the broker is accepting connections. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["tcp://localhost:1883"]],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"client_id","type":"string","kind":"scalar","description":"An identifier for the client connection.","default":""},{"name":"dynamic_client_id_suffix","type":"string","kind":"scalar","description":"Append a dynamically generated suffix to the specified `client_id` on each run of the pipeline. This can be useful when clustering Redpanda Connect producers.","is_advanced":true,"is_optional":true,"annotated_options":[["nanoid","append a nanoid of length 21 characters"]],"linter":"root = []"},{"name":"connect_timeout","type":"string","kind":"scalar","description":"The maximum amount of time to wait in order to establish a connection before the attempt is abandoned.","default":"30s","examples":["1s","500ms"],"version":"3.58.0"},{"name":"will","type":"object","kind":"scalar","description":"Set last will message in case of Redpanda Connect failure","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to enable last will messages.","is_advanced":true,"default":false},{"name":"qos","type":"int","kind":"scalar","description":"Set QoS for last will message. Valid values are: 0, 1, 2.","is_advanced":true,"default":0},{"name":"retained","type":"bool","kind":"scalar","description":"Set retained for last will message.","is_advanced":true,"default":false},{"name":"topic","type":"string","kind":"scalar","description":"Set topic for last will message.","is_advanced":true,"default":""},{"name":"payload","type":"string","kind":"scalar","description":"Set payload for last will message.","is_advanced":true,"default":""}]},{"name":"user","type":"string","kind":"scalar","description":"A username to connect with.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to connect with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"keepalive","type":"int","kind":"scalar","description":"Max seconds of inactivity before a keepalive message is sent.","is_advanced":true,"default":30},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"topic","type":"string","kind":"scalar","description":"The topic to publish messages to.","interpolated":true},{"name":"qos","type":"int","kind":"scalar","description":"The QoS value to set for each message. Has options 0, 1, 2.","default":1},{"name":"write_timeout","type":"string","kind":"scalar","description":"The maximum amount of time to wait to write data before the attempt is abandoned.","default":"3s","examples":["1s","500ms"],"version":"3.58.0"},{"name":"retained","type":"bool","kind":"scalar","description":"Set message as retained on the topic.","default":false},{"name":"retained_interpolated","type":"string","kind":"scalar","description":"Override the value of `retained` with an interpolable value, this allows it to be dynamically set based on message contents. The value must resolve to either `true` or `false`.","is_advanced":true,"is_optional":true,"interpolated":true,"version":"3.59.0"},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64}]}},{"name":"nanomsg","type":"output","status":"stable","plugin":true,"summary":"Send messages over a Nanomsg socket.","description":"Currently only PUSH and PUB sockets are supported.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.","categories":["Network"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"bind","type":"bool","kind":"scalar","description":"Whether the URLs listed should be bind (otherwise they are connected to).","default":false},{"name":"socket_type","type":"string","kind":"scalar","description":"The socket type to send with.","default":"PUSH","options":["PUSH","PUB"],"linter":"\nlet options = {\n \"push\": true,\n \"pub\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"poll_timeout","type":"string","kind":"scalar","description":"The maximum period of time to wait for a message to send before the request is abandoned and reattempted.","default":"5s"},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64}]}},{"name":"nats","type":"output","status":"stable","plugin":true,"summary":"Publish to an NATS subject.","description":"This output will interpolate functions within the subject field, you can find a list of functions xref:configuration:interpolation.adoc#bloblang-queries[here].\n\n== Connection name\n\nWhen monitoring and managing a production NATS system, it is often useful to\nknow which connection a message was send/received from. This can be achieved by\nsetting the connection name option when creating a NATS connection.\n\nRedpanda Connect will automatically set the connection name based off the label of the given\nNATS component, so that monitoring tools between NATS and Redpanda Connect can stay in sync.\n\n\n== Authentication\n\nThere are several components within Redpanda Connect which uses NATS services. You will find that each of these components\nsupport optional advanced authentication parameters for https://docs.nats.io/nats-server/configuration/securing_nats/auth_intro/nkey_auth[NKeys^]\nand https://docs.nats.io/using-nats/developer/connecting/creds[User Credentials^].\n\nSee an https://docs.nats.io/running-a-nats-service/nats_admin/security/jwt[in-depth tutorial^].\n\n=== NKey file\n\nThe NATS server can use these NKeys in several ways for authentication. The simplest is for the server to be configured\nwith a list of known public keys and for the clients to respond to the challenge by signing it with its private NKey\nconfigured in the `nkey_file` or `nkey` field.\n\nhttps://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[More details^].\n\n=== User credentials\n\nNATS server supports decentralized authentication based on JSON Web Tokens (JWT). Clients need an https://docs.nats.io/nats-server/configuration/securing_nats/jwt#json-web-tokens[user JWT^]\nand a corresponding https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[NKey secret^] when connecting to a server\nwhich is configured to use this authentication scheme.\n\nThe `user_credentials_file` field should point to a file containing both the private key and the JWT and can be\ngenerated with the https://docs.nats.io/nats-tools/nsc[nsc tool^].\n\nAlternatively, the `user_jwt` field can contain a plain text JWT and the `user_nkey_seed`can contain\nthe plain text NKey Seed.\n\nhttps://docs.nats.io/using-nats/developer/connecting/creds[More details^].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["nats://127.0.0.1:4222"],["nats://username:password@127.0.0.1:4222"]]},{"name":"subject","type":"string","kind":"scalar","description":"The subject to publish to.","interpolated":true,"examples":["foo.bar.baz"]},{"name":"headers","type":"string","kind":"map","description":"Explicit message headers to add to messages.","default":{},"interpolated":true,"examples":[{"Content-Type":"application/json","Timestamp":"${!meta(\"Timestamp\")}"}]},{"name":"metadata","type":"object","kind":"scalar","description":"Determine which (if any) metadata values should be added to messages as headers.","is_optional":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auth","type":"object","kind":"scalar","description":"Optional configuration of NATS authentication parameters.","is_advanced":true,"children":[{"name":"nkey_file","type":"string","kind":"scalar","description":"An optional file containing a NKey seed.","is_advanced":true,"is_optional":true,"examples":["./seed.nk"]},{"name":"nkey","type":"string","kind":"scalar","description":"The NKey seed.","is_advanced":true,"is_optional":true,"is_secret":true,"examples":["UDXU4RCSJNZOIQHZNWXHXORDPRTGNJAHAHFRGZNEEJCPQTT2M7NLCNF4"],"version":"4.38.0","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_credentials_file","type":"string","kind":"scalar","description":"An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.","is_advanced":true,"is_optional":true,"examples":["./user.creds"]},{"name":"user_jwt","type":"string","kind":"scalar","description":"An optional plain text user JWT (given along with the corresponding user NKey Seed).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_nkey_seed","type":"string","kind":"scalar","description":"An optional plain text user NKey Seed (given along with the corresponding user JWT).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"inject_tracing_map","type":"string","kind":"scalar","description":"EXPERIMENTAL: A xref:guides:bloblang/about.adoc[Bloblang mapping] used to inject an object containing tracing propagation information into outbound messages. The specification of the injected fields will match the format used by the service wide tracer.","is_advanced":true,"is_optional":true,"bloblang":true,"examples":["meta = @.merge(this)","root.meta.span = this"],"version":"4.23.0"}]}},{"name":"nats_jetstream","type":"output","status":"stable","plugin":true,"summary":"Write messages to a NATS JetStream subject.","description":"== Connection name\n\nWhen monitoring and managing a production NATS system, it is often useful to\nknow which connection a message was send/received from. This can be achieved by\nsetting the connection name option when creating a NATS connection.\n\nRedpanda Connect will automatically set the connection name based off the label of the given\nNATS component, so that monitoring tools between NATS and Redpanda Connect can stay in sync.\n\n\n== Authentication\n\nThere are several components within Redpanda Connect which uses NATS services. You will find that each of these components\nsupport optional advanced authentication parameters for https://docs.nats.io/nats-server/configuration/securing_nats/auth_intro/nkey_auth[NKeys^]\nand https://docs.nats.io/using-nats/developer/connecting/creds[User Credentials^].\n\nSee an https://docs.nats.io/running-a-nats-service/nats_admin/security/jwt[in-depth tutorial^].\n\n=== NKey file\n\nThe NATS server can use these NKeys in several ways for authentication. The simplest is for the server to be configured\nwith a list of known public keys and for the clients to respond to the challenge by signing it with its private NKey\nconfigured in the `nkey_file` or `nkey` field.\n\nhttps://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[More details^].\n\n=== User credentials\n\nNATS server supports decentralized authentication based on JSON Web Tokens (JWT). Clients need an https://docs.nats.io/nats-server/configuration/securing_nats/jwt#json-web-tokens[user JWT^]\nand a corresponding https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[NKey secret^] when connecting to a server\nwhich is configured to use this authentication scheme.\n\nThe `user_credentials_file` field should point to a file containing both the private key and the JWT and can be\ngenerated with the https://docs.nats.io/nats-tools/nsc[nsc tool^].\n\nAlternatively, the `user_jwt` field can contain a plain text JWT and the `user_nkey_seed`can contain\nthe plain text NKey Seed.\n\nhttps://docs.nats.io/using-nats/developer/connecting/creds[More details^].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["nats://127.0.0.1:4222"],["nats://username:password@127.0.0.1:4222"]]},{"name":"subject","type":"string","kind":"scalar","description":"A subject to write to.","interpolated":true,"examples":["foo.bar.baz","${! meta(\"kafka_topic\") }","foo.${! json(\"meta.type\") }"]},{"name":"headers","type":"string","kind":"map","description":"Explicit message headers to add to messages.","default":{},"interpolated":true,"examples":[{"Content-Type":"application/json","Timestamp":"${!meta(\"Timestamp\")}"}],"version":"4.1.0"},{"name":"metadata","type":"object","kind":"scalar","description":"Determine which (if any) metadata values should be added to messages as headers.","is_optional":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":1024},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auth","type":"object","kind":"scalar","description":"Optional configuration of NATS authentication parameters.","is_advanced":true,"children":[{"name":"nkey_file","type":"string","kind":"scalar","description":"An optional file containing a NKey seed.","is_advanced":true,"is_optional":true,"examples":["./seed.nk"]},{"name":"nkey","type":"string","kind":"scalar","description":"The NKey seed.","is_advanced":true,"is_optional":true,"is_secret":true,"examples":["UDXU4RCSJNZOIQHZNWXHXORDPRTGNJAHAHFRGZNEEJCPQTT2M7NLCNF4"],"version":"4.38.0","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_credentials_file","type":"string","kind":"scalar","description":"An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.","is_advanced":true,"is_optional":true,"examples":["./user.creds"]},{"name":"user_jwt","type":"string","kind":"scalar","description":"An optional plain text user JWT (given along with the corresponding user NKey Seed).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_nkey_seed","type":"string","kind":"scalar","description":"An optional plain text user NKey Seed (given along with the corresponding user JWT).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"inject_tracing_map","type":"string","kind":"scalar","description":"EXPERIMENTAL: A xref:guides:bloblang/about.adoc[Bloblang mapping] used to inject an object containing tracing propagation information into outbound messages. The specification of the injected fields will match the format used by the service wide tracer.","is_advanced":true,"is_optional":true,"bloblang":true,"examples":["meta = @.merge(this)","root.meta.span = this"],"version":"4.23.0"}]},"version":"3.46.0"},{"name":"nats_kv","type":"output","status":"beta","plugin":true,"summary":"Put messages in a NATS key-value bucket.","description":"\nThe field `key` supports\nxref:configuration:interpolation.adoc#bloblang-queries[interpolation functions], allowing\nyou to create a unique key for each message.\n\n== Connection name\n\nWhen monitoring and managing a production NATS system, it is often useful to\nknow which connection a message was send/received from. This can be achieved by\nsetting the connection name option when creating a NATS connection.\n\nRedpanda Connect will automatically set the connection name based off the label of the given\nNATS component, so that monitoring tools between NATS and Redpanda Connect can stay in sync.\n\n\n== Authentication\n\nThere are several components within Redpanda Connect which uses NATS services. You will find that each of these components\nsupport optional advanced authentication parameters for https://docs.nats.io/nats-server/configuration/securing_nats/auth_intro/nkey_auth[NKeys^]\nand https://docs.nats.io/using-nats/developer/connecting/creds[User Credentials^].\n\nSee an https://docs.nats.io/running-a-nats-service/nats_admin/security/jwt[in-depth tutorial^].\n\n=== NKey file\n\nThe NATS server can use these NKeys in several ways for authentication. The simplest is for the server to be configured\nwith a list of known public keys and for the clients to respond to the challenge by signing it with its private NKey\nconfigured in the `nkey_file` or `nkey` field.\n\nhttps://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[More details^].\n\n=== User credentials\n\nNATS server supports decentralized authentication based on JSON Web Tokens (JWT). Clients need an https://docs.nats.io/nats-server/configuration/securing_nats/jwt#json-web-tokens[user JWT^]\nand a corresponding https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[NKey secret^] when connecting to a server\nwhich is configured to use this authentication scheme.\n\nThe `user_credentials_file` field should point to a file containing both the private key and the JWT and can be\ngenerated with the https://docs.nats.io/nats-tools/nsc[nsc tool^].\n\nAlternatively, the `user_jwt` field can contain a plain text JWT and the `user_nkey_seed`can contain\nthe plain text NKey Seed.\n\nhttps://docs.nats.io/using-nats/developer/connecting/creds[More details^].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["nats://127.0.0.1:4222"],["nats://username:password@127.0.0.1:4222"]]},{"name":"bucket","type":"string","kind":"scalar","description":"The name of the KV bucket.","examples":["my_kv_bucket"]},{"name":"key","type":"string","kind":"scalar","description":"The key for each message.","interpolated":true,"examples":["foo","foo.bar.baz","foo.${! json(\"meta.type\") }"]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":1024},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auth","type":"object","kind":"scalar","description":"Optional configuration of NATS authentication parameters.","is_advanced":true,"children":[{"name":"nkey_file","type":"string","kind":"scalar","description":"An optional file containing a NKey seed.","is_advanced":true,"is_optional":true,"examples":["./seed.nk"]},{"name":"nkey","type":"string","kind":"scalar","description":"The NKey seed.","is_advanced":true,"is_optional":true,"is_secret":true,"examples":["UDXU4RCSJNZOIQHZNWXHXORDPRTGNJAHAHFRGZNEEJCPQTT2M7NLCNF4"],"version":"4.38.0","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_credentials_file","type":"string","kind":"scalar","description":"An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.","is_advanced":true,"is_optional":true,"examples":["./user.creds"]},{"name":"user_jwt","type":"string","kind":"scalar","description":"An optional plain text user JWT (given along with the corresponding user NKey Seed).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_nkey_seed","type":"string","kind":"scalar","description":"An optional plain text user NKey Seed (given along with the corresponding user JWT).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},"version":"4.12.0"},{"name":"nats_stream","type":"output","status":"stable","plugin":true,"summary":"Publish to a NATS Stream subject.","description":"\n[CAUTION]\n.Deprecation notice\n====\nThe NATS Streaming Server is being deprecated. Critical bug fixes and security fixes will be applied until June of 2023. NATS-enabled applications requiring persistence should use https://docs.nats.io/nats-concepts/jetstream[JetStream^].\n====\n\n\n\n== Authentication\n\nThere are several components within Redpanda Connect which uses NATS services. You will find that each of these components\nsupport optional advanced authentication parameters for https://docs.nats.io/nats-server/configuration/securing_nats/auth_intro/nkey_auth[NKeys^]\nand https://docs.nats.io/using-nats/developer/connecting/creds[User Credentials^].\n\nSee an https://docs.nats.io/running-a-nats-service/nats_admin/security/jwt[in-depth tutorial^].\n\n=== NKey file\n\nThe NATS server can use these NKeys in several ways for authentication. The simplest is for the server to be configured\nwith a list of known public keys and for the clients to respond to the challenge by signing it with its private NKey\nconfigured in the `nkey_file` or `nkey` field.\n\nhttps://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[More details^].\n\n=== User credentials\n\nNATS server supports decentralized authentication based on JSON Web Tokens (JWT). Clients need an https://docs.nats.io/nats-server/configuration/securing_nats/jwt#json-web-tokens[user JWT^]\nand a corresponding https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[NKey secret^] when connecting to a server\nwhich is configured to use this authentication scheme.\n\nThe `user_credentials_file` field should point to a file containing both the private key and the JWT and can be\ngenerated with the https://docs.nats.io/nats-tools/nsc[nsc tool^].\n\nAlternatively, the `user_jwt` field can contain a plain text JWT and the `user_nkey_seed`can contain\nthe plain text NKey Seed.\n\nhttps://docs.nats.io/using-nats/developer/connecting/creds[More details^].\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["nats://127.0.0.1:4222"],["nats://username:password@127.0.0.1:4222"]]},{"name":"cluster_id","type":"string","kind":"scalar","description":"The cluster ID to publish to."},{"name":"subject","type":"string","kind":"scalar","description":"The subject to publish to."},{"name":"client_id","type":"string","kind":"scalar","description":"The client ID to connect with.","default":""},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auth","type":"object","kind":"scalar","description":"Optional configuration of NATS authentication parameters.","is_advanced":true,"children":[{"name":"nkey_file","type":"string","kind":"scalar","description":"An optional file containing a NKey seed.","is_advanced":true,"is_optional":true,"examples":["./seed.nk"]},{"name":"nkey","type":"string","kind":"scalar","description":"The NKey seed.","is_advanced":true,"is_optional":true,"is_secret":true,"examples":["UDXU4RCSJNZOIQHZNWXHXORDPRTGNJAHAHFRGZNEEJCPQTT2M7NLCNF4"],"version":"4.38.0","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_credentials_file","type":"string","kind":"scalar","description":"An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.","is_advanced":true,"is_optional":true,"examples":["./user.creds"]},{"name":"user_jwt","type":"string","kind":"scalar","description":"An optional plain text user JWT (given along with the corresponding user NKey Seed).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_nkey_seed","type":"string","kind":"scalar","description":"An optional plain text user NKey Seed (given along with the corresponding user JWT).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"inject_tracing_map","type":"string","kind":"scalar","description":"EXPERIMENTAL: A xref:guides:bloblang/about.adoc[Bloblang mapping] used to inject an object containing tracing propagation information into outbound messages. The specification of the injected fields will match the format used by the service wide tracer.","is_advanced":true,"is_optional":true,"bloblang":true,"examples":["meta = @.merge(this)","root.meta.span = this"],"version":"4.23.0"}]}},{"name":"nsq","type":"output","status":"stable","plugin":true,"summary":"Publish to an NSQ topic.","description":"The `topic` field can be dynamically set using function interpolations described xref:configuration:interpolation.adoc#bloblang-queries[here]. When sending batched messages these interpolations are performed per message part.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"nsqd_tcp_address","type":"string","kind":"scalar","description":"The address of the target NSQD server."},{"name":"topic","type":"string","kind":"scalar","description":"The topic to publish to.","interpolated":true},{"name":"user_agent","type":"string","kind":"scalar","description":"A user agent to assume when connecting.","is_optional":true},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64}]}},{"name":"ockam_kafka","type":"output","status":"experimental","plugin":true,"summary":"Ockam","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"kafka","type":"object","kind":"scalar","children":[{"name":"seed_brokers","type":"string","kind":"array","description":"A list of broker addresses to connect to in order to establish connections. If an item of the list contains commas it will be expanded into multiple addresses.","is_optional":true,"examples":[["localhost:9092"],["foo:9092","bar:9092"],["foo:9092,bar:9092"]]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of batches to be sending in parallel at any given time.","default":10},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"partitioner","type":"string","kind":"scalar","description":"Override the default murmur2 hashing partitioner.","is_advanced":true,"is_optional":true,"annotated_options":[["least_backup","Chooses the least backed up partition (the partition with the fewest amount of buffered records). Partitions are selected per batch."],["manual","Manually select a partition for each message, requires the field `partition` to be specified."],["murmur2_hash","Kafka's default hash algorithm that uses a 32-bit murmur2 hash of the key to compute which partition the record will be on."],["round_robin","Round-robin's messages through all available partitions. This algorithm has lower throughput and causes higher CPU load on brokers, but can be useful if you want to ensure an even distribution of records to partitions."]],"linter":"\nlet options = {\n \"least_backup\": true,\n \"manual\": true,\n \"murmur2_hash\": true,\n \"round_robin\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"idempotent_write","type":"bool","kind":"scalar","description":"Enable the idempotent write producer option. This requires the `IDEMPOTENT_WRITE` permission on `CLUSTER` and can be disabled if this permission is not available.","is_advanced":true,"default":true},{"name":"compression","type":"string","kind":"scalar","description":"Optionally set an explicit compression type. The default preference is to use snappy when the broker supports it, and fall back to none if not.","is_advanced":true,"is_optional":true,"options":["lz4","snappy","gzip","none","zstd"],"linter":"\nlet options = {\n \"lz4\": true,\n \"snappy\": true,\n \"gzip\": true,\n \"none\": true,\n \"zstd\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period of time to wait for message sends before abandoning the request and retrying","is_advanced":true,"default":"10s"},{"name":"max_message_bytes","type":"string","kind":"scalar","description":"The maximum space in bytes than an individual message may take, messages larger than this value will be rejected. This field corresponds to Kafka's `max.message.bytes`.","is_advanced":true,"default":"1MiB","examples":["100MB","50mib"]},{"name":"broker_write_max_bytes","type":"string","kind":"scalar","description":"The upper bound for the number of bytes written to a broker connection in a single write. This field corresponds to Kafka's `socket.request.max.bytes`.","is_advanced":true,"default":"100MiB","examples":["128MB","50mib"]},{"name":"topic","type":"string","kind":"scalar","description":"A topic to write messages to.","interpolated":true},{"name":"key","type":"string","kind":"scalar","description":"An optional key to populate for each message.","is_optional":true,"interpolated":true},{"name":"partition","type":"string","kind":"scalar","description":"An optional explicit partition to set for each message. This field is only relevant when the `partitioner` is set to `manual`. The provided interpolation string must be a valid integer.","is_optional":true,"interpolated":true,"examples":["${! meta(\"partition\") }"]},{"name":"metadata","type":"object","kind":"scalar","description":"Determine which (if any) metadata values should be added to messages as headers.","is_optional":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"timestamp","type":"string","kind":"scalar","description":"An optional timestamp to set for each message. When left empty, the current timestamp is used.","is_advanced":true,"is_deprecated":true,"is_optional":true,"interpolated":true,"examples":["${! timestamp_unix() }","${! metadata(\"kafka_timestamp_unix\") }"]},{"name":"timestamp_ms","type":"string","kind":"scalar","description":"An optional timestamp to set for each message expressed in milliseconds. When left empty, the current timestamp is used.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["${! timestamp_unix_milli() }","${! metadata(\"kafka_timestamp_ms\") }"]}]},{"name":"disable_content_encryption","type":"bool","kind":"scalar","default":false},{"name":"enrollment_ticket","type":"string","kind":"scalar","is_optional":true},{"name":"identity_name","type":"string","kind":"scalar","is_optional":true},{"name":"allow","type":"string","kind":"scalar","is_optional":true,"default":"self"},{"name":"route_to_kafka_outlet","type":"string","kind":"scalar","default":"self"},{"name":"allow_consumer","type":"string","kind":"scalar","default":"self"},{"name":"route_to_consumer","type":"string","kind":"scalar","default":"/ip4/127.0.0.1/tcp/6262"},{"name":"encrypted_fields","type":"string","kind":"array","description":"The fields to encrypt in the kafka messages, assuming the record is a valid JSON map. By default, the whole record is encrypted.","default":[]}]}},{"name":"opensearch","type":"output","status":"stable","plugin":true,"summary":"Publishes messages into an Elasticsearch index. If the index does not exist then it is created with a dynamic mapping.","description":"\nBoth the `id` and `index` fields can be dynamically set using function interpolations described xref:configuration:interpolation.adoc#bloblang-queries[here]. When sending batched messages these interpolations are performed per message part.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services"],"examples":[{"title":"Updating Documents","summary":"When https://opensearch.org/docs/latest/api-reference/document-apis/update-document/[updating documents^] the request body should contain a combination of a `doc`, `upsert`, and/or `script` fields at the top level, this should be done via mapping processors.","config":"\noutput:\n processors:\n - mapping: |\n meta id = this.id\n root.doc = this\n opensearch:\n urls: [ TODO ]\n index: foo\n id: ${! @id }\n action: update\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["http://localhost:9200"]]},{"name":"index","type":"string","kind":"scalar","description":"The index to place messages.","interpolated":true},{"name":"action","type":"string","kind":"scalar","description":"The action to take on the document. This field must resolve to one of the following action types: `index`, `update` or `delete`.","interpolated":true},{"name":"id","type":"string","kind":"scalar","description":"The ID for indexed messages. Interpolation should be used in order to create a unique ID for each message.","interpolated":true,"examples":["${!counter()}-${!timestamp_unix()}"]},{"name":"pipeline","type":"string","kind":"scalar","description":"An optional pipeline id to preprocess incoming documents.","is_advanced":true,"default":"","interpolated":true},{"name":"routing","type":"string","kind":"scalar","description":"The routing key to use for the document.","is_advanced":true,"default":"","interpolated":true},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"aws","type":"object","kind":"scalar","description":"Enables and customises connectivity to Amazon Elastic Service.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to connect to Amazon Elastic Service.","is_advanced":true,"default":false},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]}]}},{"name":"pinecone","type":"output","status":"experimental","plugin":true,"summary":"Inserts items into a Pinecone index.","description":"\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["AI"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"host","type":"string","kind":"scalar","description":"The host for the Pinecone index.","linter":"root = if this.has_prefix(\"https://\") { [\"host field must be a FQDN not a URL (remove the https:// prefix)\"] }"},{"name":"api_key","type":"string","kind":"scalar","description":"The Pinecone api key.","is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"operation","type":"string","kind":"scalar","description":"The operation to perform against the Pinecone index.","default":"upsert-vectors","options":["update-vector","upsert-vectors","delete-vectors"],"linter":"\nlet options = {\n \"update-vector\": true,\n \"upsert-vectors\": true,\n \"delete-vectors\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"namespace","type":"string","kind":"scalar","description":"The namespace to write to - writes to the default namespace by default.","is_advanced":true,"default":"","interpolated":true},{"name":"id","type":"string","kind":"scalar","description":"The ID for the index entry in Pinecone.","interpolated":true},{"name":"vector_mapping","type":"string","kind":"scalar","description":"The mapping to extract out the vector from the document. The result must be a floating point array. Required if not a delete operation.","is_optional":true,"bloblang":true,"examples":["root = this.embeddings_vector","root = [1.2, 0.5, 0.76]"]},{"name":"metadata_mapping","type":"string","kind":"scalar","description":"An optional mapping of message to metadata in the Pinecone index entry.","is_optional":true,"bloblang":true,"examples":["root = @","root = metadata()","root = {\"summary\": this.summary, \"foo\": this.other_field}"]}]},"version":"4.31.0"},{"name":"pulsar","type":"output","status":"experimental","plugin":true,"summary":"Write messages to an Apache Pulsar server.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"A URL to connect to.","examples":["pulsar://localhost:6650","pulsar://pulsar.us-west.example.com:6650","pulsar+ssl://pulsar.us-west.example.com:6651"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"topic","type":"string","kind":"scalar","description":"The topic to publish to."},{"name":"tls","type":"object","kind":"scalar","description":"Specify the path to a custom CA certificate to trust broker TLS service.","children":[{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","default":"","examples":["./root_cas.pem"]}]},{"name":"key","type":"string","kind":"scalar","description":"The key to publish messages with.","default":"","interpolated":true},{"name":"ordering_key","type":"string","kind":"scalar","description":"The ordering key to publish messages with.","default":"","interpolated":true},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"auth","type":"object","kind":"scalar","description":"Optional configuration of Pulsar authentication methods.","is_advanced":true,"is_optional":true,"children":[{"name":"oauth2","type":"object","kind":"scalar","description":"Parameters for Pulsar OAuth2 authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether OAuth2 is enabled.","is_advanced":true,"default":false},{"name":"audience","type":"string","kind":"scalar","description":"OAuth2 audience.","is_advanced":true,"default":""},{"name":"issuer_url","type":"string","kind":"scalar","description":"OAuth2 issuer URL.","is_advanced":true,"default":"","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"scope","type":"string","kind":"scalar","description":"OAuth2 scope to request.","is_advanced":true,"default":"","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"private_key_file","type":"string","kind":"scalar","description":"The path to a file containing a private key.","is_advanced":true,"default":""}]},{"name":"token","type":"object","kind":"scalar","description":"Parameters for Pulsar Token authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether Token Auth is enabled.","is_advanced":true,"default":false},{"name":"token","type":"string","kind":"scalar","description":"Actual base64 encoded token.","is_advanced":true,"default":""}]}],"version":"3.60.0"}]},"version":"3.43.0"},{"name":"pusher","type":"output","status":"experimental","plugin":true,"summary":"Output for publishing messages to Pusher API (https://pusher.com)","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"batching","type":"object","kind":"","description":"maximum batch size is 10 (limit of the pusher library)","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"channel","type":"string","kind":"scalar","description":"Pusher channel to publish to. Interpolation functions can also be used","interpolated":true,"examples":["my_channel","${!json(\"id\")}"]},{"name":"event","type":"string","kind":"scalar","description":"Event to publish to"},{"name":"appId","type":"string","kind":"scalar","description":"Pusher app id"},{"name":"key","type":"string","kind":"scalar","description":"Pusher key"},{"name":"secret","type":"string","kind":"scalar","description":"Pusher secret"},{"name":"cluster","type":"string","kind":"scalar","description":"Pusher cluster"},{"name":"secure","type":"bool","kind":"scalar","description":"Enable SSL encryption","default":true},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of parallel message batches to have in flight at any given time.","default":1}]},"version":"4.3.0"},{"name":"qdrant","type":"output","status":"experimental","plugin":true,"summary":"Adds items to a https://qdrant.tech/[Qdrant^] collection","description":"\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["AI"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"grpc_host","type":"string","kind":"scalar","description":"The gRPC host of the Qdrant server.","examples":["localhost:6334","xyz-example.eu-central.aws.cloud.qdrant.io:6334"]},{"name":"api_token","type":"string","kind":"scalar","description":"The Qdrant API token for authentication. Defaults to an empty string.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"tls","type":"object","kind":"scalar","description":"TLS(HTTPS) config to use when connecting","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"collection_name","type":"string","kind":"scalar","description":"The name of the collection in Qdrant.","interpolated":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of the point to insert. Can be a UUID string or positive integer.","bloblang":true,"examples":["root = \"dc88c126-679f-49f5-ab85-04b77e8c2791\"","root = 832"]},{"name":"vector_mapping","type":"string","kind":"scalar","description":"The mapping to extract the vector from the document.","bloblang":true,"examples":["root = {\"dense_vector\": [0.352,0.532,0.754],\"sparse_vector\": {\"indices\": [23,325,532],\"values\": [0.352,0.532,0.532]}, \"multi_vector\": [[0.352,0.532],[0.352,0.532]]}","root = [1.2, 0.5, 0.76]","root = this.vector","root = [[0.352,0.532,0.532,0.234],[0.352,0.532,0.532,0.234]]","root = {\"some_sparse\": {\"indices\":[23,325,532],\"values\":[0.352,0.532,0.532]}}","root = {\"some_multi\": [[0.352,0.532,0.532,0.234],[0.352,0.532,0.532,0.234]]}","root = {\"some_dense\": [0.352,0.532,0.532,0.234]}"]},{"name":"payload_mapping","type":"string","kind":"scalar","description":"An optional mapping of message to payload associated with the point.","default":"root = {}","bloblang":true,"examples":["root = {\"field\": this.value, \"field_2\": 987}","root = metadata()"]}]},"version":"4.33.0"},{"name":"questdb","type":"output","status":"experimental","plugin":true,"summary":"Pushes messages to a QuestDB table","description":"Important: We recommend that the dedupe feature is enabled on the QuestDB server. Please visit https://questdb.io/docs/ for more information about deploying, configuring, and using QuestDB.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"address","type":"string","kind":"scalar","description":"Address of the QuestDB server's HTTP port (excluding protocol)","examples":["localhost:9000"]},{"name":"username","type":"string","kind":"scalar","description":"Username for HTTP basic auth","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"password","type":"string","kind":"scalar","description":"Password for HTTP basic auth","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"Bearer token for HTTP auth (takes precedence over basic auth username \u0026 password)","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"retry_timeout","type":"string","kind":"scalar","description":"The time to continue retrying after a failed HTTP request. The interval between retries is an exponential backoff starting at 10ms and doubling after each failed attempt up to a maximum of 1 second.","is_advanced":true,"is_optional":true},{"name":"request_timeout","type":"string","kind":"scalar","description":"The time to wait for a response from the server. This is in addition to the calculation derived from the request_min_throughput parameter.","is_advanced":true,"is_optional":true},{"name":"request_min_throughput","type":"int","kind":"scalar","description":"Minimum expected throughput in bytes per second for HTTP requests. If the throughput is lower than this value, the connection will time out. This is used to calculate an additional timeout on top of request_timeout. This is useful for large requests. You can set this value to 0 to disable this logic.","is_advanced":true,"is_optional":true},{"name":"table","type":"string","kind":"scalar","description":"Destination table","examples":["trades"]},{"name":"designated_timestamp_field","type":"string","kind":"scalar","description":"Name of the designated timestamp field","is_optional":true},{"name":"designated_timestamp_unit","type":"string","kind":"scalar","description":"Designated timestamp field units","is_optional":true,"default":"auto","linter":"root = if [\"nanos\",\"micros\",\"millis\",\"seconds\",\"auto\"].contains(this) != true { [ \"valid options are \\\"nanos\\\", \\\"micros\\\", \\\"millis\\\", \\\"seconds\\\", \\\"auto\\\"\" ] }"},{"name":"timestamp_string_fields","type":"string","kind":"array","description":"String fields with textual timestamps","is_optional":true},{"name":"timestamp_string_format","type":"string","kind":"scalar","description":"Timestamp format, used when parsing timestamp string fields. Specified in golang's time.Parse layout","is_optional":true,"default":"Jan _2 15:04:05.000000Z0700"},{"name":"symbols","type":"string","kind":"array","description":"Columns that should be the SYMBOL type (string values default to STRING)","is_optional":true},{"name":"doubles","type":"string","kind":"array","description":"Columns that should be double type, (int is default)","is_optional":true},{"name":"error_on_empty_messages","type":"bool","kind":"scalar","description":"Mark a message as errored if it is empty after field validation","is_optional":true,"default":false}]}},{"name":"redis_hash","type":"output","status":"stable","plugin":true,"summary":"Sets Redis hash objects using the HMSET command.","description":"\nThe field `key` supports xref:configuration:interpolation.adoc#bloblang-queries[interpolation functions], allowing you to create a unique key for each message.\n\nThe field `fields` allows you to specify an explicit map of field names to interpolated values, also evaluated per message of a batch:\n\n```yaml\noutput:\n redis_hash:\n url: tcp://localhost:6379\n key: ${!json(\"id\")}\n fields:\n topic: ${!meta(\"kafka_topic\")}\n partition: ${!meta(\"kafka_partition\")}\n content: ${!json(\"document.text\")}\n```\n\nIf the field `walk_metadata` is set to `true` then Redpanda Connect will walk all metadata fields of messages and add them to the list of hash fields to set.\n\nIf the field `walk_json_object` is set to `true` then Redpanda Connect will walk each message as a JSON object, extracting keys and the string representation of their value and adds them to the list of hash fields to set.\n\nThe order of hash field extraction is as follows:\n\n1. Metadata (if enabled)\n2. JSON object (if enabled)\n3. Explicit fields\n\nWhere latter stages will overwrite matching field names of a former stage.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target Redis server. Database is optional and is supplied as the URL path.","examples":["redis://:6379","redis://localhost:6379","redis://foousername:foopassword@redisplace:6379","redis://:foopassword@redisplace:6379","redis://localhost:6379/1","redis://localhost:6379/1,redis://localhost:6380/1"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"kind","type":"string","kind":"scalar","description":"Specifies a simple, cluster-aware, or failover-aware redis client.","is_advanced":true,"default":"simple","options":["simple","cluster","failover"],"linter":"\nlet options = {\n \"simple\": true,\n \"cluster\": true,\n \"failover\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"master","type":"string","kind":"scalar","description":"Name of the redis master when `kind` is `failover`","is_advanced":true,"default":"","examples":["mymaster"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"key","type":"string","kind":"scalar","description":"The key for each message, function interpolations should be used to create a unique key per message.","interpolated":true,"examples":["${! @.kafka_key )}","${! this.doc.id }","${! counter() }"]},{"name":"walk_metadata","type":"bool","kind":"scalar","description":"Whether all metadata fields of messages should be walked and added to the list of hash fields to set.","default":false},{"name":"walk_json_object","type":"bool","kind":"scalar","description":"Whether to walk each message as a JSON object and add each key/value pair to the list of hash fields to set.","default":false},{"name":"fields","type":"string","kind":"map","description":"A map of key/value pairs to set as hash fields.","default":{},"interpolated":true},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64}]}},{"name":"redis_list","type":"output","status":"stable","plugin":true,"summary":"Pushes messages onto the end of a Redis list (which is created if it doesn't already exist) using the RPUSH command.","description":"The field `key` supports xref:configuration:interpolation.adoc#bloblang-queries[interpolation functions], allowing you to create a unique key for each message.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target Redis server. Database is optional and is supplied as the URL path.","examples":["redis://:6379","redis://localhost:6379","redis://foousername:foopassword@redisplace:6379","redis://:foopassword@redisplace:6379","redis://localhost:6379/1","redis://localhost:6379/1,redis://localhost:6380/1"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"kind","type":"string","kind":"scalar","description":"Specifies a simple, cluster-aware, or failover-aware redis client.","is_advanced":true,"default":"simple","options":["simple","cluster","failover"],"linter":"\nlet options = {\n \"simple\": true,\n \"cluster\": true,\n \"failover\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"master","type":"string","kind":"scalar","description":"Name of the redis master when `kind` is `failover`","is_advanced":true,"default":"","examples":["mymaster"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"key","type":"string","kind":"scalar","description":"The key for each message, function interpolations can be optionally used to create a unique key per message.","interpolated":true,"examples":["some_list","${! @.kafka_key )}","${! this.doc.id }","${! counter() }"]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"command","type":"string","kind":"scalar","description":"The command used to push elements to the Redis list","is_advanced":true,"default":"rpush","options":["rpush","lpush"],"version":"4.22.0","linter":"\nlet options = {\n \"rpush\": true,\n \"lpush\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"}]}},{"name":"redis_pubsub","type":"output","status":"stable","plugin":true,"summary":"Publishes messages through the Redis PubSub model. It is not possible to guarantee that messages have been received.","description":"\nThis output will interpolate functions within the channel field, you can find a list of functions xref:configuration:interpolation.adoc#bloblang-queries[here].\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target Redis server. Database is optional and is supplied as the URL path.","examples":["redis://:6379","redis://localhost:6379","redis://foousername:foopassword@redisplace:6379","redis://:foopassword@redisplace:6379","redis://localhost:6379/1","redis://localhost:6379/1,redis://localhost:6380/1"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"kind","type":"string","kind":"scalar","description":"Specifies a simple, cluster-aware, or failover-aware redis client.","is_advanced":true,"default":"simple","options":["simple","cluster","failover"],"linter":"\nlet options = {\n \"simple\": true,\n \"cluster\": true,\n \"failover\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"master","type":"string","kind":"scalar","description":"Name of the redis master when `kind` is `failover`","is_advanced":true,"default":"","examples":["mymaster"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"channel","type":"string","kind":"scalar","description":"The channel to publish messages to.","interpolated":true},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]}},{"name":"redis_streams","type":"output","status":"stable","plugin":true,"summary":"Pushes messages to a Redis (v5.0+) Stream (which is created if it doesn't already exist) using the XADD command.","description":"\nIt's possible to specify a maximum length of the target stream by setting it to a value greater than 0, in which case this cap is applied only when Redis is able to remove a whole macro node, for efficiency.\n\nRedis stream entries are key/value pairs, as such it is necessary to specify the key to be set to the body of the message. All metadata fields of the message will also be set as key/value pairs, if there is a key collision between a metadata item and the body then the body takes precedence.\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target Redis server. Database is optional and is supplied as the URL path.","examples":["redis://:6379","redis://localhost:6379","redis://foousername:foopassword@redisplace:6379","redis://:foopassword@redisplace:6379","redis://localhost:6379/1","redis://localhost:6379/1,redis://localhost:6380/1"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"kind","type":"string","kind":"scalar","description":"Specifies a simple, cluster-aware, or failover-aware redis client.","is_advanced":true,"default":"simple","options":["simple","cluster","failover"],"linter":"\nlet options = {\n \"simple\": true,\n \"cluster\": true,\n \"failover\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"master","type":"string","kind":"scalar","description":"Name of the redis master when `kind` is `failover`","is_advanced":true,"default":"","examples":["mymaster"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"stream","type":"string","kind":"scalar","description":"The stream to add messages to.","interpolated":true},{"name":"body_key","type":"string","kind":"scalar","description":"A key to set the raw body of the message to.","default":"body"},{"name":"max_length","type":"int","kind":"scalar","description":"When greater than zero enforces a rough cap on the length of the target stream.","default":0},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"metadata","type":"object","kind":"scalar","description":"Specify criteria for which metadata values are included in the message body.","children":[{"name":"exclude_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages.","default":[]}]},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]}},{"name":"redpanda","type":"output","status":"beta","plugin":true,"summary":"A Kafka output using the https://github.com/twmb/franz-go[Franz Kafka client library^].","description":"\nWrites a batch of messages to Kafka brokers and waits for acknowledgement before propagating it back to the input.\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"seed_brokers","type":"string","kind":"array","description":"A list of broker addresses to connect to in order to establish connections. If an item of the list contains commas it will be expanded into multiple addresses.","examples":[["localhost:9092"],["foo:9092","bar:9092"],["foo:9092,bar:9092"]]},{"name":"client_id","type":"string","kind":"scalar","description":"An identifier for the client connection.","is_advanced":true,"default":"benthos"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"sasl","type":"object","kind":"array","description":"Specify one or more methods of SASL authentication. SASL is tried in order; if the broker supports the first mechanism, all connections will use that mechanism. If the first mechanism fails, the client will pick the first supported mechanism. If the broker does not support any client mechanisms, connections will fail.","is_advanced":true,"is_optional":true,"examples":[[{"mechanism":"SCRAM-SHA-512","password":"bar","username":"foo"}]],"children":[{"name":"mechanism","type":"string","kind":"scalar","description":"The SASL mechanism to use.","is_advanced":true,"annotated_options":[["AWS_MSK_IAM","AWS IAM based authentication as specified by the 'aws-msk-iam-auth' java library."],["OAUTHBEARER","OAuth Bearer based authentication."],["PLAIN","Plain text authentication."],["SCRAM-SHA-256","SCRAM based authentication as specified in RFC5802."],["SCRAM-SHA-512","SCRAM based authentication as specified in RFC5802."],["none","Disable sasl authentication"]],"linter":"\nlet options = {\n \"aws_msk_iam\": true,\n \"oauthbearer\": true,\n \"plain\": true,\n \"scram-sha-256\": true,\n \"scram-sha-512\": true,\n \"none\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"username","type":"string","kind":"scalar","description":"A username to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token to use for a single session's OAUTHBEARER authentication.","is_advanced":true,"default":""},{"name":"extensions","type":"string","kind":"map","description":"Key/value pairs to add to OAUTHBEARER authentication requests.","is_advanced":true,"is_optional":true},{"name":"aws","type":"object","kind":"scalar","description":"Contains AWS specific fields for when the `mechanism` is set to `AWS_MSK_IAM`.","is_advanced":true,"is_optional":true,"children":[{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]}]},{"name":"metadata_max_age","type":"string","kind":"scalar","description":"The maximum age of metadata before it is refreshed.","is_advanced":true,"default":"5m"},{"name":"request_timeout_overhead","type":"string","kind":"scalar","description":"The request time overhead. Uses the given time as overhead while deadlining requests. Roughly equivalent to request.timeout.ms, but grants additional time to requests that have timeout fields.","is_advanced":true,"default":"10s"},{"name":"conn_idle_timeout","type":"string","kind":"scalar","description":"The rough amount of time to allow connections to idle before they are closed.","is_advanced":true,"default":"20s"},{"name":"topic","type":"string","kind":"scalar","description":"A topic to write messages to.","interpolated":true},{"name":"key","type":"string","kind":"scalar","description":"An optional key to populate for each message.","is_optional":true,"interpolated":true},{"name":"partition","type":"string","kind":"scalar","description":"An optional explicit partition to set for each message. This field is only relevant when the `partitioner` is set to `manual`. The provided interpolation string must be a valid integer.","is_optional":true,"interpolated":true,"examples":["${! meta(\"partition\") }"]},{"name":"metadata","type":"object","kind":"scalar","description":"Determine which (if any) metadata values should be added to messages as headers.","is_optional":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"timestamp","type":"string","kind":"scalar","description":"An optional timestamp to set for each message. When left empty, the current timestamp is used.","is_advanced":true,"is_deprecated":true,"is_optional":true,"interpolated":true,"examples":["${! timestamp_unix() }","${! metadata(\"kafka_timestamp_unix\") }"]},{"name":"timestamp_ms","type":"string","kind":"scalar","description":"An optional timestamp to set for each message expressed in milliseconds. When left empty, the current timestamp is used.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["${! timestamp_unix_milli() }","${! metadata(\"kafka_timestamp_ms\") }"]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of batches to be sending in parallel at any given time.","default":256},{"name":"partitioner","type":"string","kind":"scalar","description":"Override the default murmur2 hashing partitioner.","is_advanced":true,"is_optional":true,"annotated_options":[["least_backup","Chooses the least backed up partition (the partition with the fewest amount of buffered records). Partitions are selected per batch."],["manual","Manually select a partition for each message, requires the field `partition` to be specified."],["murmur2_hash","Kafka's default hash algorithm that uses a 32-bit murmur2 hash of the key to compute which partition the record will be on."],["round_robin","Round-robin's messages through all available partitions. This algorithm has lower throughput and causes higher CPU load on brokers, but can be useful if you want to ensure an even distribution of records to partitions."]],"linter":"\nlet options = {\n \"least_backup\": true,\n \"manual\": true,\n \"murmur2_hash\": true,\n \"round_robin\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"idempotent_write","type":"bool","kind":"scalar","description":"Enable the idempotent write producer option. This requires the `IDEMPOTENT_WRITE` permission on `CLUSTER` and can be disabled if this permission is not available.","is_advanced":true,"default":true},{"name":"compression","type":"string","kind":"scalar","description":"Optionally set an explicit compression type. The default preference is to use snappy when the broker supports it, and fall back to none if not.","is_advanced":true,"is_optional":true,"options":["lz4","snappy","gzip","none","zstd"],"linter":"\nlet options = {\n \"lz4\": true,\n \"snappy\": true,\n \"gzip\": true,\n \"none\": true,\n \"zstd\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period of time to wait for message sends before abandoning the request and retrying","is_advanced":true,"default":"10s"},{"name":"max_message_bytes","type":"string","kind":"scalar","description":"The maximum space in bytes than an individual message may take, messages larger than this value will be rejected. This field corresponds to Kafka's `max.message.bytes`.","is_advanced":true,"default":"1MiB","examples":["100MB","50mib"]},{"name":"broker_write_max_bytes","type":"string","kind":"scalar","description":"The upper bound for the number of bytes written to a broker connection in a single write. This field corresponds to Kafka's `socket.request.max.bytes`.","is_advanced":true,"default":"100MiB","examples":["128MB","50mib"]}],"linter":"root = match {\n this.partitioner == \"manual\" \u0026\u0026 this.partition.or(\"\") == \"\" =\u003e \"a partition must be specified when the partitioner is set to manual\"\n this.partitioner != \"manual\" \u0026\u0026 this.partition.or(\"\") != \"\" =\u003e \"a partition cannot be specified unless the partitioner is set to manual\"\n this.timestamp.or(\"\") != \"\" \u0026\u0026 this.timestamp_ms.or(\"\") != \"\" =\u003e \"both timestamp and timestamp_ms cannot be specified simultaneously\"\n}"}},{"name":"redpanda_common","type":"output","status":"beta","plugin":true,"summary":"Sends data to a Redpanda (Kafka) broker, using credentials defined in a common top-level `redpanda` config block.","categories":["Services"],"examples":[{"title":"Simple Output","summary":"Data is generated and written to a topic bar, targetting the cluster configured within the redpanda block at the bottom. This is useful as it allows us to configure TLS and SASL only once for potentially multiple inputs and outputs.","config":"\ninput:\n generate:\n interval: 1s\n mapping: 'root.name = fake(\"name\")'\n\npipeline:\n processors:\n - mutation: |\n root.id = uuid_v4()\n root.loud_name = this.name.uppercase()\n\noutput:\n redpanda_common:\n topic: bar\n key: ${! @id }\n\nredpanda:\n seed_brokers: [ \"127.0.0.1:9092\" ]\n tls:\n enabled: true\n sasl:\n - mechanism: SCRAM-SHA-512\n password: bar\n username: foo\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"topic","type":"string","kind":"scalar","description":"A topic to write messages to.","interpolated":true},{"name":"key","type":"string","kind":"scalar","description":"An optional key to populate for each message.","is_optional":true,"interpolated":true},{"name":"partition","type":"string","kind":"scalar","description":"An optional explicit partition to set for each message. This field is only relevant when the `partitioner` is set to `manual`. The provided interpolation string must be a valid integer.","is_optional":true,"interpolated":true,"examples":["${! meta(\"partition\") }"]},{"name":"metadata","type":"object","kind":"scalar","description":"Determine which (if any) metadata values should be added to messages as headers.","is_optional":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"timestamp","type":"string","kind":"scalar","description":"An optional timestamp to set for each message. When left empty, the current timestamp is used.","is_advanced":true,"is_deprecated":true,"is_optional":true,"interpolated":true,"examples":["${! timestamp_unix() }","${! metadata(\"kafka_timestamp_unix\") }"]},{"name":"timestamp_ms","type":"string","kind":"scalar","description":"An optional timestamp to set for each message expressed in milliseconds. When left empty, the current timestamp is used.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["${! timestamp_unix_milli() }","${! metadata(\"kafka_timestamp_ms\") }"]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":10},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}],"linter":"root = match {\n this.partitioner == \"manual\" \u0026\u0026 this.partition.or(\"\") == \"\" =\u003e \"a partition must be specified when the partitioner is set to manual\"\n this.partitioner != \"manual\" \u0026\u0026 this.partition.or(\"\") != \"\" =\u003e \"a partition cannot be specified unless the partitioner is set to manual\"\n this.timestamp.or(\"\") != \"\" \u0026\u0026 this.timestamp_ms.or(\"\") != \"\" =\u003e \"both timestamp and timestamp_ms cannot be specified simultaneously\"\n}"}},{"name":"redpanda_migrator","type":"output","status":"beta","plugin":true,"summary":"A Redpanda Migrator output using the https://github.com/twmb/franz-go[Franz Kafka client library^].","description":"\nWrites a batch of messages to a Kafka broker and waits for acknowledgement before propagating it back to the input.\n\nThis output should be used in combination with a `redpanda_migrator` input identified by the label specified in\n`input_resource` which it can query for topic and ACL configurations. Once connected, the output will attempt to\ncreate all topics which the input consumes from along with their ACLs.\n\nIf the configured broker does not contain the current message topic, this output attempts to create it along with its\nACLs.\n\nACL migration adheres to the following principles:\n\n- `ALLOW WRITE` ACLs for topics are not migrated\n- `ALLOW ALL` ACLs for topics are downgraded to `ALLOW READ`\n- Only topic ACLs are migrated, group ACLs are not migrated\n","categories":["Services"],"examples":[{"title":"Transfer data","summary":"Writes messages to the configured broker and creates topics and topic ACLs if they don't exist. It also ensures that the message order is preserved.","config":"\noutput:\n redpanda_migrator:\n seed_brokers: [ \"127.0.0.1:9093\" ]\n topic: ${! metadata(\"kafka_topic\").or(throw(\"missing kafka_topic metadata\")) }\n key: ${! metadata(\"kafka_key\") }\n partitioner: manual\n partition: ${! metadata(\"kafka_partition\").or(throw(\"missing kafka_partition metadata\")) }\n timestamp_ms: ${! metadata(\"kafka_timestamp_ms\").or(timestamp_unix_milli()) }\n input_resource: redpanda_migrator_input\n max_in_flight: 1\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"seed_brokers","type":"string","kind":"array","description":"A list of broker addresses to connect to in order to establish connections. If an item of the list contains commas it will be expanded into multiple addresses.","examples":[["localhost:9092"],["foo:9092","bar:9092"],["foo:9092,bar:9092"]]},{"name":"client_id","type":"string","kind":"scalar","description":"An identifier for the client connection.","is_advanced":true,"default":"benthos"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"sasl","type":"object","kind":"array","description":"Specify one or more methods of SASL authentication. SASL is tried in order; if the broker supports the first mechanism, all connections will use that mechanism. If the first mechanism fails, the client will pick the first supported mechanism. If the broker does not support any client mechanisms, connections will fail.","is_advanced":true,"is_optional":true,"examples":[[{"mechanism":"SCRAM-SHA-512","password":"bar","username":"foo"}]],"children":[{"name":"mechanism","type":"string","kind":"scalar","description":"The SASL mechanism to use.","is_advanced":true,"annotated_options":[["AWS_MSK_IAM","AWS IAM based authentication as specified by the 'aws-msk-iam-auth' java library."],["OAUTHBEARER","OAuth Bearer based authentication."],["PLAIN","Plain text authentication."],["SCRAM-SHA-256","SCRAM based authentication as specified in RFC5802."],["SCRAM-SHA-512","SCRAM based authentication as specified in RFC5802."],["none","Disable sasl authentication"]],"linter":"\nlet options = {\n \"aws_msk_iam\": true,\n \"oauthbearer\": true,\n \"plain\": true,\n \"scram-sha-256\": true,\n \"scram-sha-512\": true,\n \"none\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"username","type":"string","kind":"scalar","description":"A username to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token to use for a single session's OAUTHBEARER authentication.","is_advanced":true,"default":""},{"name":"extensions","type":"string","kind":"map","description":"Key/value pairs to add to OAUTHBEARER authentication requests.","is_advanced":true,"is_optional":true},{"name":"aws","type":"object","kind":"scalar","description":"Contains AWS specific fields for when the `mechanism` is set to `AWS_MSK_IAM`.","is_advanced":true,"is_optional":true,"children":[{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]}]},{"name":"metadata_max_age","type":"string","kind":"scalar","description":"The maximum age of metadata before it is refreshed.","is_advanced":true,"default":"5m"},{"name":"request_timeout_overhead","type":"string","kind":"scalar","description":"The request time overhead. Uses the given time as overhead while deadlining requests. Roughly equivalent to request.timeout.ms, but grants additional time to requests that have timeout fields.","is_advanced":true,"default":"10s"},{"name":"conn_idle_timeout","type":"string","kind":"scalar","description":"The rough amount of time to allow connections to idle before they are closed.","is_advanced":true,"default":"20s"},{"name":"topic","type":"string","kind":"scalar","description":"A topic to write messages to.","interpolated":true},{"name":"key","type":"string","kind":"scalar","description":"An optional key to populate for each message.","is_optional":true,"interpolated":true},{"name":"partition","type":"string","kind":"scalar","description":"An optional explicit partition to set for each message. This field is only relevant when the `partitioner` is set to `manual`. The provided interpolation string must be a valid integer.","is_optional":true,"interpolated":true,"examples":["${! meta(\"partition\") }"]},{"name":"metadata","type":"object","kind":"scalar","description":"Determine which (if any) metadata values should be added to messages as headers.","is_optional":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"timestamp","type":"string","kind":"scalar","description":"An optional timestamp to set for each message. When left empty, the current timestamp is used.","is_advanced":true,"is_deprecated":true,"is_optional":true,"interpolated":true,"examples":["${! timestamp_unix() }","${! metadata(\"kafka_timestamp_unix\") }"]},{"name":"timestamp_ms","type":"string","kind":"scalar","description":"An optional timestamp to set for each message expressed in milliseconds. When left empty, the current timestamp is used.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["${! timestamp_unix_milli() }","${! metadata(\"kafka_timestamp_ms\") }"]},{"name":"topic_prefix","type":"string","kind":"scalar","description":"The topic prefix.","is_advanced":true,"default":"","interpolated":true},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of batches to be sending in parallel at any given time.","default":256},{"name":"input_resource","type":"string","kind":"scalar","description":"The label of the redpanda_migrator input from which to read the configurations for topics and ACLs which need to be created.","is_advanced":true,"default":"redpanda_migrator_input"},{"name":"replication_factor_override","type":"bool","kind":"scalar","description":"Use the specified replication factor when creating topics.","is_advanced":true,"default":true},{"name":"replication_factor","type":"int","kind":"scalar","description":"Replication factor for created topics. This is only used when `replication_factor_override` is set to `true`.","is_advanced":true,"default":3},{"name":"translate_schema_ids","type":"bool","kind":"scalar","description":"Translate schema IDs.","is_advanced":true,"default":false},{"name":"schema_registry_output_resource","type":"string","kind":"scalar","description":"The label of the schema_registry output to use for fetching schema IDs.","is_advanced":true,"default":"schema_registry_output"},{"name":"rack_id","type":"string","kind":"scalar","is_deprecated":true},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","is_deprecated":true,"examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","is_deprecated":true,"default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","is_deprecated":true,"default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","is_deprecated":true,"default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","is_deprecated":true,"default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_deprecated":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"partitioner","type":"string","kind":"scalar","description":"Override the default murmur2 hashing partitioner.","is_advanced":true,"is_optional":true,"annotated_options":[["least_backup","Chooses the least backed up partition (the partition with the fewest amount of buffered records). Partitions are selected per batch."],["manual","Manually select a partition for each message, requires the field `partition` to be specified."],["murmur2_hash","Kafka's default hash algorithm that uses a 32-bit murmur2 hash of the key to compute which partition the record will be on."],["round_robin","Round-robin's messages through all available partitions. This algorithm has lower throughput and causes higher CPU load on brokers, but can be useful if you want to ensure an even distribution of records to partitions."]],"linter":"\nlet options = {\n \"least_backup\": true,\n \"manual\": true,\n \"murmur2_hash\": true,\n \"round_robin\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"idempotent_write","type":"bool","kind":"scalar","description":"Enable the idempotent write producer option. This requires the `IDEMPOTENT_WRITE` permission on `CLUSTER` and can be disabled if this permission is not available.","is_advanced":true,"default":true},{"name":"compression","type":"string","kind":"scalar","description":"Optionally set an explicit compression type. The default preference is to use snappy when the broker supports it, and fall back to none if not.","is_advanced":true,"is_optional":true,"options":["lz4","snappy","gzip","none","zstd"],"linter":"\nlet options = {\n \"lz4\": true,\n \"snappy\": true,\n \"gzip\": true,\n \"none\": true,\n \"zstd\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period of time to wait for message sends before abandoning the request and retrying","is_advanced":true,"default":"10s"},{"name":"max_message_bytes","type":"string","kind":"scalar","description":"The maximum space in bytes than an individual message may take, messages larger than this value will be rejected. This field corresponds to Kafka's `max.message.bytes`.","is_advanced":true,"default":"1MiB","examples":["100MB","50mib"]},{"name":"broker_write_max_bytes","type":"string","kind":"scalar","description":"The upper bound for the number of bytes written to a broker connection in a single write. This field corresponds to Kafka's `socket.request.max.bytes`.","is_advanced":true,"default":"100MiB","examples":["128MB","50mib"]}],"linter":"root = match {\n this.partitioner == \"manual\" \u0026\u0026 this.partition.or(\"\") == \"\" =\u003e \"a partition must be specified when the partitioner is set to manual\"\n this.partitioner != \"manual\" \u0026\u0026 this.partition.or(\"\") != \"\" =\u003e \"a partition cannot be specified unless the partitioner is set to manual\"\n this.timestamp.or(\"\") != \"\" \u0026\u0026 this.timestamp_ms.or(\"\") != \"\" =\u003e \"both timestamp and timestamp_ms cannot be specified simultaneously\"\n}"},"version":"4.37.0"},{"name":"redpanda_migrator_bundle","type":"output","status":"experimental","plugin":true,"summary":"Redpanda Migrator bundle output","description":"All-in-one output which writes messages and schemas to a Kafka or Redpanda cluster. This output is meant to be used\ntogether with the `redpanda_migrator_bundle` input.\n","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"redpanda_migrator","type":"unknown","kind":"map","description":"The `redpanda_migrator` output configuration.\n"},{"name":"schema_registry","type":"unknown","kind":"map","description":"The `schema_registry` output configuration. The `subject` field must be left empty.\n"},{"name":"translate_schema_ids","type":"bool","kind":"scalar","description":"Allow the target Schema Registry instance to allocate different schema IDs for migrated schemas. This is useful\nwhen it already contains some schemas which differ from the ones being migrated.\n","default":false}]}},{"name":"redpanda_migrator_offsets","type":"output","status":"beta","plugin":true,"summary":"Redpanda Migrator consumer group offsets output using the https://github.com/twmb/franz-go[Franz Kafka client library^].","description":"This output can be used in combination with the `kafka_franz` input that is configured to read the `__consumer_offsets` topic.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"seed_brokers","type":"string","kind":"array","description":"A list of broker addresses to connect to in order to establish connections. If an item of the list contains commas it will be expanded into multiple addresses.","examples":[["localhost:9092"],["foo:9092","bar:9092"],["foo:9092,bar:9092"]]},{"name":"client_id","type":"string","kind":"scalar","description":"An identifier for the client connection.","is_advanced":true,"default":"benthos"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"sasl","type":"object","kind":"array","description":"Specify one or more methods of SASL authentication. SASL is tried in order; if the broker supports the first mechanism, all connections will use that mechanism. If the first mechanism fails, the client will pick the first supported mechanism. If the broker does not support any client mechanisms, connections will fail.","is_advanced":true,"is_optional":true,"examples":[[{"mechanism":"SCRAM-SHA-512","password":"bar","username":"foo"}]],"children":[{"name":"mechanism","type":"string","kind":"scalar","description":"The SASL mechanism to use.","is_advanced":true,"annotated_options":[["AWS_MSK_IAM","AWS IAM based authentication as specified by the 'aws-msk-iam-auth' java library."],["OAUTHBEARER","OAuth Bearer based authentication."],["PLAIN","Plain text authentication."],["SCRAM-SHA-256","SCRAM based authentication as specified in RFC5802."],["SCRAM-SHA-512","SCRAM based authentication as specified in RFC5802."],["none","Disable sasl authentication"]],"linter":"\nlet options = {\n \"aws_msk_iam\": true,\n \"oauthbearer\": true,\n \"plain\": true,\n \"scram-sha-256\": true,\n \"scram-sha-512\": true,\n \"none\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"username","type":"string","kind":"scalar","description":"A username to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to provide for PLAIN or SCRAM-* authentication.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token to use for a single session's OAUTHBEARER authentication.","is_advanced":true,"default":""},{"name":"extensions","type":"string","kind":"map","description":"Key/value pairs to add to OAUTHBEARER authentication requests.","is_advanced":true,"is_optional":true},{"name":"aws","type":"object","kind":"scalar","description":"Contains AWS specific fields for when the `mechanism` is set to `AWS_MSK_IAM`.","is_advanced":true,"is_optional":true,"children":[{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]}]},{"name":"metadata_max_age","type":"string","kind":"scalar","description":"The maximum age of metadata before it is refreshed.","is_advanced":true,"default":"5m"},{"name":"request_timeout_overhead","type":"string","kind":"scalar","description":"The request time overhead. Uses the given time as overhead while deadlining requests. Roughly equivalent to request.timeout.ms, but grants additional time to requests that have timeout fields.","is_advanced":true,"default":"10s"},{"name":"conn_idle_timeout","type":"string","kind":"scalar","description":"The rough amount of time to allow connections to idle before they are closed.","is_advanced":true,"default":"20s"},{"name":"offset_topic","type":"string","kind":"scalar","description":"Kafka offset topic.","default":"${! @kafka_offset_topic }","interpolated":true},{"name":"offset_topic_prefix","type":"string","kind":"scalar","description":"Kafka offset topic prefix.","is_advanced":true,"default":"","interpolated":true},{"name":"offset_group","type":"string","kind":"scalar","description":"Kafka offset group.","default":"${! @kafka_offset_group }","interpolated":true},{"name":"offset_partition","type":"string","kind":"scalar","description":"Kafka offset partition.","default":"${! @kafka_offset_partition }","interpolated":true},{"name":"offset_commit_timestamp","type":"string","kind":"scalar","description":"Kafka offset commit timestamp.","default":"${! @kafka_offset_commit_timestamp }","interpolated":true},{"name":"offset_metadata","type":"string","kind":"scalar","description":"Kafka offset metadata value.","default":"${! @kafka_offset_metadata }","interpolated":true},{"name":"is_high_watermark","type":"string","kind":"scalar","description":"Indicates if the update represents the high watermark of the Kafka topic partition.","default":"${! @kafka_is_high_watermark }","interpolated":true},{"name":"kafka_key","type":"string","kind":"scalar","description":"Kafka key.","is_deprecated":true,"default":"${! @kafka_key }","interpolated":true},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of batches to be sending in parallel at any given time.","is_deprecated":true,"default":1},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period of time to wait for message sends before abandoning the request and retrying","is_advanced":true,"default":"10s"},{"name":"max_message_bytes","type":"string","kind":"scalar","description":"The maximum space in bytes than an individual message may take, messages larger than this value will be rejected. This field corresponds to Kafka's `max.message.bytes`.","is_advanced":true,"default":"1MiB","examples":["100MB","50mib"]},{"name":"broker_write_max_bytes","type":"string","kind":"scalar","description":"The upper bound for the number of bytes written to a broker connection in a single write. This field corresponds to Kafka's `socket.request.max.bytes`.","is_advanced":true,"default":"100MiB","examples":["128MB","50mib"]},{"name":"max_retries","type":"int","kind":"scalar","description":"The maximum number of retries before giving up on the request. If set to zero there is no discrete limit.","is_advanced":true,"default":0},{"name":"backoff","type":"object","kind":"scalar","description":"Control time intervals between retry attempts.","is_advanced":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"1s"},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts.","is_advanced":true,"default":"5s"},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum period to wait before retry attempts are abandoned. If zero then no limit is used.","is_advanced":true,"default":"30s"}]}]},"version":"4.37.0"},{"name":"reject","type":"output","status":"stable","plugin":true,"summary":"Rejects all messages, treating them as though the output destination failed to publish them.","description":"\nThe routing of messages after this output depends on the type of input it came from. For inputs that support propagating nacks upstream such as AMQP or NATS the message will be nacked. However, for inputs that are sequential such as files or Kafka the messages will simply be reprocessed from scratch.\n\nTo learn when this output could be useful, see [the \u003c\u003cexamples\u003e\u003e.","categories":["Utility"],"examples":[{"title":"Rejecting Failed Messages","summary":"\nThis input is particularly useful for routing messages that have failed during processing, where instead of routing them to some sort of dead letter queue we wish to push the error upstream. We can do this with a switch broker:","config":"\noutput:\n switch:\n retry_until_success: false\n cases:\n - check: '!errored()'\n output:\n amqp_1:\n urls: [ amqps://guest:guest@localhost:5672/ ]\n target_address: queue:/the_foos\n\n - output:\n reject: \"processing failed due to: ${! error() }\"\n"}],"config":{"name":"","type":"string","kind":"scalar","default":""}},{"name":"reject_errored","type":"output","status":"stable","plugin":true,"summary":"Rejects messages that have failed their processing steps, resulting in nack behavior at the input level, otherwise sends them to a child output.","description":"\nThe routing of messages rejected by this output depends on the type of input it came from. For inputs that support propagating nacks upstream such as AMQP or NATS the message will be nacked. However, for inputs that are sequential such as files or Kafka the messages will simply be reprocessed from scratch.","categories":["Utility"],"examples":[{"title":"Rejecting Failed Messages","summary":"\nThe most straight forward use case for this output type is to nack messages that have failed their processing steps. In this example our mapping might fail, in which case the messages that failed are rejected and will be nacked by our input:","config":"\ninput:\n nats_jetstream:\n urls: [ nats://127.0.0.1:4222 ]\n subject: foos.pending\n\npipeline:\n processors:\n - mutation: 'root.age = this.fuzzy.age.int64()'\n\noutput:\n reject_errored:\n nats_jetstream:\n urls: [ nats://127.0.0.1:4222 ]\n subject: foos.processed\n"},{"title":"DLQing Failed Messages","summary":"\nAnother use case for this output is to send failed messages straight into a dead-letter queue. You use it within a xref:components:outputs/fallback.adoc[fallback output] that allows you to specify where these failed messages should go to next.","config":"\npipeline:\n processors:\n - mutation: 'root.age = this.fuzzy.age.int64()'\n\noutput:\n fallback:\n - reject_errored:\n http_client:\n url: http://foo:4195/post/might/become/unreachable\n retries: 3\n retry_period: 1s\n - http_client:\n url: http://bar:4196/somewhere/else\n retries: 3\n retry_period: 1s\n"}],"config":{"name":"","type":"output","kind":"scalar"}},{"name":"resource","type":"output","status":"stable","plugin":true,"summary":"Resource is an output type that channels messages to a resource output, identified by its name.","description":"Resources allow you to tidy up deeply nested configs. For example, the config:\n\n```yaml\noutput:\n broker:\n pattern: fan_out\n outputs:\n - kafka:\n addresses: [ TODO ]\n topic: foo\n - gcp_pubsub:\n project: bar\n topic: baz\n```\n\nCould also be expressed as:\n\n```yaml\noutput:\n broker:\n pattern: fan_out\n outputs:\n - resource: foo\n - resource: bar\n\noutput_resources:\n - label: foo\n kafka:\n addresses: [ TODO ]\n topic: foo\n\n - label: bar\n gcp_pubsub:\n project: bar\n topic: baz\n```\n\nYou can find out more about resources in xref:configuration:resources.adoc[]","categories":["Utility"],"config":{"name":"","type":"string","kind":"scalar","default":""}},{"name":"retry","type":"output","status":"stable","plugin":true,"summary":"Attempts to write messages to a child output and if the write fails for any reason the message is retried either until success or, if the retries or max elapsed time fields are non-zero, either is reached.","description":"\nAll messages in Redpanda Connect are always retried on an output error, but this would usually involve propagating the error back to the source of the message, whereby it would be reprocessed before reaching the output layer once again.\n\nThis output type is useful whenever we wish to avoid reprocessing a message on the event of a failed send. We might, for example, have a deduplication processor that we want to avoid reapplying to the same message more than once in the pipeline.\n\nRather than retrying the same output you may wish to retry the send using a different output target (a dead letter queue). In which case you should instead use the xref:components:outputs/fallback.adoc[`fallback`] output type.","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"max_retries","type":"int","kind":"scalar","description":"The maximum number of retries before giving up on the request. If set to zero there is no discrete limit.","is_advanced":true,"default":0},{"name":"backoff","type":"object","kind":"scalar","description":"Control time intervals between retry attempts.","is_advanced":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"default":"500ms"},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts.","is_advanced":true,"default":"3s"},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum period to wait before retry attempts are abandoned. If zero then no limit is used.","is_advanced":true,"default":"0s"}]},{"name":"output","type":"output","kind":"scalar","description":"A child output."}]}},{"name":"schema_registry","type":"output","status":"beta","plugin":true,"summary":"Publishes schemas to SchemaRegistry.","description":"\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.","categories":["Integration"],"examples":[{"title":"Write schemas","summary":"Write schemas to a Schema Registry instance and log errors for schemas which already exist.","config":"\noutput:\n fallback:\n - schema_registry:\n url: http://localhost:8082\n subject: ${! @schema_registry_subject }\n - switch:\n cases:\n - check: '@fallback_error == \"request returned status: 422\"'\n output:\n drop: {}\n processors:\n - log:\n message: |\n Subject '${! @schema_registry_subject }' version ${! @schema_registry_version } already has schema: ${! content() }\n - output:\n reject: ${! @fallback_error }\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The base URL of the schema registry service."},{"name":"subject","type":"string","kind":"scalar","description":"Subject.","interpolated":true},{"name":"backfill_dependencies","type":"bool","kind":"scalar","description":"Backfill schema references and previous versions.","is_advanced":true,"default":true},{"name":"translate_ids","type":"bool","kind":"scalar","description":"Translate schema IDs.","is_advanced":true,"default":false},{"name":"input_resource","type":"string","kind":"scalar","description":"The label of the schema_registry input from which to read source schemas.","is_advanced":true,"default":"schema_registry_input"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"oauth","type":"object","kind":"scalar","description":"Allows you to specify open authentication via OAuth version 1.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use OAuth version 1 in requests.","is_advanced":true,"default":false},{"name":"consumer_key","type":"string","kind":"scalar","description":"A value used to identify the client to the service provider.","is_advanced":true,"default":""},{"name":"consumer_secret","type":"string","kind":"scalar","description":"A secret used to establish ownership of the consumer key.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"access_token","type":"string","kind":"scalar","description":"A value used to gain access to the protected resources on behalf of the user.","is_advanced":true,"default":""},{"name":"access_token_secret","type":"string","kind":"scalar","description":"A secret provided in order to establish ownership of a given access token.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"jwt","type":"object","kind":"scalar","description":"BETA: Allows you to specify JWT authentication.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use JWT authentication in requests.","is_advanced":true,"default":false},{"name":"private_key_file","type":"string","kind":"scalar","description":"A file with the PEM encoded via PKCS1 or PKCS8 as private key.","is_advanced":true,"default":""},{"name":"signing_method","type":"string","kind":"scalar","description":"A method used to sign the token such as RS256, RS384, RS512 or EdDSA.","is_advanced":true,"default":""},{"name":"claims","type":"unknown","kind":"map","description":"A value used to identify the claims that issued the JWT.","is_advanced":true,"default":{}},{"name":"headers","type":"unknown","kind":"map","description":"Add optional key/value headers to the JWT.","is_advanced":true,"default":{}}]}]},"version":"4.32.2"},{"name":"sftp","type":"output","status":"beta","plugin":true,"summary":"Writes files to an SFTP server.","description":"In order to have a different path for each object you should use function interpolations described xref:configuration:interpolation.adoc#bloblang-queries[here].\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.","categories":["Network"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"address","type":"string","kind":"scalar","description":"The address of the server to connect to."},{"name":"path","type":"string","kind":"scalar","description":"The file to save the messages to on the server.","interpolated":true},{"name":"codec","type":"string","kind":"scalar","description":"The way in which the bytes of messages should be written out into the output data stream. It's possible to write lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter.","default":"all-bytes","examples":["lines","delim:\t","delim:foobar"],"annotated_options":[["all-bytes","Only applicable to file based outputs. Writes each message to a file in full, if the file already exists the old content is deleted."],["append","Append each message to the output stream without any delimiter or special encoding."],["delim:x","Append each message to the output stream followed by a custom delimiter."],["lines","Append each message to the output stream followed by a line break."]]},{"name":"credentials","type":"object","kind":"scalar","description":"The credentials to use to log into the target server.","children":[{"name":"username","type":"string","kind":"scalar","description":"The username to connect to the SFTP server.","default":""},{"name":"password","type":"string","kind":"scalar","description":"The password for the username to connect to the SFTP server.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"private_key_file","type":"string","kind":"scalar","description":"The private key for the username to connect to the SFTP server.","default":""},{"name":"private_key","type":"string","kind":"scalar","description":"The private key file for the username to connect to the SFTP server.","is_secret":true,"default":"","linter":"root = match { this.exists(\"private_key\") \u0026\u0026 this.exists(\"private_key_file\") =\u003e [ \"both private_key and private_key_file can't be set simultaneously\" ], }","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"private_key_pass","type":"string","kind":"scalar","description":"Optional passphrase for private key.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64}]},"version":"3.39.0"},{"name":"slack_post","type":"output","status":"experimental","plugin":true,"description":"Post a new message to a Slack channel using https://api.slack.com/methods/chat.postMessage[^chat.postMessage]","categories":null,"examples":[{"title":"Echo Slackbot","summary":"A slackbot that echo messages from other users","config":"\ninput:\n slack:\n app_token: \"${APP_TOKEN:xapp-demo}\"\n bot_token: \"${BOT_TOKEN:xoxb-demo}\"\npipeline:\n processors:\n - mutation: |\n # ignore hidden or non message events\n if this.event.type != \"message\" || (this.event.hidden | false) {\n root = deleted()\n }\n # Don't respond to our own messages\n if this.authorizations.any(auth -\u003e auth.user_id == this.event.user) {\n root = deleted()\n }\noutput:\n slack_post:\n bot_token: \"${BOT_TOKEN:xoxb-demo}\"\n channel_id: \"${!this.event.channel}\"\n thread_ts: \"${!this.event.ts}\"\n text: \"ECHO: ${!this.event.text}\"\n "}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"bot_token","type":"string","kind":"scalar","description":"The Slack Bot User OAuth token to use.","linter":"\n root = if !this.has_prefix(\"xoxb-\") { [ \"field must start with xoxb-\" ] }\n "},{"name":"channel_id","type":"string","kind":"scalar","description":"The channel ID to post messages to.","interpolated":true},{"name":"thread_ts","type":"string","kind":"scalar","description":"Optional thread timestamp to post messages to.","default":"","interpolated":true},{"name":"text","type":"string","kind":"scalar","description":"The text content of the message. Mutually exclusive with `blocks`.","default":"","interpolated":true},{"name":"blocks","type":"string","kind":"scalar","description":"A Bloblang query that should return a JSON array of Slack blocks (see https://api.slack.com/reference/block-kit/blocks[Blocks in Slack documentation]). Mutually exclusive with `text`.","is_optional":true,"bloblang":true},{"name":"markdown","type":"bool","kind":"scalar","description":"Enable markdown formatting in the message.","default":true},{"name":"unfurl_links","type":"bool","kind":"scalar","description":"Enable link unfurling in the message.","default":false},{"name":"unfurl_media","type":"bool","kind":"scalar","description":"Enable media unfurling in the message.","default":true},{"name":"link_names","type":"bool","kind":"scalar","description":"Enable link names in the message.","default":false}]}},{"name":"snowflake_put","type":"output","status":"beta","plugin":true,"summary":"Sends messages to Snowflake stages and, optionally, calls Snowpipe to load this data into one or more tables.","description":"\nIn order to use a different stage and / or Snowpipe for each message, you can use function interpolations as described in\nxref:configuration:interpolation.adoc#bloblang-queries[Bloblang queries]. When using batching, messages are grouped by the calculated\nstage and Snowpipe and are streamed to individual files in their corresponding stage and, optionally, a Snowpipe\n`insertFiles` REST API call will be made for each individual file.\n\n== Credentials\n\nTwo authentication mechanisms are supported:\n\n- User/password\n- Key Pair Authentication\n\n=== User/password\n\nThis is a basic authentication mechanism which allows you to PUT data into a stage. However, it is not compatible with\nSnowpipe.\n\n=== Key pair authentication\n\nThis authentication mechanism allows Snowpipe functionality, but it does require configuring an SSH Private Key\nbeforehand. Please consult the https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication[documentation^]\nfor details on how to set it up and assign the Public Key to your user.\n\nNote that the Snowflake documentation https://twitter.com/felipehoffa/status/1560811785606684672[used to suggest^]\nusing this command:\n\n```bash\nopenssl genrsa 2048 | openssl pkcs8 -topk8 -inform PEM -out rsa_key.p8\n```\n\nto generate an encrypted SSH private key. However, in this case, it uses an encryption algorithm called\n`pbeWithMD5AndDES-CBC`, which is part of the PKCS#5 v1.5 and is considered insecure. Due to this, Redpanda Connect does not\nsupport it and, if you wish to use password-protected keys directly, you must use PKCS#5 v2.0 to encrypt them by using\nthe following command (as the current Snowflake docs suggest):\n\n```bash\nopenssl genrsa 2048 | openssl pkcs8 -topk8 -v2 des3 -inform PEM -out rsa_key.p8\n```\n\nIf you have an existing key encrypted with PKCS#5 v1.5, you can re-encrypt it with PKCS#5 v2.0 using this command:\n\n```bash\nopenssl pkcs8 -in rsa_key_original.p8 -topk8 -v2 des3 -out rsa_key.p8\n```\n\nPlease consult the https://linux.die.net/man/1/pkcs8[pkcs8 command documentation^] for details on PKCS#5 algorithms.\n\n== Batching\n\nIt's common to want to upload messages to Snowflake as batched archives. The easiest way to do this is to batch your\nmessages at the output level and join the batch of messages with an\nxref:components:processors/archive.adoc[`archive`] and/or xref:components:processors/compress.adoc[`compress`]\nprocessor.\n\nFor the optimal batch size, please consult the Snowflake https://docs.snowflake.com/en/user-guide/data-load-considerations-prepare.html[documentation^].\n\n== Snowpipe\n\nGiven a table called `BENTHOS_TBL` with one column of type `variant`:\n\n```sql\nCREATE OR REPLACE TABLE BENTHOS_DB.PUBLIC.BENTHOS_TBL(RECORD variant)\n```\n\nand the following `BENTHOS_PIPE` Snowpipe:\n\n```sql\nCREATE OR REPLACE PIPE BENTHOS_DB.PUBLIC.BENTHOS_PIPE AUTO_INGEST = FALSE AS COPY INTO BENTHOS_DB.PUBLIC.BENTHOS_TBL FROM (SELECT * FROM @%BENTHOS_TBL) FILE_FORMAT = (TYPE = JSON COMPRESSION = AUTO)\n```\n\nyou can configure Redpanda Connect to use the implicit table stage `@%BENTHOS_TBL` as the `stage` and\n`BENTHOS_PIPE` as the `snowpipe`. In this case, you must set `compression` to `AUTO` and, if\nusing message batching, you'll need to configure an xref:components:processors/archive.adoc[`archive`] processor\nwith the `concatenate` format. Since the `compression` is set to `AUTO`, the\nhttps://github.com/snowflakedb/gosnowflake[gosnowflake^] client library will compress the messages automatically so you\ndon't need to add a xref:components:processors/compress.adoc[`compress`] processor for message batches.\n\nIf you add `STRIP_OUTER_ARRAY = TRUE` in your Snowpipe `FILE_FORMAT`\ndefinition, then you must use `json_array` instead of `concatenate` as the archive processor format.\n\nNOTE: Only Snowpipes with `FILE_FORMAT` `TYPE` `JSON` are currently supported.\n\n== Snowpipe troubleshooting\n\nSnowpipe https://docs.snowflake.com/en/user-guide/data-load-snowpipe-rest-apis.html[provides^] the `insertReport`\nand `loadHistoryScan` REST API endpoints which can be used to get information about recent Snowpipe calls. In\norder to query them, you'll first need to generate a valid JWT token for your Snowflake account. There are two methods\nfor doing so:\n\n- Using the `snowsql` https://docs.snowflake.com/en/user-guide/snowsql.html[utility^]:\n\n```bash\nsnowsql --private-key-path rsa_key.p8 --generate-jwt -a \u003caccount\u003e -u \u003cuser\u003e\n```\n\n- Using the Python `sql-api-generate-jwt` https://docs.snowflake.com/en/developer-guide/sql-api/authenticating.html#generating-a-jwt-in-python[utility^]:\n\n```bash\npython3 sql-api-generate-jwt.py --private_key_file_path=rsa_key.p8 --account=\u003caccount\u003e --user=\u003cuser\u003e\n```\n\nOnce you successfully generate a JWT token and store it into the `JWT_TOKEN` environment variable, then you can,\nfor example, query the `insertReport` endpoint using `curl`:\n\n```bash\ncurl -H \"Authorization: Bearer ${JWT_TOKEN}\" \"https://\u003caccount\u003e.snowflakecomputing.com/v1/data/pipes/\u003cdatabase\u003e.\u003cschema\u003e.\u003csnowpipe\u003e/insertReport\"\n```\n\nIf you need to pass in a valid `requestId` to any of these Snowpipe REST API endpoints, you can set a\nxref:guides:bloblang/functions.adoc#uuid_v4[uuid_v4()] string in a metadata field called\n`request_id`, log it via the xref:components:processors/log.adoc[`log`] processor and\nthen configure `request_id: ${ @request_id }` ). Alternatively, you can xref:components:logger/about.adoc[enable debug logging]\n and Redpanda Connect will print the Request IDs that it sends to Snowpipe.\n\n== General troubleshooting\n\nThe underlying https://github.com/snowflakedb/gosnowflake[`gosnowflake` driver^] requires write access to\nthe default directory to use for temporary files. Please consult the https://pkg.go.dev/os#TempDir[`os.TempDir`^]\ndocs for details on how to change this directory via environment variables.\n\nA silent failure can occur due to https://github.com/snowflakedb/gosnowflake/issues/701[this issue^], where the\nunderlying https://github.com/snowflakedb/gosnowflake[`gosnowflake` driver^] doesn't return an error and doesn't\nlog a failure if it can't figure out the current username. One way to trigger this behavior is by running Redpanda Connect in a\nDocker container with a non-existent user ID (such as `--user 1000:1000`).\n\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services"],"examples":[{"title":"Kafka / realtime brokers","summary":"Upload message batches from realtime brokers such as Kafka persisting the batch partition and offsets in the stage path and filename similarly to the https://docs.snowflake.com/en/user-guide/kafka-connector-ts.html#step-1-view-the-copy-history-for-the-table[Kafka Connector scheme^] and call Snowpipe to load them into a table. When batching is configured at the input level, it is done per-partition.","config":"\ninput:\n kafka:\n addresses:\n - localhost:9092\n topics:\n - foo\n consumer_group: benthos\n batching:\n count: 10\n period: 3s\n processors:\n - mapping: |\n meta kafka_start_offset = meta(\"kafka_offset\").from(0)\n meta kafka_end_offset = meta(\"kafka_offset\").from(-1)\n meta batch_timestamp = if batch_index() == 0 { now() }\n - mapping: |\n meta batch_timestamp = if batch_index() != 0 { meta(\"batch_timestamp\").from(0) }\n\noutput:\n snowflake_put:\n account: benthos\n user: test@benthos.dev\n private_key_file: path_to_ssh_key.pem\n role: ACCOUNTADMIN\n database: BENTHOS_DB\n warehouse: COMPUTE_WH\n schema: PUBLIC\n stage: \"@%BENTHOS_TBL\"\n path: benthos/BENTHOS_TBL/${! @kafka_partition }\n file_name: ${! @kafka_start_offset }_${! @kafka_end_offset }_${! meta(\"batch_timestamp\") }\n upload_parallel_threads: 4\n compression: NONE\n snowpipe: BENTHOS_PIPE\n"},{"title":"No compression","summary":"Upload concatenated messages into a `.json` file to a table stage without calling Snowpipe.","config":"\noutput:\n snowflake_put:\n account: benthos\n user: test@benthos.dev\n private_key_file: path_to_ssh_key.pem\n role: ACCOUNTADMIN\n database: BENTHOS_DB\n warehouse: COMPUTE_WH\n schema: PUBLIC\n stage: \"@%BENTHOS_TBL\"\n path: benthos\n upload_parallel_threads: 4\n compression: NONE\n batching:\n count: 10\n period: 3s\n processors:\n - archive:\n format: concatenate\n"},{"title":"Parquet format with snappy compression","summary":"Upload concatenated messages into a `.parquet` file to a table stage without calling Snowpipe.","config":"\noutput:\n snowflake_put:\n account: benthos\n user: test@benthos.dev\n private_key_file: path_to_ssh_key.pem\n role: ACCOUNTADMIN\n database: BENTHOS_DB\n warehouse: COMPUTE_WH\n schema: PUBLIC\n stage: \"@%BENTHOS_TBL\"\n path: benthos\n file_extension: parquet\n upload_parallel_threads: 4\n compression: NONE\n batching:\n count: 10\n period: 3s\n processors:\n - parquet_encode:\n schema:\n - name: ID\n type: INT64\n - name: CONTENT\n type: BYTE_ARRAY\n default_compression: snappy\n"},{"title":"Automatic compression","summary":"Upload concatenated messages compressed automatically into a `.gz` archive file to a table stage without calling Snowpipe.","config":"\noutput:\n snowflake_put:\n account: benthos\n user: test@benthos.dev\n private_key_file: path_to_ssh_key.pem\n role: ACCOUNTADMIN\n database: BENTHOS_DB\n warehouse: COMPUTE_WH\n schema: PUBLIC\n stage: \"@%BENTHOS_TBL\"\n path: benthos\n upload_parallel_threads: 4\n compression: AUTO\n batching:\n count: 10\n period: 3s\n processors:\n - archive:\n format: concatenate\n"},{"title":"DEFLATE compression","summary":"Upload concatenated messages compressed into a `.deflate` archive file to a table stage and call Snowpipe to load them into a table.","config":"\noutput:\n snowflake_put:\n account: benthos\n user: test@benthos.dev\n private_key_file: path_to_ssh_key.pem\n role: ACCOUNTADMIN\n database: BENTHOS_DB\n warehouse: COMPUTE_WH\n schema: PUBLIC\n stage: \"@%BENTHOS_TBL\"\n path: benthos\n upload_parallel_threads: 4\n compression: DEFLATE\n snowpipe: BENTHOS_PIPE\n batching:\n count: 10\n period: 3s\n processors:\n - archive:\n format: concatenate\n - mapping: |\n root = content().compress(\"zlib\")\n"},{"title":"RAW_DEFLATE compression","summary":"Upload concatenated messages compressed into a `.raw_deflate` archive file to a table stage and call Snowpipe to load them into a table.","config":"\noutput:\n snowflake_put:\n account: benthos\n user: test@benthos.dev\n private_key_file: path_to_ssh_key.pem\n role: ACCOUNTADMIN\n database: BENTHOS_DB\n warehouse: COMPUTE_WH\n schema: PUBLIC\n stage: \"@%BENTHOS_TBL\"\n path: benthos\n upload_parallel_threads: 4\n compression: RAW_DEFLATE\n snowpipe: BENTHOS_PIPE\n batching:\n count: 10\n period: 3s\n processors:\n - archive:\n format: concatenate\n - mapping: |\n root = content().compress(\"flate\")\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"account","type":"string","kind":"scalar","description":"Account name, which is the same as the https://docs.snowflake.com/en/user-guide/admin-account-identifier.html#where-are-account-identifiers-used[Account Identifier^].\nHowever, when using an https://docs.snowflake.com/en/user-guide/admin-account-identifier.html#using-an-account-locator-as-an-identifier[Account Locator^],\nthe Account Identifier is formatted as `\u003caccount_locator\u003e.\u003cregion_id\u003e.\u003ccloud\u003e` and this field needs to be\npopulated using the `\u003caccount_locator\u003e` part.\n"},{"name":"region","type":"string","kind":"scalar","description":"Optional region field which needs to be populated when using\nan https://docs.snowflake.com/en/user-guide/admin-account-identifier.html#using-an-account-locator-as-an-identifier[Account Locator^]\nand it must be set to the `\u003cregion_id\u003e` part of the Account Identifier\n(`\u003caccount_locator\u003e.\u003cregion_id\u003e.\u003ccloud\u003e`).\n","is_optional":true,"examples":["us-west-2"]},{"name":"cloud","type":"string","kind":"scalar","description":"Optional cloud platform field which needs to be populated\nwhen using an https://docs.snowflake.com/en/user-guide/admin-account-identifier.html#using-an-account-locator-as-an-identifier[Account Locator^]\nand it must be set to the `\u003ccloud\u003e` part of the Account Identifier\n(`\u003caccount_locator\u003e.\u003cregion_id\u003e.\u003ccloud\u003e`).\n","is_optional":true,"examples":["aws","gcp","azure"]},{"name":"user","type":"string","kind":"scalar","description":"Username."},{"name":"password","type":"string","kind":"scalar","description":"An optional password.","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"private_key","type":"string","kind":"scalar","description":"The private SSH key. `private_key_pass` is required when using encrypted keys.","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"private_key_file","type":"string","kind":"scalar","description":"The path to a file containing the private SSH key. `private_key_pass` is required when using encrypted keys.","is_optional":true},{"name":"private_key_pass","type":"string","kind":"scalar","description":"An optional private SSH key passphrase.","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"role","type":"string","kind":"scalar","description":"Role."},{"name":"database","type":"string","kind":"scalar","description":"Database."},{"name":"warehouse","type":"string","kind":"scalar","description":"Warehouse."},{"name":"schema","type":"string","kind":"scalar","description":"Schema."},{"name":"stage","type":"string","kind":"scalar","description":"Stage name. Use either one of the\n\t\thttps://docs.snowflake.com/en/user-guide/data-load-local-file-system-create-stage.html[supported^] stage types.","interpolated":true},{"name":"path","type":"string","kind":"scalar","description":"Stage path.","default":"","interpolated":true},{"name":"file_name","type":"string","kind":"scalar","description":"Stage file name. Will be equal to the Request ID if not set or empty.","is_optional":true,"default":"","interpolated":true,"version":"v4.12.0"},{"name":"file_extension","type":"string","kind":"scalar","description":"Stage file extension. Will be derived from the configured `compression` if not set or empty.","is_optional":true,"default":"","interpolated":true,"examples":["csv","parquet"],"version":"v4.12.0"},{"name":"upload_parallel_threads","type":"int","kind":"scalar","description":"Specifies the number of threads to use for uploading files.","is_advanced":true,"default":4,"linter":"root = if this \u003c 1 || this \u003e 99 { [ \"upload_parallel_threads must be between 1 and 99\" ] }"},{"name":"compression","type":"string","kind":"scalar","description":"Compression type.","default":"AUTO","annotated_options":[["AUTO","Compression (gzip) is applied automatically by the output and messages must contain plain-text JSON. Default `file_extension`: `gz`."],["DEFLATE","Messages must be pre-compressed using the zlib algorithm (with zlib header, RFC1950). Default `file_extension`: `deflate`."],["GZIP","Messages must be pre-compressed using the gzip algorithm. Default `file_extension`: `gz`."],["NONE","No compression is applied and messages must contain plain-text JSON. Default `file_extension`: `json`."],["RAW_DEFLATE","Messages must be pre-compressed using the flate algorithm (without header, RFC1951). Default `file_extension`: `raw_deflate`."],["ZSTD","Messages must be pre-compressed using the Zstandard algorithm. Default `file_extension`: `zst`."]],"linter":"\nlet options = {\n \"auto\": true,\n \"deflate\": true,\n \"gzip\": true,\n \"none\": true,\n \"raw_deflate\": true,\n \"zstd\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"request_id","type":"string","kind":"scalar","description":"Request ID. Will be assigned a random UUID (v4) string if not set or empty.","is_optional":true,"default":"","interpolated":true,"version":"v4.12.0"},{"name":"snowpipe","type":"string","kind":"scalar","description":"An optional Snowpipe name. Use the `\u003csnowpipe\u003e` part from `\u003cdatabase\u003e.\u003cschema\u003e.\u003csnowpipe\u003e`. `private_key` or `private_key_file` must be set when using this feature.","is_optional":true,"interpolated":true},{"name":"client_session_keep_alive","type":"bool","kind":"scalar","description":"Enable Snowflake keepalive mechanism to prevent the client session from expiring after 4 hours (error 390114).","is_advanced":true,"default":false},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of parallel message batches to have in flight at any given time.","default":1}],"linter":"root = match {\n (!this.exists(\"password\") || this.password == \"\") \u0026\u0026 (!this.exists(\"private_key\") || this.private_key == \"\") \u0026\u0026 (!this.exists(\"private_key_file\") || this.private_key_file == \"\") =\u003e [ \"either `password` or `private_key` or `private_key_file` must be set\" ],\n this.exists(\"password\") \u0026\u0026 this.password != \"\" \u0026\u0026 (this.exists(\"private_key\") \u0026\u0026 this.private_key != \"\" || this.exists(\"private_key_file\") \u0026\u0026 this.private_key_file != \"\") =\u003e [ \"only one of `password`, `private_key` and `private_key_file` can be set\" ],\n this.exists(\"snowpipe\") \u0026\u0026 this.snowpipe != \"\" \u0026\u0026 !((this.exists(\"private_key\") \u0026\u0026 this.private_key != \"\") || (this.exists(\"private_key_file\") \u0026\u0026 this.private_key_file != \"\")) =\u003e [ \"either `private_key` or `private_key_file` must be set when using `snowpipe`\" ],\n}"},"version":"4.0.0"},{"name":"snowflake_streaming","type":"output","status":"experimental","plugin":true,"summary":"Ingest data into Snowflake using Snowpipe Streaming.","description":"\nIngest data into Snowflake using Snowpipe Streaming.\n\n[%header,format=dsv]\n|===\nSnowflake column type:Allowed format in Redpanda Connect\nCHAR, VARCHAR:string\nBINARY:[]byte\nNUMBER:any numeric type, string\nFLOAT:any numeric type\nBOOLEAN:bool,any numeric type,string parsable according to `strconv.ParseBool`\nTIME,DATE,TIMESTAMP:unix or RFC 3339 with nanoseconds timestamps\nVARIANT,ARRAY,OBJECT:any data type is converted into JSON\nGEOGRAPHY,GEOMETRY: Not supported\n|===\n\nFor TIMESTAMP, TIME and DATE columns, you can parse different string formats using a bloblang `mapping`.\n\nAuthentication can be configured using a https://docs.snowflake.com/en/user-guide/key-pair-auth[RSA Key Pair^].\n\nThere are https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview#limitations[limitations^] of what data types can be loaded into Snowflake using this method.\n\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].\n\nIt is recommended that each batches results in at least 16MiB of compressed output being written to Snowflake.\nYou can monitor the output batch size using the `snowflake_compressed_output_size_bytes` metric.\n","categories":["Services"],"examples":[{"title":"Exactly once CDC into Snowflake","summary":"How to send data from a PostgreSQL table into Snowflake exactly once using Postgres Logical Replication.\n\nNOTE: If attempting to do exactly-once it's important that rows are delivered in order to the output. Be sure to read the documentation for offset_token first.\nRemoving the offset_token is a safer option that will instruct Redpanda Connect to use its default at-least-once delivery model instead.","config":"\ninput:\n postgres_cdc:\n dsn: postgres://foouser:foopass@localhost:5432/foodb\n schema: \"public\"\n slot_name: \"my_repl_slot\"\n tables: [\"my_pg_table\"]\n # We want very large batches - each batch will be sent to Snowflake individually\n # so to optimize query performance we want as big of files as we have memory for\n batching:\n count: 50000\n period: 45s\n # Prevent multiple batches from being in flight at once, so that we never send\n # a batch while another batch is being retried, this is important to ensure that\n # the Snowflake Snowpipe Streaming channel does not see older data - as it will\n # assume that the older data is already committed.\n checkpoint_limit: 1\noutput:\n snowflake_streaming:\n # We use the log sequence number in the WAL from Postgres to ensure we\n # only upload data exactly once, these are already lexicographically\n # ordered.\n offset_token: \"${!@lsn}\"\n # Since we're sending a single ordered log, we can only send one thing\n # at a time to ensure that we're properly incrementing our offset_token\n # and only using a single channel at a time.\n max_in_flight: 1\n account: \"MYSNOW-ACCOUNT\"\n user: MYUSER\n role: ACCOUNTADMIN\n database: \"MYDATABASE\"\n schema: \"PUBLIC\"\n table: \"MY_PG_TABLE\"\n private_key_file: \"my/private/key.p8\"\n"},{"title":"Ingesting data exactly once from Redpanda","summary":"How to ingest data from Redpanda with consumer groups, decode the schema using the schema registry, then write the corresponding data into Snowflake exactly once.\n\nNOTE: If attempting to do exactly-once its important that records are delivered in order to the output and correctly partitioned. Be sure to read the documentation for \nchannel_name and offset_token first. Removing the offset_token is a safer option that will instruct Redpanda Connect to use its default at-least-once delivery model instead.","config":"\ninput:\n redpanda_common:\n topics: [\"my_topic_going_to_snow\"]\n consumer_group: \"redpanda_connect_to_snowflake\"\n # We want very large batches - each batch will be sent to Snowflake individually\n # so to optimize query performance we want as big of files as we have memory for\n fetch_max_bytes: 100MiB\n fetch_min_bytes: 50MiB\n partition_buffer_bytes: 100MiB\npipeline:\n processors:\n - schema_registry_decode:\n url: \"redpanda.example.com:8081\"\n basic_auth:\n enabled: true\n username: MY_USER_NAME\n password: \"${TODO}\"\noutput:\n fallback:\n - snowflake_streaming:\n # To ensure that we write an ordered stream each partition in kafka gets its own\n # channel.\n channel_name: \"partition-${!@kafka_partition}\"\n # Ensure that our offsets are lexicographically sorted in string form by padding with\n # leading zeros\n offset_token: offset-${!\"%016X\".format(@kafka_offset)}\n account: \"MYSNOW-ACCOUNT\"\n user: MYUSER\n role: ACCOUNTADMIN\n database: \"MYDATABASE\"\n schema: \"PUBLIC\"\n table: \"MYTABLE\"\n private_key_file: \"my/private/key.p8\"\n schema_evolution:\n enabled: true\n # In order to prevent delivery orders from messing with the order of delivered records\n # it's important that failures are immediately sent to a dead letter queue and not retried\n # to Snowflake. See the ordering documentation for the \"redpanda\" input for more details.\n - retry:\n output:\n redpanda_common:\n topic: \"dead_letter_queue\"\n"},{"title":"HTTP Server to push data to Snowflake","summary":"This example demonstrates how to create an HTTP server input that can recieve HTTP PUT requests\nwith JSON payloads, that are buffered locally then written to Snowflake in batches.\n\nNOTE: This example uses a buffer to respond to the HTTP request immediately, so it's possible that failures to deliver data could result in data loss.\nSee the documentation about xref:components:buffers/memory.adoc[buffers] for more information, or remove the buffer entirely to respond to the HTTP request only once the data is written to Snowflake.","config":"\ninput:\n http_server:\n path: /snowflake\nbuffer:\n memory:\n # Max inflight data before applying backpressure\n limit: 524288000 # 50MiB\n # Batching policy, influences how large the generated files sent to Snowflake are\n batch_policy:\n enabled: true\n byte_size: 33554432 # 32MiB\n period: \"10s\"\noutput:\n snowflake_streaming:\n account: \"MYSNOW-ACCOUNT\"\n user: MYUSER\n role: ACCOUNTADMIN\n database: \"MYDATABASE\"\n schema: \"PUBLIC\"\n table: \"MYTABLE\"\n private_key_file: \"my/private/key.p8\"\n # By default there is only a single channel per output table allowed\n # if we want to have multiple Redpanda Connect streams writing data\n # then we need a unique channel prefix per stream. We'll use the host\n # name to get unique prefixes in this example.\n channel_prefix: \"snowflake-channel-for-${HOST}\"\n schema_evolution:\n enabled: true\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"account","type":"string","kind":"scalar","description":"The Snowflake https://docs.snowflake.com/en/user-guide/admin-account-identifier.html#using-an-account-locator-as-an-identifier[Account name^]. Which should be formatted as `\u003corgname\u003e-\u003caccount_name\u003e` where `\u003corgname\u003e` is the name of your Snowflake organization and `\u003caccount_name\u003e` is the unique name of your account within your organization.\n","examples":["ORG-ACCOUNT"]},{"name":"url","type":"string","kind":"scalar","description":"Override the default URL used to connect to Snowflake which is https://ORG-ACCOUNT.snowflakecomputing.com","is_advanced":true,"is_optional":true,"examples":["https://org-account.privatelink.snowflakecomputing.com"]},{"name":"user","type":"string","kind":"scalar","description":"The user to run the Snowpipe Stream as. See https://docs.snowflake.com/en/user-guide/admin-user-management[Snowflake Documentation^] on how to create a user."},{"name":"role","type":"string","kind":"scalar","description":"The role for the `user` field. The role must have the https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview#required-access-privileges[required privileges^] to call the Snowpipe Streaming APIs. See https://docs.snowflake.com/en/user-guide/admin-user-management#user-roles[Snowflake Documentation^] for more information about roles.","examples":["ACCOUNTADMIN"]},{"name":"database","type":"string","kind":"scalar","description":"The Snowflake database to ingest data into.","examples":["MY_DATABASE"]},{"name":"schema","type":"string","kind":"scalar","description":"The Snowflake schema to ingest data into.","examples":["PUBLIC"]},{"name":"table","type":"string","kind":"scalar","description":"The Snowflake table to ingest data into.","interpolated":true,"examples":["MY_TABLE"]},{"name":"private_key","type":"string","kind":"scalar","description":"The PEM encoded private RSA key to use for authenticating with Snowflake. Either this or `private_key_file` must be specified.","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"private_key_file","type":"string","kind":"scalar","description":"The file to load the private RSA key from. This should be a `.p8` PEM encoded file. Either this or `private_key` must be specified.","is_optional":true},{"name":"private_key_pass","type":"string","kind":"scalar","description":"The RSA key passphrase if the RSA key is encrypted.","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"mapping","type":"string","kind":"scalar","description":"A bloblang mapping to execute on each message.","is_optional":true,"bloblang":true},{"name":"init_statement","type":"string","kind":"scalar","description":"\nOptional SQL statements to execute immediately upon the first connection. This is a useful way to initialize tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n","is_optional":true,"examples":["\nCREATE TABLE IF NOT EXISTS mytable (amount NUMBER);\n","\nALTER TABLE t1 ALTER COLUMN c1 DROP NOT NULL;\nALTER TABLE t1 ADD COLUMN a2 NUMBER;\n"]},{"name":"schema_evolution","type":"object","kind":"scalar","description":"Options to control schema evolution within the pipeline as new columns are added to the pipeline.","is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether schema evolution is enabled."},{"name":"ignore_nulls","type":"bool","kind":"scalar","description":"If `true`, then new columns that are `null` are ignored and schema evolution is not triggered. If `false` then null columns trigger schema migrations in Snowflake. NOTE: unless you already know what type this column will be in advance, it's highly encouraged to ignore null values.","is_advanced":true,"default":true},{"name":"new_column_type_mapping","type":"string","kind":"scalar","description":"\nThe mapping function from Redpanda Connect type to column type in Snowflake. Overriding this can allow for customization of the datatype if there is specific information that you know about the data types in use. This mapping should result in the `root` variable being assigned a string with the data type for the new column in Snowflake.\n\n The input to this mapping is either the output of `processors` if specified, otherwise it is an object with the value and the name of the new column, the original message and table being written too. The metadata is unchanged from the original message that caused the schema to change. For example: `{\"value\": 42.3, \"name\":\"new_data_field\", \"message\": {\"existing_data_field\": 42, \"new_data_field\": \"foo\"}, \"db\": MY_DATABASE\", \"schema\": \"MY_SCHEMA\", \"table\": \"MY_TABLE\"}","is_deprecated":true,"is_optional":true,"bloblang":true},{"name":"processors","type":"processor","kind":"array","description":"\nA series of processors to execute when new columns are added to the table. Specifying this can support running side effects when the schema evolves or enriching the message with additional data to guide the schema changes. For example, one could read the schema the message was produced with from the schema registry and use that to decide which type the new column in Snowflake should be.\n\n The input to these processors is an object with the value and the name of the new column, the original message and table being written too. The metadata is unchanged from the original message that caused the schema to change. For example: `{\"value\": 42.3, \"name\":\"new_data_field\", \"message\": {\"existing_data_field\": 42, \"new_data_field\": \"foo\"}, \"db\": MY_DATABASE\", \"schema\": \"MY_SCHEMA\", \"table\": \"MY_TABLE\"}`. The output of these series of processors should be a single message, where the contents of the message is a string indicating the column data type to use (FLOAT, VARIANT, NUMBER(38, 0), etc. An ALTER TABLE statement will then be executed on the table in Snowflake to add the column with the corresponding data type.","is_advanced":true,"is_optional":true,"examples":[[{"mapping":"root = match this.value.type() {\n this == \"string\" =\u003e \"STRING\"\n this == \"bytes\" =\u003e \"BINARY\"\n this == \"number\" =\u003e \"DOUBLE\"\n this == \"bool\" =\u003e \"BOOLEAN\"\n this == \"timestamp\" =\u003e \"TIMESTAMP\"\n _ =\u003e \"VARIANT\"\n}"}]]}]},{"name":"parallelism","type":"int","kind":"scalar","description":"The maximum amount of parallelism to use when building the output for Snowflake. The metric to watch to see if you need to change this is `snowflake_build_output_latency_ns`.","is_advanced":true,"is_deprecated":true,"is_optional":true},{"name":"build_options","type":"object","kind":"scalar","description":"Options to optimize the time to build output data that is sent to Snowflake. The metric to watch to see if you need to change this is `snowflake_build_output_latency_ns`.","is_advanced":true,"children":[{"name":"parallelism","type":"int","kind":"scalar","description":"The maximum amount of parallelism to use.","is_advanced":true,"default":1,"linter":"root = if this \u003c 1 { [\"parallelism must be positive\"] }"},{"name":"chunk_size","type":"int","kind":"scalar","description":"The number of rows to chunk for parallelization.","is_advanced":true,"default":50000,"linter":"root = if this \u003c 1 { [\"chunk_size must be positive\"] }"}]},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":4},{"name":"channel_prefix","type":"string","kind":"scalar","description":"The prefix to use when creating a channel name.\nDuplicate channel names will result in errors and prevent multiple instances of Redpanda Connect from writing at the same time.\nBy default if neither `channel_prefix` or `channel_name is specified then the output will create a channel name that is based on the table FQN so there will only be a single stream per table.\n\nAt most `max_in_flight` channels will be opened.\n\nThis option is mutually exclusive with `channel_name`.\n\nNOTE: There is a limit of 10,000 streams per table - if using more than 10k streams please reach out to Snowflake support.","is_advanced":true,"is_optional":true,"examples":["channel-${HOST}"]},{"name":"channel_name","type":"string","kind":"scalar","description":"The channel name to use.\nDuplicate channel names will result in errors and prevent multiple instances of Redpanda Connect from writing at the same time.\nNote that batches are assumed to all contain messages for the same channel, so this interpolation is only executed on the first\nmessage in each batch. It's recommended to batch at the input level to ensure that batches contain messages for the same channel\nif using an input that is partitioned (such as an Apache Kafka topic).\n\nThis option is mutually exclusive with `channel_prefix`.\n\nNOTE: There is a limit of 10,000 streams per table - if using more than 10k streams please reach out to Snowflake support.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["partition-${!@kafka_partition}"]},{"name":"offset_token","type":"string","kind":"scalar","description":"The offset token to use for exactly once delivery of data in the pipeline. When data is sent on a channel, each message in a batch's offset token\nis compared to the latest token for a channel. If the offset token is lexicographically less than the latest in the channel, it's assumed the message is a duplicate and\nis dropped. This means it is *very important* to have ordered delivery to the output, any out of order messages to the output will be seen as duplicates and dropped.\nSpecifically this means that retried messages could be seen as duplicates if later messages have succeeded in the meantime, so in most circumstances a dead letter queue\noutput should be employed for failed messages.\n\nNOTE: It's assumed that messages within a batch are in increasing order by offset token, additionally if you're using a numeric value as an offset token, make sure to pad\n the value so that it's lexicographically ordered in its string representation, since offset tokens are compared in string form.\n\nFor more information about offset tokens, see https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview#offset-tokens[^Snowflake Documentation]","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["offset-${!\"%016X\".format(@kafka_offset)}","postgres-${!@lsn}"]},{"name":"commit_timeout","type":"string","kind":"scalar","description":"The max duration to wait until the data has been asynchronously committed to Snowflake.","is_advanced":true,"default":"60s","examples":["10s","10m"]}],"linter":"root = match {\n this.exists(\"channel_prefix\") \u0026\u0026 this.exists(\"channel_name\") =\u003e [ \"both `channel_prefix` and `channel_name` can't be set simultaneously\" ],\n}"},"version":"4.39.0"},{"name":"socket","type":"output","status":"stable","plugin":true,"summary":"Connects to a (tcp/udp/unix) server and sends a continuous stream of data, dividing messages according to the specified codec.","categories":["Network"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"network","type":"string","kind":"scalar","description":"A network type to connect as.","options":["unix","tcp","udp"],"linter":"\nlet options = {\n \"unix\": true,\n \"tcp\": true,\n \"udp\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"address","type":"string","kind":"scalar","description":"The address to connect to.","examples":["/tmp/benthos.sock","127.0.0.1:6000"]},{"name":"codec","type":"string","kind":"scalar","description":"The way in which the bytes of messages should be written out into the output data stream. It's possible to write lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter.","default":"lines","examples":["lines","delim:\t","delim:foobar"],"annotated_options":[["all-bytes","Only applicable to file based outputs. Writes each message to a file in full, if the file already exists the old content is deleted."],["append","Append each message to the output stream without any delimiter or special encoding."],["lines","Append each message to the output stream followed by a line break."],["delim:x","Append each message to the output stream followed by a custom delimiter."]]}]}},{"name":"splunk_hec","type":"output","status":"beta","plugin":true,"summary":"Publishes messages to a Splunk HTTP Endpoint Collector (HEC).","description":"\n\n== Performance\n\nThis output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.\n\nThis output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"Full HTTP Endpoint Collector (HEC) URL.","examples":["https://foobar.splunkcloud.com/services/collector/event"]},{"name":"token","type":"string","kind":"scalar","description":"A bot token used for authentication.","is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"gzip","type":"bool","kind":"scalar","description":"Enable gzip compression","default":false},{"name":"event_host","type":"string","kind":"scalar","description":"Set the host value to assign to the event data. Overrides existing host field if present.","is_optional":true},{"name":"event_source","type":"string","kind":"scalar","description":"Set the source value to assign to the event data. Overrides existing source field if present.","is_optional":true},{"name":"event_sourcetype","type":"string","kind":"scalar","description":"Set the sourcetype value to assign to the event data. Overrides existing sourcetype field if present.","is_optional":true},{"name":"event_index","type":"string","kind":"scalar","description":"Set the index value to assign to the event data. Overrides existing index field if present.","is_optional":true},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]},{"name":"skip_cert_verify","type":"bool","kind":"scalar","is_deprecated":true,"is_optional":true},{"name":"batching_count","type":"int","kind":"scalar","is_deprecated":true,"is_optional":true},{"name":"batching_period","type":"string","kind":"scalar","is_deprecated":true,"is_optional":true},{"name":"batching_byte_size","type":"int","kind":"scalar","is_deprecated":true,"is_optional":true},{"name":"rate_limit","type":"string","kind":"scalar","is_deprecated":true,"is_optional":true}]},"version":"4.30.0"},{"name":"sql","type":"output","status":"deprecated","plugin":true,"summary":"Executes an arbitrary SQL query for each message.","description":"\n== Alternatives\n\nFor basic inserts use the xref:components:outputs/sql.adoc[`sql_insert`] output. For more complex queries use the xref:components:outputs/sql_raw.adoc[`sql_raw`] output.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"driver","type":"string","kind":"scalar","description":"A database \u003c\u003cdrivers, driver\u003e\u003e to use.","options":["mysql","postgres","clickhouse","mssql","sqlite","oracle","snowflake","trino","gocosmos","spanner"],"linter":"\nlet options = {\n \"mysql\": true,\n \"postgres\": true,\n \"clickhouse\": true,\n \"mssql\": true,\n \"sqlite\": true,\n \"oracle\": true,\n \"snowflake\": true,\n \"trino\": true,\n \"gocosmos\": true,\n \"spanner\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"data_source_name","type":"string","kind":"scalar","description":"Data source name."},{"name":"query","type":"string","kind":"scalar","description":"The query to execute. The style of placeholder to use depends on the driver, some drivers require question marks (`?`) whereas others expect incrementing dollar signs (`$1`, `$2`, and so on) or colons (`:1`, `:2` and so on). The style to use is outlined in this table:\n\n| Driver | Placeholder Style |\n|---|---|\n| `clickhouse` | Dollar sign |\n| `mysql` | Question mark |\n| `postgres` | Dollar sign |\n| `mssql` | Question mark |\n| `sqlite` | Question mark |\n| `oracle` | Colon |\n| `snowflake` | Question mark |\n| `trino` | Question mark |\n| `gocosmos` | Colon |\n","examples":["INSERT INTO footable (foo, bar, baz) VALUES (?, ?, ?);"]},{"name":"args_mapping","type":"string","kind":"scalar","description":"An optional xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `query`.","is_optional":true,"bloblang":true,"examples":["root = [ this.cat.meow, this.doc.woofs[0] ]","root = [ meta(\"user.id\") ]"]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of inserts to run in parallel.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]},"version":"3.65.0"},{"name":"sql_insert","type":"output","status":"stable","plugin":true,"summary":"Inserts a row into an SQL database for each message.","categories":["Services"],"examples":[{"title":"Table Insert (MySQL)","summary":"\nHere we insert rows into a database by populating the columns id, name and topic with values extracted from messages and metadata:","config":"\noutput:\n sql_insert:\n driver: mysql\n dsn: foouser:foopassword@tcp(localhost:3306)/foodb\n table: footable\n columns: [ id, name, topic ]\n args_mapping: |\n root = [\n this.user.id,\n this.user.name,\n meta(\"kafka_topic\"),\n ]\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"driver","type":"string","kind":"scalar","description":"A database \u003c\u003cdrivers, driver\u003e\u003e to use.","options":["mysql","postgres","clickhouse","mssql","sqlite","oracle","snowflake","trino","gocosmos","spanner"],"linter":"\nlet options = {\n \"mysql\": true,\n \"postgres\": true,\n \"clickhouse\": true,\n \"mssql\": true,\n \"sqlite\": true,\n \"oracle\": true,\n \"snowflake\": true,\n \"trino\": true,\n \"gocosmos\": true,\n \"spanner\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"dsn","type":"string","kind":"scalar","description":"A Data Source Name to identify the target database.\n\n==== Drivers\n\n:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community, spanner=community\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n|===\n| Driver | Data Source Name Format\n\n| `clickhouse` \n| https://github.com/ClickHouse/clickhouse-go#dsn[`clickhouse://[username[:password\\]@\\][netloc\\][:port\\]/dbname[?param1=value1\u0026...\u0026paramN=valueN\\]`^] \n\n| `mysql` \n| `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` \n\n| `postgres` \n| `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` \n\n| `mssql` \n| `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` \n\n| `sqlite` \n| `file:/path/to/filename.db[?param\u0026=value1\u0026...]` \n\n| `oracle` \n| `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` \n\n| `snowflake` \n| `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` \n\n| `trino` \n| https://github.com/trinodb/trino-go-client#dsn-data-source-name[`http[s\\]://user[:pass\\]@host[:port\\][?parameters\\]`^] \n\n| `gocosmos` \n| https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage[`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e\\][;Version=\u003ccosmosdb-api-version\u003e\\][;DefaultDb/Db=\u003cdb-name\u003e\\][;AutoId=\u003ctrue/false\u003e\\][;InsecureSkipVerify=\u003ctrue/false\u003e\\]`^] \n\n| `spanner` \n| projects/[PROJECT]/instances/[INSTANCE]/databases/[DATABASE] \n|===\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String[the docs^] for more details. For https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication[key pair authentication^], the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe https://pkg.go.dev/github.com/microsoft/gocosmos[`gocosmos`^] driver is still experimental, but it has support for https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys[hierarchical partition keys^] as well as https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query[cross-partition queries^]. Please refer to the https://github.com/microsoft/gocosmos/blob/main/SQL.md[SQL notes^] for details.","examples":["clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60","foouser:foopassword@tcp(localhost:3306)/foodb","postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable","oracle://foouser:foopass@localhost:1521/service_name"]},{"name":"table","type":"string","kind":"scalar","description":"The table to insert to.","examples":["foo"]},{"name":"columns","type":"string","kind":"array","description":"A list of columns to insert.","examples":[["foo","bar","baz"]]},{"name":"args_mapping","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of columns specified.","bloblang":true,"examples":["root = [ this.cat.meow, this.doc.woofs[0] ]","root = [ meta(\"user.id\") ]"]},{"name":"prefix","type":"string","kind":"scalar","description":"An optional prefix to prepend to the insert query (before INSERT).","is_advanced":true,"is_optional":true},{"name":"suffix","type":"string","kind":"scalar","description":"An optional suffix to append to the insert query.","is_advanced":true,"is_optional":true,"examples":["ON CONFLICT (name) DO NOTHING"]},{"name":"options","type":"string","kind":"array","description":"A list of keyword options to add before the INTO clause of the query.","is_advanced":true,"is_optional":true,"examples":[["DELAYED","IGNORE"]]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of inserts to run in parallel.","default":64},{"name":"init_files","type":"string","kind":"array","description":"\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":[["./init/*.sql"],["./foo.sql","./bar.sql"]],"version":"4.10.0"},{"name":"init_statement","type":"string","kind":"scalar","description":"\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":["\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n"],"version":"4.10.0"},{"name":"conn_max_idle_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time.","is_advanced":true,"is_optional":true},{"name":"conn_max_life_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age.","is_advanced":true,"is_optional":true},{"name":"conn_max_idle","type":"int","kind":"scalar","description":"An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release.","is_advanced":true,"is_optional":true,"default":2},{"name":"conn_max_open","type":"int","kind":"scalar","description":"An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited).","is_advanced":true,"is_optional":true},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]},"version":"3.59.0"},{"name":"sql_raw","type":"output","status":"stable","plugin":true,"summary":"Executes an arbitrary SQL query for each message.","categories":["Services"],"examples":[{"title":"Table Insert (MySQL)","summary":"\nHere we insert rows into a database by populating the columns id, name and topic with values extracted from messages and metadata:","config":"\noutput:\n sql_raw:\n driver: mysql\n dsn: foouser:foopassword@tcp(localhost:3306)/foodb\n query: \"INSERT INTO footable (id, name, topic) VALUES (?, ?, ?);\"\n args_mapping: |\n root = [\n this.user.id,\n this.user.name,\n meta(\"kafka_topic\"),\n ]\n"},{"title":"Dynamically Creating Tables (PostgreSQL)","summary":"Here we dynamically create output tables transactionally with inserting a record into the newly created table.","config":"\noutput:\n processors:\n - mapping: |\n root = this\n # Prevent SQL injection when using unsafe_dynamic_query\n meta table_name = \"\\\"\" + metadata(\"table_name\").replace_all(\"\\\"\", \"\\\"\\\"\") + \"\\\"\"\n sql_raw:\n driver: postgres\n dsn: postgres://localhost/postgres\n unsafe_dynamic_query: true\n queries:\n - query: |\n CREATE TABLE IF NOT EXISTS ${!metadata(\"table_name\")} (id varchar primary key, document jsonb);\n - query: |\n INSERT INTO ${!metadata(\"table_name\")} (id, document) VALUES ($1, $2)\n ON CONFLICT (id) DO UPDATE SET document = EXCLUDED.document;\n args_mapping: |\n root = [ this.id, this.document.string() ]\n\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"driver","type":"string","kind":"scalar","description":"A database \u003c\u003cdrivers, driver\u003e\u003e to use.","options":["mysql","postgres","clickhouse","mssql","sqlite","oracle","snowflake","trino","gocosmos","spanner"],"linter":"\nlet options = {\n \"mysql\": true,\n \"postgres\": true,\n \"clickhouse\": true,\n \"mssql\": true,\n \"sqlite\": true,\n \"oracle\": true,\n \"snowflake\": true,\n \"trino\": true,\n \"gocosmos\": true,\n \"spanner\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"dsn","type":"string","kind":"scalar","description":"A Data Source Name to identify the target database.\n\n==== Drivers\n\n:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community, spanner=community\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n|===\n| Driver | Data Source Name Format\n\n| `clickhouse` \n| https://github.com/ClickHouse/clickhouse-go#dsn[`clickhouse://[username[:password\\]@\\][netloc\\][:port\\]/dbname[?param1=value1\u0026...\u0026paramN=valueN\\]`^] \n\n| `mysql` \n| `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` \n\n| `postgres` \n| `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` \n\n| `mssql` \n| `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` \n\n| `sqlite` \n| `file:/path/to/filename.db[?param\u0026=value1\u0026...]` \n\n| `oracle` \n| `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` \n\n| `snowflake` \n| `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` \n\n| `trino` \n| https://github.com/trinodb/trino-go-client#dsn-data-source-name[`http[s\\]://user[:pass\\]@host[:port\\][?parameters\\]`^] \n\n| `gocosmos` \n| https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage[`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e\\][;Version=\u003ccosmosdb-api-version\u003e\\][;DefaultDb/Db=\u003cdb-name\u003e\\][;AutoId=\u003ctrue/false\u003e\\][;InsecureSkipVerify=\u003ctrue/false\u003e\\]`^] \n\n| `spanner` \n| projects/[PROJECT]/instances/[INSTANCE]/databases/[DATABASE] \n|===\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String[the docs^] for more details. For https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication[key pair authentication^], the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe https://pkg.go.dev/github.com/microsoft/gocosmos[`gocosmos`^] driver is still experimental, but it has support for https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys[hierarchical partition keys^] as well as https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query[cross-partition queries^]. Please refer to the https://github.com/microsoft/gocosmos/blob/main/SQL.md[SQL notes^] for details.","examples":["clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60","foouser:foopassword@tcp(localhost:3306)/foodb","postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable","oracle://foouser:foopass@localhost:1521/service_name"]},{"name":"query","type":"string","kind":"scalar","description":"The query to execute. The style of placeholder to use depends on the driver, some drivers require question marks (`?`) whereas others expect incrementing dollar signs (`$1`, `$2`, and so on) or colons (`:1`, `:2` and so on). The style to use is outlined in this table:\n\n| Driver | Placeholder Style |\n|---|---|\n| `clickhouse` | Dollar sign |\n| `mysql` | Question mark |\n| `postgres` | Dollar sign |\n| `mssql` | Question mark |\n| `sqlite` | Question mark |\n| `oracle` | Colon |\n| `snowflake` | Question mark |\n| `trino` | Question mark |\n| `gocosmos` | Colon |\n","is_optional":true,"examples":["INSERT INTO footable (foo, bar, baz) VALUES (?, ?, ?);"]},{"name":"unsafe_dynamic_query","type":"bool","kind":"scalar","description":"Whether to enable xref:configuration:interpolation.adoc#bloblang-queries[interpolation functions] in the query. Great care should be made to ensure your queries are defended against injection attacks.","is_advanced":true,"default":false},{"name":"args_mapping","type":"string","kind":"scalar","description":"An optional xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `query`.","is_optional":true,"bloblang":true,"examples":["root = [ this.cat.meow, this.doc.woofs[0] ]","root = [ meta(\"user.id\") ]"]},{"name":"queries","type":"object","kind":"array","description":"A list of statements to run in addition to `query`. When specifying multiple statements, they are all executed within a transaction.","is_optional":true,"children":[{"name":"query","type":"string","kind":"scalar","description":"The query to execute. The style of placeholder to use depends on the driver, some drivers require question marks (`?`) whereas others expect incrementing dollar signs (`$1`, `$2`, and so on) or colons (`:1`, `:2` and so on). The style to use is outlined in this table:\n\n| Driver | Placeholder Style |\n|---|---|\n| `clickhouse` | Dollar sign |\n| `mysql` | Question mark |\n| `postgres` | Dollar sign |\n| `mssql` | Question mark |\n| `sqlite` | Question mark |\n| `oracle` | Colon |\n| `snowflake` | Question mark |\n| `trino` | Question mark |\n| `gocosmos` | Colon |\n"},{"name":"args_mapping","type":"string","kind":"scalar","description":"An optional xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `query`.","is_optional":true,"bloblang":true,"examples":["root = [ this.cat.meow, this.doc.woofs[0] ]","root = [ meta(\"user.id\") ]"]}]},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of statements to execute in parallel.","default":64},{"name":"init_files","type":"string","kind":"array","description":"\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":[["./init/*.sql"],["./foo.sql","./bar.sql"]],"version":"4.10.0"},{"name":"init_statement","type":"string","kind":"scalar","description":"\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":["\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n"],"version":"4.10.0"},{"name":"conn_max_idle_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time.","is_advanced":true,"is_optional":true},{"name":"conn_max_life_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age.","is_advanced":true,"is_optional":true},{"name":"conn_max_idle","type":"int","kind":"scalar","description":"An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release.","is_advanced":true,"is_optional":true,"default":2},{"name":"conn_max_open","type":"int","kind":"scalar","description":"An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited).","is_advanced":true,"is_optional":true},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}],"linter":"root = match {\n !this.exists(\"queries\") \u0026\u0026 !this.exists(\"query\") =\u003e [ \"either `query` or `queries` is required\" ],\n }"},"version":"3.65.0"},{"name":"stdout","type":"output","status":"stable","plugin":true,"summary":"Prints messages to stdout as a continuous stream of data.","categories":["Local"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"codec","type":"string","kind":"scalar","description":"The way in which the bytes of messages should be written out into the output data stream. It's possible to write lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter.","default":"lines","examples":["lines","delim:\t","delim:foobar"],"annotated_options":[["all-bytes","Only applicable to file based outputs. Writes each message to a file in full, if the file already exists the old content is deleted."],["append","Append each message to the output stream without any delimiter or special encoding."],["lines","Append each message to the output stream followed by a line break."],["delim:x","Append each message to the output stream followed by a custom delimiter."]],"version":"3.46.0"}]}},{"name":"subprocess","type":"output","status":"beta","plugin":true,"summary":"Executes a command, runs it as a subprocess, and writes messages to it over stdin.","description":"\nMessages are written according to a specified codec. The process is expected to terminate gracefully when stdin is closed.\n\nIf the subprocess exits unexpectedly then Redpanda Connect will log anything printed to stderr and will log the exit code, and will attempt to execute the command again until success.\n\nThe execution environment of the subprocess is the same as the Redpanda Connect instance, including environment variables and the current working directory.","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"name","type":"string","kind":"scalar","description":"The command to execute as a subprocess."},{"name":"args","type":"string","kind":"array","description":"A list of arguments to provide the command.","default":[]},{"name":"codec","type":"string","kind":"scalar","description":"The way in which messages should be written to the subprocess.","default":"lines","options":["lines"],"linter":"\nlet options = {\n \"lines\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"}]}},{"name":"switch","type":"output","status":"stable","plugin":true,"summary":"The switch output type allows you to route messages to different outputs based on their contents.","description":"Messages that do not pass the check of a single output case are effectively dropped. In order to prevent this outcome set the field \u003c\u003cstrict_mode, `strict_mode`\u003e\u003e to `true`, in which case messages that do not pass at least one case are considered failed and will be nacked and/or reprocessed depending on your input.","categories":["Utility"],"examples":[{"title":"Basic Multiplexing","summary":"\nThe most common use for a switch output is to multiplex messages across a range of output destinations. The following config checks the contents of the field `type` of messages and sends `foo` type messages to an `amqp_1` output, `bar` type messages to a `gcp_pubsub` output, and everything else to a `redis_streams` output.\n\nOutputs can have their own processors associated with them, and in this example the `redis_streams` output has a processor that enforces the presence of a type field before sending it.","config":"\noutput:\n switch:\n cases:\n - check: this.type == \"foo\"\n output:\n amqp_1:\n urls: [ amqps://guest:guest@localhost:5672/ ]\n target_address: queue:/the_foos\n\n - check: this.type == \"bar\"\n output:\n gcp_pubsub:\n project: dealing_with_mike\n topic: mikes_bars\n\n - output:\n redis_streams:\n url: tcp://localhost:6379\n stream: everything_else\n processors:\n - mapping: |\n root = this\n root.type = this.type | \"unknown\"\n"},{"title":"Control Flow","summary":"\nThe `continue` field allows messages that have passed a case to be tested against the next one also. This can be useful when combining non-mutually-exclusive case checks.\n\nIn the following example a message that passes both the check of the first case as well as the second will be routed to both.","config":"\noutput:\n switch:\n cases:\n - check: 'this.user.interests.contains(\"walks\").catch(false)'\n output:\n amqp_1:\n urls: [ amqps://guest:guest@localhost:5672/ ]\n target_address: queue:/people_what_think_good\n continue: true\n\n - check: 'this.user.dislikes.contains(\"videogames\").catch(false)'\n output:\n gcp_pubsub:\n project: people\n topic: that_i_dont_want_to_hang_with\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"retry_until_success","type":"bool","kind":"scalar","description":"\nIf a selected output fails to send a message this field determines whether it is reattempted indefinitely. If set to false the error is instead propagated back to the input level.\n\nIf a message can be routed to \u003e1 outputs it is usually best to set this to true in order to avoid duplicate messages being routed to an output.","default":false},{"name":"strict_mode","type":"bool","kind":"scalar","description":"This field determines whether an error should be reported if no condition is met. If set to true, an error is propagated back to the input level. The default behavior is false, which will drop the message.","is_advanced":true,"default":false},{"name":"cases","type":"object","kind":"array","description":"A list of switch cases, outlining outputs that can be routed to.","examples":[[{"check":"this.urls.contains(\"http://benthos.dev\")","continue":true,"output":{"cache":{"key":"${!json(\"id\")}","target":"foo"}}},{"output":{"s3":{"bucket":"bar","path":"${!json(\"id\")}"}}}]],"children":[{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should be routed to the case output. If left empty the case always passes.","default":"","bloblang":true,"examples":["this.type == \"foo\"","this.contents.urls.contains(\"https://benthos.dev/\")"]},{"name":"output","type":"output","kind":"scalar","description":"An xref:components:outputs/about.adoc[output] for messages that pass the check to be routed to."},{"name":"continue","type":"bool","kind":"scalar","description":"Indicates whether, if this case passes for a message, the next case should also be tested.","is_advanced":true,"default":false}]}],"linter":"if this.exists(\"retry_until_success\") \u0026\u0026 this.retry_until_success {\n if this.cases.or([]).any(oconf -\u003e oconf.output.type.or(\"\") == \"reject\" || oconf.output.reject.type() == \"string\" ) {\n \"a 'switch' output with a 'reject' case output must have the field 'switch.retry_until_success' set to 'false', otherwise the 'reject' child output will result in infinite retries\"\n }\n}"}},{"name":"sync_response","type":"output","status":"stable","plugin":true,"summary":"Returns the final message payload back to the input origin of the message, where it is dealt with according to that specific input type.","description":"\nFor most inputs this mechanism is ignored entirely, in which case the sync response is dropped without penalty. It is therefore safe to use this output even when combining input types that might not have support for sync responses. An example of an input able to utilize this is the `http_server`.\n\nIt is safe to combine this output with others using broker types. For example, with the `http_server` input we could send the payload to a Kafka topic and also send a modified payload back with:\n\n```yaml\ninput:\n http_server:\n path: /post\noutput:\n broker:\n pattern: fan_out\n outputs:\n - kafka:\n addresses: [ TODO:9092 ]\n topic: foo_topic\n - sync_response: {}\n processors:\n - mapping: 'root = content().uppercase()'\n```\n\nUsing the above example and posting the message 'hello world' to the endpoint `/post` Redpanda Connect would send it unchanged to the topic `foo_topic` and also respond with 'HELLO WORLD'.\n\nFor more information please read xref:guides:sync_responses.adoc[synchronous responses].","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","default":{}}},{"name":"timeplus","type":"output","status":"experimental","plugin":true,"summary":"Sends message to a Timeplus Enterprise stream via ingest endpoint","description":"\nThis output can send message to Timeplus Enterprise Cloud, Timeplus Enterprise (self-hosted) or directly to timeplusd.\n\nThis output accepts structured message only. It also expects all message contains the same keys and matches the schema of the destination stream. If the upstream source or pipeline returns\nunstructured message such as string, please refer to the \"Unstructured message\" example.","categories":["Services"],"examples":[{"title":"To Timeplus Enterprise Cloud","summary":"You will need to create API Key on Timeplus Enterprise Cloud Web console first and then set the `apikey` field.","config":"\noutput:\n timeplus:\n workspace: my_workspace_id\n stream: mystream\n apikey: \u003cYour API Key\u003e"},{"title":"To Timeplus Enterprise (self-hosted)","summary":"For self-housted Timeplus Enterprise, you will need to specify the username and password as well as the URL of the App server","config":"\noutput:\n timeplus:\n url: http://localhost:8000\n workspace: my_workspace_id\n stream: mystream\n username: username\n password: pw"},{"title":"To Timeplusd","summary":"This output writes to Timeplusd via HTTP so make sure you specify the HTTP port of the Timeplusd.","config":"\noutput:\n timeplus:\n url: http://localhost:3218\n stream: mystream\n username: username\n password: pw"},{"title":"Unstructured message","summary":"If the upstream source or pipeline returns unstructured message such as string, you can leverage the output processors to wrap it into a stucture message and then pass it to the output. This example create a strcutre mesasge with `raw` field and store the original string content into this field. You can modify the name of this `raw` field to whatever you want. Please make sure the destiation stream contains such field","config":"\noutput:\n timeplus:\n workspace: my_workspace_id\n stream: mystream\n apikey: \u003cApi key genereated on web console\u003e\n\n processors:\n - mapping: |\n root = {}\n root.raw = content().string()"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"target","type":"string","kind":"scalar","description":"The destination type, either Timeplus Enterprise or timeplusd","default":"timeplus","options":["timeplus","timeplusd"],"linter":"\nlet options = {\n \"timeplus\": true,\n \"timeplusd\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"url","type":"string","kind":"scalar","description":"The url should always include schema and host.","default":"https://us-west-2.timeplus.cloud","examples":["http://localhost:8000","http://127.0.0.1:3218"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"workspace","type":"string","kind":"scalar","description":"ID of the workspace. Required if target is `timeplus`.","is_optional":true},{"name":"stream","type":"string","kind":"scalar","description":"The name of the stream. Make sure the schema of the stream matches the input"},{"name":"apikey","type":"string","kind":"scalar","description":"The API key. Required if you are sending message to Timeplus Enterprise Cloud","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"username","type":"string","kind":"scalar","description":"The username. Required if you are sending message to Timeplus Enterprise (self-hosted) or timeplusd","is_optional":true},{"name":"password","type":"string","kind":"scalar","description":"The password. Required if you are sending message to Timeplus Enterprise (self-hosted) or timeplusd","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"max_in_flight","type":"int","kind":"scalar","description":"The maximum number of messages to have in flight at a given time. Increase this to improve throughput.","default":64},{"name":"batching","type":"object","kind":"","description":"\nAllows you to configure a xref:configuration:batching.adoc[batching policy].","examples":[{"byte_size":5000,"count":0,"period":"1s"},{"count":10,"period":"1s"},{"check":"this.contains(\"END BATCH\")","count":0,"period":"1m"}],"children":[{"name":"count","type":"int","kind":"scalar","description":"A number of messages at which the batch should be flushed. If `0` disables count based batching.","default":0},{"name":"byte_size","type":"int","kind":"scalar","description":"An amount of bytes at which the batch should be flushed. If `0` disables size based batching.","default":0},{"name":"period","type":"string","kind":"scalar","description":"A period in which an incomplete batch should be flushed regardless of its size.","default":"","examples":["1s","1m","500ms"]},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.","default":"","bloblang":true,"examples":["this.type == \"end_of_transaction\""]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.","is_advanced":true,"is_optional":true,"examples":[[{"archive":{"format":"concatenate"}}],[{"archive":{"format":"lines"}}],[{"archive":{"format":"json_array"}}]]}]}]}},{"name":"websocket","type":"output","status":"stable","plugin":true,"summary":"Sends messages to an HTTP server via a websocket connection.","categories":["Network"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL to connect to.","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"proxy_url","type":"string","kind":"scalar","description":"An optional HTTP proxy URL.","is_advanced":true,"is_optional":true,"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"oauth","type":"object","kind":"scalar","description":"Allows you to specify open authentication via OAuth version 1.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use OAuth version 1 in requests.","is_advanced":true,"default":false},{"name":"consumer_key","type":"string","kind":"scalar","description":"A value used to identify the client to the service provider.","is_advanced":true,"default":""},{"name":"consumer_secret","type":"string","kind":"scalar","description":"A secret used to establish ownership of the consumer key.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"access_token","type":"string","kind":"scalar","description":"A value used to gain access to the protected resources on behalf of the user.","is_advanced":true,"default":""},{"name":"access_token_secret","type":"string","kind":"scalar","description":"A secret provided in order to establish ownership of a given access token.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"jwt","type":"object","kind":"scalar","description":"BETA: Allows you to specify JWT authentication.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use JWT authentication in requests.","is_advanced":true,"default":false},{"name":"private_key_file","type":"string","kind":"scalar","description":"A file with the PEM encoded via PKCS1 or PKCS8 as private key.","is_advanced":true,"default":""},{"name":"signing_method","type":"string","kind":"scalar","description":"A method used to sign the token such as RS256, RS384, RS512 or EdDSA.","is_advanced":true,"default":""},{"name":"claims","type":"unknown","kind":"map","description":"A value used to identify the claims that issued the JWT.","is_advanced":true,"default":{}},{"name":"headers","type":"unknown","kind":"map","description":"Add optional key/value headers to the JWT.","is_advanced":true,"default":{}}]}]}}],"processors":[{"name":"archive","type":"processor","status":"stable","plugin":true,"summary":"Archives all the messages of a batch into a single message according to the selected archive format.","description":"\nSome archive formats (such as tar, zip) treat each archive item (message part) as a file with a path. Since message parts only contain raw data a unique path must be generated for each part. This can be done by using function interpolations on the 'path' field as described in xref:configuration:interpolation.adoc#bloblang-queries[Bloblang queries]. For types that aren't file based (such as binary) the file field is ignored.\n\nThe resulting archived message adopts the metadata of the _first_ message part of the batch.\n\nThe functionality of this processor depends on being applied across messages that are batched. You can find out more about batching xref:configuration:batching.adoc[in this doc].","categories":["Parsing","Utility"],"examples":[{"title":"Tar Archive","summary":"\nIf we had JSON messages in a batch each of the form:\n\n```json\n{\"doc\":{\"id\":\"foo\",\"body\":\"hello world 1\"}}\n```\n\nAnd we wished to tar archive them, setting their filenames to their respective unique IDs (with the extension `.json`), our config might look like\nthis:","config":"\npipeline:\n processors:\n - archive:\n format: tar\n path: ${!json(\"doc.id\")}.json\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"format","type":"string","kind":"scalar","description":"The archiving format to apply.","annotated_options":[["binary","Archive messages to a https://github.com/redpanda-data/benthos/blob/main/internal/message/message.go#L96[binary blob format^]."],["concatenate","Join the raw contents of each message into a single binary message."],["json_array","Attempt to parse each message as a JSON document and append the result to an array, which becomes the contents of the resulting message."],["lines","Join the raw contents of each message and insert a line break between each one."],["tar","Archive messages to a unix standard tape archive."],["zip","Archive messages to a zip file."]],"linter":"\nlet options = {\n \"binary\": true,\n \"concatenate\": true,\n \"json_array\": true,\n \"lines\": true,\n \"tar\": true,\n \"zip\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"path","type":"string","kind":"scalar","description":"The path to set for each message in the archive (when applicable).","default":"","interpolated":true,"examples":["${!count(\"files\")}-${!timestamp_unix_nano()}.txt","${!meta(\"kafka_key\")}-${!json(\"id\")}.json"]}]}},{"name":"avro","type":"processor","status":"beta","plugin":true,"summary":"Performs Avro based operations on messages based on a schema.","description":"\nWARNING: If you are consuming or generating messages using a schema registry service then it is likely this processor will fail as those services require messages to be prefixed with the identifier of the schema version being used. Instead, try the xref:components:processors/schema_registry_encode.adoc[`schema_registry_encode`] and xref:components:processors/schema_registry_decode.adoc[`schema_registry_decode`] processors.\n\n== Operators\n\n=== `to_json`\n\nConverts Avro documents into a JSON structure. This makes it easier to\nmanipulate the contents of the document within Benthos. The encoding field\nspecifies how the source documents are encoded.\n\n=== `from_json`\n\nAttempts to convert JSON documents into Avro documents according to the\nspecified encoding.","categories":["Parsing"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"operator","type":"string","kind":"scalar","description":"The \u003c\u003coperators, operator\u003e\u003e to execute","options":["to_json","from_json"],"linter":"\nlet options = {\n \"to_json\": true,\n \"from_json\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"encoding","type":"string","kind":"scalar","description":"An Avro encoding format to use for conversions to and from a schema.","default":"textual","options":["textual","binary","single"],"linter":"\nlet options = {\n \"textual\": true,\n \"binary\": true,\n \"single\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"schema","type":"string","kind":"scalar","description":"A full Avro schema to use.","default":""},{"name":"schema_path","type":"string","kind":"scalar","description":"The path of a schema document to apply. Use either this or the `schema` field.","default":"","examples":["file://path/to/spec.avsc","http://localhost:8081/path/to/spec/versions/1"]}]}},{"name":"awk","type":"processor","status":"stable","plugin":true,"summary":"Executes an AWK program on messages. This processor is very powerful as it offers a range of \u003c\u003cawk-functions,custom functions\u003e\u003e for querying and mutating message contents and metadata.","description":"\nWorks by feeding message contents as the program input based on a chosen \u003c\u003ccodecs,codec\u003e\u003e and replaces the contents of each message with the result. If the result is empty (nothing is printed by the program) then the original message contents remain unchanged.\n\nComes with a wide range of \u003c\u003cawk-functions,custom functions\u003e\u003e for accessing message metadata, json fields, printing logs, etc. These functions can be overridden by functions within the program.\n\nCheck out the \u003c\u003cexamples,examples section\u003e\u003e in order to see how this processor can be used.\n\nThis processor uses https://github.com/benhoyt/goawk[GoAWK^], in order to understand the differences in how the program works you can read more about it in https://github.com/benhoyt/goawk#differences-from-awk[goawk.differences^].","categories":["Mapping"],"footnotes":"\n== Codecs\n\nThe chosen codec determines how the contents of the message are fed into the\nprogram. Codecs only impact the input string and variables initialized for your\nprogram, they do not change the range of custom functions available.\n\n=== `none`\n\nAn empty string is fed into the program. Functions can still be used in order to\nextract and mutate metadata and message contents.\n\nThis is useful for when your program only uses functions and doesn't need the\nfull text of the message to be parsed by the program, as it is significantly\nfaster.\n\n=== `text`\n\nThe full contents of the message are fed into the program as a string, allowing\nyou to reference tokenized segments of the message with variables ($0, $1, etc).\nCustom functions can still be used with this codec.\n\nThis is the default codec as it behaves most similar to typical usage of the awk\ncommand line tool.\n\n=== `json`\n\nAn empty string is fed into the program, and variables are automatically\ninitialized before execution of your program by walking the flattened JSON\nstructure. Each value is converted into a variable by taking its full path,\ne.g. the object:\n\n```json\n{\n\t\"foo\": {\n\t\t\"bar\": {\n\t\t\t\"value\": 10\n\t\t},\n\t\t\"created_at\": \"2018-12-18T11:57:32\"\n\t}\n}\n```\n\nWould result in the following variable declarations:\n\n```\nfoo_bar_value = 10\nfoo_created_at = \"2018-12-18T11:57:32\"\n```\n\nCustom functions can also still be used with this codec.\n\n== AWK functions\n\n=== `json_get`\n\nSignature: `json_get(path)`\n\nAttempts to find a JSON value in the input message payload by a\nxref:configuration:field_paths.adoc[dot separated path] and returns it as a string.\n\n=== `json_set`\n\nSignature: `json_set(path, value)`\n\nAttempts to set a JSON value in the input message payload identified by a\nxref:configuration:field_paths.adoc[dot separated path], the value argument will be interpreted\nas a string.\n\nIn order to set non-string values use one of the following typed varieties:\n\n- `json_set_int(path, value)`\n- `json_set_float(path, value)`\n- `json_set_bool(path, value)`\n\n=== `json_append`\n\nSignature: `json_append(path, value)`\n\nAttempts to append a value to an array identified by a\nxref:configuration:field_paths.adoc[dot separated path]. If the target does not\nexist it will be created. If the target exists but is not already an array then\nit will be converted into one, with its original contents set to the first\nelement of the array.\n\nThe value argument will be interpreted as a string. In order to append\nnon-string values use one of the following typed varieties:\n\n- `json_append_int(path, value)`\n- `json_append_float(path, value)`\n- `json_append_bool(path, value)`\n\n=== `json_delete`\n\nSignature: `json_delete(path)`\n\nAttempts to delete a JSON field from the input message payload identified by a\nxref:configuration:field_paths.adoc[dot separated path].\n\n=== `json_length`\n\nSignature: `json_length(path)`\n\nReturns the size of the string or array value of JSON field from the input\nmessage payload identified by a xref:configuration:field_paths.adoc[dot separated path].\n\nIf the target field does not exist, or is not a string or array type, then zero\nis returned. In order to explicitly check the type of a field use `json_type`.\n\n=== `json_type`\n\nSignature: `json_type(path)`\n\nReturns the type of a JSON field from the input message payload identified by a\nxref:configuration:field_paths.adoc[dot separated path].\n\nPossible values are: \"string\", \"int\", \"float\", \"bool\", \"undefined\", \"null\",\n\"array\", \"object\".\n\n=== `create_json_object`\n\nSignature: `create_json_object(key1, val1, key2, val2, ...)`\n\nGenerates a valid JSON object of key value pair arguments. The arguments are\nvariadic, meaning any number of pairs can be listed. The value will always\nresolve to a string regardless of the value type. E.g. the following call:\n\n`create_json_object(\"a\", \"1\", \"b\", 2, \"c\", \"3\")`\n\nWould result in this string:\n\n`\\{\"a\":\"1\",\"b\":\"2\",\"c\":\"3\"}`\n\n=== `create_json_array`\n\nSignature: `create_json_array(val1, val2, ...)`\n\nGenerates a valid JSON array of value arguments. The arguments are variadic,\nmeaning any number of values can be listed. The value will always resolve to a\nstring regardless of the value type. E.g. the following call:\n\n`create_json_array(\"1\", 2, \"3\")`\n\nWould result in this string:\n\n`[\"1\",\"2\",\"3\"]`\n\n=== `metadata_set`\n\nSignature: `metadata_set(key, value)`\n\nSet a metadata key for the message to a value. The value will always resolve to\na string regardless of the value type.\n\n=== `metadata_get`\n\nSignature: `metadata_get(key) string`\n\nGet the value of a metadata key from the message.\n\n=== `timestamp_unix`\n\nSignature: `timestamp_unix() int`\n\nReturns the current unix timestamp (the number of seconds since 01-01-1970).\n\n=== `timestamp_unix`\n\nSignature: `timestamp_unix(date) int`\n\nAttempts to parse a date string by detecting its format and returns the\nequivalent unix timestamp (the number of seconds since 01-01-1970).\n\n=== `timestamp_unix`\n\nSignature: `timestamp_unix(date, format) int`\n\nAttempts to parse a date string according to a format and returns the equivalent\nunix timestamp (the number of seconds since 01-01-1970).\n\nThe format is defined by showing how the reference time, defined to be\n`Mon Jan 2 15:04:05 -0700 MST 2006` would be displayed if it were the value.\n\n=== `timestamp_unix_nano`\n\nSignature: `timestamp_unix_nano() int`\n\nReturns the current unix timestamp in nanoseconds (the number of nanoseconds\nsince 01-01-1970).\n\n=== `timestamp_unix_nano`\n\nSignature: `timestamp_unix_nano(date) int`\n\nAttempts to parse a date string by detecting its format and returns the\nequivalent unix timestamp in nanoseconds (the number of nanoseconds since\n01-01-1970).\n\n=== `timestamp_unix_nano`\n\nSignature: `timestamp_unix_nano(date, format) int`\n\nAttempts to parse a date string according to a format and returns the equivalent\nunix timestamp in nanoseconds (the number of nanoseconds since 01-01-1970).\n\nThe format is defined by showing how the reference time, defined to be\n`Mon Jan 2 15:04:05 -0700 MST 2006` would be displayed if it were the value.\n\n=== `timestamp_format`\n\nSignature: `timestamp_format(unix, format) string`\n\nFormats a unix timestamp. The format is defined by showing how the reference\ntime, defined to be `Mon Jan 2 15:04:05 -0700 MST 2006` would be displayed if it\nwere the value.\n\nThe format is optional, and if omitted RFC3339 (`2006-01-02T15:04:05Z07:00`)\nwill be used.\n\n=== `timestamp_format_nano`\n\nSignature: `timestamp_format_nano(unixNano, format) string`\n\nFormats a unix timestamp in nanoseconds. The format is defined by showing how\nthe reference time, defined to be `Mon Jan 2 15:04:05 -0700 MST 2006` would be\ndisplayed if it were the value.\n\nThe format is optional, and if omitted RFC3339 (`2006-01-02T15:04:05Z07:00`)\nwill be used.\n\n=== `print_log`\n\nSignature: `print_log(message, level)`\n\nPrints a Redpanda Connect log message at a particular log level. The log level is\noptional, and if omitted the level `INFO` will be used.\n\n=== `base64_encode`\n\nSignature: `base64_encode(data)`\n\nEncodes the input data to a base64 string.\n\n=== `base64_decode`\n\nSignature: `base64_decode(data)`\n\nAttempts to base64-decode the input data and returns the decoded string if\nsuccessful. It will emit an error otherwise.\n\n","examples":[{"title":"JSON Mapping and Arithmetic","summary":"\nBecause AWK is a full programming language it's much easier to map documents and perform arithmetic with it than with other Redpanda Connect processors. For example, if we were expecting documents of the form:\n\n```json\n{\"doc\":{\"val1\":5,\"val2\":10},\"id\":\"1\",\"type\":\"add\"}\n{\"doc\":{\"val1\":5,\"val2\":10},\"id\":\"2\",\"type\":\"multiply\"}\n```\n\nAnd we wished to perform the arithmetic specified in the `type` field,\non the values `val1` and `val2` and, finally, map the result into the\ndocument, giving us the following resulting documents:\n\n```json\n{\"doc\":{\"result\":15,\"val1\":5,\"val2\":10},\"id\":\"1\",\"type\":\"add\"}\n{\"doc\":{\"result\":50,\"val1\":5,\"val2\":10},\"id\":\"2\",\"type\":\"multiply\"}\n```\n\nWe can do that with the following:","config":"\npipeline:\n processors:\n - awk:\n codec: none\n program: |\n function map_add_vals() {\n json_set_int(\"doc.result\", json_get(\"doc.val1\") + json_get(\"doc.val2\"));\n }\n function map_multiply_vals() {\n json_set_int(\"doc.result\", json_get(\"doc.val1\") * json_get(\"doc.val2\"));\n }\n function map_unknown(type) {\n json_set(\"error\",\"unknown document type\");\n print_log(\"Document type not recognised: \" type, \"ERROR\");\n }\n {\n type = json_get(\"type\");\n if (type == \"add\")\n map_add_vals();\n else if (type == \"multiply\")\n map_multiply_vals();\n else\n map_unknown(type);\n }\n"},{"title":"Stuff With Arrays","summary":"\nIt's possible to iterate JSON arrays by appending an index value to the path, this can be used to do things like removing duplicates from arrays. For example, given the following input document:\n\n```json\n{\"path\":{\"to\":{\"foos\":[\"one\",\"two\",\"three\",\"two\",\"four\"]}}}\n```\n\nWe could create a new array `foos_unique` from `foos` giving us the result:\n\n```json\n{\"path\":{\"to\":{\"foos\":[\"one\",\"two\",\"three\",\"two\",\"four\"],\"foos_unique\":[\"one\",\"two\",\"three\",\"four\"]}}}\n```\n\nWith the following config:","config":"\npipeline:\n processors:\n - awk:\n codec: none\n program: |\n {\n array_path = \"path.to.foos\"\n array_len = json_length(array_path)\n\n for (i = 0; i \u003c array_len; i++) {\n ele = json_get(array_path \".\" i)\n if ( ! ( ele in seen ) ) {\n json_append(array_path \"_unique\", ele)\n seen[ele] = 1\n }\n }\n }\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"codec","type":"string","kind":"scalar","description":"A \u003c\u003ccodecs,codec\u003e\u003e defines how messages should be inserted into the AWK program as variables. The codec does not change which \u003c\u003cawk-functions,custom Redpanda Connect functions\u003e\u003e are available. The `text` codec is the closest to a typical AWK use case.","options":["none","text","json"],"linter":"\nlet options = {\n \"none\": true,\n \"text\": true,\n \"json\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"program","type":"string","kind":"scalar","description":"An AWK program to execute"}]}},{"name":"aws_bedrock_chat","type":"processor","status":"experimental","plugin":true,"summary":"Generates responses to messages in a chat conversation, using the AWS Bedrock API.","description":"This processor sends prompts to your chosen large language model (LLM) and generates text from the responses, using the AWS Bedrock API.\nFor more information, see the https://docs.aws.amazon.com/bedrock/latest/userguide[AWS Bedrock documentation^].","categories":["AI"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]},{"name":"model","type":"string","kind":"scalar","description":"The model ID to use. For a full list see the https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids.html[AWS Bedrock documentation^].","examples":["amazon.titan-text-express-v1","anthropic.claude-3-5-sonnet-20240620-v1:0","cohere.command-text-v14","meta.llama3-1-70b-instruct-v1:0","mistral.mistral-large-2402-v1:0"]},{"name":"prompt","type":"string","kind":"scalar","description":"The prompt you want to generate a response for. By default, the processor submits the entire payload as a string.","is_optional":true},{"name":"system_prompt","type":"string","kind":"scalar","description":"The system prompt to submit to the AWS Bedrock LLM.","is_optional":true},{"name":"max_tokens","type":"int","kind":"scalar","description":"The maximum number of tokens to allow in the generated response.","is_optional":true},{"name":"temperature","type":"float","kind":"scalar","description":"The likelihood of the model selecting higher-probability options while generating a response. A lower value makes the model omre likely to choose higher-probability options, while a higher value makes the model more likely to choose lower-probability options.","is_optional":true,"linter":"root = if this \u003c 0 || this \u003e 1 { [\"field must be between 0.0-1.0\"] }"},{"name":"stop","type":"string","kind":"array","description":"A list of stop sequences. A stop sequence is a sequence of characters that causes the model to stop generating the response.","is_advanced":true,"is_optional":true},{"name":"top_p","type":"float","kind":"scalar","description":"The percentage of most-likely candidates that the model considers for the next token. For example, if you choose a value of 0.8, the model selects from the top 80% of the probability distribution of tokens that could be next in the sequence. ","is_advanced":true,"is_optional":true,"linter":"root = if this \u003c 0 || this \u003e 1 { [\"field must be between 0.0-1.0\"] }"}]},"version":"4.34.0"},{"name":"aws_bedrock_embeddings","type":"processor","status":"experimental","plugin":true,"summary":"Computes vector embeddings on text, using the AWS Bedrock API.","description":"This processor sends text to your chosen large language model (LLM) and computes vector embeddings, using the AWS Bedrock API.\nFor more information, see the https://docs.aws.amazon.com/bedrock/latest/userguide[AWS Bedrock documentation^].","categories":["AI"],"examples":[{"title":"Store embedding vectors in Clickhouse","summary":"Compute embeddings for some generated data and store it within https://clickhouse.com/[Clickhouse^]","config":"input:\n generate:\n interval: 1s\n mapping: |\n root = {\"text\": fake(\"paragraph\")}\npipeline:\n processors:\n - branch:\n request_map: |\n root = this.text\n processors:\n - aws_bedrock_embeddings:\n model: amazon.titan-embed-text-v1\n result_map: |\n root.embeddings = this\noutput:\n sql_insert:\n driver: clickhouse\n dsn: \"clickhouse://localhost:9000\"\n table: searchable_text\n columns: [\"id\", \"text\", \"vector\"]\n args_mapping: \"root = [uuid_v4(), this.text, this.embeddings]\"\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]},{"name":"model","type":"string","kind":"scalar","description":"The model ID to use. For a full list see the https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids.html[AWS Bedrock documentation^].","examples":["amazon.titan-embed-text-v1","amazon.titan-embed-text-v2:0","cohere.embed-english-v3","cohere.embed-multilingual-v3"]},{"name":"text","type":"string","kind":"scalar","description":"The prompt you want to generate a response for. By default, the processor submits the entire payload as a string.","is_optional":true}]},"version":"4.37.0"},{"name":"aws_dynamodb_partiql","type":"processor","status":"experimental","plugin":true,"summary":"Executes a PartiQL expression against a DynamoDB table for each message.","description":"Both writes or reads are supported, when the query is a read the contents of the message will be replaced with the result. This processor is more efficient when messages are pre-batched as the whole batch will be executed in a single call.","categories":["Integration"],"examples":[{"title":"Insert","summary":"The following example inserts rows into the table footable with the columns foo, bar and baz populated with values extracted from messages:","config":"\npipeline:\n processors:\n - aws_dynamodb_partiql:\n query: \"INSERT INTO footable VALUE {'foo':'?','bar':'?','baz':'?'}\"\n args_mapping: |\n root = [\n { \"S\": this.foo },\n { \"S\": meta(\"kafka_topic\") },\n { \"S\": this.document.content },\n ]\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"query","type":"string","kind":"scalar","description":"A PartiQL query to execute for each message."},{"name":"unsafe_dynamic_query","type":"bool","kind":"scalar","description":"Whether to enable dynamic queries that support interpolation functions.","is_advanced":true,"default":false},{"name":"args_mapping","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] that, for each message, creates a list of arguments to use with the query.","default":"","bloblang":true},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]},"version":"3.48.0"},{"name":"aws_lambda","type":"processor","status":"stable","plugin":true,"summary":"Invokes an AWS lambda for each message. The contents of the message is the payload of the request, and the result of the invocation will become the new contents of the message.","description":"The `rate_limit` field can be used to specify a rate limit xref:components:rate_limits/about.adoc[resource] to cap the rate of requests across parallel components service wide.\n\nIn order to map or encode the payload to a specific request body, and map the response back into the original payload instead of replacing it entirely, you can use the xref:components:processors/branch.adoc[`branch` processor].\n\n== Error handling\n\nWhen Redpanda Connect is unable to connect to the AWS endpoint or is otherwise unable to invoke the target lambda function it will retry the request according to the configured number of retries. Once these attempts have been exhausted the failed message will continue through the pipeline with it's contents unchanged, but flagged as having failed, allowing you to use xref:configuration:error_handling.adoc[standard processor error handling patterns].\n\nHowever, if the invocation of the function is successful but the function itself throws an error, then the message will have it's contents updated with a JSON payload describing the reason for the failure, and a metadata field `lambda_function_error` will be added to the message allowing you to detect and handle function errors with a xref:components:processors/branch.adoc[`branch`]:\n\n```yaml\npipeline:\n processors:\n - branch:\n processors:\n - aws_lambda:\n function: foo\n result_map: |\n root = if meta().exists(\"lambda_function_error\") {\n throw(\"Invocation failed due to %v: %v\".format(this.errorType, this.errorMessage))\n } else {\n this\n }\noutput:\n switch:\n retry_until_success: false\n cases:\n - check: errored()\n output:\n reject: ${! error() }\n - output:\n resource: somewhere_else\n```\n\n== Credentials\n\nBy default Redpanda Connect will use a shared credentials file when connecting to AWS services. It's also possible to set them explicitly at the component level, allowing you to transfer data across accounts. You can find out more in xref:guides:cloud/aws.adoc[].","categories":["Integration"],"examples":[{"title":"Branched Invoke","summary":"\nThis example uses a xref:components:processors/branch.adoc[`branch` processor] to map a new payload for triggering a lambda function with an ID and username from the original message, and the result of the lambda is discarded, meaning the original message is unchanged.","config":"\npipeline:\n processors:\n - branch:\n request_map: '{\"id\":this.doc.id,\"username\":this.user.name}'\n processors:\n - aws_lambda:\n function: trigger_user_update\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"parallel","type":"bool","kind":"scalar","description":"Whether messages of a batch should be dispatched in parallel.","default":false},{"name":"function","type":"string","kind":"scalar","description":"The function to invoke."},{"name":"rate_limit","type":"string","kind":"scalar","description":"An optional xref:components:rate_limits/about.adoc[`rate_limit`] to throttle invocations by.","is_advanced":true,"default":""},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period of time to wait before abandoning an invocation.","is_advanced":true,"default":"5s"},{"name":"retries","type":"int","kind":"scalar","description":"The maximum number of retry attempts for each message.","is_advanced":true,"default":3}]},"version":"3.36.0"},{"name":"azure_cosmosdb","type":"processor","status":"experimental","plugin":true,"summary":"Creates or updates messages as JSON documents in https://learn.microsoft.com/en-us/azure/cosmos-db/introduction[Azure CosmosDB^].","description":"\nWhen creating documents, each message must have the `id` property (case-sensitive) set (or use `auto_id: true`). It is the unique name that identifies the document, that is, no two documents share the same `id` within a logical partition. The `id` field must not exceed 255 characters. https://learn.microsoft.com/en-us/rest/api/cosmos-db/documents[See details^].\n\nThe `partition_keys` field must resolve to the same value(s) across the entire message batch.\n\n\n== Credentials\n\nYou can use one of the following authentication mechanisms:\n\n- Set the `endpoint` field and the `account_key` field\n- Set only the `endpoint` field to use https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity#DefaultAzureCredential[DefaultAzureCredential^]\n- Set the `connection_string` field\n\n\n== Metadata\n\nThis component adds the following metadata fields to each message:\n```\n- activity_id\n- request_charge\n```\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].\n\n\n== Batching\n\nCosmosDB limits the maximum batch size to 100 messages and the payload must not exceed 2MB (https://learn.microsoft.com/en-us/azure/cosmos-db/concepts-limits#per-request-limits[details here^]).\n","categories":["Azure"],"footnotes":"\n\n== CosmosDB emulator\n\nIf you wish to run the CosmosDB emulator that is referenced in the documentation https://learn.microsoft.com/en-us/azure/cosmos-db/linux-emulator[here^], the following Docker command should do the trick:\n\n```bash\n\u003e docker run --rm -it -p 8081:8081 --name=cosmosdb -e AZURE_COSMOS_EMULATOR_PARTITION_COUNT=10 -e AZURE_COSMOS_EMULATOR_ENABLE_DATA_PERSISTENCE=false mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator\n```\n\nNote: `AZURE_COSMOS_EMULATOR_PARTITION_COUNT` controls the number of partitions that will be supported by the emulator. The bigger the value, the longer it takes for the container to start up.\n\nAdditionally, instead of installing the container self-signed certificate which is exposed via `https://localhost:8081/_explorer/emulator.pem`, you can run https://mitmproxy.org/[mitmproxy^] like so:\n\n```bash\n\u003e mitmproxy -k --mode \"reverse:https://localhost:8081\"\n```\n\nThen you can access the CosmosDB UI via `http://localhost:8080/_explorer/index.html` and use `http://localhost:8080` as the CosmosDB endpoint.\n","examples":[{"title":"Patch documents","summary":"Query documents from a container and patch them.","config":"\ninput:\n azure_cosmosdb:\n endpoint: http://localhost:8080\n account_key: C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==\n database: blobbase\n container: blobfish\n partition_keys_map: root = \"AbyssalPlain\"\n query: SELECT * FROM blobfish\n\n processors:\n - mapping: |\n root = \"\"\n meta habitat = json(\"habitat\")\n meta id = this.id\n - azure_cosmosdb:\n endpoint: http://localhost:8080\n account_key: C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==\n database: testdb\n container: blobfish\n partition_keys_map: root = json(\"habitat\")\n item_id: ${! meta(\"id\") }\n operation: Patch\n patch_operations:\n # Add a new /diet field\n - operation: Add\n path: /diet\n value_map: root = json(\"diet\")\n # Remove the first location from the /locations array field\n - operation: Remove\n path: /locations/0\n # Add new location at the end of the /locations array field\n - operation: Add\n path: /locations/-\n value_map: root = \"Challenger Deep\"\n # Return the updated document\n enable_content_response_on_write: true\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"endpoint","type":"string","kind":"scalar","description":"CosmosDB endpoint.","is_optional":true,"examples":["https://localhost:8081"]},{"name":"account_key","type":"string","kind":"scalar","description":"Account key.","is_optional":true,"is_secret":true,"examples":["C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw=="],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"connection_string","type":"string","kind":"scalar","description":"Connection string.","is_optional":true,"is_secret":true,"examples":["AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==;"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"database","type":"string","kind":"scalar","description":"Database.","examples":["testdb"]},{"name":"container","type":"string","kind":"scalar","description":"Container.","examples":["testcontainer"]},{"name":"partition_keys_map","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to a single partition key value or an array of partition key values of type string, integer or boolean. Currently, hierarchical partition keys are not supported so only one value may be provided.","bloblang":true,"examples":["root = \"blobfish\"","root = 41","root = true","root = null","root = json(\"blobfish\").depth"]},{"name":"operation","type":"string","kind":"scalar","description":"Operation.","default":"Create","annotated_options":[["Create","Create operation."],["Delete","Delete operation."],["Patch","Patch operation."],["Read","Read operation."],["Replace","Replace operation."],["Upsert","Upsert operation."]],"linter":"\nlet options = {\n \"create\": true,\n \"delete\": true,\n \"patch\": true,\n \"read\": true,\n \"replace\": true,\n \"upsert\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"patch_operations","type":"object","kind":"array","description":"Patch operations to be performed when `operation: Patch` .","is_advanced":true,"is_optional":true,"children":[{"name":"operation","type":"string","kind":"scalar","description":"Operation.","is_advanced":true,"default":"Add","annotated_options":[["Add","Add patch operation."],["Increment","Increment patch operation."],["Remove","Remove patch operation."],["Replace","Replace patch operation."],["Set","Set patch operation."]],"linter":"\nlet options = {\n \"add\": true,\n \"increment\": true,\n \"remove\": true,\n \"replace\": true,\n \"set\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"path","type":"string","kind":"scalar","description":"Path.","is_advanced":true,"examples":["/foo/bar/baz"]},{"name":"value_map","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to a value of any type that is supported by CosmosDB.","is_advanced":true,"is_optional":true,"bloblang":true,"examples":["root = \"blobfish\"","root = 41","root = true","root = json(\"blobfish\").depth","root = [1, 2, 3]"]}]},{"name":"patch_condition","type":"string","kind":"scalar","description":"Patch operation condition.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["from c where not is_defined(c.blobfish)"]},{"name":"auto_id","type":"bool","kind":"scalar","description":"Automatically set the item `id` field to a random UUID v4. If the `id` field is already set, then it will not be overwritten. Setting this to `false` can improve performance, since the messages will not have to be parsed.","is_advanced":true,"default":true},{"name":"item_id","type":"string","kind":"scalar","description":"ID of item to replace or delete. Only used by the Replace and Delete operations","is_optional":true,"interpolated":true,"examples":["${! json(\"id\") }"]},{"name":"enable_content_response_on_write","type":"bool","kind":"scalar","description":"Enable content response on write operations. To save some bandwidth, set this to false if you don't need to receive the updated message(s) from the server, in which case the processor will not modify the content of the messages which are fed into it. Applies to every operation except Read.","is_advanced":true,"default":true}],"linter":"root = []\nlet hasEndpoint = this.endpoint.or(\"\") != \"\"\nlet hasConnectionString = this.connection_string.or(\"\") != \"\"\n\nroot.\"-\" = if !$hasEndpoint \u0026\u0026 !$hasConnectionString {\n \"Either `endpoint` or `connection_string` must be set.\"\n}\n\nlet hasItemID = this.item_id.or(\"\") != \"\"\nlet hasPatchOperations = this.patch_operations.length().or(0) \u003e 0\nlet hasPatchCondition = this.patch_condition.or(\"\") != \"\"\n\nroot.\"-\" = if !$hasItemID \u0026\u0026 (this.operation == \"Replace\" || this.operation == \"Delete\" || this.operation == \"Read\" || this.operation == \"Patch\") {\n \"The `item_id` field must be set for Replace, Delete, Read and Patch operations.\"\n}\n\nroot.\"-\" = if this.operation == \"Patch\" \u0026\u0026 !$hasPatchOperations {\n \"At least one `patch_operations` must be set when `operation: Patch`.\"\n}\n\nroot.\"-\" = if $hasPatchCondition \u0026\u0026 (!$hasPatchOperations || this.operation != \"Patch\") {\n \"The `patch_condition` field only applies to `Patch` operations and it requires one or more `patch_operations`.\"\n}\n\nroot.\"-\" = if this.operation == \"Patch\" \u0026\u0026 this.patch_operations.any(o -\u003e o.operation != \"Remove\" \u0026\u0026 o.value_map.or(\"\") == \"\") {\n \"The `patch_operations` `value_map` field must be set when `operation` is not `Remove`.\"\n}\n\nroot.\"-\" = if this.operation == \"Patch\" \u0026\u0026 this.patch_operations.any(o -\u003e o.operation == \"Remove\" \u0026\u0026 o.value_map.or(\"\") != \"\") {\n \"The `patch_operations` `value_map` field must not be set when `operation` is `Remove`.\"\n}\n"},"version":"v4.25.0"},{"name":"benchmark","type":"processor","status":"experimental","plugin":true,"summary":"Logs basic throughput statistics of messages that pass through this processor.","description":"Logs messages per second and bytes per second of messages that are processed at a regular interval. A summary of the amount of messages processed over the entire lifetime of the processor will also be printed when the processor shuts down.","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"interval","type":"string","kind":"scalar","description":"How often to emit rolling statistics. If set to 0, only a summary will be logged when the processor shuts down.","default":"5s"},{"name":"count_bytes","type":"bool","kind":"scalar","description":"Whether or not to measure the number of bytes per second of throughput. Counting the number of bytes requires serializing structured data, which can cause an unnecessary performance hit if serialization is not required elsewhere in the pipeline.","default":true}]}},{"name":"bloblang","type":"processor","status":"stable","plugin":true,"summary":"Executes a xref:guides:bloblang/about.adoc[Bloblang] mapping on messages.","description":"\nBloblang is a powerful language that enables a wide range of mapping, transformation and filtering tasks. For more information see xref:guides:bloblang/about.adoc[].\n\nIf your mapping is large and you'd prefer for it to live in a separate file then you can execute a mapping directly from a file with the expression `from \"\u003cpath\u003e\"`, where the path must be absolute, or relative from the location that Redpanda Connect is executed from.\n\n== Component rename\n\nThis processor was recently renamed to the xref:components:processors/mapping.adoc[`mapping` processor] in order to make the purpose of the processor more prominent. It is still valid to use the existing `bloblang` name but eventually it will be deprecated and replaced by the new name in example configs.","categories":["Mapping","Parsing"],"footnotes":"\n== Error handling\n\nBloblang mappings can fail, in which case the message remains unchanged, errors are logged, and the message is flagged as having failed, allowing you to use\nxref:configuration:error_handling.adoc[standard processor error handling patterns].\n\nHowever, Bloblang itself also provides powerful ways of ensuring your mappings do not fail by specifying desired fallback behavior, which you can read about in xref:guides:bloblang/about#error-handling.adoc[Error handling].","examples":[{"title":"Mapping","summary":"\nGiven JSON documents containing an array of fans:\n\n```json\n{\n \"id\":\"foo\",\n \"description\":\"a show about foo\",\n \"fans\":[\n {\"name\":\"bev\",\"obsession\":0.57},\n {\"name\":\"grace\",\"obsession\":0.21},\n {\"name\":\"ali\",\"obsession\":0.89},\n {\"name\":\"vic\",\"obsession\":0.43}\n ]\n}\n```\n\nWe can reduce the fans to only those with an obsession score above 0.5, giving us:\n\n```json\n{\n \"id\":\"foo\",\n \"description\":\"a show about foo\",\n \"fans\":[\n {\"name\":\"bev\",\"obsession\":0.57},\n {\"name\":\"ali\",\"obsession\":0.89}\n ]\n}\n```\n\nWith the following config:","config":"\npipeline:\n processors:\n - bloblang: |\n root = this\n root.fans = this.fans.filter(fan -\u003e fan.obsession \u003e 0.5)\n"},{"title":"More Mapping","summary":"\nWhen receiving JSON documents of the form:\n\n```json\n{\n \"locations\": [\n {\"name\": \"Seattle\", \"state\": \"WA\"},\n {\"name\": \"New York\", \"state\": \"NY\"},\n {\"name\": \"Bellevue\", \"state\": \"WA\"},\n {\"name\": \"Olympia\", \"state\": \"WA\"}\n ]\n}\n```\n\nWe could collapse the location names from the state of Washington into a field `Cities`:\n\n```json\n{\"Cities\": \"Bellevue, Olympia, Seattle\"}\n```\n\nWith the following config:","config":"\npipeline:\n processors:\n - bloblang: |\n root.Cities = this.locations.\n filter(loc -\u003e loc.state == \"WA\").\n map_each(loc -\u003e loc.name).\n sort().join(\", \")\n"}],"config":{"name":"","type":"string","kind":"scalar","default":"","bloblang":true}},{"name":"bounds_check","type":"processor","status":"stable","plugin":true,"summary":"Removes messages (and batches) that do not fit within certain size boundaries.","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"max_part_size","type":"int","kind":"scalar","description":"The maximum size of a message to allow (in bytes)","default":1073741824},{"name":"min_part_size","type":"int","kind":"scalar","description":"The minimum size of a message to allow (in bytes)","default":1},{"name":"max_parts","type":"int","kind":"scalar","description":"The maximum size of message batches to allow (in message count)","is_advanced":true,"default":100},{"name":"min_parts","type":"int","kind":"scalar","description":"The minimum size of message batches to allow (in message count)","is_advanced":true,"default":1}]}},{"name":"branch","type":"processor","status":"stable","plugin":true,"summary":"The `branch` processor allows you to create a new request message via a xref:guides:bloblang/about.adoc[Bloblang mapping], execute a list of processors on the request messages, and, finally, map the result back into the source message using another mapping.","description":"\nThis is useful for preserving the original message contents when using processors that would otherwise replace the entire contents.\n\n== Metadata\n\nMetadata fields that are added to messages during branch processing will not be automatically copied into the resulting message. In order to do this you should explicitly declare in your `result_map` either a wholesale copy with `meta = metadata()`, or selective copies with `meta foo = metadata(\"bar\")` and so on. It is also possible to reference the metadata of the origin message in the `result_map` using the xref:guides:bloblang/about.adoc#metadata[`@` operator].\n\n== Error handling\n\nIf the `request_map` fails the child processors will not be executed. If the child processors themselves result in an (uncaught) error then the `result_map` will not be executed. If the `result_map` fails the message will remain unchanged. Under any of these conditions standard xref:configuration:error_handling.adoc[error handling methods] can be used in order to filter, DLQ or recover the failed messages.\n\n== Conditional branching\n\nIf the root of your request map is set to `deleted()` then the branch processors are skipped for the given message, this allows you to conditionally branch messages.","categories":["Composition"],"examples":[{"title":"HTTP Request","summary":"\nThis example strips the request message into an empty body, grabs an HTTP payload, and places the result back into the original message at the path `image.pull_count`:","config":"\npipeline:\n processors:\n - branch:\n request_map: 'root = \"\"'\n processors:\n - http:\n url: https://hub.docker.com/v2/repositories/jeffail/benthos\n verb: GET\n headers:\n Content-Type: application/json\n result_map: root.image.pull_count = this.pull_count\n\n# Example input: {\"id\":\"foo\",\"some\":\"pre-existing data\"}\n# Example output: {\"id\":\"foo\",\"some\":\"pre-existing data\",\"image\":{\"pull_count\":1234}}\n"},{"title":"Non Structured Results","summary":"\nWhen the result of your branch processors is unstructured and you wish to simply set a resulting field to the raw output use the content function to obtain the raw bytes of the resulting message and then coerce it into your value type of choice:","config":"\npipeline:\n processors:\n - branch:\n request_map: 'root = this.document.id'\n processors:\n - cache:\n resource: descriptions_cache\n key: ${! content() }\n operator: get\n result_map: root.document.description = content().string()\n\n# Example input: {\"document\":{\"id\":\"foo\",\"content\":\"hello world\"}}\n# Example output: {\"document\":{\"id\":\"foo\",\"content\":\"hello world\",\"description\":\"this is a cool doc\"}}\n"},{"title":"Lambda Function","summary":"\nThis example maps a new payload for triggering a lambda function with an ID and username from the original message, and the result of the lambda is discarded, meaning the original message is unchanged.","config":"\npipeline:\n processors:\n - branch:\n request_map: '{\"id\":this.doc.id,\"username\":this.user.name}'\n processors:\n - aws_lambda:\n function: trigger_user_update\n\n# Example input: {\"doc\":{\"id\":\"foo\",\"body\":\"hello world\"},\"user\":{\"name\":\"fooey\"}}\n# Output matches the input, which is unchanged\n"},{"title":"Conditional Caching","summary":"\nThis example caches a document by a message ID only when the type of the document is a foo:","config":"\npipeline:\n processors:\n - branch:\n request_map: |\n meta id = this.id\n root = if this.type == \"foo\" {\n this.document\n } else {\n deleted()\n }\n processors:\n - cache:\n resource: TODO\n operator: set\n key: ${! @id }\n value: ${! content() }\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"request_map","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] that describes how to create a request payload suitable for the child processors of this branch. If left empty then the branch will begin with an exact copy of the origin message (including metadata).","default":"","bloblang":true,"examples":["root = {\n\t\"id\": this.doc.id,\n\t\"content\": this.doc.body.text\n}","root = if this.type == \"foo\" {\n\tthis.foo.request\n} else {\n\tdeleted()\n}"]},{"name":"processors","type":"processor","kind":"array","description":"A list of processors to apply to mapped requests. When processing message batches the resulting batch must match the size and ordering of the input batch, therefore filtering, grouping should not be performed within these processors."},{"name":"result_map","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] that describes how the resulting messages from branched processing should be mapped back into the original payload. If left empty the origin message will remain unchanged (including metadata).","default":"","bloblang":true,"examples":["meta foo_code = metadata(\"code\")\nroot.foo_result = this","meta = metadata()\nroot.bar.body = this.body\nroot.bar.id = this.user.id","root.raw_result = content().string()","root.enrichments.foo = if metadata(\"request_failed\") != null {\n throw(metadata(\"request_failed\"))\n} else {\n this\n}","# Retain only the updated metadata fields which were present in the origin message\nmeta = metadata().filter(v -\u003e @.get(v.key) != null)"]}]}},{"name":"cache","type":"processor","status":"stable","plugin":true,"summary":"Performs operations against a xref:components:caches/about.adoc[cache resource] for each message, allowing you to store or retrieve data within message payloads.","description":"\nFor use cases where you wish to cache the result of processors consider using the xref:components:processors/cached.adoc[`cached` processor] instead.\n\nThis processor will interpolate functions within the `key` and `value` fields individually for each message. This allows you to specify dynamic keys and values based on the contents of the message payloads and metadata. You can find a list of functions in xref:configuration:interpolation.adoc#bloblang-queries[Bloblang queries].","categories":["Integration"],"footnotes":"\n== Operators\n\n=== `set`\n\nSet a key in the cache to a value. If the key already exists the contents are\noverridden.\n\n=== `add`\n\nSet a key in the cache to a value. If the key already exists the action fails\nwith a 'key already exists' error, which can be detected with\nxref:configuration:error_handling.adoc[processor error handling].\n\n=== `get`\n\nRetrieve the contents of a cached key and replace the original message payload\nwith the result. If the key does not exist the action fails with an error, which\ncan be detected with xref:configuration:error_handling.adoc[processor error handling].\n\n=== `delete`\n\nDelete a key and its contents from the cache. If the key does not exist the\naction is a no-op and will not fail with an error.\n\n=== `exists`\n\nCheck if a given key exists in the cache and replace the original message payload\nwith `true` or `false`.","examples":[{"title":"Deduplication","summary":"\nDeduplication can be done using the add operator with a key extracted from the message payload, since it fails when a key already exists we can remove the duplicates using a xref:components:processors/mapping.adoc[`mapping` processor]:","config":"\npipeline:\n processors:\n - cache:\n resource: foocache\n operator: add\n key: '${! json(\"message.id\") }'\n value: \"storeme\"\n - mapping: root = if errored() { deleted() }\n\ncache_resources:\n - label: foocache\n redis:\n url: tcp://TODO:6379\n"},{"title":"Deduplication Batch-Wide","summary":"\nSometimes it's necessary to deduplicate a batch of messages (also known as a window) by a single identifying value. This can be done by introducing a xref:components:processors/branch.adoc[`branch` processor], which executes the cache only once on behalf of the batch, in this case with a value make from a field extracted from the first and last messages of the batch:","config":"\npipeline:\n processors:\n # Try and add one message to a cache that identifies the whole batch\n - branch:\n request_map: |\n root = if batch_index() == 0 {\n json(\"id\").from(0) + json(\"meta.tail_id\").from(-1)\n } else { deleted() }\n processors:\n - cache:\n resource: foocache\n operator: add\n key: ${! content() }\n value: t\n # Delete all messages if we failed\n - mapping: |\n root = if errored().from(0) {\n deleted()\n }\n"},{"title":"Hydration","summary":"\nIt's possible to enrich payloads with content previously stored in a cache by using the xref:components:processors/branch.adoc[`branch`] processor:","config":"\npipeline:\n processors:\n - branch:\n processors:\n - cache:\n resource: foocache\n operator: get\n key: '${! json(\"message.document_id\") }'\n result_map: 'root.message.document = this'\n\n # NOTE: If the data stored in the cache is not valid JSON then use\n # something like this instead:\n # result_map: 'root.message.document = content().string()'\n\ncache_resources:\n - label: foocache\n memcached:\n addresses: [ \"TODO:11211\" ]\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"resource","type":"string","kind":"scalar","description":"The xref:components:caches/about.adoc[`cache` resource] to target with this processor."},{"name":"operator","type":"string","kind":"scalar","description":"The \u003c\u003coperators, operation\u003e\u003e to perform with the cache.","options":["set","add","get","delete","exists"],"linter":"\nlet options = {\n \"set\": true,\n \"add\": true,\n \"get\": true,\n \"delete\": true,\n \"exists\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"key","type":"string","kind":"scalar","description":"A key to use with the cache.","interpolated":true},{"name":"value","type":"string","kind":"scalar","description":"A value to use with the cache (when applicable).","is_optional":true,"interpolated":true},{"name":"ttl","type":"string","kind":"scalar","description":"The TTL of each individual item as a duration string. After this period an item will be eligible for removal during the next compaction. Not all caches support per-key TTLs, those that do will have a configuration field `default_ttl`, and those that do not will fall back to their generally configured TTL setting.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["60s","5m","36h"],"version":"3.33.0"}]}},{"name":"cached","type":"processor","status":"experimental","plugin":true,"summary":"Cache the result of applying one or more processors to messages identified by a key. If the key already exists within the cache the contents of the message will be replaced with the cached result instead of applying the processors. This component is therefore useful in situations where an expensive set of processors need only be executed periodically.","description":"The format of the data when stored within the cache is a custom and versioned schema chosen to balance performance and storage space. It is therefore not possible to point this processor to a cache that is pre-populated with data that this processor has not created itself.","categories":["Utility"],"examples":[{"title":"Cached Enrichment","summary":"In the following example we want to we enrich messages consumed from Kafka with data specific to the origin topic partition, we do this by placing an `http` processor within a `branch`, where the HTTP URL contains interpolation functions with the topic and partition in the path.\n\nHowever, it would be inefficient to make this HTTP request for every single message as the result is consistent for all data of a given topic partition. We can solve this by placing our enrichment call within a `cached` processor where the key contains the topic and partition, resulting in messages that originate from the same topic/partition combination using the cached result of the prior.","config":"\npipeline:\n processors:\n - branch:\n processors:\n - cached:\n key: '${! meta(\"kafka_topic\") }-${! meta(\"kafka_partition\") }'\n cache: foo_cache\n processors:\n - mapping: 'root = \"\"'\n - http:\n url: http://example.com/enrichment/${! meta(\"kafka_topic\") }/${! meta(\"kafka_partition\") }\n verb: GET\n result_map: 'root.enrichment = this'\n\ncache_resources:\n - label: foo_cache\n memory:\n # Disable compaction so that cached items never expire\n compaction_interval: \"\"\n"},{"title":"Periodic Global Enrichment","summary":"In the following example we enrich all messages with the same data obtained from a static URL with an `http` processor within a `branch`. However, we expect the data from this URL to change roughly every 10 minutes, so we configure a `cached` processor with a static key (since this request is consistent for all messages) and a TTL of `10m`.","config":"\npipeline:\n processors:\n - branch:\n request_map: 'root = \"\"'\n processors:\n - cached:\n key: static_foo\n cache: foo_cache\n ttl: 10m\n processors:\n - http:\n url: http://example.com/get/foo.json\n verb: GET\n result_map: 'root.foo = this'\n\ncache_resources:\n - label: foo_cache\n memory: {}\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"cache","type":"string","kind":"scalar","description":"The cache resource to read and write processor results from."},{"name":"skip_on","type":"string","kind":"scalar","description":"A condition that can be used to skip caching the results from the processors.","is_optional":true,"bloblang":true,"examples":["errored()"]},{"name":"key","type":"string","kind":"scalar","description":"A key to be resolved for each message, if the key already exists in the cache then the cached result is used, otherwise the processors are applied and the result is cached under this key. The key could be static and therefore apply generally to all messages or it could be an interpolated expression that is potentially unique for each message.","interpolated":true,"examples":["my_foo_result","${! this.document.id }","${! meta(\"kafka_key\") }","${! meta(\"kafka_topic\") }"]},{"name":"ttl","type":"string","kind":"scalar","description":"An optional expiry period to set for each cache entry. Some caches only have a general TTL and will therefore ignore this setting.","is_optional":true,"interpolated":true},{"name":"processors","type":"processor","kind":"array","description":"The list of processors whose result will be cached."}]},"version":"4.3.0"},{"name":"catch","type":"processor","status":"stable","plugin":true,"summary":"Applies a list of child processors _only_ when a previous processing step has failed.","description":"\nBehaves similarly to the xref:components:processors/for_each.adoc[`for_each`] processor, where a list of child processors are applied to individual messages of a batch. However, processors are only applied to messages that failed a processing step prior to the catch.\n\nFor example, with the following config:\n\n```yaml\npipeline:\n processors:\n - resource: foo\n - catch:\n - resource: bar\n - resource: baz\n```\n\nIf the processor `foo` fails for a particular message, that message will be fed into the processors `bar` and `baz`. Messages that do not fail for the processor `foo` will skip these processors.\n\nWhen messages leave the catch block their fail flags are cleared. This processor is useful for when it's possible to recover failed messages, or when special actions (such as logging/metrics) are required before dropping them.\n\nMore information about error handling can be found in xref:configuration:error_handling.adoc[].","categories":["Composition"],"config":{"name":"","type":"processor","kind":"array","default":[]}},{"name":"cohere_chat","type":"processor","status":"experimental","plugin":true,"summary":"Generates responses to messages in a chat conversation, using the Cohere API.","description":"\nThis processor sends the contents of user prompts to the Cohere API, which generates responses. By default, the processor submits the entire payload of each message as a string, unless you use the `prompt` configuration field to customize it.\n\nTo learn more about chat completion, see the https://docs.cohere.com/docs/chat-api[Cohere API documentation^].","categories":["AI"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"base_url","type":"string","kind":"scalar","description":"The base URL to use for API requests.","default":"https://api.cohere.com"},{"name":"api_key","type":"string","kind":"scalar","description":"The API key for the Cohere API.","is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"model","type":"string","kind":"scalar","description":"The name of the Cohere model to use.","examples":["command-r-plus","command-r","command","command-light"]},{"name":"prompt","type":"string","kind":"scalar","description":"The user prompt you want to generate a response for. By default, the processor submits the entire payload as a string.","is_optional":true,"interpolated":true},{"name":"system_prompt","type":"string","kind":"scalar","description":"The system prompt to submit along with the user prompt.","is_optional":true,"interpolated":true},{"name":"max_tokens","type":"int","kind":"scalar","description":"The maximum number of tokens that can be generated in the chat completion.","is_optional":true},{"name":"temperature","type":"float","kind":"scalar","description":"What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic.\n\nWe generally recommend altering this or top_p but not both.","is_optional":true,"linter":"root = if this \u003e 2 || this \u003c 0 { [ \"field must be between 0 and 2\" ] }"},{"name":"response_format","type":"string","kind":"scalar","description":"Specify the model's output format. If `json_schema` is specified, then additionally a `json_schema` or `schema_registry` must be configured.","default":"text","options":["text","json","json_schema"],"linter":"\nlet options = {\n \"text\": true,\n \"json\": true,\n \"json_schema\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"json_schema","type":"string","kind":"scalar","description":"The JSON schema to use when responding in `json_schema` format. To learn more about what JSON schema is supported see the https://docs.cohere.com/docs/structured-outputs-json[Cohere documentation^].","is_optional":true},{"name":"schema_registry","type":"object","kind":"scalar","description":"The schema registry to dynamically load schemas from when responding in `json_schema` format. Schemas themselves must be in JSON format. To learn more about what JSON schema is supported see the https://docs.cohere.com/docs/structured-outputs-json[Cohere documentation^].","is_advanced":true,"is_optional":true,"children":[{"name":"url","type":"string","kind":"scalar","description":"The base URL of the schema registry service.","is_advanced":true,"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"subject","type":"string","kind":"scalar","description":"The subject name to fetch the schema for.","is_advanced":true},{"name":"refresh_interval","type":"string","kind":"scalar","description":"The refresh rate for getting the latest schema. If not specified the schema does not refresh.","is_advanced":true,"is_optional":true},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"oauth","type":"object","kind":"scalar","description":"Allows you to specify open authentication via OAuth version 1.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use OAuth version 1 in requests.","is_advanced":true,"default":false},{"name":"consumer_key","type":"string","kind":"scalar","description":"A value used to identify the client to the service provider.","is_advanced":true,"default":""},{"name":"consumer_secret","type":"string","kind":"scalar","description":"A secret used to establish ownership of the consumer key.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"access_token","type":"string","kind":"scalar","description":"A value used to gain access to the protected resources on behalf of the user.","is_advanced":true,"default":""},{"name":"access_token_secret","type":"string","kind":"scalar","description":"A secret provided in order to establish ownership of a given access token.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"jwt","type":"object","kind":"scalar","description":"BETA: Allows you to specify JWT authentication.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use JWT authentication in requests.","is_advanced":true,"default":false},{"name":"private_key_file","type":"string","kind":"scalar","description":"A file with the PEM encoded via PKCS1 or PKCS8 as private key.","is_advanced":true,"default":""},{"name":"signing_method","type":"string","kind":"scalar","description":"A method used to sign the token such as RS256, RS384, RS512 or EdDSA.","is_advanced":true,"default":""},{"name":"claims","type":"unknown","kind":"map","description":"A value used to identify the claims that issued the JWT.","is_advanced":true,"default":{}},{"name":"headers","type":"unknown","kind":"map","description":"Add optional key/value headers to the JWT.","is_advanced":true,"default":{}}]}]},{"name":"top_p","type":"float","kind":"scalar","description":"An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.\n\nWe generally recommend altering this or temperature but not both.","is_advanced":true,"is_optional":true,"linter":"root = if this \u003e 1 || this \u003c 0 { [ \"field must be between 0 and 1\" ] }"},{"name":"frequency_penalty","type":"float","kind":"scalar","description":"Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim.","is_advanced":true,"is_optional":true,"linter":"root = if this \u003e 2 || this \u003c -2 { [ \"field must be less than 2 and greater than -2\" ] }"},{"name":"presence_penalty","type":"float","kind":"scalar","description":"Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics.","is_advanced":true,"is_optional":true,"linter":"root = if this \u003e 2 || this \u003c -2 { [ \"field must be less than 2 and greater than -2\" ] }"},{"name":"seed","type":"int","kind":"scalar","description":"If specified, our system will make a best effort to sample deterministically, such that repeated requests with the same seed and parameters should return the same result. Determinism is not guaranteed.","is_advanced":true,"is_optional":true},{"name":"stop","type":"string","kind":"array","description":"Up to 4 sequences where the API will stop generating further tokens.","is_advanced":true,"is_optional":true},{"name":"max_tool_calls","type":"int","kind":"scalar","description":"Maximum number of tool calls the model can do.","default":10},{"name":"tools","type":"object","kind":"array","description":"The tools to allow the LLM to invoke. This allows building subpipelines that the LLM can choose to invoke to execute agentic-like actions.","default":[],"children":[{"name":"name","type":"string","kind":"scalar","description":"The name of this tool."},{"name":"description","type":"string","kind":"scalar","description":"A description of this tool, the LLM uses this to decide if the tool should be used."},{"name":"parameters","type":"object","kind":"scalar","description":"The parameters the LLM needs to provide to invoke this tool.","children":[{"name":"required","type":"string","kind":"array","description":"The required parameters for this pipeline.","default":[]},{"name":"properties","type":"object","kind":"map","description":"The properties for the processor's input data","children":[{"name":"type","type":"string","kind":"scalar","description":"The type of this parameter."},{"name":"description","type":"string","kind":"scalar","description":"A description of this parameter."},{"name":"enum","type":"string","kind":"array","description":"Specifies that this parameter is an enum and only these specific values should be used.","default":[]}]}]},{"name":"processors","type":"processor","kind":"array","description":"The pipeline to execute when the LLM uses this tool.","is_optional":true}]}],"linter":"\n root = match {\n this.exists(\"json_schema\") \u0026\u0026 this.exists(\"schema_registry\") =\u003e [\"cannot set both `json_schema` and `schema_registry`\"]\n this.response_format == \"json_schema\" \u0026\u0026 !this.exists(\"json_schema\") \u0026\u0026 !this.exists(\"schema_registry\") =\u003e [\"schema must be specified using either `json_schema` or `schema_registry`\"]\n }\n "},"version":"4.37.0"},{"name":"cohere_embeddings","type":"processor","status":"experimental","plugin":true,"summary":"Generates vector embeddings to represent input text, using the Cohere API.","description":"\nThis processor sends text strings to the Cohere API, which generates vector embeddings. By default, the processor submits the entire payload of each message as a string, unless you use the `text_mapping` configuration field to customize it.\n\nTo learn more about vector embeddings, see the https://docs.cohere.com/docs/embeddings[Cohere API documentation^].","categories":["AI"],"examples":[{"title":"Store embedding vectors in Qdrant","summary":"Compute embeddings for some generated data and store it within xrefs:component:outputs/qdrant.adoc[Qdrant]","config":"input:\n generate:\n interval: 1s\n mapping: |\n root = {\"text\": fake(\"paragraph\")}\npipeline:\n processors:\n - cohere_embeddings:\n model: embed-english-v3\n api_key: \"${COHERE_API_KEY}\"\n text_mapping: \"root = this.text\"\noutput:\n qdrant:\n grpc_host: localhost:6334\n collection_name: \"example_collection\"\n id: \"root = uuid_v4()\"\n vector_mapping: \"root = this\""}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"base_url","type":"string","kind":"scalar","description":"The base URL to use for API requests.","default":"https://api.cohere.com"},{"name":"api_key","type":"string","kind":"scalar","description":"The API key for the Cohere API.","is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"model","type":"string","kind":"scalar","description":"The name of the Cohere model to use.","examples":["embed-english-v3.0","embed-english-light-v3.0","embed-multilingual-v3.0","embed-multilingual-light-v3.0"]},{"name":"text_mapping","type":"string","kind":"scalar","description":"The text you want to generate a vector embedding for. By default, the processor submits the entire payload as a string.","is_optional":true,"bloblang":true},{"name":"input_type","type":"string","kind":"scalar","description":"Specifies the type of input passed to the model.","default":"search_document","annotated_options":[["classification","Used for embeddings passed through a text classifier."],["clustering","Used for the embeddings run through a clustering algorithm."],["search_document","Used for embeddings stored in a vector database for search use-cases."],["search_query","Used for embeddings of search queries run against a vector DB to find relevant documents."]],"linter":"\nlet options = {\n \"classification\": true,\n \"clustering\": true,\n \"search_document\": true,\n \"search_query\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"dimensions","type":"int","kind":"scalar","description":"The number of dimensions of the output embedding. This is only available for embed-v4 and newer models. Possible values are 256, 512, 1024, and 1536.","is_optional":true}]},"version":"4.37.0"},{"name":"cohere_rerank","type":"processor","status":"experimental","plugin":true,"summary":"Generates vector embeddings to represent input text, using the Cohere API.","description":"\nThis processor sends document strings to the Cohere API, which reranks them based on the relevance to the query.\n\nTo learn more about reranking, see the https://docs.cohere.com/docs/rerank-2[Cohere API documentation^].\n\nThe output of this processor is an array of strings that are ordered by their relevance.\n\n== Metadata\n\nrelevance_scores: an array of scores for each document, indicating how relevant it is to the query. The scores are in the same order as the documents in the input. The higher the score, the more relevant the document is to the query.\n\n\t\t","categories":["AI"],"examples":[{"title":"Rerank some documents based on a query","summary":"Rerank some documents based on a query","config":"input:\n generate:\n interval: 1s\n mapping: |\n root = {\n \"query\": fake(\"sentence\"),\n \"docs\": [fake(\"paragraph\"), fake(\"paragraph\"), fake(\"paragraph\")],\n }\npipeline:\n processors:\n - cohere_rerank:\n model: rerank-v3.5\n api_key: \"${COHERE_API_KEY}\"\n query: \"${!this.query}\"\n documents: \"root = this.docs\"\noutput:\n stdout: {}"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"base_url","type":"string","kind":"scalar","description":"The base URL to use for API requests.","default":"https://api.cohere.com"},{"name":"api_key","type":"string","kind":"scalar","description":"The API key for the Cohere API.","is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"model","type":"string","kind":"scalar","description":"The name of the Cohere model to use.","examples":["rerank-v3.5"]},{"name":"query","type":"string","kind":"scalar","description":"The search query","interpolated":true},{"name":"documents","type":"string","kind":"scalar","description":"A list of texts that will be compared to the query. For optimal performance Cohere recommends against sending more than 1000 documents in a single request. NOTE: structured data should be formatted as YAML for best performance.","bloblang":true},{"name":"top_n","type":"int","kind":"scalar","description":"The number of documents to return, if 0 all documents are returned.","default":0},{"name":"max_tokens_per_doc","type":"int","kind":"scalar","description":"Long documents will be automatically truncated to the specified number of tokens.","default":4096}]},"version":"4.37.0"},{"name":"command","type":"processor","status":"experimental","plugin":true,"summary":"Executes a command for each message.","description":"\nThe specified command is executed for each message processed, with the raw bytes of the message being fed into the stdin of the command process, and the resulting message having its contents replaced with the stdout of it.\n\n== Performance\n\nSince this processor executes a new process for each message performance will likely be an issue for high throughput streams. If this is the case then consider using the xref:components:processors/subprocess.adoc[`subprocess` processor] instead as it keeps the underlying process alive long term and uses codecs to insert and extract inputs and outputs to it via stdin/stdout.\n\n== Error handling\n\nIf a non-zero error code is returned by the command then an error containing the entirety of stderr (or a generic message if nothing is written) is set on the message. These failed messages will continue through the pipeline unchanged, but can be dropped or placed in a dead letter queue according to your config, you can read about xref:configuration:error_handling.adoc[these patterns].\n\nIf the command is successful but stderr is written to then a metadata field `command_stderr` is populated with its contents.\n","categories":["Integration"],"examples":[{"title":"Cron Scheduled Command","summary":"This example uses a xref:components:inputs/generate.adoc[`generate` input] to trigger a command on a cron schedule:","config":"\ninput:\n generate:\n interval: '0,30 */2 * * * *'\n mapping: 'root = \"\"' # Empty string as we do not need to pipe anything to stdin\n processors:\n - command:\n name: df\n args_mapping: '[ \"-h\" ]'\n"},{"title":"Dynamic Command Execution","summary":"This example config takes structured messages of the form `{\"command\":\"echo\",\"args\":[\"foo\"]}` and uses their contents to execute the contained command and arguments dynamically, replacing its contents with the command result printed to stdout:","config":"\npipeline:\n processors:\n - command:\n name: ${! this.command }\n args_mapping: 'this.args'\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"name","type":"string","kind":"scalar","description":"The name of the command to execute.","interpolated":true,"examples":["bash","go","${! @command }"]},{"name":"args_mapping","type":"string","kind":"scalar","description":"An optional xref:guides:bloblang/about.adoc[Bloblang mapping] that, when specified, should resolve into an array of arguments to pass to the command. Command arguments are expressed this way in order to support dynamic behavior.","is_optional":true,"bloblang":true,"examples":["[ \"-c\", this.script_path ]"]}]},"version":"4.21.0"},{"name":"compress","type":"processor","status":"stable","plugin":true,"summary":"Compresses messages according to the selected algorithm. Supported compression algorithms are: [flate gzip lz4 pgzip snappy zlib]","description":"The 'level' field might not apply to all algorithms.","categories":["Parsing"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"algorithm","type":"string","kind":"scalar","description":"The compression algorithm to use.","options":["flate","gzip","lz4","pgzip","snappy","zlib"]},{"name":"level","type":"int","kind":"scalar","description":"The level of compression to use. May not be applicable to all algorithms.","default":-1}]}},{"name":"couchbase","type":"processor","status":"experimental","plugin":true,"summary":"Performs operations against Couchbase for each message, allowing you to store or retrieve data within message payloads.","description":"When inserting, replacing or upserting documents, each must have the `content` property set.","categories":["Integration"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"Couchbase connection string.","examples":["couchbase://localhost:11210"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"username","type":"string","kind":"scalar","description":"Username to connect to the cluster.","is_optional":true},{"name":"password","type":"string","kind":"scalar","description":"Password to connect to the cluster.","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"bucket","type":"string","kind":"scalar","description":"Couchbase bucket."},{"name":"collection","type":"string","kind":"scalar","description":"Bucket collection.","is_advanced":true,"is_optional":true,"default":"_default"},{"name":"transcoder","type":"string","kind":"scalar","description":"Couchbase transcoder to use.","is_advanced":true,"default":"legacy","annotated_options":[["json","JSONTranscoder implements the default transcoding behavior and applies JSON transcoding to all values. This will apply the following behavior to the value: binary ([]byte) -\u003e error. default -\u003e JSON value, JSON Flags."],["legacy","LegacyTranscoder implements the behavior for a backward-compatible transcoder. This transcoder implements behavior matching that of gocb v1.This will apply the following behavior to the value: binary ([]byte) -\u003e binary bytes, Binary expectedFlags. string -\u003e string bytes, String expectedFlags. default -\u003e JSON value, JSON expectedFlags."],["raw","RawBinaryTranscoder implements passthrough behavior of raw binary data. This transcoder does not apply any serialization. This will apply the following behavior to the value: binary ([]byte) -\u003e binary bytes, binary expectedFlags. default -\u003e error."],["rawjson","RawJSONTranscoder implements passthrough behavior of JSON data. This transcoder does not apply any serialization. It will forward data across the network without incurring unnecessary parsing costs. This will apply the following behavior to the value: binary ([]byte) -\u003e JSON bytes, JSON expectedFlags. string -\u003e JSON bytes, JSON expectedFlags. default -\u003e error."],["rawstring","RawStringTranscoder implements passthrough behavior of raw string data. This transcoder does not apply any serialization. This will apply the following behavior to the value: string -\u003e string bytes, string expectedFlags. default -\u003e error."]],"linter":"\nlet options = {\n \"json\": true,\n \"legacy\": true,\n \"raw\": true,\n \"rawjson\": true,\n \"rawstring\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"timeout","type":"string","kind":"scalar","description":"Operation timeout.","is_advanced":true,"default":"15s"},{"name":"id","type":"string","kind":"scalar","description":"Document id.","interpolated":true,"examples":["${! json(\"id\") }"]},{"name":"content","type":"string","kind":"scalar","description":"Document content.","is_optional":true,"bloblang":true},{"name":"operation","type":"string","kind":"scalar","description":"Couchbase operation to perform.","default":"get","annotated_options":[["get","fetch a document."],["insert","insert a new document."],["remove","delete a document."],["replace","replace the contents of a document."],["upsert","creates a new document if it does not exist, if it does exist then it updates it."]],"linter":"\nlet options = {\n \"get\": true,\n \"insert\": true,\n \"remove\": true,\n \"replace\": true,\n \"upsert\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"}],"linter":"root = if ((this.operation == \"insert\" || this.operation == \"replace\" || this.operation == \"upsert\") \u0026\u0026 !this.exists(\"content\")) { [ \"content must be set for insert, replace and upsert operations.\" ] }"},"version":"4.11.0"},{"name":"crash","type":"processor","status":"beta","plugin":true,"summary":"Crashes the process using a fatal log message. The log message can be set using function interpolations described in xref:configuration:interpolation.adoc#bloblang-queries[Bloblang queries] which allows you to log the contents and metadata of messages.","categories":["Utility"],"config":{"name":"","type":"string","kind":"scalar","interpolated":true}},{"name":"decompress","type":"processor","status":"stable","plugin":true,"summary":"Decompresses messages according to the selected algorithm. Supported decompression algorithms are: [bzip2 flate gzip lz4 pgzip snappy zlib]","categories":["Parsing"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"algorithm","type":"string","kind":"scalar","description":"The decompression algorithm to use.","options":["bzip2","flate","gzip","lz4","pgzip","snappy","zlib"]}]}},{"name":"dedupe","type":"processor","status":"stable","plugin":true,"summary":"Deduplicates messages by storing a key value in a cache using the `add` operator. If the key already exists within the cache it is dropped.","description":"\nCaches must be configured as resources, for more information check out the xref:components:caches/about.adoc[cache documentation].\n\nWhen using this processor with an output target that might fail you should always wrap the output within an indefinite xref:components:outputs/retry.adoc[`retry`] block. This ensures that during outages your messages aren't reprocessed after failures, which would result in messages being dropped.\n\n== Batch deduplication\n\nThis processor enacts on individual messages only, in order to perform a deduplication on behalf of a batch (or window) of messages instead use the xref:components:processors/cache.adoc#examples[`cache` processor].\n\n== Delivery guarantees\n\nPerforming deduplication on a stream using a distributed cache voids any at-least-once guarantees that it previously had. This is because the cache will preserve message signatures even if the message fails to leave the Redpanda Connect pipeline, which would cause message loss in the event of an outage at the output sink followed by a restart of the Redpanda Connect instance (or a server crash, etc).\n\nThis problem can be mitigated by using an in-memory cache and distributing messages to horizontally scaled Redpanda Connect pipelines partitioned by the deduplication key. However, in situations where at-least-once delivery guarantees are important it is worth avoiding deduplication in favour of implement idempotent behavior at the edge of your stream pipelines.","categories":["Utility"],"examples":[{"title":"Deduplicate based on Kafka key","summary":"The following configuration demonstrates a pipeline that deduplicates messages based on the Kafka key.","config":"\npipeline:\n processors:\n - dedupe:\n cache: keycache\n key: ${! meta(\"kafka_key\") }\n\ncache_resources:\n - label: keycache\n memory:\n default_ttl: 60s\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"cache","type":"string","kind":"scalar","description":"The xref:components:caches/about.adoc[`cache` resource] to target with this processor."},{"name":"key","type":"string","kind":"scalar","description":"An interpolated string yielding the key to deduplicate by for each message.","interpolated":true,"examples":["${! meta(\"kafka_key\") }","${! content().hash(\"xxhash64\") }"]},{"name":"drop_on_err","type":"bool","kind":"scalar","description":"Whether messages should be dropped when the cache returns a general error such as a network issue.","default":true}]}},{"name":"for_each","type":"processor","status":"stable","plugin":true,"summary":"A processor that applies a list of child processors to messages of a batch as though they were each a batch of one message.","description":"\nThis is useful for forcing batch wide processors such as xref:components:processors/dedupe.adoc[`dedupe`] or interpolations such as the `value` field of the `metadata` processor to execute on individual message parts of a batch instead.\n\nPlease note that most processors already process per message of a batch, and this processor is not needed in those cases.","categories":["Composition"],"config":{"name":"","type":"processor","kind":"array","default":[]}},{"name":"gcp_bigquery_select","type":"processor","status":"experimental","plugin":true,"summary":"Executes a `SELECT` query against BigQuery and replaces messages with the rows returned.","categories":["Integration"],"examples":[{"title":"Word count","summary":"\nGiven a stream of English terms, enrich the messages with the word count from Shakespeare's public works:","config":"\npipeline:\n processors:\n - branch:\n processors:\n - gcp_bigquery_select:\n project: test-project\n table: bigquery-public-data.samples.shakespeare\n columns:\n - word\n - sum(word_count) as total_count\n where: word = ?\n suffix: |\n GROUP BY word\n ORDER BY total_count DESC\n LIMIT 10\n args_mapping: root = [ this.term ]\n result_map: |\n root.count = this.get(\"0.total_count\")\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"project","type":"string","kind":"scalar","description":"GCP project where the query job will execute."},{"name":"credentials_json","type":"string","kind":"scalar","description":"An optional field to set Google Service Account Credentials json.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"table","type":"string","kind":"scalar","description":"Fully-qualified BigQuery table name to query.","examples":["bigquery-public-data.samples.shakespeare"]},{"name":"columns","type":"string","kind":"array","description":"A list of columns to query."},{"name":"where","type":"string","kind":"scalar","description":"An optional where clause to add. Placeholder arguments are populated with the `args_mapping` field. Placeholders should always be question marks (`?`).","is_optional":true,"examples":["type = ? and created_at \u003e ?","user_id = ?"]},{"name":"job_labels","type":"string","kind":"map","description":"A list of labels to add to the query job.","default":{}},{"name":"args_mapping","type":"string","kind":"scalar","description":"An optional xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `where`.","is_optional":true,"bloblang":true,"examples":["root = [ \"article\", now().ts_format(\"2006-01-02\") ]"]},{"name":"prefix","type":"string","kind":"scalar","description":"An optional prefix to prepend to the select query (before SELECT).","is_optional":true},{"name":"suffix","type":"string","kind":"scalar","description":"An optional suffix to append to the select query.","is_optional":true}]},"version":"3.64.0"},{"name":"gcp_vertex_ai_chat","type":"processor","status":"experimental","plugin":true,"summary":"Generates responses to messages in a chat conversation, using the Vertex AI API.","description":"This processor sends prompts to your chosen large language model (LLM) and generates text from the responses, using the Vertex AI API.\n\nFor more information, see the https://cloud.google.com/vertex-ai/docs[Vertex AI documentation^].","categories":["AI"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"project","type":"string","kind":"scalar","description":"GCP project ID to use"},{"name":"credentials_json","type":"string","kind":"scalar","description":"An optional field to set google Service Account Credentials json.","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"location","type":"string","kind":"scalar","description":"The location of the model if using a fined tune model. For base models this can be omitted","is_optional":true,"examples":["us-central1"]},{"name":"model","type":"string","kind":"scalar","description":"The name of the LLM to use. For a full list of models, see the https://console.cloud.google.com/vertex-ai/model-garden[Vertex AI Model Garden].","examples":["gemini-1.5-pro-001","gemini-1.5-flash-001"]},{"name":"prompt","type":"string","kind":"scalar","description":"The prompt you want to generate a response for. By default, the processor submits the entire payload as a string.","is_optional":true,"interpolated":true},{"name":"system_prompt","type":"string","kind":"scalar","description":"The system prompt to submit to the Vertex AI LLM.","is_advanced":true,"is_optional":true,"interpolated":true},{"name":"attachment","type":"string","kind":"scalar","description":"Additional data like an image to send with the prompt to the model. The result of the mapping must be a byte array, and the content type is automatically detected.","is_optional":true,"bloblang":true,"examples":["root = this.image.decode(\"base64\") # decode base64 encoded image"],"version":"4.38.0"},{"name":"temperature","type":"float","kind":"scalar","description":"Controls the randomness of predications.","is_optional":true,"linter":"root = if this \u003c 0 || this \u003e 2 { [\"field must be between 0.0-2.0\"] }"},{"name":"max_tokens","type":"int","kind":"scalar","description":"The maximum number of output tokens to generate per message.","is_optional":true},{"name":"response_format","type":"string","kind":"scalar","description":"The response format of generated type, the model must also be prompted to output the appropriate response type.","default":"text","options":["text","json"],"linter":"\nlet options = {\n \"text\": true,\n \"json\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"top_p","type":"float","kind":"scalar","description":"If specified, nucleus sampling will be used.","is_advanced":true,"is_optional":true,"linter":"root = if this \u003c 0 || this \u003e 1 { [\"field must be between 0.0-1.0\"] }"},{"name":"top_k","type":"int","kind":"scalar","description":"If specified top-k sampling will be used.","is_advanced":true,"is_optional":true,"linter":"root = if this \u003c 1 || this \u003e 40 { [\"field must be between 1-40\"] }"},{"name":"stop","type":"string","kind":"array","description":"Stop sequences to when the model will stop generating further tokens.","is_advanced":true,"is_optional":true},{"name":"presence_penalty","type":"float","kind":"scalar","description":"Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics.","is_advanced":true,"is_optional":true,"linter":"root = if this \u003c -2 || this \u003e 2 { [\"field must be greater than -2.0 and less than 2.0\"] }"},{"name":"frequency_penalty","type":"float","kind":"scalar","description":"Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim.","is_advanced":true,"is_optional":true,"linter":"root = if this \u003c -2 || this \u003e 2 { [\"field must be greater than -2.0 and less than 2.0\"] }"}]},"version":"4.34.0"},{"name":"gcp_vertex_ai_embeddings","type":"processor","status":"experimental","plugin":true,"summary":"Generates vector embeddings to represent input text, using the Vertex AI API.","description":"This processor sends text strings to the Vertex AI API, which generates vector embeddings. By default, the processor submits the entire payload of each message as a string, unless you use the `text` configuration field to customize it.\n\nFor more information, see the https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings[Vertex AI documentation^].","categories":["AI"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"project","type":"string","kind":"scalar","description":"GCP project ID to use"},{"name":"credentials_json","type":"string","kind":"scalar","description":"An optional field to set google Service Account Credentials json.","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"location","type":"string","kind":"scalar","description":"The location of the model.","default":"us-central1"},{"name":"model","type":"string","kind":"scalar","description":"The name of the LLM to use. For a full list of models, see the https://console.cloud.google.com/vertex-ai/model-garden[Vertex AI Model Garden].","examples":["text-embedding-004","text-multilingual-embedding-002"]},{"name":"task_type","type":"string","kind":"scalar","description":"The way to optimize embeddings that the model generates for specific use cases.","default":"RETRIEVAL_DOCUMENT","annotated_options":[["CLASSIFICATION","optimize for being able classify texts according to preset labels"],["CLUSTERING","optimize for clustering texts based on their similarities"],["FACT_VERIFICATION","optimize for queries that are proving or disproving a fact such as \"apples grow underground\""],["QUESTION_ANSWERING","optimize for search proper questions such as \"Why is the sky blue?\""],["RETRIEVAL_DOCUMENT","optimize for documents that will be searched (also known as a corpus)"],["RETRIEVAL_QUERY","optimize for queries such as \"What is the best fish recipe?\" or \"best restaurant in Chicago\""],["SEMANTIC_SIMILARITY","optimize for text similarity"]],"linter":"\nlet options = {\n \"classification\": true,\n \"clustering\": true,\n \"fact_verification\": true,\n \"question_answering\": true,\n \"retrieval_document\": true,\n \"retrieval_query\": true,\n \"semantic_similarity\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"text","type":"string","kind":"scalar","description":"The text you want to compute vector embeddings for. By default, the processor submits the entire payload as a string.","is_optional":true,"interpolated":true},{"name":"output_dimensions","type":"int","kind":"scalar","description":"The maximum length for the output embedding size. If set, the output embeddings will be truncated to this size.","is_optional":true}]},"version":"4.37.0"},{"name":"google_drive_download","type":"processor","status":"experimental","plugin":true,"summary":"Downloads files from Google Drive","description":"\nCan download a file from Google Drive based on a file ID.\n== Authentication\nBy default, this connector will use Google Application Default Credentials (ADC) to authenticate with Google APIs.\n\nTo use this mechanism locally, the following gcloud commands can be used:\n\n\t# Login for the application default credentials and add scopes for readonly drive access\n\tgcloud auth application-default login --scopes='openid,https://www.googleapis.com/auth/userinfo.email,https://www.googleapis.com/auth/cloud-platform,https://www.googleapis.com/auth/drive.readonly'\n\t# When logging in with a user account, you may need to set the quota project for the application default credentials\n\tgcloud auth application-default set-quota-project \u003cproject-id\u003e\n\nOtherwise if using a service account, you can create a JSON key for the service account and set it in the `credentials_json` field.\nIn order for a service account to access files in Google Drive either files need to be explicitly shared with the service account email, otherwise https://support.google.com/a/answer/162106[^domain wide delegation] can be used to share all files within a Google Workspace.\n","categories":["Unstructured"],"examples":[{"title":"Download files from Google Drive","summary":"This examples downloads all the files from Google Drive","config":"\npipeline:\n processors:\n - google_drive_search:\n query: \"name = 'Test Doc'\"\n - google_drive_download:\n file_id: \"${!this.id}\"\n mime_type: \"${!this.mimeType}\"\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"credentials_json","type":"string","kind":"scalar","description":"A service account credentials JSON file. If left unset then the application default credentials are used.","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"file_id","type":"string","kind":"scalar","description":"The file ID of the file to download.","interpolated":true},{"name":"mime_type","type":"string","kind":"scalar","description":"The mime type of the file in drive.","interpolated":true},{"name":"export_mime_types","type":"string","kind":"map","description":"A map of Google Drive MIME types to their export formats. The key is the MIME type, and the value is the export format. See https://developers.google.com/workspace/drive/api/guides/ref-export-formats[^Google Drive API Documentation] for a list of supported export types","is_advanced":true,"default":{"application/vnd.google-apps.document":"text/markdown","application/vnd.google-apps.drawing":"image/png","application/vnd.google-apps.presentation":"application/pdf","application/vnd.google-apps.script":"application/vnd.google-apps.script+json","application/vnd.google-apps.spreadsheet":"text/csv"},"examples":[{"application/vnd.google-apps.document":"application/pdf","application/vnd.google-apps.drawing":"application/pdf","application/vnd.google-apps.presentation":"application/pdf","application/vnd.google-apps.spreadsheet":"application/pdf"},{"application/vnd.google-apps.document":"application/vnd.openxmlformats-officedocument.wordprocessingml.document","application/vnd.google-apps.drawing":"image/svg+xml","application/vnd.google-apps.presentation":"application/vnd.openxmlformats-officedocument.presentationml.presentation","application/vnd.google-apps.spreadsheet":"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}]}]}},{"name":"google_drive_list_labels","type":"processor","status":"experimental","plugin":true,"summary":"Lists labels for a file in Google Drive","description":"\nCan list all labels from Google Drive.\n\t\t== Authentication\nBy default, this connector will use Google Application Default Credentials (ADC) to authenticate with Google APIs.\n\nTo use this mechanism locally, the following gcloud commands can be used:\n\n\t# Login for the application default credentials and add scopes for readonly drive access\n\tgcloud auth application-default login --scopes='openid,https://www.googleapis.com/auth/userinfo.email,https://www.googleapis.com/auth/cloud-platform,https://www.googleapis.com/auth/drive.labels.readonly'\n\t# When logging in with a user account, you may need to set the quota project for the application default credentials\n\tgcloud auth application-default set-quota-project \u003cproject-id\u003e\n\nOtherwise if using a service account, you can create a JSON key for the service account and set it in the `credentials_json` field.\nIn order for a service account to access files in Google Drive either files need to be explicitly shared with the service account email, otherwise https://support.google.com/a/answer/162106[^domain wide delegation] can be used to share all files within a Google Workspace.\n","categories":["Unstructured"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"credentials_json","type":"string","kind":"scalar","description":"A service account credentials JSON file. If left unset then the application default credentials are used.","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}},{"name":"google_drive_search","type":"processor","status":"experimental","plugin":true,"summary":"Searches Google Drive for files matching the provided query.","description":"\nThis processor searches for files in Google Drive using the provided query.\n\nSearch results are emitted as message batch, where each message is a https://developers.google.com/workspace/drive/api/reference/rest/v3/files#File[^Google Drive File]\n\n== Authentication\nBy default, this connector will use Google Application Default Credentials (ADC) to authenticate with Google APIs.\n\nTo use this mechanism locally, the following gcloud commands can be used:\n\n\t# Login for the application default credentials and add scopes for readonly drive access\n\tgcloud auth application-default login --scopes='openid,https://www.googleapis.com/auth/userinfo.email,https://www.googleapis.com/auth/cloud-platform,https://www.googleapis.com/auth/drive.readonly'\n\t# When logging in with a user account, you may need to set the quota project for the application default credentials\n\tgcloud auth application-default set-quota-project \u003cproject-id\u003e\n\nOtherwise if using a service account, you can create a JSON key for the service account and set it in the `credentials_json` field.\nIn order for a service account to access files in Google Drive either files need to be explicitly shared with the service account email, otherwise https://support.google.com/a/answer/162106[^domain wide delegation] can be used to share all files within a Google Workspace.\n","categories":["Unstructured"],"examples":[{"title":"Search \u0026 download files from Google Drive","summary":"This examples downloads all the files from Google Drive that are returned in the query","config":"\ninput:\n stdin: {}\npipeline:\n processors:\n - google_drive_search:\n query: \"${!content().string()}\"\n - mutation: 'meta path = this.name'\n - google_drive_download:\n file_id: \"${!this.id}\"\n mime_type: \"${!this.mimeType}\"\noutput:\n file:\n path: \"${!@path}\"\n codec: all-bytes\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"credentials_json","type":"string","kind":"scalar","description":"A service account credentials JSON file. If left unset then the application default credentials are used.","is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"query","type":"string","kind":"scalar","description":"The search query to use for finding files in Google Drive. Supports the same query format as the Google Drive UI.","interpolated":true},{"name":"projection","type":"string","kind":"array","description":"The partial fields to include in the result.","default":["id","name","mimeType","size","labelInfo"]},{"name":"include_label_ids","type":"string","kind":"scalar","description":"A comma delimited list of label IDs to include in the result","default":"","interpolated":true},{"name":"max_results","type":"int","kind":"scalar","description":"The maximum number of results to return.","default":64}]}},{"name":"grok","type":"processor","status":"stable","plugin":true,"summary":"Parses messages into a structured format by attempting to apply a list of Grok expressions, the first expression to result in at least one value replaces the original message with a JSON object containing the values.","description":"\nType hints within patterns are respected, therefore with the pattern `%\\{WORD:first},%{INT:second:int}` and a payload of `foo,1` the resulting payload would be `\\{\"first\":\"foo\",\"second\":1}`.\n\n== Performance\n\nThis processor currently uses the https://golang.org/s/re2syntax[Go RE2^] regular expression engine, which is guaranteed to run in time linear to the size of the input. However, this property often makes it less performant than PCRE based implementations of grok. For more information, see https://swtch.com/~rsc/regexp/regexp1.html.","categories":["Parsing"],"footnotes":"\n== Default patterns\n\nFor summary of the default patterns on offer, see https://github.com/Jeffail/grok/blob/master/patterns.go#L5.","examples":[{"title":"VPC Flow Logs","summary":"\nGrok can be used to parse unstructured logs such as VPC flow logs that look like this:\n\n```text\n2 123456789010 eni-1235b8ca123456789 172.31.16.139 172.31.16.21 20641 22 6 20 4249 1418530010 1418530070 ACCEPT OK\n```\n\nInto structured objects that look like this:\n\n```json\n{\"accountid\":\"123456789010\",\"action\":\"ACCEPT\",\"bytes\":4249,\"dstaddr\":\"172.31.16.21\",\"dstport\":22,\"end\":1418530070,\"interfaceid\":\"eni-1235b8ca123456789\",\"logstatus\":\"OK\",\"packets\":20,\"protocol\":6,\"srcaddr\":\"172.31.16.139\",\"srcport\":20641,\"start\":1418530010,\"version\":2}\n```\n\nWith the following config:","config":"\npipeline:\n processors:\n - grok:\n expressions:\n - '%{VPCFLOWLOG}'\n pattern_definitions:\n VPCFLOWLOG: '%{NUMBER:version:int} %{NUMBER:accountid} %{NOTSPACE:interfaceid} %{NOTSPACE:srcaddr} %{NOTSPACE:dstaddr} %{NOTSPACE:srcport:int} %{NOTSPACE:dstport:int} %{NOTSPACE:protocol:int} %{NOTSPACE:packets:int} %{NOTSPACE:bytes:int} %{NUMBER:start:int} %{NUMBER:end:int} %{NOTSPACE:action} %{NOTSPACE:logstatus}'\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"expressions","type":"string","kind":"array","description":"One or more Grok expressions to attempt against incoming messages. The first expression to match at least one value will be used to form a result."},{"name":"pattern_definitions","type":"string","kind":"map","description":"A map of pattern definitions that can be referenced within `patterns`.","default":{}},{"name":"pattern_paths","type":"string","kind":"array","description":"A list of paths to load Grok patterns from. This field supports wildcards, including super globs (double star).","default":[]},{"name":"named_captures_only","type":"bool","kind":"scalar","description":"Whether to only capture values from named patterns.","is_advanced":true,"default":true},{"name":"use_default_patterns","type":"bool","kind":"scalar","description":"Whether to use a \u003c\u003cdefault-patterns, default set of patterns\u003e\u003e.","is_advanced":true,"default":true},{"name":"remove_empty_values","type":"bool","kind":"scalar","description":"Whether to remove values that are empty from the resulting structure.","is_advanced":true,"default":true}]}},{"name":"group_by","type":"processor","status":"stable","plugin":true,"summary":"Splits a xref:configuration:batching.adoc[batch of messages] into N batches, where each resulting batch contains a group of messages determined by a xref:guides:bloblang/about.adoc[Bloblang query].","description":"\nOnce the groups are established a list of processors are applied to their respective grouped batch, which can be used to label the batch as per their grouping. Messages that do not pass the check of any specified group are placed in their own group.\n\nThe functionality of this processor depends on being applied across messages that are batched. You can find out more about batching xref:configuration:batching.adoc[in this doc].","categories":["Composition"],"examples":[{"title":"Grouped Processing","summary":"Imagine we have a batch of messages that we wish to split into a group of foos and everything else, which should be sent to different output destinations based on those groupings. We also need to send the foos as a tar gzip archive. For this purpose we can use the `group_by` processor with a xref:components:outputs/switch.adoc[`switch`] output:","config":"\npipeline:\n processors:\n - group_by:\n - check: content().contains(\"this is a foo\")\n processors:\n - archive:\n format: tar\n - compress:\n algorithm: gzip\n - mapping: 'meta grouping = \"foo\"'\n\noutput:\n switch:\n cases:\n - check: meta(\"grouping\") == \"foo\"\n output:\n gcp_pubsub:\n project: foo_prod\n topic: only_the_foos\n - output:\n gcp_pubsub:\n project: somewhere_else\n topic: no_foos_here\n"}],"config":{"name":"","type":"object","kind":"array","children":[{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message belongs to a given group.","bloblang":true,"examples":["this.type == \"foo\"","this.contents.urls.contains(\"https://benthos.dev/\")","true"]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to execute on the newly formed group.","default":[]}]}},{"name":"group_by_value","type":"processor","status":"stable","plugin":true,"summary":"Splits a batch of messages into N batches, where each resulting batch contains a group of messages determined by a xref:configuration:interpolation.adoc#bloblang-queries[function interpolated string] evaluated per message.","description":"\nThis allows you to group messages using arbitrary fields within their content or metadata, process them individually, and send them to unique locations as per their group.\n\nThe functionality of this processor depends on being applied across messages that are batched. You can find out more about batching xref:configuration:batching.adoc[in this doc].","categories":["Composition"],"footnotes":"\n== Examples\n\nIf we were consuming Kafka messages and needed to group them by their key, archive the groups, and send them to S3 with the key as part of the path we could achieve that with the following:\n\n```yaml\npipeline:\n processors:\n - group_by_value:\n value: ${! meta(\"kafka_key\") }\n - archive:\n format: tar\n - compress:\n algorithm: gzip\noutput:\n aws_s3:\n bucket: TODO\n path: docs/${! meta(\"kafka_key\") }/${! count(\"files\") }-${! timestamp_unix_nano() }.tar.gz\n```","config":{"name":"","type":"object","kind":"scalar","children":[{"name":"value","type":"string","kind":"scalar","description":"The interpolated string to group based on.","interpolated":true,"examples":["${! meta(\"kafka_key\") }","${! json(\"foo.bar\") }-${! meta(\"baz\") }"]}]}},{"name":"http","type":"processor","status":"stable","plugin":true,"summary":"Performs an HTTP request using a message batch as the request body, and replaces the original message parts with the body of the response.","description":"\nThe `rate_limit` field can be used to specify a rate limit xref:components:rate_limits/about.adoc[resource] to cap the rate of requests across all parallel components service wide.\n\nThe URL and header values of this type can be dynamically set using function interpolations described xref:configuration:interpolation.adoc#bloblang-queries[here].\n\nIn order to map or encode the payload to a specific request body, and map the response back into the original payload instead of replacing it entirely, you can use the xref:components:processors/branch.adoc[`branch` processor].\n\n== Response codes\n\nRedpanda Connect considers any response code between 200 and 299 inclusive to indicate a successful response, you can add more success status codes with the field `successful_on`.\n\nWhen a request returns a response code within the `backoff_on` field it will be retried after increasing intervals.\n\nWhen a request returns a response code within the `drop_on` field it will not be reattempted and is immediately considered a failed request.\n\n== Add metadata\n\nIf the request returns an error response code this processor sets a metadata field `http_status_code` on the resulting message.\n\nUse the field `extract_headers` to specify rules for which other headers should be copied into the resulting message from the response.\n\n== Error handling\n\nWhen all retry attempts for a message are exhausted the processor cancels the attempt. These failed messages will continue through the pipeline unchanged, but can be dropped or placed in a dead letter queue according to your config, you can read about xref:configuration:error_handling.adoc[these patterns].","categories":["Integration"],"examples":[{"title":"Branched Request","summary":"This example uses a xref:components:processors/branch.adoc[`branch` processor] to strip the request message into an empty body, grab an HTTP payload, and place the result back into the original message at the path `repo.status`:","config":"\npipeline:\n processors:\n - branch:\n request_map: 'root = \"\"'\n processors:\n - http:\n url: https://hub.docker.com/v2/repositories/jeffail/benthos\n verb: GET\n headers:\n Content-Type: application/json\n result_map: 'root.repo.status = this'\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL to connect to.","interpolated":true},{"name":"verb","type":"string","kind":"scalar","description":"A verb to connect with","default":"POST","examples":["POST","GET","DELETE"]},{"name":"headers","type":"string","kind":"map","description":"A map of headers to add to the request.","default":{},"interpolated":true,"examples":[{"Content-Type":"application/octet-stream","traceparent":"${! tracing_span().traceparent }"}]},{"name":"metadata","type":"object","kind":"scalar","description":"Specify optional matching rules to determine which metadata keys should be added to the HTTP request as headers.","is_advanced":true,"is_optional":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","is_advanced":true,"default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","is_advanced":true,"default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"dump_request_log_level","type":"string","kind":"scalar","description":"EXPERIMENTAL: Optionally set a level at which the request and response payload of each request made will be logged.","is_advanced":true,"default":"","options":["TRACE","DEBUG","INFO","WARN","ERROR","FATAL",""],"version":"4.12.0","linter":"\nlet options = {\n \"trace\": true,\n \"debug\": true,\n \"info\": true,\n \"warn\": true,\n \"error\": true,\n \"fatal\": true,\n \"\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"oauth","type":"object","kind":"scalar","description":"Allows you to specify open authentication via OAuth version 1.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use OAuth version 1 in requests.","is_advanced":true,"default":false},{"name":"consumer_key","type":"string","kind":"scalar","description":"A value used to identify the client to the service provider.","is_advanced":true,"default":""},{"name":"consumer_secret","type":"string","kind":"scalar","description":"A secret used to establish ownership of the consumer key.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"access_token","type":"string","kind":"scalar","description":"A value used to gain access to the protected resources on behalf of the user.","is_advanced":true,"default":""},{"name":"access_token_secret","type":"string","kind":"scalar","description":"A secret provided in order to establish ownership of a given access token.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"oauth2","type":"object","kind":"scalar","description":"Allows you to specify open authentication via OAuth version 2 using the client credentials token flow.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use OAuth version 2 in requests.","is_advanced":true,"default":false},{"name":"client_key","type":"string","kind":"scalar","description":"A value used to identify the client to the token provider.","is_advanced":true,"default":""},{"name":"client_secret","type":"string","kind":"scalar","description":"A secret used to establish ownership of the client key.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token_url","type":"string","kind":"scalar","description":"The URL of the token provider.","is_advanced":true,"default":"","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"scopes","type":"string","kind":"array","description":"A list of optional requested permissions.","is_advanced":true,"default":[],"version":"3.45.0"},{"name":"endpoint_params","type":"unknown","kind":"map","description":"A list of optional endpoint parameters, values should be arrays of strings.","is_advanced":true,"is_optional":true,"default":{},"examples":[{"bar":["woof"],"foo":["meow","quack"]}],"version":"4.21.0","linter":"\nroot = if this.type() == \"object\" {\n this.values().map_each(ele -\u003e if ele.type() != \"array\" {\n \"field must be an object containing arrays of strings, got %s (%v)\".format(ele.format_json(no_indent: true), ele.type())\n } else {\n ele.map_each(str -\u003e if str.type() != \"string\" {\n \"field values must be strings, got %s (%v)\".format(str.format_json(no_indent: true), str.type())\n } else { deleted() })\n }).\n flatten()\n}\n"}]},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"jwt","type":"object","kind":"scalar","description":"BETA: Allows you to specify JWT authentication.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use JWT authentication in requests.","is_advanced":true,"default":false},{"name":"private_key_file","type":"string","kind":"scalar","description":"A file with the PEM encoded via PKCS1 or PKCS8 as private key.","is_advanced":true,"default":""},{"name":"signing_method","type":"string","kind":"scalar","description":"A method used to sign the token such as RS256, RS384, RS512 or EdDSA.","is_advanced":true,"default":""},{"name":"claims","type":"unknown","kind":"map","description":"A value used to identify the claims that issued the JWT.","is_advanced":true,"default":{}},{"name":"headers","type":"unknown","kind":"map","description":"Add optional key/value headers to the JWT.","is_advanced":true,"default":{}}]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"extract_headers","type":"object","kind":"scalar","description":"Specify which response headers should be added to resulting messages as metadata. Header keys are lowercased before matching, so ensure that your patterns target lowercased versions of the header keys that you expect.","is_advanced":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","is_advanced":true,"default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","is_advanced":true,"default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"rate_limit","type":"string","kind":"scalar","description":"An optional xref:components:rate_limits/about.adoc[rate limit] to throttle requests by.","is_optional":true},{"name":"timeout","type":"string","kind":"scalar","description":"A static timeout to apply to requests.","default":"5s"},{"name":"retry_period","type":"string","kind":"scalar","description":"The base period to wait between failed requests.","is_advanced":true,"default":"1s"},{"name":"max_retry_backoff","type":"string","kind":"scalar","description":"The maximum period to wait between failed requests.","is_advanced":true,"default":"300s"},{"name":"retries","type":"int","kind":"scalar","description":"The maximum number of retry attempts to make.","is_advanced":true,"default":3},{"name":"follow_redirects","type":"bool","kind":"scalar","description":"Whether or not to transparently follow redirects, i.e. responses with 300-399 status codes. If disabled, the response message will contain the body, status, and headers from the redirect response and the processor will not make a request to the URL set in the Location header of the response.","is_advanced":true,"default":true},{"name":"backoff_on","type":"int","kind":"array","description":"A list of status codes whereby the request should be considered to have failed and retries should be attempted, but the period between them should be increased gradually.","is_advanced":true,"default":[429]},{"name":"drop_on","type":"int","kind":"array","description":"A list of status codes whereby the request should be considered to have failed but retries should not be attempted. This is useful for preventing wasted retries for requests that will never succeed. Note that with these status codes the _request_ is dropped, but _message_ that caused the request will not be dropped.","is_advanced":true,"default":[]},{"name":"successful_on","type":"int","kind":"array","description":"A list of status codes whereby the attempt should be considered successful, this is useful for dropping requests that return non-2XX codes indicating that the message has been dealt with, such as a 303 See Other or a 409 Conflict. All 2XX codes are considered successful unless they are present within `backoff_on` or `drop_on`, regardless of this field.","is_advanced":true,"default":[]},{"name":"proxy_url","type":"string","kind":"scalar","description":"An optional HTTP proxy URL.","is_advanced":true,"is_optional":true},{"name":"disable_http2","type":"bool","kind":"scalar","description":"Whether or not to disable disable HTTP/2","is_advanced":true,"default":false,"version":"4.44.0"},{"name":"batch_as_multipart","type":"bool","kind":"scalar","description":"Send message batches as a single request using https://www.w3.org/Protocols/rfc1341/7_2_Multipart.html[RFC1341^].","is_advanced":true,"default":false},{"name":"parallel","type":"bool","kind":"scalar","description":"When processing batched messages, whether to send messages of the batch in parallel, otherwise they are sent serially.","default":false}]}},{"name":"insert_part","type":"processor","status":"stable","plugin":true,"summary":"Insert a new message into a batch at an index. If the specified index is greater than the length of the existing batch it will be appended to the end.","description":"\nThe index can be negative, and if so the message will be inserted from the end counting backwards starting from -1. E.g. if index = -1 then the new message will become the last of the batch, if index = -2 then the new message will be inserted before the last message, and so on. If the negative index is greater than the length of the existing batch it will be inserted at the beginning.\n\nThe new message will have metadata copied from the first pre-existing message of the batch.\n\nThis processor will interpolate functions within the 'content' field, you can find a list of functions xref:configuration:interpolation.adoc#bloblang-queries[here].","categories":["Composition"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"index","type":"int","kind":"scalar","description":"The index within the batch to insert the message at.","default":-1},{"name":"content","type":"string","kind":"scalar","description":"The content of the message being inserted.","default":"","interpolated":true}]}},{"name":"javascript","type":"processor","status":"experimental","plugin":true,"summary":"Executes a provided JavaScript code block or file for each message.","description":"\nThe https://github.com/dop251/goja[execution engine^] behind this processor provides full ECMAScript 5.1 support (including regex and strict mode). Most of the ECMAScript 6 spec is implemented but this is a work in progress.\n\nImports via `require` should work similarly to NodeJS, and access to the console is supported which will print via the Redpanda Connect logger. More caveats can be found on https://github.com/dop251/goja#known-incompatibilities-and-caveats[GitHub^].\n\nThis processor is implemented using the https://github.com/dop251/goja[github.com/dop251/goja^] library.","categories":["Mapping"],"footnotes":"\n== Runtime\n\nIn order to optimize code execution JS runtimes are created on demand (in order to support parallel execution) and are reused across invocations. Therefore, it is important to understand that global state created by your programs will outlive individual invocations. In order for your programs to avoid failing after the first invocation ensure that you do not define variables at the global scope.\n\nAlthough technically possible, it is recommended that you do not rely on the global state for maintaining state across invocations as the pooling nature of the runtimes will prevent deterministic behavior. We aim to support deterministic strategies for mutating global state in the future.\n\n== Functions\n\n### `benthos.v0_fetch`\n\nExecutes an HTTP request synchronously and returns the result as an object of the form `{\"status\":200,\"body\":\"foo\"}`.\n\n#### Parameters\n\n**`url`** \u0026lt;string\u0026gt; The URL to fetch \n**`headers`** \u0026lt;object(string,string)\u0026gt; An object of string/string key/value pairs to add the request as headers. \n**`method`** \u0026lt;string\u0026gt; The method of the request. \n**`body`** \u0026lt;(optional) string\u0026gt; A body to send. \n\n#### Examples\n\n```javascript\nlet result = benthos.v0_fetch(\"http://example.com\", {}, \"GET\", \"\")\nbenthos.v0_msg_set_structured(result);\n```\n\n### `benthos.v0_msg_as_string`\n\nObtain the raw contents of the processed message as a string.\n\n#### Examples\n\n```javascript\nlet contents = benthos.v0_msg_as_string();\n```\n\n### `benthos.v0_msg_as_structured`\n\nObtain the root of the processed message as a structured value. If the message is not valid JSON or has not already been expanded into a structured form this function will throw an error.\n\n#### Examples\n\n```javascript\nlet foo = benthos.v0_msg_as_structured().foo;\n```\n\n### `benthos.v0_msg_exists_meta`\n\nCheck that a metadata key exists.\n\n#### Parameters\n\n**`name`** \u0026lt;string\u0026gt; The metadata key to search for. \n\n#### Examples\n\n```javascript\nif (benthos.v0_msg_exists_meta(\"kafka_key\")) {}\n```\n\n### `benthos.v0_msg_get_meta`\n\nGet the value of a metadata key from the processed message.\n\n#### Parameters\n\n**`name`** \u0026lt;string\u0026gt; The metadata key to search for. \n\n#### Examples\n\n```javascript\nlet key = benthos.v0_msg_get_meta(\"kafka_key\");\n```\n\n### `benthos.v0_msg_set_meta`\n\nSet a metadata key on the processed message to a value.\n\n#### Parameters\n\n**`name`** \u0026lt;string\u0026gt; The metadata key to set. \n**`value`** \u0026lt;anything\u0026gt; The value to set it to. \n\n#### Examples\n\n```javascript\nbenthos.v0_msg_set_meta(\"thing\", \"hello world\");\n```\n\n### `benthos.v0_msg_set_string`\n\nSet the contents of the processed message to a given string.\n\n#### Parameters\n\n**`value`** \u0026lt;string\u0026gt; The value to set it to. \n\n#### Examples\n\n```javascript\nbenthos.v0_msg_set_string(\"hello world\");\n```\n\n### `benthos.v0_msg_set_structured`\n\nSet the root of the processed message to a given value of any type.\n\n#### Parameters\n\n**`value`** \u0026lt;anything\u0026gt; The value to set it to. \n\n#### Examples\n\n```javascript\nbenthos.v0_msg_set_structured({\n \"foo\": \"a thing\",\n \"bar\": \"something else\",\n \"baz\": 1234\n});\n```\n\n","examples":[{"title":"Simple mutation","summary":"In this example we define a simple function that performs a basic mutation against messages, treating their contents as raw strings.","config":"\npipeline:\n processors:\n - javascript:\n code: 'benthos.v0_msg_set_string(benthos.v0_msg_as_string() + \"hello world\");'\n"},{"title":"Structured mutation","summary":"In this example we define a function that performs basic mutations against a structured message. Note that we encapsulate the logic within an anonymous function that is called for each invocation, this is required in order to avoid duplicate variable declarations in the global state.","config":"\npipeline:\n processors:\n - javascript:\n code: |\n (() =\u003e {\n let thing = benthos.v0_msg_as_structured();\n thing.num_keys = Object.keys(thing).length;\n delete thing[\"b\"];\n benthos.v0_msg_set_structured(thing);\n })();\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"code","type":"string","kind":"scalar","description":"An inline JavaScript program to run. One of `code` or `file` must be defined.","is_optional":true},{"name":"file","type":"string","kind":"scalar","description":"A file containing a JavaScript program to run. One of `code` or `file` must be defined.","is_optional":true},{"name":"global_folders","type":"string","kind":"array","description":"List of folders that will be used to load modules from if the requested JS module is not found elsewhere.","default":[]}],"linter":"\nlet codeLen = (this.code | \"\").length()\nlet fileLen = (this.file | \"\").length()\nroot = if $codeLen == 0 \u0026\u0026 $fileLen == 0 {\n \"either the code or file field must be specified\"\n} else if $codeLen \u003e 0 \u0026\u0026 $fileLen \u003e 0 {\n \"cannot specify both the code and file fields\"\n}"},"version":"4.14.0"},{"name":"jmespath","type":"processor","status":"stable","plugin":true,"summary":"Executes a http://jmespath.org/[JMESPath query] on JSON documents and replaces the message with the resulting document.","description":"\n[TIP]\n.Try out Bloblang\n====\nFor better performance and improved capabilities try native Redpanda Connect mapping with the xref:components:processors/mapping.adoc[`mapping` processor].\n====\n","categories":["Mapping"],"examples":[{"title":"Mapping","summary":"\nWhen receiving JSON documents of the form:\n\n```json\n{\n \"locations\": [\n {\"name\": \"Seattle\", \"state\": \"WA\"},\n {\"name\": \"New York\", \"state\": \"NY\"},\n {\"name\": \"Bellevue\", \"state\": \"WA\"},\n {\"name\": \"Olympia\", \"state\": \"WA\"}\n ]\n}\n```\n\nWe could collapse the location names from the state of Washington into a field `Cities`:\n\n```json\n{\"Cities\": \"Bellevue, Olympia, Seattle\"}\n```\n\nWith the following config:","config":"\npipeline:\n processors:\n - jmespath:\n query: \"locations[?state == 'WA'].name | sort(@) | {Cities: join(', ', @)}\"\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"query","type":"string","kind":"scalar","description":"The JMESPath query to apply to messages."}]}},{"name":"jq","type":"processor","status":"stable","plugin":true,"summary":"Transforms and filters messages using jq queries.","description":"\n[TIP]\n.Try out Bloblang\n====\nFor better performance and improved capabilities try out native Redpanda Connect mapping with the xref:components:processors/mapping.adoc[`mapping` processor].\n====\n\nThe provided query is executed on each message, targeting either the contents as a structured JSON value or as a raw string using the field `raw`, and the message is replaced with the query result.\n\nMessage metadata is also accessible within the query from the variable `$metadata`.\n\nThis processor uses the https://github.com/itchyny/gojq[gojq library^], and therefore does not require jq to be installed as a dependency. However, this also means there are some https://github.com/itchyny/gojq#difference-to-jq[differences in how these queries are executed^] versus the jq cli.\n\nIf the query does not emit any value then the message is filtered, if the query returns multiple values then the resulting message will be an array containing all values.\n\nThe full query syntax is described in https://stedolan.github.io/jq/manual/[jq's documentation^].\n\n== Error handling\n\nQueries can fail, in which case the message remains unchanged, errors are logged, and the message is flagged as having failed, allowing you to use xref:configuration:error_handling.adoc[standard processor error handling patterns].","categories":["Mapping"],"examples":[{"title":"Mapping","summary":"\nWhen receiving JSON documents of the form:\n\n```json\n{\n \"locations\": [\n {\"name\": \"Seattle\", \"state\": \"WA\"},\n {\"name\": \"New York\", \"state\": \"NY\"},\n {\"name\": \"Bellevue\", \"state\": \"WA\"},\n {\"name\": \"Olympia\", \"state\": \"WA\"}\n ]\n}\n```\n\nWe could collapse the location names from the state of Washington into a field `Cities`:\n\n```json\n{\"Cities\": \"Bellevue, Olympia, Seattle\"}\n```\n\nWith the following config:","config":"\npipeline:\n processors:\n - jq:\n query: '{Cities: .locations | map(select(.state == \"WA\").name) | sort | join(\", \") }'\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"query","type":"string","kind":"scalar","description":"The jq query to filter and transform messages with."},{"name":"raw","type":"bool","kind":"scalar","description":"Whether to process the input as a raw string instead of as JSON.","is_advanced":true,"default":false},{"name":"output_raw","type":"bool","kind":"scalar","description":"Whether to output raw text (unquoted) instead of JSON strings when the emitted values are string types.","is_advanced":true,"default":false}]}},{"name":"json_schema","type":"processor","status":"stable","plugin":true,"summary":"Checks messages against a provided JSONSchema definition but does not change the payload under any circumstances. If a message does not match the schema it can be caught using xref:configuration:error_handling.adoc[error handling methods].","description":"Please refer to the https://json-schema.org/[JSON Schema website^] for information and tutorials regarding the syntax of the schema.","categories":["Mapping"],"footnotes":"\n== Examples\n\nWith the following JSONSchema document:\n\n```json\n{\n\t\"$id\": \"https://example.com/person.schema.json\",\n\t\"$schema\": \"http://json-schema.org/draft-07/schema#\",\n\t\"title\": \"Person\",\n\t\"type\": \"object\",\n\t\"properties\": {\n\t \"firstName\": {\n\t\t\"type\": \"string\",\n\t\t\"description\": \"The person's first name.\"\n\t },\n\t \"lastName\": {\n\t\t\"type\": \"string\",\n\t\t\"description\": \"The person's last name.\"\n\t },\n\t \"age\": {\n\t\t\"description\": \"Age in years which must be equal to or greater than zero.\",\n\t\t\"type\": \"integer\",\n\t\t\"minimum\": 0\n\t }\n\t}\n}\n```\n\nAnd the following Redpanda Connect configuration:\n\n```yaml\npipeline:\n processors:\n - json_schema:\n schema_path: \"file://path_to_schema.json\"\n - catch:\n - log:\n level: ERROR\n message: \"Schema validation failed due to: ${!error()}\"\n - mapping: 'root = deleted()' # Drop messages that fail\n```\n\nIf a payload being processed looked like:\n\n```json\n{\"firstName\":\"John\",\"lastName\":\"Doe\",\"age\":-21}\n```\n\nThen a log message would appear explaining the fault and the payload would be\ndropped.","config":{"name":"","type":"object","kind":"scalar","children":[{"name":"schema","type":"string","kind":"scalar","description":"A schema to apply. Use either this or the `schema_path` field.","is_optional":true},{"name":"schema_path","type":"string","kind":"scalar","description":"The path of a schema document to apply. Use either this or the `schema` field.","is_optional":true}]}},{"name":"log","type":"processor","status":"stable","plugin":true,"summary":"Prints a log event for each message. Messages always remain unchanged. The log message can be set using function interpolations described in xref:configuration:interpolation.adoc#bloblang-queries[Bloblang queries] which allows you to log the contents and metadata of messages.","description":"\nThe `level` field determines the log level of the printed events and can be any of the following values: TRACE, DEBUG, INFO, WARN, ERROR.\n\n== Structured fields\n\nIt's also possible add custom fields to logs when the format is set to a structured form such as `json` or `logfmt` with the config field \u003c\u003cfields_mapping, `fields_mapping`\u003e\u003e:\n\n```yaml\npipeline:\n processors:\n - log:\n level: DEBUG\n message: hello world\n fields_mapping: |\n root.reason = \"cus I wana\"\n root.id = this.id\n root.age = this.user.age\n root.kafka_topic = meta(\"kafka_topic\")\n```\n","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"level","type":"string","kind":"scalar","description":"The log level to use.","default":"INFO","options":["ERROR","WARN","INFO","DEBUG","TRACE"]},{"name":"fields_mapping","type":"string","kind":"scalar","description":"An optional xref:guides:bloblang/about.adoc[Bloblang mapping] that can be used to specify extra fields to add to the log. If log fields are also added with the `fields` field then those values will override matching keys from this mapping.","is_optional":true,"bloblang":true,"examples":["root.reason = \"cus I wana\"\nroot.id = this.id\nroot.age = this.user.age.number()\nroot.kafka_topic = meta(\"kafka_topic\")"]},{"name":"message","type":"string","kind":"scalar","description":"The message to print.","default":"","interpolated":true},{"name":"fields","type":"string","kind":"map","description":"A map of fields to print along with the log message.","is_deprecated":true,"is_optional":true,"interpolated":true}]}},{"name":"mapping","type":"processor","status":"stable","plugin":true,"summary":"Executes a xref:guides:bloblang/about.adoc[Bloblang] mapping on messages, creating a new document that replaces (or filters) the original message.","description":"\nBloblang is a powerful language that enables a wide range of mapping, transformation and filtering tasks. For more information, see xref:guides:bloblang/about.adoc[].\n\nIf your mapping is large and you'd prefer for it to live in a separate file then you can execute a mapping directly from a file with the expression `from \"\u003cpath\u003e\"`, where the path must be absolute, or relative from the location that Redpanda Connect is executed from.\n\nNote: This processor is equivalent to the xref:components:processors/bloblang.adoc#component-rename[Bloblang] one. The latter will be deprecated in a future release.\n\n== Input document immutability\n\nMapping operates by creating an entirely new object during assignments, this has the advantage of treating the original referenced document as immutable and therefore queryable at any stage of your mapping. For example, with the following mapping:\n\n```coffeescript\nroot.id = this.id\nroot.invitees = this.invitees.filter(i -\u003e i.mood \u003e= 0.5)\nroot.rejected = this.invitees.filter(i -\u003e i.mood \u003c 0.5)\n```\n\nNotice that we mutate the value of `invitees` in the resulting document by filtering out objects with a lower mood. However, even after doing so we're still able to reference the unchanged original contents of this value from the input document in order to populate a second field. Within this mapping we also have the flexibility to reference the mutable mapped document by using the keyword `root` (i.e. `root.invitees`) on the right-hand side instead.\n\nMapping documents is advantageous in situations where the result is a document with a dramatically different shape to the input document, since we are effectively rebuilding the document in its entirety and might as well keep a reference to the unchanged input document throughout. However, in situations where we are only performing minor alterations to the input document, the rest of which is unchanged, it might be more efficient to use the xref:components:processors/mutation.adoc[`mutation` processor] instead.\n\n== Error handling\n\nBloblang mappings can fail, in which case the message remains unchanged, errors are logged, and the message is flagged as having failed, allowing you to use xref:configuration:error_handling.adoc[standard processor error handling patterns].\n\nHowever, Bloblang itself also provides powerful ways of ensuring your mappings do not fail by specifying desired xref:guides:bloblang/about.adoc#error-handling[fallback behavior].\n\t\t\t","categories":["Mapping","Parsing"],"examples":[{"title":"Mapping","summary":"\nGiven JSON documents containing an array of fans:\n\n```json\n{\n \"id\":\"foo\",\n \"description\":\"a show about foo\",\n \"fans\":[\n {\"name\":\"bev\",\"obsession\":0.57},\n {\"name\":\"grace\",\"obsession\":0.21},\n {\"name\":\"ali\",\"obsession\":0.89},\n {\"name\":\"vic\",\"obsession\":0.43}\n ]\n}\n```\n\nWe can reduce the documents down to just the ID and only those fans with an obsession score above 0.5, giving us:\n\n```json\n{\n \"id\":\"foo\",\n \"fans\":[\n {\"name\":\"bev\",\"obsession\":0.57},\n {\"name\":\"ali\",\"obsession\":0.89}\n ]\n}\n```\n\nWith the following config:","config":"\npipeline:\n processors:\n - mapping: |\n root.id = this.id\n root.fans = this.fans.filter(fan -\u003e fan.obsession \u003e 0.5)\n"},{"title":"More Mapping","summary":"\nWhen receiving JSON documents of the form:\n\n```json\n{\n \"locations\": [\n {\"name\": \"Seattle\", \"state\": \"WA\"},\n {\"name\": \"New York\", \"state\": \"NY\"},\n {\"name\": \"Bellevue\", \"state\": \"WA\"},\n {\"name\": \"Olympia\", \"state\": \"WA\"}\n ]\n}\n```\n\nWe could collapse the location names from the state of Washington into a field `Cities`:\n\n```json\n{\"Cities\": \"Bellevue, Olympia, Seattle\"}\n```\n\nWith the following config:","config":"\npipeline:\n processors:\n - mapping: |\n root.Cities = this.locations.\n filter(loc -\u003e loc.state == \"WA\").\n map_each(loc -\u003e loc.name).\n sort().join(\", \")\n"}],"config":{"name":"","type":"string","kind":"scalar","bloblang":true},"version":"4.5.0"},{"name":"metric","type":"processor","status":"stable","plugin":true,"summary":"Emit custom metrics by extracting values from messages.","description":"\nThis processor works by evaluating an xref:configuration:interpolation.adoc#bloblang-queries[interpolated field `value`] for each message and updating a emitted metric according to the \u003c\u003ctypes, type\u003e\u003e.\n\nCustom metrics such as these are emitted along with Redpanda Connect internal metrics, where you can customize where metrics are sent, which metric names are emitted and rename them as/when appropriate. For more information see the xref:components:metrics/about.adoc[metrics docs].","categories":["Utility"],"footnotes":"\n== Types\n\n=== `counter`\n\nIncrements a counter by exactly 1, the contents of `value` are ignored\nby this type.\n\n=== `counter_by`\n\nIf the contents of `value` can be parsed as a positive integer value\nthen the counter is incremented by this value.\n\nFor example, the following configuration will increment the value of the\n`count.custom.field` metric by the contents of `field.some.value`:\n\n```yaml\npipeline:\n processors:\n - metric:\n type: counter_by\n name: CountCustomField\n value: ${!json(\"field.some.value\")}\n```\n\n=== `gauge`\n\nIf the contents of `value` can be parsed as a positive integer value\nthen the gauge is set to this value.\n\nFor example, the following configuration will set the value of the\n`gauge.custom.field` metric to the contents of `field.some.value`:\n\n```yaml\npipeline:\n processors:\n - metric:\n type: gauge\n name: GaugeCustomField\n value: ${!json(\"field.some.value\")}\n```\n\n=== `timing`\n\nEquivalent to `gauge` where instead the metric is a timing. It is recommended that timing values are recorded in nanoseconds in order to be consistent with standard Redpanda Connect timing metrics, as in some cases these values are automatically converted into other units such as when exporting timings as histograms with Prometheus metrics.","examples":[{"title":"Counter","summary":"In this example we emit a counter metric called `Foos`, which increments for every message processed, and we label the metric with some metadata about where the message came from and a field from the document that states what type it is. We also configure our metrics to emit to CloudWatch, and explicitly only allow our custom metric and some internal Redpanda Connect metrics to emit.","config":"\npipeline:\n processors:\n - metric:\n name: Foos\n type: counter\n labels:\n topic: ${! meta(\"kafka_topic\") }\n partition: ${! meta(\"kafka_partition\") }\n type: ${! json(\"document.type\").or(\"unknown\") }\n\nmetrics:\n mapping: |\n root = if ![\n \"Foos\",\n \"input_received\",\n \"output_sent\"\n ].contains(this) { deleted() }\n aws_cloudwatch:\n namespace: ProdConsumer\n"},{"title":"Gauge","summary":"In this example we emit a gauge metric called `FooSize`, which is given a value extracted from JSON messages at the path `foo.size`. We then also configure our Prometheus metric exporter to only emit this custom metric and nothing else. We also label the metric with some metadata.","config":"\npipeline:\n processors:\n - metric:\n name: FooSize\n type: gauge\n labels:\n topic: ${! meta(\"kafka_topic\") }\n value: ${! json(\"foo.size\") }\n\nmetrics:\n mapping: 'if this != \"FooSize\" { deleted() }'\n prometheus: {}\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"type","type":"string","kind":"scalar","description":"The metric \u003c\u003ctypes, type\u003e\u003e to create.","options":["counter","counter_by","gauge","timing"],"linter":"\nlet options = {\n \"counter\": true,\n \"counter_by\": true,\n \"gauge\": true,\n \"timing\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"name","type":"string","kind":"scalar","description":"The name of the metric to create, this must be unique across all Redpanda Connect components otherwise it will overwrite those other metrics."},{"name":"labels","type":"string","kind":"map","description":"A map of label names and values that can be used to enrich metrics. Labels are not supported by some metric destinations, in which case the metrics series are combined.","is_optional":true,"interpolated":true,"examples":[{"topic":"${! meta(\"kafka_topic\") }","type":"${! json(\"doc.type\") }"}]},{"name":"value","type":"string","kind":"scalar","description":"For some metric types specifies a value to set, increment. Certain metrics exporters such as Prometheus support floating point values, but those that do not will cast a floating point value into an integer.","default":"","interpolated":true}]}},{"name":"mongodb","type":"processor","status":"experimental","plugin":true,"summary":"Performs operations against MongoDB for each message, allowing you to store or retrieve data within message payloads.","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target MongoDB server.","examples":["mongodb://localhost:27017"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"database","type":"string","kind":"scalar","description":"The name of the target MongoDB database."},{"name":"username","type":"string","kind":"scalar","description":"The username to connect to the database.","default":""},{"name":"password","type":"string","kind":"scalar","description":"The password to connect to the database.","is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"app_name","type":"string","kind":"scalar","description":"The client application name.","is_advanced":true,"default":"benthos","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"collection","type":"string","kind":"scalar","description":"The name of the target collection."},{"name":"operation","type":"string","kind":"scalar","description":"The mongodb operation to perform.","default":"insert-one","options":["insert-one","delete-one","delete-many","replace-one","update-one","find-one","aggregate"],"linter":"\nlet options = {\n \"insert-one\": true,\n \"delete-one\": true,\n \"delete-many\": true,\n \"replace-one\": true,\n \"update-one\": true,\n \"find-one\": true,\n \"aggregate\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"write_concern","type":"object","kind":"scalar","description":"The write concern settings for the mongo connection.","children":[{"name":"w","type":"string","kind":"scalar","description":"W requests acknowledgement that write operations propagate to the specified number of mongodb instances. Can be the string \"majority\" to wait for a calculated majority of nodes to acknowledge the write operation, or an integer value specifying an minimum number of nodes to acknowledge the operation, or a string specifying the name of a custom write concern configured in the cluster.","default":"majority"},{"name":"j","type":"bool","kind":"scalar","description":"J requests acknowledgement from MongoDB that write operations are written to the journal.","default":false},{"name":"w_timeout","type":"string","kind":"scalar","description":"The write concern timeout.","default":""}]},{"name":"document_map","type":"string","kind":"scalar","description":"A bloblang map representing a document to store within MongoDB, expressed as https://www.mongodb.com/docs/manual/reference/mongodb-extended-json/[extended JSON in canonical form^]. The document map is required for the operations insert-one, replace-one, update-one and aggregate.","default":"","bloblang":true,"examples":["root.a = this.foo\nroot.b = this.bar"]},{"name":"filter_map","type":"string","kind":"scalar","description":"A bloblang map representing a filter for a MongoDB command, expressed as https://www.mongodb.com/docs/manual/reference/mongodb-extended-json/[extended JSON in canonical form^]. The filter map is required for all operations except insert-one. It is used to find the document(s) for the operation. For example in a delete-one case, the filter map should have the fields required to locate the document to delete.","default":"","bloblang":true,"examples":["root.a = this.foo\nroot.b = this.bar"]},{"name":"hint_map","type":"string","kind":"scalar","description":"A bloblang map representing the hint for the MongoDB command, expressed as https://www.mongodb.com/docs/manual/reference/mongodb-extended-json/[extended JSON in canonical form^]. This map is optional and is used with all operations except insert-one. It is used to improve performance of finding the documents in the mongodb.","default":"","bloblang":true,"examples":["root.a = this.foo\nroot.b = this.bar"]},{"name":"upsert","type":"bool","kind":"scalar","description":"The upsert setting is optional and only applies for update-one and replace-one operations. If the filter specified in filter_map matches, the document is updated or replaced accordingly, otherwise it is created.","default":false,"version":"3.60.0"},{"name":"json_marshal_mode","type":"string","kind":"scalar","description":"The json_marshal_mode setting is optional and controls the format of the output message.","is_advanced":true,"default":"canonical","annotated_options":[["canonical","A string format that emphasizes type preservation at the expense of readability and interoperability. That is, conversion from canonical to BSON will generally preserve type information except in certain specific cases. "],["relaxed","A string format that emphasizes readability and interoperability at the expense of type preservation. That is, conversion from relaxed format to BSON can lose type information."]],"version":"3.60.0","linter":"\nlet options = {\n \"canonical\": true,\n \"relaxed\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"max_retries","type":"int","kind":"scalar","description":"The maximum number of retries before giving up on the request. If set to zero there is no discrete limit.","is_advanced":true,"is_deprecated":true,"default":3},{"name":"backoff","type":"object","kind":"scalar","description":"Control time intervals between retry attempts.","is_advanced":true,"is_deprecated":true,"children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","is_advanced":true,"is_deprecated":true,"default":"1s"},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts.","is_advanced":true,"is_deprecated":true,"default":"5s"},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum period to wait before retry attempts are abandoned. If zero then no limit is used.","is_advanced":true,"is_deprecated":true,"default":"30s"}]}]},"version":"3.43.0"},{"name":"msgpack","type":"processor","status":"beta","plugin":true,"summary":"Converts messages to or from the https://msgpack.org/[MessagePack^] format.","categories":["Parsing"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"operator","type":"string","kind":"scalar","description":"The operation to perform on messages.","annotated_options":[["from_json","Convert JSON messages to MessagePack format"],["to_json","Convert MessagePack messages to JSON format"]],"linter":"\nlet options = {\n \"from_json\": true,\n \"to_json\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"}]},"version":"3.59.0"},{"name":"mutation","type":"processor","status":"stable","plugin":true,"summary":"Executes a xref:guides:bloblang/about.adoc[Bloblang] mapping and directly transforms the contents of messages, mutating (or deleting) them.","description":"\nBloblang is a powerful language that enables a wide range of mapping, transformation and filtering tasks. For more information, see xref:guides:bloblang/about.adoc[].\n\nIf your mapping is large and you'd prefer for it to live in a separate file then you can execute a mapping directly from a file with the expression `from \"\u003cpath\u003e\"`, where the path must be absolute, or relative from the location that Redpanda Connect is executed from.\n\n== Input document mutability\n\nA mutation is a mapping that transforms input documents directly, this has the advantage of reducing the need to copy the data fed into the mapping. However, this also means that the referenced document is mutable and therefore changes throughout the mapping. For example, with the following Bloblang:\n\n```coffeescript\nroot.rejected = this.invitees.filter(i -\u003e i.mood \u003c 0.5)\nroot.invitees = this.invitees.filter(i -\u003e i.mood \u003e= 0.5)\n```\n\nNotice that we create a field `rejected` by copying the array field `invitees` and filtering out objects with a high mood. We then overwrite the field `invitees` by filtering out objects with a low mood, resulting in two array fields that are each a subset of the original. If we were to reverse the ordering of these assignments like so:\n\n```coffeescript\nroot.invitees = this.invitees.filter(i -\u003e i.mood \u003e= 0.5)\nroot.rejected = this.invitees.filter(i -\u003e i.mood \u003c 0.5)\n```\n\nThen the new field `rejected` would be empty as we have already mutated `invitees` to exclude the objects that it would be populated by. We can solve this problem either by carefully ordering our assignments or by capturing the original array using a variable (`let invitees = this.invitees`).\n\nMutations are advantageous over a standard mapping in situations where the result is a document with mostly the same shape as the input document, since we can avoid unnecessarily copying data from the referenced input document. However, in situations where we are creating an entirely new document shape it can be more convenient to use the traditional xref:components:processors/mapping.adoc[`mapping` processor] instead.\n\n== Error handling\n\nBloblang mappings can fail, in which case the error is logged and the message is flagged as having failed, allowing you to use xref:configuration:error_handling.adoc[standard processor error handling patterns].\n\nHowever, Bloblang itself also provides powerful ways of ensuring your mappings do not fail by specifying desired xref:guides:bloblang/about.adoc#error-handling[fallback behavior].\n\t\t\t","categories":["Mapping","Parsing"],"examples":[{"title":"Mapping","summary":"\nGiven JSON documents containing an array of fans:\n\n```json\n{\n \"id\":\"foo\",\n \"description\":\"a show about foo\",\n \"fans\":[\n {\"name\":\"bev\",\"obsession\":0.57},\n {\"name\":\"grace\",\"obsession\":0.21},\n {\"name\":\"ali\",\"obsession\":0.89},\n {\"name\":\"vic\",\"obsession\":0.43}\n ]\n}\n```\n\nWe can reduce the documents down to just the ID and only those fans with an obsession score above 0.5, giving us:\n\n```json\n{\n \"id\":\"foo\",\n \"fans\":[\n {\"name\":\"bev\",\"obsession\":0.57},\n {\"name\":\"ali\",\"obsession\":0.89}\n ]\n}\n```\n\nWith the following config:","config":"\npipeline:\n processors:\n - mutation: |\n root.description = deleted()\n root.fans = this.fans.filter(fan -\u003e fan.obsession \u003e 0.5)\n"},{"title":"More Mapping","summary":"\nWhen receiving JSON documents of the form:\n\n```json\n{\n \"locations\": [\n {\"name\": \"Seattle\", \"state\": \"WA\"},\n {\"name\": \"New York\", \"state\": \"NY\"},\n {\"name\": \"Bellevue\", \"state\": \"WA\"},\n {\"name\": \"Olympia\", \"state\": \"WA\"}\n ]\n}\n```\n\nWe could collapse the location names from the state of Washington into a field `Cities`:\n\n```json\n{\"Cities\": \"Bellevue, Olympia, Seattle\"}\n```\n\nWith the following config:","config":"\npipeline:\n processors:\n - mutation: |\n root.Cities = this.locations.\n filter(loc -\u003e loc.state == \"WA\").\n map_each(loc -\u003e loc.name).\n sort().join(\", \")\n"}],"config":{"name":"","type":"string","kind":"scalar","bloblang":true},"version":"4.5.0"},{"name":"nats_kv","type":"processor","status":"beta","plugin":true,"summary":"Perform operations on a NATS key-value bucket.","description":"\n== KV operations\n\nThe NATS KV processor supports a multitude of KV operations via the \u003c\u003coperation\u003e\u003e field. Along with `get`, `put`, and `delete`, this processor supports atomic operations like `update` and `create`, as well as utility operations like `purge`, `history`, and `keys`.\n\n== Metadata\n\nThis processor adds the following metadata fields to each message, depending on the chosen `operation`:\n\n=== get, get_revision\n``` text\n- nats_kv_key\n- nats_kv_bucket\n- nats_kv_revision\n- nats_kv_delta\n- nats_kv_operation\n- nats_kv_created\n```\n\n=== create, update, delete, purge\n``` text\n- nats_kv_key\n- nats_kv_bucket\n- nats_kv_revision\n- nats_kv_operation\n```\n\n=== keys\n``` text\n- nats_kv_bucket\n```\n\n== Connection name\n\nWhen monitoring and managing a production NATS system, it is often useful to\nknow which connection a message was send/received from. This can be achieved by\nsetting the connection name option when creating a NATS connection.\n\nRedpanda Connect will automatically set the connection name based off the label of the given\nNATS component, so that monitoring tools between NATS and Redpanda Connect can stay in sync.\n\n\n== Authentication\n\nThere are several components within Redpanda Connect which uses NATS services. You will find that each of these components\nsupport optional advanced authentication parameters for https://docs.nats.io/nats-server/configuration/securing_nats/auth_intro/nkey_auth[NKeys^]\nand https://docs.nats.io/using-nats/developer/connecting/creds[User Credentials^].\n\nSee an https://docs.nats.io/running-a-nats-service/nats_admin/security/jwt[in-depth tutorial^].\n\n=== NKey file\n\nThe NATS server can use these NKeys in several ways for authentication. The simplest is for the server to be configured\nwith a list of known public keys and for the clients to respond to the challenge by signing it with its private NKey\nconfigured in the `nkey_file` or `nkey` field.\n\nhttps://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[More details^].\n\n=== User credentials\n\nNATS server supports decentralized authentication based on JSON Web Tokens (JWT). Clients need an https://docs.nats.io/nats-server/configuration/securing_nats/jwt#json-web-tokens[user JWT^]\nand a corresponding https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[NKey secret^] when connecting to a server\nwhich is configured to use this authentication scheme.\n\nThe `user_credentials_file` field should point to a file containing both the private key and the JWT and can be\ngenerated with the https://docs.nats.io/nats-tools/nsc[nsc tool^].\n\nAlternatively, the `user_jwt` field can contain a plain text JWT and the `user_nkey_seed`can contain\nthe plain text NKey Seed.\n\nhttps://docs.nats.io/using-nats/developer/connecting/creds[More details^].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["nats://127.0.0.1:4222"],["nats://username:password@127.0.0.1:4222"]]},{"name":"bucket","type":"string","kind":"scalar","description":"The name of the KV bucket.","examples":["my_kv_bucket"]},{"name":"operation","type":"string","kind":"scalar","description":"The operation to perform on the KV bucket.","annotated_options":[["create","Adds the key/value pair if it does not exist. Returns an error if it already exists."],["delete","Deletes the key/value pair, but keeps historical values."],["get","Returns the latest value for `key`."],["get_revision","Returns the value of `key` for the specified `revision`."],["history","Returns historical values of `key` as an array of objects containing the following fields: `key`, `value`, `bucket`, `revision`, `delta`, `operation`, `created`."],["keys","Returns the keys in the `bucket` which match the `keys_filter` as an array of strings."],["purge","Deletes the key/value pair and all historical values."],["put","Places a new value for the key into the store."],["update","Updates the value for `key` only if the `revision` matches the latest revision."]],"linter":"\nlet options = {\n \"create\": true,\n \"delete\": true,\n \"get\": true,\n \"get_revision\": true,\n \"history\": true,\n \"keys\": true,\n \"purge\": true,\n \"put\": true,\n \"update\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"key","type":"string","kind":"scalar","description":"The key for each message. Supports https://docs.nats.io/nats-concepts/subjects#wildcards[wildcards^] for the `history` and `keys` operations.","interpolated":true,"examples":["foo","foo.bar.baz","foo.*","foo.\u003e","foo.${! json(\"meta.type\") }"],"linter":"if this == \"\" {[ \"'key' must be set to a non-empty string\" ]}"},{"name":"revision","type":"string","kind":"scalar","description":"The revision of the key to operate on. Used for `get_revision` and `update` operations.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["42","${! @nats_kv_revision }"]},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period to wait on an operation before aborting and returning an error.","is_advanced":true,"default":"5s"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auth","type":"object","kind":"scalar","description":"Optional configuration of NATS authentication parameters.","is_advanced":true,"children":[{"name":"nkey_file","type":"string","kind":"scalar","description":"An optional file containing a NKey seed.","is_advanced":true,"is_optional":true,"examples":["./seed.nk"]},{"name":"nkey","type":"string","kind":"scalar","description":"The NKey seed.","is_advanced":true,"is_optional":true,"is_secret":true,"examples":["UDXU4RCSJNZOIQHZNWXHXORDPRTGNJAHAHFRGZNEEJCPQTT2M7NLCNF4"],"version":"4.38.0","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_credentials_file","type":"string","kind":"scalar","description":"An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.","is_advanced":true,"is_optional":true,"examples":["./user.creds"]},{"name":"user_jwt","type":"string","kind":"scalar","description":"An optional plain text user JWT (given along with the corresponding user NKey Seed).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_nkey_seed","type":"string","kind":"scalar","description":"An optional plain text user NKey Seed (given along with the corresponding user JWT).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}],"linter":"root = match {\n [\"get_revision\", \"update\"].contains(this.operation) \u0026\u0026 !this.exists(\"revision\") =\u003e [ \"'revision' must be set when operation is '\" + this.operation + \"'\" ],\n ![\"get_revision\", \"update\"].contains(this.operation) \u0026\u0026 this.exists(\"revision\") =\u003e [ \"'revision' cannot be set when operation is '\" + this.operation + \"'\" ],\n }"},"version":"4.12.0"},{"name":"nats_request_reply","type":"processor","status":"experimental","plugin":true,"summary":"Sends a message to a NATS subject and expects a reply, from a NATS subscriber acting as a responder, back.","description":"\n== Metadata\n\nThis input adds the following metadata fields to each message:\n\n```text\n- nats_subject\n- nats_sequence_stream\n- nats_sequence_consumer\n- nats_num_delivered\n- nats_num_pending\n- nats_domain\n- nats_timestamp_unix_nano\n```\n\nYou can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].\n\n== Connection name\n\nWhen monitoring and managing a production NATS system, it is often useful to\nknow which connection a message was send/received from. This can be achieved by\nsetting the connection name option when creating a NATS connection.\n\nRedpanda Connect will automatically set the connection name based off the label of the given\nNATS component, so that monitoring tools between NATS and Redpanda Connect can stay in sync.\n\n\n== Authentication\n\nThere are several components within Redpanda Connect which uses NATS services. You will find that each of these components\nsupport optional advanced authentication parameters for https://docs.nats.io/nats-server/configuration/securing_nats/auth_intro/nkey_auth[NKeys^]\nand https://docs.nats.io/using-nats/developer/connecting/creds[User Credentials^].\n\nSee an https://docs.nats.io/running-a-nats-service/nats_admin/security/jwt[in-depth tutorial^].\n\n=== NKey file\n\nThe NATS server can use these NKeys in several ways for authentication. The simplest is for the server to be configured\nwith a list of known public keys and for the clients to respond to the challenge by signing it with its private NKey\nconfigured in the `nkey_file` or `nkey` field.\n\nhttps://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[More details^].\n\n=== User credentials\n\nNATS server supports decentralized authentication based on JSON Web Tokens (JWT). Clients need an https://docs.nats.io/nats-server/configuration/securing_nats/jwt#json-web-tokens[user JWT^]\nand a corresponding https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth[NKey secret^] when connecting to a server\nwhich is configured to use this authentication scheme.\n\nThe `user_credentials_file` field should point to a file containing both the private key and the JWT and can be\ngenerated with the https://docs.nats.io/nats-tools/nsc[nsc tool^].\n\nAlternatively, the `user_jwt` field can contain a plain text JWT and the `user_nkey_seed`can contain\nthe plain text NKey Seed.\n\nhttps://docs.nats.io/using-nats/developer/connecting/creds[More details^].","categories":["Services"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"urls","type":"string","kind":"array","description":"A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.","examples":[["nats://127.0.0.1:4222"],["nats://username:password@127.0.0.1:4222"]]},{"name":"subject","type":"string","kind":"scalar","description":"A subject to write to.","interpolated":true,"examples":["foo.bar.baz","${! meta(\"kafka_topic\") }","foo.${! json(\"meta.type\") }"]},{"name":"inbox_prefix","type":"string","kind":"scalar","description":"Set an explicit inbox prefix for the response subject","is_advanced":true,"is_optional":true,"examples":["_INBOX_joe"]},{"name":"headers","type":"string","kind":"map","description":"Explicit message headers to add to messages.","default":{},"interpolated":true,"examples":[{"Content-Type":"application/json","Timestamp":"${!meta(\"Timestamp\")}"}]},{"name":"metadata","type":"object","kind":"scalar","description":"Determine which (if any) metadata values should be added to messages as headers.","is_optional":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"timeout","type":"string","kind":"scalar","description":"A duration string is a possibly signed sequence of decimal numbers, each with optional fraction and a unit suffix, such as 300ms, -1.5h or 2h45m. Valid time units are ns, us (or Β΅s), ms, s, m, h.","is_optional":true,"default":"3s"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"auth","type":"object","kind":"scalar","description":"Optional configuration of NATS authentication parameters.","is_advanced":true,"children":[{"name":"nkey_file","type":"string","kind":"scalar","description":"An optional file containing a NKey seed.","is_advanced":true,"is_optional":true,"examples":["./seed.nk"]},{"name":"nkey","type":"string","kind":"scalar","description":"The NKey seed.","is_advanced":true,"is_optional":true,"is_secret":true,"examples":["UDXU4RCSJNZOIQHZNWXHXORDPRTGNJAHAHFRGZNEEJCPQTT2M7NLCNF4"],"version":"4.38.0","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_credentials_file","type":"string","kind":"scalar","description":"An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.","is_advanced":true,"is_optional":true,"examples":["./user.creds"]},{"name":"user_jwt","type":"string","kind":"scalar","description":"An optional plain text user JWT (given along with the corresponding user NKey Seed).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"user_nkey_seed","type":"string","kind":"scalar","description":"An optional plain text user NKey Seed (given along with the corresponding user JWT).","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},"version":"4.27.0"},{"name":"noop","type":"processor","status":"stable","plugin":true,"summary":"Noop is a processor that does nothing, the message passes through unchanged. Why? Sometimes doing nothing is the braver option.","categories":null,"config":{"name":"","type":"object","kind":"scalar","default":{}}},{"name":"ollama_chat","type":"processor","status":"experimental","plugin":true,"summary":"Generates responses to messages in a chat conversation, using the Ollama API.","description":"This processor sends prompts to your chosen Ollama large language model (LLM) and generates text from the responses, using the Ollama API.\n\nBy default, the processor starts and runs a locally installed Ollama server. Alternatively, to use an already running Ollama server, add your server details to the `server_address` field. You can https://ollama.com/download[download and install Ollama from the Ollama website^].\n\nFor more information, see the https://github.com/ollama/ollama/tree/main/docs[Ollama documentation^].","categories":["AI"],"examples":[{"title":"Use Llava to analyze an image","summary":"This example fetches image URLs from stdin and has a multimodal LLM describe the image.","config":"\ninput:\n stdin:\n scanner:\n lines: {}\npipeline:\n processors:\n - http:\n verb: GET\n url: \"${!content().string()}\"\n - ollama_chat:\n model: llava\n prompt: \"Describe the following image\"\n image: \"root = content()\"\noutput:\n stdout:\n codec: lines\n"},{"title":"Use subpipelines as tool calls","summary":"This example allows llama3.2 to execute a subpipeline as a tool call to get more data.","config":"\ninput:\n generate:\n count: 1\n mapping: |\n root = \"What is the weather like in Chicago?\"\npipeline:\n processors:\n - ollama_chat:\n model: llama3.2\n prompt: \"${!content().string()}\"\n tools:\n - name: GetWeather\n description: \"Retrieve the weather for a specific city\"\n parameters:\n required: [\"city\"]\n properties:\n city:\n type: string\n description: the city to lookup the weather for\n processors:\n - http:\n verb: GET\n url: 'https://wttr.in/${!this.city}?T'\n headers:\n # Spoof curl user-ageent to get a plaintext text\n User-Agent: curl/8.11.1\noutput:\n stdout: {}\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"model","type":"string","kind":"scalar","description":"The name of the Ollama LLM to use. For a full list of models, see the https://ollama.com/models[Ollama website].","examples":["llama3.1","gemma2","qwen2","phi3"]},{"name":"prompt","type":"string","kind":"scalar","description":"The prompt you want to generate a response for. By default, the processor submits the entire payload as a string.","is_optional":true,"interpolated":true},{"name":"system_prompt","type":"string","kind":"scalar","description":"The system prompt to submit to the Ollama LLM.","is_advanced":true,"is_optional":true,"interpolated":true},{"name":"image","type":"string","kind":"scalar","description":"The image to submit along with the prompt to the model. The result should be a byte array.","is_optional":true,"bloblang":true,"examples":["root = this.image.decode(\"base64\") # decode base64 encoded image"],"version":"4.38.0"},{"name":"response_format","type":"string","kind":"scalar","description":"The format of the response that the Ollama model generates. If specifying JSON output, then the `prompt` should specify that the output should be in JSON as well.","default":"text","options":["text","json"],"linter":"\nlet options = {\n \"text\": true,\n \"json\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"max_tokens","type":"int","kind":"scalar","description":"The maximum number of tokens to predict and output. Limiting the amount of output means that requests are processed faster and have a fixed limit on the cost.","is_optional":true},{"name":"temperature","type":"int","kind":"scalar","description":"The temperature of the model. Increasing the temperature makes the model answer more creatively.","is_optional":true,"linter":"root = if this \u003e 2 || this \u003c 0 { [ \"field must be between 0.0 and 2.0\" ] }"},{"name":"num_keep","type":"int","kind":"scalar","description":"Specify the number of tokens from the initial prompt to retain when the model resets its internal context. By default, this value is set to `4`. Use `-1` to retain all tokens from the initial prompt.","is_advanced":true,"is_optional":true},{"name":"seed","type":"int","kind":"scalar","description":"Sets the random number seed to use for generation. Setting this to a specific number will make the model generate the same text for the same prompt.","is_advanced":true,"is_optional":true,"examples":[42]},{"name":"top_k","type":"int","kind":"scalar","description":"Reduces the probability of generating nonsense. A higher value, for example `100`, will give more diverse answers. A lower value, for example `10`, will be more conservative.","is_advanced":true,"is_optional":true},{"name":"top_p","type":"float","kind":"scalar","description":"Works together with `top-k`. A higher value, for example 0.95, will lead to more diverse text. A lower value, for example 0.5, will generate more focused and conservative text.","is_advanced":true,"is_optional":true,"linter":"root = if this \u003e 1 || this \u003c 0 { [ \"field must be between 0.0 and 1.0\" ] }"},{"name":"repeat_penalty","type":"float","kind":"scalar","description":"Sets how strongly to penalize repetitions. A higher value, for example 1.5, will penalize repetitions more strongly. A lower value, for example 0.9, will be more lenient.","is_advanced":true,"is_optional":true,"linter":"root = if this \u003e 2 || this \u003c -2 { [ \"field must be between -2.0 and 2.0\" ] }"},{"name":"presence_penalty","type":"float","kind":"scalar","description":"Positive values penalize new tokens if they have appeared in the text so far. This increases the model's likelihood to talk about new topics.","is_advanced":true,"is_optional":true,"linter":"root = if this \u003e 2 || this \u003c -2 { [ \"field must be between -2.0 and 2.0\" ] }"},{"name":"frequency_penalty","type":"float","kind":"scalar","description":"Positive values penalize new tokens based on the frequency of their appearance in the text so far. This decreases the model's likelihood to repeat the same line verbatim.","is_advanced":true,"is_optional":true,"linter":"root = if this \u003e 2 || this \u003c -2 { [ \"field must be between -2.0 and 2.0\" ] }"},{"name":"stop","type":"string","kind":"array","description":"Sets the stop sequences to use. When this pattern is encountered the LLM stops generating text and returns the final response.","is_advanced":true,"is_optional":true},{"name":"save_prompt_metadata","type":"bool","kind":"scalar","description":"If enabled the prompt is saved as @prompt metadata on the output message. If system_prompt is used it's also saved as @system_prompt","default":false},{"name":"history","type":"string","kind":"scalar","description":"Historical messages to include in the chat request. The result of the bloblang query should be an array of objects of the form of [{\"role\": \"\", \"content\":\"\"}].","is_optional":true,"bloblang":true},{"name":"max_tool_calls","type":"int","kind":"scalar","description":"The maximum number of sequential tool calls.","is_advanced":true,"default":3,"linter":"root = if this \u003c= 0 { [\"field must be greater than zero\"] }"},{"name":"tools","type":"object","kind":"array","description":"The tools to allow the LLM to invoke. This allows building subpipelines that the LLM can choose to invoke to execute agentic-like actions.","default":[],"children":[{"name":"name","type":"string","kind":"scalar","description":"The name of this tool."},{"name":"description","type":"string","kind":"scalar","description":"A description of this tool, the LLM uses this to decide if the tool should be used."},{"name":"parameters","type":"object","kind":"scalar","description":"The parameters the LLM needs to provide to invoke this tool.","children":[{"name":"required","type":"string","kind":"array","description":"The required parameters for this pipeline.","default":[]},{"name":"properties","type":"object","kind":"map","description":"The properties for the processor's input data","children":[{"name":"type","type":"string","kind":"scalar","description":"The type of this parameter."},{"name":"description","type":"string","kind":"scalar","description":"A description of this parameter."},{"name":"enum","type":"string","kind":"array","description":"Specifies that this parameter is an enum and only these specific values should be used.","default":[]}]}]},{"name":"processors","type":"processor","kind":"array","description":"The pipeline to execute when the LLM uses this tool.","is_optional":true}]},{"name":"runner","type":"object","kind":"scalar","description":"Options for the model runner that are used when the model is first loaded into memory.","is_optional":true,"children":[{"name":"context_size","type":"int","kind":"scalar","description":"Sets the size of the context window used to generate the next token. Using a larger context window uses more memory and takes longer to processor.","is_optional":true},{"name":"batch_size","type":"int","kind":"scalar","description":"The maximum number of requests to process in parallel.","is_optional":true},{"name":"gpu_layers","type":"int","kind":"scalar","description":"This option allows offloading some layers to the GPU for computation. This generally results in increased performance. By default, the runtime decides the number of layers dynamically.","is_advanced":true,"is_optional":true},{"name":"threads","type":"int","kind":"scalar","description":"Set the number of threads to use during generation. For optimal performance, it is recommended to set this value to the number of physical CPU cores your system has. By default, the runtime decides the optimal number of threads.","is_advanced":true,"is_optional":true},{"name":"use_mmap","type":"bool","kind":"scalar","description":"Map the model into memory. This is only support on unix systems and allows loading only the necessary parts of the model as needed.","is_advanced":true,"is_optional":true},{"name":"use_mlock","type":"bool","kind":"scalar","description":"Lock the model in memory, preventing it from being swapped out when memory-mapped. This option can improve performance but reduces some of the advantages of memory-mapping because it uses more RAM to run and can slow down load times as the model loads into RAM.","is_advanced":true,"is_optional":true}]},{"name":"server_address","type":"string","kind":"scalar","description":"The address of the Ollama server to use. Leave the field blank and the processor starts and runs a local Ollama server or specify the address of your own local or remote server.","is_optional":true,"examples":["http://127.0.0.1:11434"]},{"name":"cache_directory","type":"string","kind":"scalar","description":"If `server_address` is not set - the directory to download the ollama binary and use as a model cache.","is_advanced":true,"is_optional":true,"examples":["/opt/cache/connect/ollama"]},{"name":"download_url","type":"string","kind":"scalar","description":"If `server_address` is not set - the URL to download the ollama binary from. Defaults to the offical Ollama GitHub release for this platform.","is_advanced":true,"is_optional":true}]},"version":"4.32.0"},{"name":"ollama_embeddings","type":"processor","status":"experimental","plugin":true,"summary":"Generates vector embeddings from text, using the Ollama API.","description":"This processor sends text to your chosen Ollama large language model (LLM) and creates vector embeddings, using the Ollama API. Vector embeddings are long arrays of numbers that represent values or objects, in this case text. \n\nBy default, the processor starts and runs a locally installed Ollama server. Alternatively, to use an already running Ollama server, add your server details to the `server_address` field. You can https://ollama.com/download[download and install Ollama from the Ollama website^].\n\nFor more information, see the https://github.com/ollama/ollama/tree/main/docs[Ollama documentation^].","categories":["AI"],"examples":[{"title":"Store embedding vectors in Qdrant","summary":"Compute embeddings for some generated data and store it within xrefs:component:outputs/qdrant.adoc[Qdrant]","config":"input:\n generate:\n interval: 1s\n mapping: |\n root = {\"text\": fake(\"paragraph\")}\npipeline:\n processors:\n - ollama_embeddings:\n model: snowflake-artic-embed\n text: \"${!this.text}\"\noutput:\n qdrant:\n grpc_host: localhost:6334\n collection_name: \"example_collection\"\n id: \"root = uuid_v4()\"\n vector_mapping: \"root = this\"\n"},{"title":"Store embedding vectors in Clickhouse","summary":"Compute embeddings for some generated data and store it within https://clickhouse.com/[Clickhouse^]","config":"input:\n generate:\n interval: 1s\n mapping: |\n root = {\"text\": fake(\"paragraph\")}\npipeline:\n processors:\n - branch:\n processors:\n - ollama_embeddings:\n model: snowflake-artic-embed\n text: \"${!this.text}\"\n result_map: |\n root.embeddings = this\noutput:\n sql_insert:\n driver: clickhouse\n dsn: \"clickhouse://localhost:9000\"\n table: searchable_text\n columns: [\"id\", \"text\", \"vector\"]\n args_mapping: \"root = [uuid_v4(), this.text, this.embeddings]\"\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"model","type":"string","kind":"scalar","description":"The name of the Ollama LLM to use. For a full list of models, see the https://ollama.com/models[Ollama website].","examples":["nomic-embed-text","mxbai-embed-large","snowflake-artic-embed","all-minilm"]},{"name":"text","type":"string","kind":"scalar","description":"The text you want to create vector embeddings for. By default, the processor submits the entire payload as a string.","is_optional":true,"interpolated":true},{"name":"runner","type":"object","kind":"scalar","description":"Options for the model runner that are used when the model is first loaded into memory.","is_optional":true,"children":[{"name":"context_size","type":"int","kind":"scalar","description":"Sets the size of the context window used to generate the next token. Using a larger context window uses more memory and takes longer to processor.","is_optional":true},{"name":"batch_size","type":"int","kind":"scalar","description":"The maximum number of requests to process in parallel.","is_optional":true},{"name":"gpu_layers","type":"int","kind":"scalar","description":"This option allows offloading some layers to the GPU for computation. This generally results in increased performance. By default, the runtime decides the number of layers dynamically.","is_advanced":true,"is_optional":true},{"name":"threads","type":"int","kind":"scalar","description":"Set the number of threads to use during generation. For optimal performance, it is recommended to set this value to the number of physical CPU cores your system has. By default, the runtime decides the optimal number of threads.","is_advanced":true,"is_optional":true},{"name":"use_mmap","type":"bool","kind":"scalar","description":"Map the model into memory. This is only support on unix systems and allows loading only the necessary parts of the model as needed.","is_advanced":true,"is_optional":true},{"name":"use_mlock","type":"bool","kind":"scalar","description":"Lock the model in memory, preventing it from being swapped out when memory-mapped. This option can improve performance but reduces some of the advantages of memory-mapping because it uses more RAM to run and can slow down load times as the model loads into RAM.","is_advanced":true,"is_optional":true}]},{"name":"server_address","type":"string","kind":"scalar","description":"The address of the Ollama server to use. Leave the field blank and the processor starts and runs a local Ollama server or specify the address of your own local or remote server.","is_optional":true,"examples":["http://127.0.0.1:11434"]},{"name":"cache_directory","type":"string","kind":"scalar","description":"If `server_address` is not set - the directory to download the ollama binary and use as a model cache.","is_advanced":true,"is_optional":true,"examples":["/opt/cache/connect/ollama"]},{"name":"download_url","type":"string","kind":"scalar","description":"If `server_address` is not set - the URL to download the ollama binary from. Defaults to the offical Ollama GitHub release for this platform.","is_advanced":true,"is_optional":true}]},"version":"4.32.0"},{"name":"ollama_moderation","type":"processor","status":"experimental","plugin":true,"summary":"Generates responses to messages in a chat conversation, using the Ollama API.","description":"This processor checks LLM response safety using either `llama-guard3` or `shieldgemma`. If you want to check if a given prompt is safe, then that can be done with the `ollama_chat` processor - this processor is for response classification only.\n\nBy default, the processor starts and runs a locally installed Ollama server. Alternatively, to use an already running Ollama server, add your server details to the `server_address` field. You can https://ollama.com/download[download and install Ollama from the Ollama website^].\n\nFor more information, see the https://github.com/ollama/ollama/tree/main/docs[Ollama documentation^].","categories":["AI"],"examples":[{"title":"Use Llama Guard 3 classify a LLM response","summary":"This example uses Llama Guard 3 to check if another model responded with a safe or unsafe content.","config":"\ninput:\n stdin:\n scanner:\n lines: {}\npipeline:\n processors:\n - ollama_chat:\n model: llava\n prompt: \"${!content().string()}\"\n save_prompt_metadata: true\n - ollama_moderation:\n model: llama-guard3\n prompt: \"${!@prompt}\"\n response: \"${!content().string()}\"\n - mapping: |\n root.response = content().string()\n root.is_safe = @safe\noutput:\n stdout:\n codec: lines\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"model","type":"string","kind":"scalar","description":"The name of the Ollama LLM to use.","examples":["llama-guard3","shieldgemma"],"annotated_options":[["llama-guard3","When using llama-guard3, two pieces of metadata is added: @safe with the value of `yes` or `no` and the second being @category for the safety category violation. For more information see the https://ollama.com/library/llama-guard3[Llama Guard 3 Model Card]."],["shieldgemma","When using shieldgemma, the model output is a single piece of metadata of @safe with a value of `yes` or `no` if the response is not in violation of its defined safety policies."]],"linter":"\nlet options = {\n \"llama-guard3\": true,\n \"shieldgemma\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"prompt","type":"string","kind":"scalar","description":"The input prompt that was used with the LLM. If using `ollama_chat` the you can use `save_prompt_metadata` to safe the prompt as metadata.","interpolated":true},{"name":"response","type":"string","kind":"scalar","description":"The LLM's response to classify if it contains safe or unsafe content.","interpolated":true},{"name":"runner","type":"object","kind":"scalar","description":"Options for the model runner that are used when the model is first loaded into memory.","is_optional":true,"children":[{"name":"context_size","type":"int","kind":"scalar","description":"Sets the size of the context window used to generate the next token. Using a larger context window uses more memory and takes longer to processor.","is_optional":true},{"name":"batch_size","type":"int","kind":"scalar","description":"The maximum number of requests to process in parallel.","is_optional":true},{"name":"gpu_layers","type":"int","kind":"scalar","description":"This option allows offloading some layers to the GPU for computation. This generally results in increased performance. By default, the runtime decides the number of layers dynamically.","is_advanced":true,"is_optional":true},{"name":"threads","type":"int","kind":"scalar","description":"Set the number of threads to use during generation. For optimal performance, it is recommended to set this value to the number of physical CPU cores your system has. By default, the runtime decides the optimal number of threads.","is_advanced":true,"is_optional":true},{"name":"use_mmap","type":"bool","kind":"scalar","description":"Map the model into memory. This is only support on unix systems and allows loading only the necessary parts of the model as needed.","is_advanced":true,"is_optional":true},{"name":"use_mlock","type":"bool","kind":"scalar","description":"Lock the model in memory, preventing it from being swapped out when memory-mapped. This option can improve performance but reduces some of the advantages of memory-mapping because it uses more RAM to run and can slow down load times as the model loads into RAM.","is_advanced":true,"is_optional":true}]},{"name":"server_address","type":"string","kind":"scalar","description":"The address of the Ollama server to use. Leave the field blank and the processor starts and runs a local Ollama server or specify the address of your own local or remote server.","is_optional":true,"examples":["http://127.0.0.1:11434"]},{"name":"cache_directory","type":"string","kind":"scalar","description":"If `server_address` is not set - the directory to download the ollama binary and use as a model cache.","is_advanced":true,"is_optional":true,"examples":["/opt/cache/connect/ollama"]},{"name":"download_url","type":"string","kind":"scalar","description":"If `server_address` is not set - the URL to download the ollama binary from. Defaults to the offical Ollama GitHub release for this platform.","is_advanced":true,"is_optional":true}]},"version":"4.42.0"},{"name":"openai_chat_completion","type":"processor","status":"experimental","plugin":true,"summary":"Generates responses to messages in a chat conversation, using the OpenAI API.","description":"\nThis processor sends the contents of user prompts to the OpenAI API, which generates responses. By default, the processor submits the entire payload of each message as a string, unless you use the `prompt` configuration field to customize it.\n\nTo learn more about chat completion, see the https://platform.openai.com/docs/guides/chat-completions[OpenAI API documentation^].","categories":["AI"],"examples":[{"title":"Use GPT-4o analyze an image","summary":"This example fetches image URLs from stdin and has GPT-4o describe the image.","config":"\ninput:\n stdin:\n scanner:\n lines: {}\npipeline:\n processors:\n - http:\n verb: GET\n url: \"${!content().string()}\"\n - openai_chat_completion:\n model: gpt-4o\n api_key: TODO\n prompt: \"Describe the following image\"\n image: \"root = content()\"\noutput:\n stdout:\n codec: lines\n"},{"title":"Provide historical chat history","summary":"This pipeline provides a historical chat history to GPT-4o using a cache.","config":"\ninput:\n stdin:\n scanner:\n lines: {}\npipeline:\n processors:\n - mapping: |\n root.prompt = content().string()\n - branch:\n processors:\n - cache:\n resource: mem\n operator: get\n key: history\n - catch:\n - mapping: 'root = []'\n result_map: 'root.history = this'\n - branch:\n processors:\n - openai_chat_completion:\n model: gpt-4o\n api_key: TODO\n prompt: \"${!this.prompt}\"\n history: 'root = this.history'\n result_map: 'root.response = content().string()'\n - mutation: |\n root.history = this.history.concat([\n {\"role\": \"user\", \"content\": this.prompt},\n {\"role\": \"assistant\", \"content\": this.response},\n ])\n - cache:\n resource: mem\n operator: set\n key: history\n value: '${!this.history}'\n - mapping: |\n root = this.response\noutput:\n stdout:\n codec: lines\n\ncache_resources:\n - label: mem \n memory: {}\n"},{"title":"Use GPT-4o to call a tool","summary":"This example asks GPT-4o to respond with the weather by invoking an HTTP processor to get the forecast.","config":"\ninput:\n generate:\n count: 1\n mapping: |\n root = \"What is the weather like in Chicago?\"\npipeline:\n processors:\n - openai_chat_completion:\n model: gpt-4o\n api_key: \"${OPENAI_API_KEY}\"\n prompt: \"${!content().string()}\"\n tools:\n - name: GetWeather\n description: \"Retrieve the weather for a specific city\"\n parameters:\n required: [\"city\"]\n properties:\n city:\n type: string\n description: the city to look up the weather for\n processors:\n - http:\n verb: GET\n url: 'https://wttr.in/${!this.city}?T'\n headers:\n User-Agent: curl/8.11.1 # Returns a text string from the weather website\noutput:\n stdout: {}\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"server_address","type":"string","kind":"scalar","description":"The Open API endpoint that the processor sends requests to. Update the default value to use another OpenAI compatible service.","default":"https://api.openai.com/v1"},{"name":"api_key","type":"string","kind":"scalar","description":"The API key for OpenAI API.","is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"model","type":"string","kind":"scalar","description":"The name of the OpenAI model to use.","examples":["gpt-4o","gpt-4o-mini","gpt-4","gpt4-turbo"]},{"name":"prompt","type":"string","kind":"scalar","description":"The user prompt you want to generate a response for. By default, the processor submits the entire payload as a string.","is_optional":true,"interpolated":true},{"name":"system_prompt","type":"string","kind":"scalar","description":"The system prompt to submit along with the user prompt.","is_optional":true,"interpolated":true},{"name":"history","type":"string","kind":"scalar","description":"The history of the prior conversation. A bloblang query that should result in an array of objects of the form: [{\"role\": \"user\", \"content\": \"\u003ctext\u003e\"}, {\"role\":\"assistant\", \"content\":\"\u003ctext\u003e\"}]","is_optional":true,"bloblang":true},{"name":"image","type":"string","kind":"scalar","description":"An image to send along with the prompt. The mapping result must be a byte array.","is_optional":true,"bloblang":true,"examples":["root = this.image.decode(\"base64\") # decode base64 encoded image"],"version":"4.38.0"},{"name":"max_tokens","type":"int","kind":"scalar","description":"The maximum number of tokens that can be generated in the chat completion.","is_optional":true},{"name":"temperature","type":"float","kind":"scalar","description":"What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic.\n\nWe generally recommend altering this or top_p but not both.","is_optional":true,"linter":"root = if this \u003e 2 || this \u003c 0 { [ \"field must be between 0 and 2\" ] }"},{"name":"user","type":"string","kind":"scalar","description":"A unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse.","is_optional":true,"interpolated":true},{"name":"response_format","type":"string","kind":"scalar","description":"Specify the model's output format. If `json_schema` is specified, then additionally a `json_schema` or `schema_registry` must be configured.","default":"text","options":["text","json","json_schema"],"linter":"\nlet options = {\n \"text\": true,\n \"json\": true,\n \"json_schema\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"json_schema","type":"object","kind":"scalar","description":"The JSON schema to use when responding in `json_schema` format. To learn more about what JSON schema is supported see the https://platform.openai.com/docs/guides/structured-outputs/supported-schemas[OpenAI documentation^].","is_optional":true,"children":[{"name":"name","type":"string","kind":"scalar","description":"The name of the schema."},{"name":"description","type":"string","kind":"scalar","description":"Additional description of the schema for the LLM.","is_advanced":true,"is_optional":true},{"name":"schema","type":"string","kind":"scalar","description":"The JSON schema for the LLM to use when generating the output."}]},{"name":"schema_registry","type":"object","kind":"scalar","description":"The schema registry to dynamically load schemas from when responding in `json_schema` format. Schemas themselves must be in JSON format. To learn more about what JSON schema is supported see the https://platform.openai.com/docs/guides/structured-outputs/supported-schemas[OpenAI documentation^].","is_advanced":true,"is_optional":true,"children":[{"name":"url","type":"string","kind":"scalar","description":"The base URL of the schema registry service.","is_advanced":true,"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"name_prefix","type":"string","kind":"scalar","description":"The prefix of the name for this schema, the schema ID is used as a suffix.","is_advanced":true,"default":"schema_registry_id_"},{"name":"subject","type":"string","kind":"scalar","description":"The subject name to fetch the schema for.","is_advanced":true},{"name":"refresh_interval","type":"string","kind":"scalar","description":"The refresh rate for getting the latest schema. If not specified the schema does not refresh.","is_advanced":true,"is_optional":true},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"oauth","type":"object","kind":"scalar","description":"Allows you to specify open authentication via OAuth version 1.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use OAuth version 1 in requests.","is_advanced":true,"default":false},{"name":"consumer_key","type":"string","kind":"scalar","description":"A value used to identify the client to the service provider.","is_advanced":true,"default":""},{"name":"consumer_secret","type":"string","kind":"scalar","description":"A secret used to establish ownership of the consumer key.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"access_token","type":"string","kind":"scalar","description":"A value used to gain access to the protected resources on behalf of the user.","is_advanced":true,"default":""},{"name":"access_token_secret","type":"string","kind":"scalar","description":"A secret provided in order to establish ownership of a given access token.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"jwt","type":"object","kind":"scalar","description":"BETA: Allows you to specify JWT authentication.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use JWT authentication in requests.","is_advanced":true,"default":false},{"name":"private_key_file","type":"string","kind":"scalar","description":"A file with the PEM encoded via PKCS1 or PKCS8 as private key.","is_advanced":true,"default":""},{"name":"signing_method","type":"string","kind":"scalar","description":"A method used to sign the token such as RS256, RS384, RS512 or EdDSA.","is_advanced":true,"default":""},{"name":"claims","type":"unknown","kind":"map","description":"A value used to identify the claims that issued the JWT.","is_advanced":true,"default":{}},{"name":"headers","type":"unknown","kind":"map","description":"Add optional key/value headers to the JWT.","is_advanced":true,"default":{}}]}]},{"name":"top_p","type":"float","kind":"scalar","description":"An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.\n\nWe generally recommend altering this or temperature but not both.","is_advanced":true,"is_optional":true,"linter":"root = if this \u003e 1 || this \u003c 0 { [ \"field must be between 0 and 1\" ] }"},{"name":"frequency_penalty","type":"float","kind":"scalar","description":"Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim.","is_advanced":true,"is_optional":true,"linter":"root = if this \u003e 2 || this \u003c -2 { [ \"field must be less than 2 and greater than -2\" ] }"},{"name":"presence_penalty","type":"float","kind":"scalar","description":"Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics.","is_advanced":true,"is_optional":true,"linter":"root = if this \u003e 2 || this \u003c -2 { [ \"field must be less than 2 and greater than -2\" ] }"},{"name":"seed","type":"int","kind":"scalar","description":"If specified, our system will make a best effort to sample deterministically, such that repeated requests with the same seed and parameters should return the same result. Determinism is not guaranteed.","is_advanced":true,"is_optional":true},{"name":"stop","type":"string","kind":"array","description":"Up to 4 sequences where the API will stop generating further tokens.","is_advanced":true,"is_optional":true},{"name":"tools","type":"object","kind":"array","description":"The tools to allow the LLM to invoke. This allows building subpipelines that the LLM can choose to invoke to execute agentic-like actions.","children":[{"name":"name","type":"string","kind":"scalar","description":"The name of this tool."},{"name":"description","type":"string","kind":"scalar","description":"A description of this tool, the LLM uses this to decide if the tool should be used."},{"name":"parameters","type":"object","kind":"scalar","description":"The parameters the LLM needs to provide to invoke this tool.","default":[],"children":[{"name":"required","type":"string","kind":"array","description":"The required parameters for this pipeline.","default":[]},{"name":"properties","type":"object","kind":"map","description":"The properties for the processor's input data","children":[{"name":"type","type":"string","kind":"scalar","description":"The type of this parameter."},{"name":"description","type":"string","kind":"scalar","description":"A description of this parameter."},{"name":"enum","type":"string","kind":"array","description":"Specifies that this parameter is an enum and only these specific values should be used.","default":[]}]}]},{"name":"processors","type":"processor","kind":"array","description":"The pipeline to execute when the LLM uses this tool.","is_optional":true}]}],"linter":"\n root = match {\n this.exists(\"json_schema\") \u0026\u0026 this.exists(\"schema_registry\") =\u003e [\"cannot set both `json_schema` and `schema_registry`\"]\n this.response_format == \"json_schema\" \u0026\u0026 !this.exists(\"json_schema\") \u0026\u0026 !this.exists(\"schema_registry\") =\u003e [\"schema must be specified using either `json_schema` or `schema_registry`\"]\n }\n "},"version":"4.32.0"},{"name":"openai_embeddings","type":"processor","status":"experimental","plugin":true,"summary":"Generates vector embeddings to represent input text, using the OpenAI API.","description":"\nThis processor sends text strings to the OpenAI API, which generates vector embeddings. By default, the processor submits the entire payload of each message as a string, unless you use the `text_mapping` configuration field to customize it.\n\nTo learn more about vector embeddings, see the https://platform.openai.com/docs/guides/embeddings[OpenAI API documentation^].","categories":["AI"],"examples":[{"title":"Store embedding vectors in Pinecone","summary":"Compute embeddings for some generated data and store it within xrefs:component:outputs/pinecone.adoc[Pinecone]","config":"input:\n generate:\n interval: 1s\n mapping: |\n root = {\"text\": fake(\"paragraph\")}\npipeline:\n processors:\n - openai_embeddings:\n model: text-embedding-3-large\n api_key: \"${OPENAI_API_KEY}\"\n text_mapping: \"root = this.text\"\noutput:\n pinecone:\n host: \"${PINECONE_HOST}\"\n api_key: \"${PINECONE_API_KEY}\"\n id: \"root = uuid_v4()\"\n vector_mapping: \"root = this\""}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"server_address","type":"string","kind":"scalar","description":"The Open API endpoint that the processor sends requests to. Update the default value to use another OpenAI compatible service.","default":"https://api.openai.com/v1"},{"name":"api_key","type":"string","kind":"scalar","description":"The API key for OpenAI API.","is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"model","type":"string","kind":"scalar","description":"The name of the OpenAI model to use.","examples":["text-embedding-3-large","text-embedding-3-small","text-embedding-ada-002"]},{"name":"text_mapping","type":"string","kind":"scalar","description":"The text you want to generate a vector embedding for. By default, the processor submits the entire payload as a string.","is_optional":true,"bloblang":true},{"name":"dimensions","type":"int","kind":"scalar","description":"The number of dimensions the resulting output embeddings should have. Only supported in `text-embedding-3` and later models.","is_optional":true}]},"version":"4.32.0"},{"name":"openai_image_generation","type":"processor","status":"experimental","plugin":true,"summary":"Generates an image from a text description and other attributes, using OpenAI API.","description":"\nThis processor sends an image description and other attributes, such as image size and quality to the OpenAI API, which generates an image. By default, the processor submits the entire payload of each message as a string, unless you use the `prompt` configuration field to customize it.\n\nTo learn more about image generation, see the https://platform.openai.com/docs/guides/images[OpenAI API documentation^].","categories":["AI"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"server_address","type":"string","kind":"scalar","description":"The Open API endpoint that the processor sends requests to. Update the default value to use another OpenAI compatible service.","default":"https://api.openai.com/v1"},{"name":"api_key","type":"string","kind":"scalar","description":"The API key for OpenAI API.","is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"model","type":"string","kind":"scalar","description":"The name of the OpenAI model to use.","examples":["dall-e-3","dall-e-2"]},{"name":"prompt","type":"string","kind":"scalar","description":"A text description of the image you want to generate. The `prompt` field accepts a maximum of 1000 characters for `dall-e-2` and 4000 characters for `dall-e-3`.","is_optional":true,"bloblang":true},{"name":"quality","type":"string","kind":"scalar","description":"The quality of the image to generate. Use `hd` to create images with finer details and greater consistency across the image. This parameter is only supported for `dall-e-3` models.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["standard","hd"]},{"name":"size","type":"string","kind":"scalar","description":"The size of the generated image. Choose from `256x256`, `512x512`, or `1024x1024` for `dall-e-2`. Choose from `1024x1024`, `1792x1024`, or `1024x1792` for `dall-e-3` models.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["1024x1024","512x512","1792x1024","1024x1792"]},{"name":"style","type":"string","kind":"scalar","description":"The style of the generated image. Choose from `vivid` or `natural`. Vivid causes the model to lean towards generating hyperreal and dramatic images. Natural causes the model to produce more natural, less hyperreal looking images. This parameter is only supported for `dall-e-3`.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["vivid","natural"]}]},"version":"4.32.0"},{"name":"openai_speech","type":"processor","status":"experimental","plugin":true,"summary":"Generates audio from a text description and other attributes, using OpenAI API.","description":"\nThis processor sends a text description and other attributes, such as a voice type and format to the OpenAI API, which generates audio. By default, the processor submits the entire payload of each message as a string, unless you use the `input` configuration field to customize it.\n\nTo learn more about turning text into spoken audio, see the https://platform.openai.com/docs/guides/text-to-speech[OpenAI API documentation^].","categories":["AI"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"server_address","type":"string","kind":"scalar","description":"The Open API endpoint that the processor sends requests to. Update the default value to use another OpenAI compatible service.","default":"https://api.openai.com/v1"},{"name":"api_key","type":"string","kind":"scalar","description":"The API key for OpenAI API.","is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"model","type":"string","kind":"scalar","description":"The name of the OpenAI model to use.","examples":["tts-1","tts-1-hd"]},{"name":"input","type":"string","kind":"scalar","description":"A text description of the audio you want to generate. The `input` field accepts a maximum of 4096 characters.","is_optional":true,"bloblang":true},{"name":"voice","type":"string","kind":"scalar","description":"The type of voice to use when generating the audio.","interpolated":true,"examples":["alloy","echo","fable","onyx","nova","shimmer"]},{"name":"response_format","type":"string","kind":"scalar","description":"The format to generate audio in. Default is `mp3`.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["mp3","opus","aac","flac","wav","pcm"]}]},"version":"4.32.0"},{"name":"openai_transcription","type":"processor","status":"experimental","plugin":true,"summary":"Generates a transcription of spoken audio in the input language, using the OpenAI API.","description":"\nThis processor sends an audio file object along with the input language to OpenAI API to generate a transcription. By default, the processor submits the entire payload of each message as a string, unless you use the `file` configuration field to customize it.\n\nTo learn more about audio transcription, see the: https://platform.openai.com/docs/guides/speech-to-text[OpenAI API documentation^].","categories":["AI"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"server_address","type":"string","kind":"scalar","description":"The Open API endpoint that the processor sends requests to. Update the default value to use another OpenAI compatible service.","default":"https://api.openai.com/v1"},{"name":"api_key","type":"string","kind":"scalar","description":"The API key for OpenAI API.","is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"model","type":"string","kind":"scalar","description":"The name of the OpenAI model to use.","examples":["whisper-1"]},{"name":"file","type":"string","kind":"scalar","description":"The audio file object (not file name) to transcribe, in one of the following formats: `flac`, `mp3`, `mp4`, `mpeg`, `mpga`, `m4a`, `ogg`, `wav`, or `webm`.","bloblang":true},{"name":"language","type":"string","kind":"scalar","description":"The language of the input audio. Supplying the input language in ISO-639-1 format improves accuracy and latency.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["en","fr","de","zh"]},{"name":"prompt","type":"string","kind":"scalar","description":"Optional text to guide the model's style or continue a previous audio segment. The prompt should match the audio language.","is_advanced":true,"is_optional":true,"interpolated":true}]},"version":"4.32.0"},{"name":"openai_translation","type":"processor","status":"experimental","plugin":true,"summary":"Translates spoken audio into English, using the OpenAI API.","description":"\nThis processor sends an audio file object to OpenAI API to generate a translation. By default, the processor submits the entire payload of each message as a string, unless you use the `file` configuration field to customize it.\n\nTo learn more about translation, see the https://platform.openai.com/docs/guides/speech-to-text[OpenAI API documentation^].","categories":["AI"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"server_address","type":"string","kind":"scalar","description":"The Open API endpoint that the processor sends requests to. Update the default value to use another OpenAI compatible service.","default":"https://api.openai.com/v1"},{"name":"api_key","type":"string","kind":"scalar","description":"The API key for OpenAI API.","is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"model","type":"string","kind":"scalar","description":"The name of the OpenAI model to use.","examples":["whisper-1"]},{"name":"file","type":"string","kind":"scalar","description":"The audio file object (not file name) to translate, in one of the following formats: `flac`, `mp3`, `mp4`, `mpeg`, `mpga`, `m4a`, `ogg`, `wav`, or `webm`.","is_optional":true,"bloblang":true},{"name":"prompt","type":"string","kind":"scalar","description":"Optional text to guide the model's style or continue a previous audio segment. The prompt should match the audio language.","is_advanced":true,"is_optional":true,"interpolated":true}]},"version":"4.32.0"},{"name":"parallel","type":"processor","status":"stable","plugin":true,"summary":"A processor that applies a list of child processors to messages of a batch as though they were each a batch of one message (similar to the xref:components:processors/for_each.adoc[`for_each`] processor), but where each message is processed in parallel.","description":"\nThe field `cap`, if greater than zero, caps the maximum number of parallel processing threads.\n\nThe functionality of this processor depends on being applied across messages that are batched. You can find out more about batching in xref:configuration:batching.adoc[].","categories":["Composition"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"cap","type":"int","kind":"scalar","description":"The maximum number of messages to have processing at a given time.","default":0},{"name":"processors","type":"processor","kind":"array","description":"A list of child processors to apply."}]}},{"name":"parquet","type":"processor","status":"deprecated","plugin":true,"summary":"Converts batches of documents to or from https://parquet.apache.org/docs/[Parquet files^].","description":"\n== Alternatives\n\nThis processor is now deprecated, it's recommended that you use the new xref:components:processors/parquet_decode.adoc[`parquet_decode`] and xref:components:processors/parquet_encode.adoc[`parquet_encode`] processors as they provide a number of advantages, the most important of which is better error messages for when schemas are mismatched or files could not be consumed.\n\n== Troubleshooting\n\nThis processor is experimental and the error messages that it provides are often vague and unhelpful. An error message of the form `interface \\{} is nil, not \u003cvalue type\u003e` implies that a field of the given type was expected but not found in the processed message when writing parquet files.\n\nUnfortunately the name of the field will sometimes be missing from the error, in which case it's worth double checking the schema you provided to make sure that there are no typos in the field names, and if that doesn't reveal the issue it can help to mark fields as OPTIONAL in the schema and gradually change them back to REQUIRED until the error returns.\n\n== Define the schema\n\nThe schema must be specified as a JSON string, containing an object that describes the fields expected at the root of each document. Each field can itself have more fields defined, allowing for nested structures:\n\n```json\n{\n \"Tag\": \"name=root, repetitiontype=REQUIRED\",\n \"Fields\": [\n {\"Tag\": \"name=name, inname=NameIn, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=REQUIRED\"},\n {\"Tag\": \"name=age, inname=Age, type=INT32, repetitiontype=REQUIRED\"},\n {\"Tag\": \"name=id, inname=Id, type=INT64, repetitiontype=REQUIRED\"},\n {\"Tag\": \"name=weight, inname=Weight, type=FLOAT, repetitiontype=REQUIRED\"},\n {\n \"Tag\": \"name=favPokemon, inname=FavPokemon, type=LIST, repetitiontype=OPTIONAL\",\n \"Fields\": [\n {\"Tag\": \"name=name, inname=PokeName, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=REQUIRED\"},\n {\"Tag\": \"name=coolness, inname=Coolness, type=FLOAT, repetitiontype=REQUIRED\"}\n ]\n }\n ]\n}\n```\n\nA schema can be derived from a source file using https://github.com/xitongsys/parquet-go/tree/master/tool/parquet-tools:\n\n```sh\n./parquet-tools -cmd schema -file foo.parquet\n```","categories":["Parsing"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"operator","type":"string","kind":"scalar","description":"Determines whether the processor converts messages into a parquet file or expands parquet files into messages. Converting into JSON allows subsequent processors and mappings to convert the data into any other format.","annotated_options":[["from_json","Compress a batch of JSON documents into a file."],["to_json","Expand a file into one or more JSON messages."]],"linter":"\nlet options = {\n \"from_json\": true,\n \"to_json\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"compression","type":"string","kind":"scalar","description":"The type of compression to use when writing parquet files, this field is ignored when consuming parquet files.","default":"snappy","options":["uncompressed","snappy","gzip","lz4","zstd"],"linter":"\nlet options = {\n \"uncompressed\": true,\n \"snappy\": true,\n \"gzip\": true,\n \"lz4\": true,\n \"zstd\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"schema_file","type":"string","kind":"scalar","description":"A file path containing a schema used to describe the parquet files being generated or consumed, the format of the schema is a JSON document detailing the tag and fields of documents. The schema can be found at: https://pkg.go.dev/github.com/xitongsys/parquet-go#readme-json. Either a `schema_file` or `schema` field must be specified when creating Parquet files via the `from_json` operator.","is_optional":true,"examples":["schemas/foo.json"]},{"name":"schema","type":"string","kind":"scalar","description":"A schema used to describe the parquet files being generated or consumed, the format of the schema is a JSON document detailing the tag and fields of documents. The schema can be found at: https://pkg.go.dev/github.com/xitongsys/parquet-go#readme-json. Either a `schema_file` or `schema` field must be specified when creating Parquet files via the `from_json` operator.","is_optional":true,"examples":["{\n \"Tag\": \"name=root, repetitiontype=REQUIRED\",\n \"Fields\": [\n {\"Tag\":\"name=name,inname=NameIn,type=BYTE_ARRAY,convertedtype=UTF8, repetitiontype=REQUIRED\"},\n {\"Tag\":\"name=age,inname=Age,type=INT32,repetitiontype=REQUIRED\"}\n ]\n}"]}],"linter":"\nroot = if this.operator == \"from_json\" \u0026\u0026 (this.schema | this.schema_file | \"\") == \"\" {\n\t\"a schema or schema_file must be specified when the operator is set to from_json\"\n}"},"version":"3.62.0"},{"name":"parquet_decode","type":"processor","status":"experimental","plugin":true,"summary":"Decodes https://parquet.apache.org/docs/[Parquet files^] into a batch of structured messages.","description":"\nThis processor uses https://github.com/parquet-go/parquet-go[https://github.com/parquet-go/parquet-go^], which is itself experimental. Therefore changes could be made into how this processor functions outside of major version releases.","categories":["Parsing"],"examples":[{"title":"Reading Parquet Files from AWS S3","summary":"In this example we consume files from AWS S3 as they're written by listening onto an SQS queue for upload events. We make sure to use the `to_the_end` scanner which means files are read into memory in full, which then allows us to use a `parquet_decode` processor to expand each file into a batch of messages. Finally, we write the data out to local files as newline delimited JSON.","config":"\ninput:\n aws_s3:\n bucket: TODO\n prefix: foos/\n scanner:\n to_the_end: {}\n sqs:\n url: TODO\n processors:\n - parquet_decode: {}\n\noutput:\n file:\n codec: lines\n path: './foos/${! meta(\"s3_key\") }.jsonl'\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"byte_array_as_string","type":"bool","kind":"scalar","description":"Whether to extract BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY values as strings rather than byte slices in all cases. Values with a logical type of UTF8 will automatically be extracted as strings irrespective of this field. Enabling this field makes serializing the data as JSON more intuitive as `[]byte` values are serialized as base64 encoded strings by default.","is_deprecated":true,"default":false},{"name":"handle_logical_types","type":"string","kind":"scalar","description":"Whether to be smart about decoding logical types. In the Parquet format, logical types are stored as one of the standard physical types with some additional metadata describing the logical type. For example, UUIDs are stored in a FIXED_LEN_BYTE_ARRAY physical type, but there is metadata in the schema denoting that it is a UUID. By default, this logical type metadata will be ignored and values will be decoded directly from the physical type, which isn't always desirable. By enabling this option, logical types will be given special treatment and will decode into more useful values. The value for this field specifies a version, i.e. v0, v1... Any given version enables the logical type handling for that version and all versions below it, which allows the handling of new logical types to be introduced without breaking existing pipelines. We recommend enabling the newest version available of this feature when creating new pipelines.","default":"v1","examples":["v2"],"annotated_options":[["v1","No special handling of logical types"],["v2","\n- TIMESTAMP - decodes as an RFC3339 string describing the time. If the `isAdjustedToUTC` flag is set to true in the parquet file, the time zone will be set to UTC. If it is set to false the time zone will be set to local time.\n- UUID - decodes as a string, i.e. `00112233-4455-6677-8899-aabbccddeeff`."]],"linter":"\nlet options = {\n \"v1\": true,\n \"v2\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"}]},"version":"4.4.0"},{"name":"parquet_encode","type":"processor","status":"experimental","plugin":true,"summary":"Encodes https://parquet.apache.org/docs/[Parquet files^] from a batch of structured messages.","description":"\nThis processor uses https://github.com/parquet-go/parquet-go[https://github.com/parquet-go/parquet-go^], which is itself experimental. Therefore changes could be made into how this processor functions outside of major version releases.\n","categories":["Parsing"],"examples":[{"title":"Writing Parquet Files to AWS S3","summary":"In this example we use the batching mechanism of an `aws_s3` output to collect a batch of messages in memory, which then converts it to a parquet file and uploads it.","config":"\noutput:\n aws_s3:\n bucket: TODO\n path: 'stuff/${! timestamp_unix() }-${! uuid_v4() }.parquet'\n batching:\n count: 1000\n period: 10s\n processors:\n - parquet_encode:\n schema:\n - name: id\n type: INT64\n - name: weight\n type: DOUBLE\n - name: content\n type: BYTE_ARRAY\n default_compression: zstd\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"schema","type":"object","kind":"array","description":"Parquet schema.","children":[{"name":"name","type":"string","kind":"scalar","description":"The name of the column."},{"name":"type","type":"string","kind":"scalar","description":"The type of the column, only applicable for leaf columns with no child fields. Some logical types can be specified here such as UTF8.","is_optional":true,"options":["BOOLEAN","INT32","INT64","FLOAT","DOUBLE","BYTE_ARRAY","UTF8","TIMESTAMP","BSON","ENUM","JSON","UUID"],"linter":"\nlet options = {\n \"boolean\": true,\n \"int32\": true,\n \"int64\": true,\n \"float\": true,\n \"double\": true,\n \"byte_array\": true,\n \"utf8\": true,\n \"timestamp\": true,\n \"bson\": true,\n \"enum\": true,\n \"json\": true,\n \"uuid\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"repeated","type":"bool","kind":"scalar","description":"Whether the field is repeated.","default":false},{"name":"optional","type":"bool","kind":"scalar","description":"Whether the field is optional.","default":false},{"name":"fields","type":"unknown","kind":"array","description":"A list of child fields.","is_optional":true,"examples":[[{"name":"foo","type":"INT64"},{"name":"bar","type":"BYTE_ARRAY"}]]}]},{"name":"default_compression","type":"string","kind":"scalar","description":"The default compression type to use for fields.","default":"uncompressed","options":["uncompressed","snappy","gzip","brotli","zstd","lz4raw"],"linter":"\nlet options = {\n \"uncompressed\": true,\n \"snappy\": true,\n \"gzip\": true,\n \"brotli\": true,\n \"zstd\": true,\n \"lz4raw\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"default_encoding","type":"string","kind":"scalar","description":"The default encoding type to use for fields. A custom default encoding is only necessary when consuming data with libraries that do not support `DELTA_LENGTH_BYTE_ARRAY` and is therefore best left unset where possible.","is_advanced":true,"default":"DELTA_LENGTH_BYTE_ARRAY","options":["DELTA_LENGTH_BYTE_ARRAY","PLAIN"],"version":"4.11.0","linter":"\nlet options = {\n \"delta_length_byte_array\": true,\n \"plain\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"}]},"version":"4.4.0"},{"name":"parse_log","type":"processor","status":"stable","plugin":true,"summary":"Parses common log \u003c\u003cformats\u003e\u003e into \u003c\u003ccodecs, structured data\u003e\u003e. This is easier and often much faster than xref:components:processors/grok.adoc[`grok`].","categories":["Parsing"],"footnotes":"\n== Codecs\n\nCurrently the only supported structured data codec is `json`.\n\n== Formats\n\n=== `syslog_rfc5424`\n\nAttempts to parse a log following the https://tools.ietf.org/html/rfc5424[Syslog RFC5424^] spec. The resulting structured document may contain any of the following fields:\n\n- `message` (string)\n- `timestamp` (string, RFC3339)\n- `facility` (int)\n- `severity` (int)\n- `priority` (int)\n- `version` (int)\n- `hostname` (string)\n- `procid` (string)\n- `appname` (string)\n- `msgid` (string)\n- `structureddata` (object)\n\n=== `syslog_rfc3164`\n\nAttempts to parse a log following the https://tools.ietf.org/html/rfc3164[Syslog rfc3164] spec. The resulting structured document may contain any of the following fields:\n\n- `message` (string)\n- `timestamp` (string, RFC3339)\n- `facility` (int)\n- `severity` (int)\n- `priority` (int)\n- `hostname` (string)\n- `procid` (string)\n- `appname` (string)\n- `msgid` (string)\n","config":{"name":"","type":"object","kind":"scalar","children":[{"name":"format","type":"string","kind":"scalar","description":"A common log \u003c\u003cformats, format\u003e\u003e to parse.","options":["syslog_rfc5424","syslog_rfc3164"],"linter":"\nlet options = {\n \"syslog_rfc5424\": true,\n \"syslog_rfc3164\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"best_effort","type":"bool","kind":"scalar","description":"Still returns partially parsed messages even if an error occurs.","is_advanced":true,"default":true},{"name":"allow_rfc3339","type":"bool","kind":"scalar","description":"Also accept timestamps in rfc3339 format while parsing. Applicable to format `syslog_rfc3164`.","is_advanced":true,"default":true},{"name":"default_year","type":"string","kind":"scalar","description":"Sets the strategy used to set the year for rfc3164 timestamps. Applicable to format `syslog_rfc3164`. When set to `current` the current year will be set, when set to an integer that value will be used. Leave this field empty to not set a default year at all.","is_advanced":true,"default":"current"},{"name":"default_timezone","type":"string","kind":"scalar","description":"Sets the strategy to decide the timezone for rfc3164 timestamps. Applicable to format `syslog_rfc3164`. This value should follow the https://golang.org/pkg/time/#LoadLocation[time.LoadLocation^] format.","is_advanced":true,"default":"UTC"},{"name":"codec","type":"string","kind":"scalar","is_deprecated":true}]}},{"name":"processors","type":"processor","status":"stable","plugin":true,"summary":"A processor grouping several sub-processors.","description":"This processor is useful in situations where you want to collect several processors under a single resource identifier, whether it is for making your configuration easier to read and navigate, or for improving the testability of your configuration. The behavior of child processors will match exactly the behavior they would have under any other processors block.","categories":["Composition"],"examples":[{"title":"Grouped Processing","summary":"Imagine we have a collection of processors who cover a specific functionality. We could use this processor to group them together and make it easier to read and mock during testing by giving the whole block a label:","config":"\npipeline:\n processors:\n - label: my_super_feature\n processors:\n - log:\n message: \"Let's do something cool\"\n - archive:\n format: json_array\n - mapping: root.items = this\n"}],"config":{"name":"","type":"processor","kind":"array","default":[]}},{"name":"protobuf","type":"processor","status":"stable","plugin":true,"summary":"\nPerforms conversions to or from a protobuf message. This processor uses reflection, meaning conversions can be made directly from the target .proto files.\n","description":"\nThe main functionality of this processor is to map to and from JSON documents, you can read more about JSON mapping of protobuf messages here: https://developers.google.com/protocol-buffers/docs/proto3#json[https://developers.google.com/protocol-buffers/docs/proto3#json^]\n\nUsing reflection for processing protobuf messages in this way is less performant than generating and using native code. Therefore when performance is critical it is recommended that you use Redpanda Connect plugins instead for processing protobuf messages natively, you can find an example of Redpanda Connect plugins at https://github.com/benthosdev/benthos-plugin-example[https://github.com/benthosdev/benthos-plugin-example^]\n\n== Operators\n\n=== `to_json`\n\nConverts protobuf messages into a generic JSON structure. This makes it easier to manipulate the contents of the document within Benthos.\n\n=== `from_json`\n\nAttempts to create a target protobuf message from a generic JSON structure.\n","categories":["Parsing"],"examples":[{"title":"JSON to Protobuf","summary":"\nIf we have the following protobuf definition within a directory called `testing/schema`:\n\n```protobuf\nsyntax = \"proto3\";\npackage testing;\n\nimport \"google/protobuf/timestamp.proto\";\n\nmessage Person {\n string first_name = 1;\n string last_name = 2;\n string full_name = 3;\n int32 age = 4;\n int32 id = 5; // Unique ID number for this person.\n string email = 6;\n\n google.protobuf.Timestamp last_updated = 7;\n}\n```\n\nAnd a stream of JSON documents of the form:\n\n```json\n{\n\t\"firstName\": \"caleb\",\n\t\"lastName\": \"quaye\",\n\t\"email\": \"caleb@myspace.com\"\n}\n```\n\nWe can convert the documents into protobuf messages with the following config:","config":"\npipeline:\n processors:\n - protobuf:\n operator: from_json\n message: testing.Person\n import_paths: [ testing/schema ]\n"},{"title":"Protobuf to JSON","summary":"\nIf we have the following protobuf definition within a directory called `testing/schema`:\n\n```protobuf\nsyntax = \"proto3\";\npackage testing;\n\nimport \"google/protobuf/timestamp.proto\";\n\nmessage Person {\n string first_name = 1;\n string last_name = 2;\n string full_name = 3;\n int32 age = 4;\n int32 id = 5; // Unique ID number for this person.\n string email = 6;\n\n google.protobuf.Timestamp last_updated = 7;\n}\n```\n\nAnd a stream of protobuf messages of the type `Person`, we could convert them into JSON documents of the format:\n\n```json\n{\n\t\"firstName\": \"caleb\",\n\t\"lastName\": \"quaye\",\n\t\"email\": \"caleb@myspace.com\"\n}\n```\n\nWith the following config:","config":"\npipeline:\n processors:\n - protobuf:\n operator: to_json\n message: testing.Person\n import_paths: [ testing/schema ]\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"operator","type":"string","kind":"scalar","description":"The \u003c\u003coperators, operator\u003e\u003e to execute","options":["to_json","from_json"],"linter":"\nlet options = {\n \"to_json\": true,\n \"from_json\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"message","type":"string","kind":"scalar","description":"The fully qualified name of the protobuf message to convert to/from."},{"name":"discard_unknown","type":"bool","kind":"scalar","description":"If `true`, the `from_json` operator discards fields that are unknown to the schema.","default":false},{"name":"use_proto_names","type":"bool","kind":"scalar","description":"If `true`, the `to_json` operator deserializes fields exactly as named in schema file.","default":false},{"name":"import_paths","type":"string","kind":"array","description":"A list of directories containing .proto files, including all definitions required for parsing the target message. If left empty the current directory is used. Each directory listed will be walked with all found .proto files imported.","default":[]},{"name":"use_enum_numbers","type":"bool","kind":"scalar","description":"If `true`, the `to_json` operator deserializes enums as numerical values instead of string names.","default":false}]}},{"name":"rate_limit","type":"processor","status":"stable","plugin":true,"summary":"Throttles the throughput of a pipeline according to a specified xref:components:rate_limits/about.adoc[`rate_limit`] resource. Rate limits are shared across components and therefore apply globally to all processing pipelines.","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"resource","type":"string","kind":"scalar","description":"The target xref:components:rate_limits/about.adoc[`rate_limit` resource]."}]}},{"name":"redis","type":"processor","status":"stable","plugin":true,"summary":"Performs actions against Redis that aren't possible using a xref:components:processors/cache.adoc[`cache`] processor. Actions are\nperformed for each message and the message contents are replaced with the result. In order to merge the result into the original message compose this processor within a xref:components:processors/branch.adoc[`branch` processor].","categories":["Integration"],"examples":[{"title":"Querying Cardinality","summary":"If given payloads containing a metadata field `set_key` it's possible to query and store the cardinality of the set for each message using a xref:components:processors/branch.adoc[`branch` processor] in order to augment rather than replace the message contents:","config":"\npipeline:\n processors:\n - branch:\n processors:\n - redis:\n url: TODO\n command: scard\n args_mapping: 'root = [ meta(\"set_key\") ]'\n result_map: 'root.cardinality = this'\n"},{"title":"Running Total","summary":"If we have JSON data containing number of friends visited during covid 19:\n\n```json\n{\"name\":\"ash\",\"month\":\"feb\",\"year\":2019,\"friends_visited\":10}\n{\"name\":\"ash\",\"month\":\"apr\",\"year\":2019,\"friends_visited\":-2}\n{\"name\":\"bob\",\"month\":\"feb\",\"year\":2019,\"friends_visited\":3}\n{\"name\":\"bob\",\"month\":\"apr\",\"year\":2019,\"friends_visited\":1}\n```\n\nWe can add a field that contains the running total number of friends visited:\n\n```json\n{\"name\":\"ash\",\"month\":\"feb\",\"year\":2019,\"friends_visited\":10,\"total\":10}\n{\"name\":\"ash\",\"month\":\"apr\",\"year\":2019,\"friends_visited\":-2,\"total\":8}\n{\"name\":\"bob\",\"month\":\"feb\",\"year\":2019,\"friends_visited\":3,\"total\":3}\n{\"name\":\"bob\",\"month\":\"apr\",\"year\":2019,\"friends_visited\":1,\"total\":4}\n```\n\nUsing the `incrby` command:","config":"\npipeline:\n processors:\n - branch:\n processors:\n - redis:\n url: TODO\n command: incrby\n args_mapping: 'root = [ this.name, this.friends_visited ]'\n result_map: 'root.total = this'\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target Redis server. Database is optional and is supplied as the URL path.","examples":["redis://:6379","redis://localhost:6379","redis://foousername:foopassword@redisplace:6379","redis://:foopassword@redisplace:6379","redis://localhost:6379/1","redis://localhost:6379/1,redis://localhost:6380/1"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"kind","type":"string","kind":"scalar","description":"Specifies a simple, cluster-aware, or failover-aware redis client.","is_advanced":true,"default":"simple","options":["simple","cluster","failover"],"linter":"\nlet options = {\n \"simple\": true,\n \"cluster\": true,\n \"failover\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"master","type":"string","kind":"scalar","description":"Name of the redis master when `kind` is `failover`","is_advanced":true,"default":"","examples":["mymaster"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"command","type":"string","kind":"scalar","description":"The command to execute.","is_optional":true,"interpolated":true,"examples":["scard","incrby","${! meta(\"command\") }"],"version":"4.3.0"},{"name":"args_mapping","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of arguments required for the specified Redis command.","is_optional":true,"bloblang":true,"examples":["root = [ this.key ]","root = [ meta(\"kafka_key\"), this.count ]"],"version":"4.3.0"},{"name":"operator","type":"string","kind":"scalar","description":"The operator to apply.","is_deprecated":true,"is_optional":true,"annotated_options":[["incrby","Increments the number stored at `key` by the message content. If the key does not exist, it is set to `0` before performing the operation. Returns the value of `key` after the increment."],["keys","Returns an array of strings containing all the keys that match the pattern specified by the `key` field."],["sadd","Adds a new member to a set. Returns `1` if the member was added."],["scard","Returns the cardinality of a set, or `0` if the key does not exist."]],"linter":"\nlet options = {\n \"incrby\": true,\n \"keys\": true,\n \"sadd\": true,\n \"scard\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"key","type":"string","kind":"scalar","description":"A key to use for the target operator.","is_deprecated":true,"is_optional":true,"interpolated":true},{"name":"retries","type":"int","kind":"scalar","description":"The maximum number of retries before abandoning a request.","is_advanced":true,"default":3},{"name":"retry_period","type":"string","kind":"scalar","description":"The time to wait before consecutive retry attempts.","is_advanced":true,"default":"500ms"}],"linter":"root = match {\n this.exists(\"operator\") == this.exists(\"command\") =\u003e [ \"one of 'operator' (old style) or 'command' (new style) fields must be specified\" ]\n this.exists(\"args_mapping\") \u0026\u0026 this.exists(\"operator\") =\u003e [ \"field args_mapping is invalid with an operator set\" ],\n}"}},{"name":"redis_script","type":"processor","status":"beta","plugin":true,"summary":"Performs actions against Redis using https://redis.io/docs/manual/programmability/eval-intro/[LUA scripts^].","description":"Actions are performed for each message and the message contents are replaced with the result.\n\nIn order to merge the result into the original message compose this processor within a xref:components:processors/branch.adoc[`branch` processor].","categories":["Integration"],"examples":[{"title":"Running a script","summary":"The following example will use a script execution to get next element from a sorted set and set its score with timestamp unix nano value.","config":"\npipeline:\n processors:\n - redis_script:\n url: TODO\n script: |\n local value = redis.call(\"ZRANGE\", KEYS[1], '0', '0')\n\n if next(elements) == nil then\n return ''\n end\n\n redis.call(\"ZADD\", \"XX\", KEYS[1], ARGV[1], value)\n\n return value\n keys_mapping: 'root = [ meta(\"key\") ]'\n args_mapping: 'root = [ timestamp_unix_nano() ]'\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target Redis server. Database is optional and is supplied as the URL path.","examples":["redis://:6379","redis://localhost:6379","redis://foousername:foopassword@redisplace:6379","redis://:foopassword@redisplace:6379","redis://localhost:6379/1","redis://localhost:6379/1,redis://localhost:6380/1"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"kind","type":"string","kind":"scalar","description":"Specifies a simple, cluster-aware, or failover-aware redis client.","is_advanced":true,"default":"simple","options":["simple","cluster","failover"],"linter":"\nlet options = {\n \"simple\": true,\n \"cluster\": true,\n \"failover\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"master","type":"string","kind":"scalar","description":"Name of the redis master when `kind` is `failover`","is_advanced":true,"default":"","examples":["mymaster"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"script","type":"string","kind":"scalar","description":"A script to use for the target operator. It has precedence over the 'command' field.","examples":["return redis.call('set', KEYS[1], ARGV[1])"]},{"name":"args_mapping","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of arguments required for the specified Redis script.","bloblang":true,"examples":["root = [ this.key ]","root = [ meta(\"kafka_key\"), \"hardcoded_value\" ]"]},{"name":"keys_mapping","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of keys matching in size to the number of arguments required for the specified Redis script.","bloblang":true,"examples":["root = [ this.key ]","root = [ meta(\"kafka_key\"), this.count ]"]},{"name":"retries","type":"int","kind":"scalar","description":"The maximum number of retries before abandoning a request.","is_advanced":true,"default":3},{"name":"retry_period","type":"string","kind":"scalar","description":"The time to wait before consecutive retry attempts.","is_advanced":true,"default":"500ms"}]},"version":"4.11.0"},{"name":"redpanda_data_transform","type":"processor","status":"experimental","plugin":true,"summary":"Executes a Redpanda Data Transform as a processor","description":"\nThis processor executes a Redpanda Data Transform WebAssembly module, calling OnRecordWritten for each message being processed.\n\nYou can find out about how transforms work here: https://docs.redpanda.com/current/develop/data-transforms/how-transforms-work/[https://docs.redpanda.com/current/develop/data-transforms/how-transforms-work/^]\n","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"module_path","type":"string","kind":"scalar","description":"The path of the target WASM module to execute."},{"name":"input_key","type":"string","kind":"scalar","description":"An optional key to populate for each message.","is_optional":true,"interpolated":true},{"name":"output_key","type":"string","kind":"scalar","description":"An optional name of metadata for an output message key.","is_optional":true},{"name":"input_headers","type":"object","kind":"scalar","description":"Determine which (if any) metadata values should be added to messages as headers.","is_optional":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"output_metadata","type":"object","kind":"scalar","description":"Determine which (if any) message headers should be added to the output as metadata.","is_optional":true,"children":[{"name":"include_prefixes","type":"string","kind":"array","description":"Provide a list of explicit metadata key prefixes to match against.","default":[],"examples":[["foo_","bar_"],["kafka_"],["content-"]]},{"name":"include_patterns","type":"string","kind":"array","description":"Provide a list of explicit metadata key regular expression (re2) patterns to match against.","default":[],"examples":[[".*"],["_timestamp_unix$"]]}]},{"name":"timestamp","type":"string","kind":"scalar","description":"An optional timestamp to set for each message. When left empty, the current timestamp is used.","is_advanced":true,"is_optional":true,"interpolated":true,"examples":["${! timestamp_unix() }","${! metadata(\"kafka_timestamp_ms\") }"]},{"name":"timeout","type":"string","kind":"scalar","description":"The maximum period of time for a message to be processed","is_advanced":true,"default":"10s"},{"name":"max_memory_pages","type":"int","kind":"scalar","description":"The maximum amount of wasm memory pages (64KiB) that an individual wasm module instance can use","is_advanced":true,"default":1600}]},"version":"4.31.0"},{"name":"resource","type":"processor","status":"stable","plugin":true,"summary":"Resource is a processor type that runs a processor resource identified by its label.","description":"\nThis processor allows you to reference the same configured processor resource in multiple places, and can also tidy up large nested configs. For example, the config:\n\n```yaml\npipeline:\n processors:\n - mapping: |\n root.message = this\n root.meta.link_count = this.links.length()\n root.user.age = this.user.age.number()\n```\n\nIs equivalent to:\n\n```yaml\npipeline:\n processors:\n - resource: foo_proc\n\nprocessor_resources:\n - label: foo_proc\n mapping: |\n root.message = this\n root.meta.link_count = this.links.length()\n root.user.age = this.user.age.number()\n```\n\nYou can find out more about resources in xref:configuration:resources.adoc[]","categories":["Utility"],"config":{"name":"","type":"string","kind":"scalar","default":""}},{"name":"retry","type":"processor","status":"beta","plugin":true,"summary":"Attempts to execute a series of child processors until success.","description":"\nExecutes child processors and if a resulting message is errored then, after a specified backoff period, the same original message will be attempted again through those same processors. If the child processors result in more than one message then the retry mechanism will kick in if _any_ of the resulting messages are errored.\n\nIt is important to note that any mutations performed on the message during these child processors will be discarded for the next retry, and therefore it is safe to assume that each execution of the child processors will always be performed on the data as it was when it first reached the retry processor.\n\nBy default the retry backoff has a specified \u003c\u003cbackoffmax_elapsed_time,`max_elapsed_time`\u003e\u003e, if this time period is reached during retries and an error still occurs these errored messages will proceed through to the next processor after the retry (or your outputs). Normal xref:configuration:error_handling.adoc[error handling patterns] can be used on these messages.\n\nIn order to avoid permanent loops any error associated with messages as they first enter a retry processor will be cleared.\n\n== Metadata\n\nThis processor adds the following metadata fields to each message:\n\n```text\n- retry_count - The number of retry attempts.\n- backoff_duration - The total time elapsed while performing retries.\n```\n\n[CAUTION]\n.Batching\n====\nIf you wish to wrap a batch-aware series of processors then take a look at the \u003c\u003cbatching, batching section\u003e\u003e.\n====\n","categories":["Composition"],"footnotes":"\n== Batching\n\nWhen messages are batched the child processors of a retry are executed for each individual message in isolation, performed serially by default but in parallel when the field \u003c\u003cparallel, `parallel`\u003e\u003e is set to `true`. This is an intentional limitation of the retry processor and is done in order to ensure that errors are correctly associated with a given input message. Otherwise, the archiving, expansion, grouping, filtering and so on of the child processors could obfuscate this relationship.\n\nIf the target behavior of your retried processors is \"batch aware\", in that you wish to perform some processing across the entire batch of messages and repeat it in the event of errors, you can use an xref:components:processors/archive.adoc[`archive` processor] to collapse the batch into an individual message. Then, within these child processors either perform your batch aware processing on the archive, or use an xref:components:processors/unarchive.adoc[`unarchive` processor] in order to expand the single message back out into a batch.\n\nFor example, if the retry processor were being used to wrap an HTTP request where the payload data is a batch archived into a JSON array it should look something like this:\n\n```yaml\npipeline:\n processors:\n - archive:\n format: json_array\n - retry:\n processors:\n - http:\n url: example.com/nope\n verb: POST\n - unarchive:\n format: json_array\n```\n","examples":[{"title":"Stop ignoring me Taz","summary":"\nHere we have a config where I generate animal noises and send them to Taz via HTTP. Taz has a tendency to stop his servers whenever I dispatch my animals upon him, and therefore these HTTP requests sometimes fail. However, I have the retry processor and with this super power I can specify a back off policy and it will ensure that for each animal noise the HTTP processor is attempted until either it succeeds or my Redpanda Connect instance is stopped.\n\nI even go as far as to zero-out the maximum elapsed time field, which means that for each animal noise I will wait indefinitely, because I really really want Taz to receive every single animal noise that he is entitled to.","config":"\ninput:\n generate:\n interval: 1s\n mapping: 'root.noise = [ \"woof\", \"meow\", \"moo\", \"quack\" ].index(random_int(min: 0, max: 3))'\n\npipeline:\n processors:\n - retry:\n backoff:\n initial_interval: 100ms\n max_interval: 5s\n max_elapsed_time: 0s\n processors:\n - http:\n url: 'http://example.com/try/not/to/dox/taz'\n verb: POST\n\noutput:\n # Drop everything because it's junk data, I don't want it lol\n drop: {}\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"backoff","type":"object","kind":"scalar","description":"Determine time intervals and cut offs for retry attempts.","children":[{"name":"initial_interval","type":"string","kind":"scalar","description":"The initial period to wait between retry attempts.","default":"500ms","examples":["50ms","1s"]},{"name":"max_interval","type":"string","kind":"scalar","description":"The maximum period to wait between retry attempts","default":"10s","examples":["5s","1m"]},{"name":"max_elapsed_time","type":"string","kind":"scalar","description":"The maximum overall period of time to spend on retry attempts before the request is aborted. Setting this value to a zeroed duration (such as `0s`) will result in unbounded retries.","default":"1m","examples":["1m","1h"]}]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to execute on each message."},{"name":"parallel","type":"bool","kind":"scalar","description":"When processing batches of messages these batches are ignored and the processors apply to each message sequentially. However, when this field is set to `true` each message will be processed in parallel. Caution should be made to ensure that batch sizes do not surpass a point where this would cause resource (CPU, memory, API limits) contention.","default":false},{"name":"max_retries","type":"int","kind":"scalar","description":"The maximum number of retry attempts before the request is aborted. Setting this value to `0` will result in unbounded number of retries.","default":0}]},"version":"4.27.0"},{"name":"schema_registry_decode","type":"processor","status":"beta","plugin":true,"summary":"Automatically decodes and validates messages with schemas from a Confluent Schema Registry service.","description":"\nDecodes messages automatically from a schema stored within a https://docs.confluent.io/platform/current/schema-registry/index.html[Confluent Schema Registry service^] by extracting a schema ID from the message and obtaining the associated schema from the registry. If a message fails to match against the schema then it will remain unchanged and the error can be caught using xref:configuration:error_handling.adoc[error handling methods].\n\nAvro, Protobuf and Json schemas are supported, all are capable of expanding from schema references as of v4.22.0.\n\n== Avro JSON format\n\nThis processor creates documents formatted as https://avro.apache.org/docs/current/specification/_print/#json-encoding[Avro JSON^] when decoding with Avro schemas. In this format the value of a union is encoded in JSON as follows:\n\n- if its type is `null`, then it is encoded as a JSON `null`;\n- otherwise it is encoded as a JSON object with one name/value pair whose name is the type's name and whose value is the recursively encoded value. For Avro's named types (record, fixed or enum) the user-specified name is used, for other types the type name is used.\n\nFor example, the union schema `[\"null\",\"string\",\"Foo\"]`, where `Foo` is a record name, would encode:\n\n- `null` as `null`;\n- the string `\"a\"` as `{\"string\": \"a\"}`; and\n- a `Foo` instance as `{\"Foo\": {...}}`, where `{...}` indicates the JSON encoding of a `Foo` instance.\n\nHowever, it is possible to instead create documents in https://pkg.go.dev/github.com/linkedin/goavro/v2#NewCodecForStandardJSONFull[standard/raw JSON format^] by setting the field \u003c\u003cavro_raw_json, `avro_raw_json`\u003e\u003e to `true`.\n\n== Protobuf format\n\nThis processor decodes protobuf messages to JSON documents, you can read more about JSON mapping of protobuf messages here: https://developers.google.com/protocol-buffers/docs/proto3#json\n\n== Metadata\n\nThis processor also adds the following metadata to each outgoing message:\n\nschema_id: the ID of the schema in the schema registry that was associated with the message.\n","categories":["Parsing","Integration"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"avro_raw_json","type":"bool","kind":"scalar","description":"Whether Avro messages should be decoded into normal JSON (\"json that meets the expectations of regular internet json\") rather than https://avro.apache.org/docs/current/specification/_print/#json-encoding[Avro JSON^]. If `true` the schema returned from the subject should be decoded as https://pkg.go.dev/github.com/linkedin/goavro/v2#NewCodecForStandardJSONFull[standard json^] instead of as https://pkg.go.dev/github.com/linkedin/goavro/v2#NewCodec[avro json^]. There is a https://github.com/linkedin/goavro/blob/5ec5a5ee7ec82e16e6e2b438d610e1cab2588393/union.go#L224-L249[comment in goavro^], the https://github.com/linkedin/goavro[underlining library used for avro serialization^], that explains in more detail the difference between the standard json and avro json.","is_advanced":true,"is_deprecated":true,"default":false},{"name":"avro","type":"object","kind":"scalar","description":"Configuration for how to decode schemas that are of type AVRO.","children":[{"name":"raw_unions","type":"bool","kind":"scalar","description":"Whether avro messages should be decoded into normal JSON (\"json that meets the expectations of regular internet json\") rather than https://avro.apache.org/docs/current/specification/_print/#json-encoding[JSON as specified in the Avro Spec^].\n\nFor example, if there is a union schema `[\"null\", \"string\", \"Foo\"]` where `Foo` is a record name, with raw_unions as false (the default) you get:\n- `null` as `null`;\n- the string `\"a\"` as `{\"string\": \"a\"}`; and\n- a `Foo` instance as `{\"Foo\": {...}}`, where `{...}` indicates the JSON encoding of a `Foo` instance.\n\nWhen raw_unions is set to true then the above union schema is decoded as the following:\n- `null` as `null`;\n- the string `\"a\"` as `\"a\"`; and\n- a `Foo` instance as `{...}`, where `{...}` indicates the JSON encoding of a `Foo` instance.\n","is_optional":true},{"name":"preserve_logical_types","type":"bool","kind":"scalar","description":"Whether logical types should be preserved or transformed back into their primitive type. By default, decimals are decoded as raw bytes and timestamps are decoded as plain integers. Setting this field to true keeps decimal types as numbers in bloblang and timestamps as time values.","default":false},{"name":"mapping","type":"string","kind":"scalar","description":"A custom mapping to apply to Avro schemas JSON representation. This is useful to transform custom types emitted by other tools into standard avro.","is_advanced":true,"is_optional":true,"bloblang":true,"examples":["\nmap isDebeziumTimestampType {\n root = this.type == \"long\" \u0026\u0026 this.\"connect.name\" == \"io.debezium.time.Timestamp\" \u0026\u0026 !this.exists(\"logicalType\")\n}\nmap debeziumTimestampToAvroTimestamp {\n let mapped_fields = this.fields.or([]).map_each(item -\u003e item.apply(\"debeziumTimestampToAvroTimestamp\"))\n root = match {\n this.type == \"record\" =\u003e this.assign({\"fields\": $mapped_fields})\n this.type.type() == \"array\" =\u003e this.assign({\"type\": this.type.map_each(item -\u003e item.apply(\"debeziumTimestampToAvroTimestamp\"))})\n # Add a logical type so that it's decoded as a timestamp instead of a long.\n this.type.type() == \"object\" \u0026\u0026 this.type.apply(\"isDebeziumTimestampType\") =\u003e this.merge({\"type\":{\"logicalType\": \"timestamp-millis\"}})\n _ =\u003e this\n }\n}\nroot = this.apply(\"debeziumTimestampToAvroTimestamp\")\n"]}]},{"name":"url","type":"string","kind":"scalar","description":"The base URL of the schema registry service.","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"oauth","type":"object","kind":"scalar","description":"Allows you to specify open authentication via OAuth version 1.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use OAuth version 1 in requests.","is_advanced":true,"default":false},{"name":"consumer_key","type":"string","kind":"scalar","description":"A value used to identify the client to the service provider.","is_advanced":true,"default":""},{"name":"consumer_secret","type":"string","kind":"scalar","description":"A secret used to establish ownership of the consumer key.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"access_token","type":"string","kind":"scalar","description":"A value used to gain access to the protected resources on behalf of the user.","is_advanced":true,"default":""},{"name":"access_token_secret","type":"string","kind":"scalar","description":"A secret provided in order to establish ownership of a given access token.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}],"version":"4.7.0"},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}],"version":"4.7.0"},{"name":"jwt","type":"object","kind":"scalar","description":"BETA: Allows you to specify JWT authentication.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use JWT authentication in requests.","is_advanced":true,"default":false},{"name":"private_key_file","type":"string","kind":"scalar","description":"A file with the PEM encoded via PKCS1 or PKCS8 as private key.","is_advanced":true,"default":""},{"name":"signing_method","type":"string","kind":"scalar","description":"A method used to sign the token such as RS256, RS384, RS512 or EdDSA.","is_advanced":true,"default":""},{"name":"claims","type":"unknown","kind":"map","description":"A value used to identify the claims that issued the JWT.","is_advanced":true,"default":{}},{"name":"headers","type":"unknown","kind":"map","description":"Add optional key/value headers to the JWT.","is_advanced":true,"default":{}}],"version":"4.7.0"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]}]}},{"name":"schema_registry_encode","type":"processor","status":"beta","plugin":true,"summary":"Automatically encodes and validates messages with schemas from a Confluent Schema Registry service.","description":"\nEncodes messages automatically from schemas obtains from a https://docs.confluent.io/platform/current/schema-registry/index.html[Confluent Schema Registry service^] by polling the service for the latest schema version for target subjects.\n\nIf a message fails to encode under the schema then it will remain unchanged and the error can be caught using xref:configuration:error_handling.adoc[error handling methods].\n\nAvro, Protobuf and Json schemas are supported, all are capable of expanding from schema references as of v4.22.0.\n\n== Avro JSON format\n\nBy default this processor expects documents formatted as https://avro.apache.org/docs/current/specification/_print/#json-encoding[Avro JSON^] when encoding with Avro schemas. In this format the value of a union is encoded in JSON as follows:\n\n- if its type is `null`, then it is encoded as a JSON `null`;\n- otherwise it is encoded as a JSON object with one name/value pair whose name is the type's name and whose value is the recursively encoded value. For Avro's named types (record, fixed or enum) the user-specified name is used, for other types the type name is used.\n\nFor example, the union schema `[\"null\",\"string\",\"Foo\"]`, where `Foo` is a record name, would encode:\n\n- `null` as `null`;\n- the string `\"a\"` as `\\{\"string\": \"a\"}`; and\n- a `Foo` instance as `\\{\"Foo\": {...}}`, where `{...}` indicates the JSON encoding of a `Foo` instance.\n\nHowever, it is possible to instead consume documents in https://pkg.go.dev/github.com/linkedin/goavro/v2#NewCodecForStandardJSONFull[standard/raw JSON format^] by setting the field \u003c\u003cavro_raw_json, `avro_raw_json`\u003e\u003e to `true`.\n\n=== Known issues\n\nImportant! There is an outstanding issue in the https://github.com/linkedin/goavro[avro serializing library^] that Redpanda Connect uses which means it https://github.com/linkedin/goavro/issues/252[doesn't encode logical types correctly^]. It's still possible to encode logical types that are in-line with the spec if `avro_raw_json` is set to true, though now of course non-logical types will not be in-line with the spec.\n\n== Protobuf format\n\nThis processor encodes protobuf messages either from any format parsed within Redpanda Connect (encoded as JSON by default), or from raw JSON documents, you can read more about JSON mapping of protobuf messages here: https://developers.google.com/protocol-buffers/docs/proto3#json\n\n=== Multiple message support\n\nWhen a target subject presents a protobuf schema that contains multiple messages it becomes ambiguous which message definition a given input data should be encoded against. In such scenarios Redpanda Connect will attempt to encode the data against each of them and select the first to successfully match against the data, this process currently *ignores all nested message definitions*. In order to speed up this exhaustive search the last known successful message will be attempted first for each subsequent input.\n\nWe will be considering alternative approaches in future so please https://redpanda.com/slack[get in touch^] with thoughts and feedback.\n","categories":["Parsing","Integration"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The base URL of the schema registry service.","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"subject","type":"string","kind":"scalar","description":"The schema subject to derive schemas from.","interpolated":true,"examples":["foo","${! meta(\"kafka_topic\") }"]},{"name":"refresh_period","type":"string","kind":"scalar","description":"The period after which a schema is refreshed for each subject, this is done by polling the schema registry service.","default":"10m","examples":["60s","1h"]},{"name":"avro_raw_json","type":"bool","kind":"scalar","description":"Whether messages encoded in Avro format should be parsed as normal JSON (\"json that meets the expectations of regular internet json\") rather than https://avro.apache.org/docs/current/specification/_print/#json-encoding[Avro JSON^]. If `true` the schema returned from the subject should be parsed as https://pkg.go.dev/github.com/linkedin/goavro/v2#NewCodecForStandardJSONFull[standard json^] instead of as https://pkg.go.dev/github.com/linkedin/goavro/v2#NewCodec[avro json^]. There is a https://github.com/linkedin/goavro/blob/5ec5a5ee7ec82e16e6e2b438d610e1cab2588393/union.go#L224-L249[comment in goavro^], the https://github.com/linkedin/goavro[underlining library used for avro serialization^], that explains in more detail the difference between standard json and avro json.","is_advanced":true,"default":false,"version":"3.59.0"},{"name":"oauth","type":"object","kind":"scalar","description":"Allows you to specify open authentication via OAuth version 1.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use OAuth version 1 in requests.","is_advanced":true,"default":false},{"name":"consumer_key","type":"string","kind":"scalar","description":"A value used to identify the client to the service provider.","is_advanced":true,"default":""},{"name":"consumer_secret","type":"string","kind":"scalar","description":"A secret used to establish ownership of the consumer key.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"access_token","type":"string","kind":"scalar","description":"A value used to gain access to the protected resources on behalf of the user.","is_advanced":true,"default":""},{"name":"access_token_secret","type":"string","kind":"scalar","description":"A secret provided in order to establish ownership of a given access token.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}],"version":"4.7.0"},{"name":"basic_auth","type":"object","kind":"scalar","description":"Allows you to specify basic authentication.","is_advanced":true,"is_optional":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use basic authentication in requests.","is_advanced":true,"default":false},{"name":"username","type":"string","kind":"scalar","description":"A username to authenticate as.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password to authenticate with.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}],"version":"4.7.0"},{"name":"jwt","type":"object","kind":"scalar","description":"BETA: Allows you to specify JWT authentication.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to use JWT authentication in requests.","is_advanced":true,"default":false},{"name":"private_key_file","type":"string","kind":"scalar","description":"A file with the PEM encoded via PKCS1 or PKCS8 as private key.","is_advanced":true,"default":""},{"name":"signing_method","type":"string","kind":"scalar","description":"A method used to sign the token such as RS256, RS384, RS512 or EdDSA.","is_advanced":true,"default":""},{"name":"claims","type":"unknown","kind":"map","description":"A value used to identify the claims that issued the JWT.","is_advanced":true,"default":{}},{"name":"headers","type":"unknown","kind":"map","description":"Add optional key/value headers to the JWT.","is_advanced":true,"default":{}}],"version":"4.7.0"},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]}]},"version":"3.58.0"},{"name":"select_parts","type":"processor","status":"stable","plugin":true,"summary":"Cherry pick a set of messages from a batch by their index. Indexes larger than the number of messages are simply ignored.","description":"\nThe selected parts are added to the new message batch in the same order as the selection array. E.g. with 'parts' set to [ 2, 0, 1 ] and the message parts [ '0', '1', '2', '3' ], the output will be [ '2', '0', '1' ].\n\nIf none of the selected parts exist in the input batch (resulting in an empty output message) the batch is dropped entirely.\n\nMessage indexes can be negative, and if so the part will be selected from the end counting backwards starting from -1. E.g. if index = -1 then the selected part will be the last part of the message, if index = -2 then the part before the last element with be selected, and so on.\n\nThis processor is only applicable to xref:configuration:batching.adoc[batched messages].","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"parts","type":"int","kind":"array","description":"An array of message indexes of a batch. Indexes can be negative, and if so the part will be selected from the end counting backwards starting from -1.","default":[]}]}},{"name":"sentry_capture","type":"processor","status":"experimental","plugin":true,"summary":"Captures log events from messages and submits them to https://sentry.io/[Sentry^].","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"dsn","type":"string","kind":"scalar","description":"The DSN address to send sentry events to. If left empty, then SENTRY_DSN is used.","default":""},{"name":"message","type":"string","kind":"scalar","description":"A message to set on the sentry event","interpolated":true,"examples":["webhook event received","failed to find product in database: ${! error() }"]},{"name":"context","type":"string","kind":"scalar","description":"A mapping that must evaluate to an object-of-objects or `deleted()`. If this mapping produces a value, then it is set on a sentry event as additional context.","is_optional":true,"bloblang":true,"examples":["root = {\"order\": {\"product_id\": \"P93174\", \"quantity\": 5}}","root = deleted()"]},{"name":"tags","type":"string","kind":"map","description":"Sets key/value string tags on an event. Unlike context, these are indexed and searchable on Sentry but have length limitations.","is_optional":true,"interpolated":true},{"name":"environment","type":"string","kind":"scalar","description":"The environment to be sent with events. If left empty, then SENTRY_ENVIRONMENT is used.","default":""},{"name":"release","type":"string","kind":"scalar","description":"The version of the code deployed to an environment. If left empty, then the Sentry client will attempt to detect the release from the environment.","default":""},{"name":"level","type":"string","kind":"scalar","description":"Sets the level on sentry events similar to logging levels.","default":"INFO","options":["DEBUG","INFO","WARN","ERROR","FATAL"],"linter":"\nlet options = {\n \"debug\": true,\n \"info\": true,\n \"warn\": true,\n \"error\": true,\n \"fatal\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"transport_mode","type":"string","kind":"scalar","description":"Determines how events are sent. A sync transport will block when sending each event until a response is received from the Sentry server. The recommended async transport will enqueue events in a buffer and send them in the background.","default":"async","options":["async","sync"],"linter":"\nlet options = {\n \"async\": true,\n \"sync\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"flush_timeout","type":"string","kind":"scalar","description":"The duration to wait when closing the processor to flush any remaining enqueued events.","default":"5s"},{"name":"sampling_rate","type":"float","kind":"scalar","description":"The rate at which events are sent to the server. A value of 0 disables capturing sentry events entirely. A value of 1 results in sending all events to Sentry. Any value in between results sending some percentage of events.","default":1,"linter":"root = if this \u003c 0 || this \u003e 1 { [\"sampling rate must be between 0.0 and 1.0\" ] }"}]},"version":"4.16.0"},{"name":"slack_thread","type":"processor","status":"experimental","plugin":true,"description":"Read a thread using the https://api.slack.com/methods/conversations.replies[^Slack API]","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"bot_token","type":"string","kind":"scalar","description":"The Slack Bot User OAuth token to use.","linter":"\n root = if !this.has_prefix(\"xoxb-\") { [ \"field must start with xoxb-\" ] }\n "},{"name":"channel_id","type":"string","kind":"scalar","description":"The channel ID to read messages from.","interpolated":true},{"name":"thread_ts","type":"string","kind":"scalar","description":"The thread timestamp to read the full thread of.","interpolated":true}]}},{"name":"sleep","type":"processor","status":"stable","plugin":true,"summary":"Sleep for a period of time specified as a duration string for each message. This processor will interpolate functions within the `duration` field, you can find a list of functions xref:configuration:interpolation.adoc#bloblang-queries[here].","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"duration","type":"string","kind":"scalar","description":"The duration of time to sleep for each execution.","interpolated":true}]}},{"name":"split","type":"processor","status":"stable","plugin":true,"summary":"Breaks message batches (synonymous with multiple part messages) into smaller batches. The size of the resulting batches are determined either by a discrete size or, if the field `byte_size` is non-zero, then by total size in bytes (which ever limit is reached first).","description":"\nThis processor is for breaking batches down into smaller ones. In order to break a single message out into multiple messages use the xref:components:processors/unarchive.adoc[`unarchive` processor].\n\nIf there is a remainder of messages after splitting a batch the remainder is also sent as a single batch. For example, if your target size was 10, and the processor received a batch of 95 message parts, the result would be 9 batches of 10 messages followed by a batch of 5 messages.","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"size","type":"int","kind":"scalar","description":"The target number of messages.","default":1},{"name":"byte_size","type":"int","kind":"scalar","description":"An optional target of total message bytes.","default":0}]}},{"name":"sql","type":"processor","status":"deprecated","plugin":true,"summary":"Runs an arbitrary SQL query against a database and (optionally) returns the result as an array of objects, one for each row returned.","description":"\nIf the query fails to execute then the message will remain unchanged and the error can be caught using xref:configuration:error_handling.adoc[error handling methods].\n\n== Alternatives\n\nFor basic inserts or select queries use either the xref:components:processors/sql_insert.adoc[`sql_insert`] or the xref:components:processors/sql_select.adoc[`sql_select`] processor. For more complex queries use the xref:components:processors/sql_raw.adoc[`sql_raw`] processor.","categories":["Integration"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"driver","type":"string","kind":"scalar","description":"A database \u003c\u003cdrivers, driver\u003e\u003e to use.","options":["mysql","postgres","clickhouse","mssql","sqlite","oracle","snowflake","trino","gocosmos","spanner"],"linter":"\nlet options = {\n \"mysql\": true,\n \"postgres\": true,\n \"clickhouse\": true,\n \"mssql\": true,\n \"sqlite\": true,\n \"oracle\": true,\n \"snowflake\": true,\n \"trino\": true,\n \"gocosmos\": true,\n \"spanner\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"data_source_name","type":"string","kind":"scalar","description":"Data source name."},{"name":"query","type":"string","kind":"scalar","description":"The query to execute. The style of placeholder to use depends on the driver, some drivers require question marks (`?`) whereas others expect incrementing dollar signs (`$1`, `$2`, and so on) or colons (`:1`, `:2` and so on). The style to use is outlined in this table:\n\n| Driver | Placeholder Style |\n|---|---|\n| `clickhouse` | Dollar sign |\n| `mysql` | Question mark |\n| `postgres` | Dollar sign |\n| `mssql` | Question mark |\n| `sqlite` | Question mark |\n| `oracle` | Colon |\n| `snowflake` | Question mark |\n| `trino` | Question mark |\n| `gocosmos` | Colon |\n","examples":["INSERT INTO footable (foo, bar, baz) VALUES (?, ?, ?);"]},{"name":"unsafe_dynamic_query","type":"bool","kind":"scalar","description":"Whether to enable xref:configuration:interpolation.adoc#bloblang-queries[interpolation functions] in the query. Great care should be made to ensure your queries are defended against injection attacks.","is_advanced":true,"default":false},{"name":"args_mapping","type":"string","kind":"scalar","description":"An optional xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `query`.","is_optional":true,"bloblang":true,"examples":["root = [ this.cat.meow, this.doc.woofs[0] ]","root = [ meta(\"user.id\") ]"]},{"name":"result_codec","type":"string","kind":"scalar","description":"Result codec.","default":"none"}]},"version":"3.65.0"},{"name":"sql_insert","type":"processor","status":"stable","plugin":true,"summary":"Inserts rows into an SQL database for each message, and leaves the message unchanged.","description":"\nIf the insert fails to execute then the message will still remain unchanged and the error can be caught using xref:configuration:error_handling.adoc[error handling methods].","categories":["Integration"],"examples":[{"title":"Table Insert (MySQL)","summary":"\nHere we insert rows into a database by populating the columns id, name and topic with values extracted from messages and metadata:","config":"\npipeline:\n processors:\n - sql_insert:\n driver: mysql\n dsn: foouser:foopassword@tcp(localhost:3306)/foodb\n table: footable\n columns: [ id, name, topic ]\n args_mapping: |\n root = [\n this.user.id,\n this.user.name,\n meta(\"kafka_topic\"),\n ]\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"driver","type":"string","kind":"scalar","description":"A database \u003c\u003cdrivers, driver\u003e\u003e to use.","options":["mysql","postgres","clickhouse","mssql","sqlite","oracle","snowflake","trino","gocosmos","spanner"],"linter":"\nlet options = {\n \"mysql\": true,\n \"postgres\": true,\n \"clickhouse\": true,\n \"mssql\": true,\n \"sqlite\": true,\n \"oracle\": true,\n \"snowflake\": true,\n \"trino\": true,\n \"gocosmos\": true,\n \"spanner\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"dsn","type":"string","kind":"scalar","description":"A Data Source Name to identify the target database.\n\n==== Drivers\n\n:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community, spanner=community\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n|===\n| Driver | Data Source Name Format\n\n| `clickhouse` \n| https://github.com/ClickHouse/clickhouse-go#dsn[`clickhouse://[username[:password\\]@\\][netloc\\][:port\\]/dbname[?param1=value1\u0026...\u0026paramN=valueN\\]`^] \n\n| `mysql` \n| `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` \n\n| `postgres` \n| `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` \n\n| `mssql` \n| `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` \n\n| `sqlite` \n| `file:/path/to/filename.db[?param\u0026=value1\u0026...]` \n\n| `oracle` \n| `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` \n\n| `snowflake` \n| `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` \n\n| `trino` \n| https://github.com/trinodb/trino-go-client#dsn-data-source-name[`http[s\\]://user[:pass\\]@host[:port\\][?parameters\\]`^] \n\n| `gocosmos` \n| https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage[`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e\\][;Version=\u003ccosmosdb-api-version\u003e\\][;DefaultDb/Db=\u003cdb-name\u003e\\][;AutoId=\u003ctrue/false\u003e\\][;InsecureSkipVerify=\u003ctrue/false\u003e\\]`^] \n\n| `spanner` \n| projects/[PROJECT]/instances/[INSTANCE]/databases/[DATABASE] \n|===\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String[the docs^] for more details. For https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication[key pair authentication^], the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe https://pkg.go.dev/github.com/microsoft/gocosmos[`gocosmos`^] driver is still experimental, but it has support for https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys[hierarchical partition keys^] as well as https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query[cross-partition queries^]. Please refer to the https://github.com/microsoft/gocosmos/blob/main/SQL.md[SQL notes^] for details.","examples":["clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60","foouser:foopassword@tcp(localhost:3306)/foodb","postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable","oracle://foouser:foopass@localhost:1521/service_name"]},{"name":"table","type":"string","kind":"scalar","description":"The table to insert to.","examples":["foo"]},{"name":"columns","type":"string","kind":"array","description":"A list of columns to insert.","examples":[["foo","bar","baz"]]},{"name":"args_mapping","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of columns specified.","bloblang":true,"examples":["root = [ this.cat.meow, this.doc.woofs[0] ]","root = [ meta(\"user.id\") ]"]},{"name":"prefix","type":"string","kind":"scalar","description":"An optional prefix to prepend to the insert query (before INSERT).","is_advanced":true,"is_optional":true},{"name":"suffix","type":"string","kind":"scalar","description":"An optional suffix to append to the insert query.","is_advanced":true,"is_optional":true,"examples":["ON CONFLICT (name) DO NOTHING"]},{"name":"options","type":"string","kind":"array","description":"A list of keyword options to add before the INTO clause of the query.","is_advanced":true,"is_optional":true,"examples":[["DELAYED","IGNORE"]]},{"name":"init_files","type":"string","kind":"array","description":"\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":[["./init/*.sql"],["./foo.sql","./bar.sql"]],"version":"4.10.0"},{"name":"init_statement","type":"string","kind":"scalar","description":"\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":["\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n"],"version":"4.10.0"},{"name":"conn_max_idle_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time.","is_advanced":true,"is_optional":true},{"name":"conn_max_life_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age.","is_advanced":true,"is_optional":true},{"name":"conn_max_idle","type":"int","kind":"scalar","description":"An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release.","is_advanced":true,"is_optional":true,"default":2},{"name":"conn_max_open","type":"int","kind":"scalar","description":"An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited).","is_advanced":true,"is_optional":true}]},"version":"3.59.0"},{"name":"sql_raw","type":"processor","status":"stable","plugin":true,"summary":"Runs an arbitrary SQL query against a database and (optionally) returns the result as an array of objects, one for each row returned.","description":"\nIf the query fails to execute then the message will remain unchanged and the error can be caught using xref:configuration:error_handling.adoc[error handling methods].","categories":["Integration"],"examples":[{"title":"Table Insert (MySQL)","summary":"The following example inserts rows into the table footable with the columns foo, bar and baz populated with values extracted from messages.","config":"\npipeline:\n processors:\n - sql_raw:\n driver: mysql\n dsn: foouser:foopassword@tcp(localhost:3306)/foodb\n query: \"INSERT INTO footable (foo, bar, baz) VALUES (?, ?, ?);\"\n args_mapping: '[ document.foo, document.bar, meta(\"kafka_topic\") ]'\n exec_only: true\n"},{"title":"Table Query (PostgreSQL)","summary":"Here we query a database for columns of footable that share a `user_id` with the message field `user.id`. A xref:components:processors/branch.adoc[`branch` processor] is used in order to insert the resulting array into the original message at the path `foo_rows`.","config":"\npipeline:\n processors:\n - branch:\n processors:\n - sql_raw:\n driver: postgres\n dsn: postgres://foouser:foopass@localhost:5432/testdb?sslmode=disable\n query: \"SELECT * FROM footable WHERE user_id = $1;\"\n args_mapping: '[ this.user.id ]'\n result_map: 'root.foo_rows = this'\n"},{"title":"Dynamically Creating Tables (PostgreSQL)","summary":"Here we query a database for columns of footable that share a `user_id` with the message field `user.id`. A xref:components:processors/branch.adoc[`branch` processor] is used in order to insert the resulting array into the original message at the path `foo_rows`.","config":"\npipeline:\n processors:\n - mapping: |\n root = this\n # Prevent SQL injection when using unsafe_dynamic_query\n meta table_name = \"\\\"\" + metadata(\"table_name\").replace_all(\"\\\"\", \"\\\"\\\"\") + \"\\\"\"\n - sql_raw:\n driver: postgres\n dsn: postgres://localhost/postgres\n unsafe_dynamic_query: true\n queries:\n - query: |\n CREATE TABLE IF NOT EXISTS ${!metadata(\"table_name\")} (id varchar primary key, document jsonb);\n - query: |\n INSERT INTO ${!metadata(\"table_name\")} (id, document) VALUES ($1, $2)\n ON CONFLICT (id) DO UPDATE SET document = EXCLUDED.document;\n args_mapping: |\n root = [ this.id, this.document.string() ]\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"driver","type":"string","kind":"scalar","description":"A database \u003c\u003cdrivers, driver\u003e\u003e to use.","options":["mysql","postgres","clickhouse","mssql","sqlite","oracle","snowflake","trino","gocosmos","spanner"],"linter":"\nlet options = {\n \"mysql\": true,\n \"postgres\": true,\n \"clickhouse\": true,\n \"mssql\": true,\n \"sqlite\": true,\n \"oracle\": true,\n \"snowflake\": true,\n \"trino\": true,\n \"gocosmos\": true,\n \"spanner\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"dsn","type":"string","kind":"scalar","description":"A Data Source Name to identify the target database.\n\n==== Drivers\n\n:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community, spanner=community\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n|===\n| Driver | Data Source Name Format\n\n| `clickhouse` \n| https://github.com/ClickHouse/clickhouse-go#dsn[`clickhouse://[username[:password\\]@\\][netloc\\][:port\\]/dbname[?param1=value1\u0026...\u0026paramN=valueN\\]`^] \n\n| `mysql` \n| `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` \n\n| `postgres` \n| `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` \n\n| `mssql` \n| `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` \n\n| `sqlite` \n| `file:/path/to/filename.db[?param\u0026=value1\u0026...]` \n\n| `oracle` \n| `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` \n\n| `snowflake` \n| `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` \n\n| `trino` \n| https://github.com/trinodb/trino-go-client#dsn-data-source-name[`http[s\\]://user[:pass\\]@host[:port\\][?parameters\\]`^] \n\n| `gocosmos` \n| https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage[`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e\\][;Version=\u003ccosmosdb-api-version\u003e\\][;DefaultDb/Db=\u003cdb-name\u003e\\][;AutoId=\u003ctrue/false\u003e\\][;InsecureSkipVerify=\u003ctrue/false\u003e\\]`^] \n\n| `spanner` \n| projects/[PROJECT]/instances/[INSTANCE]/databases/[DATABASE] \n|===\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String[the docs^] for more details. For https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication[key pair authentication^], the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe https://pkg.go.dev/github.com/microsoft/gocosmos[`gocosmos`^] driver is still experimental, but it has support for https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys[hierarchical partition keys^] as well as https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query[cross-partition queries^]. Please refer to the https://github.com/microsoft/gocosmos/blob/main/SQL.md[SQL notes^] for details.","examples":["clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60","foouser:foopassword@tcp(localhost:3306)/foodb","postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable","oracle://foouser:foopass@localhost:1521/service_name"]},{"name":"query","type":"string","kind":"scalar","description":"The query to execute. The style of placeholder to use depends on the driver, some drivers require question marks (`?`) whereas others expect incrementing dollar signs (`$1`, `$2`, and so on) or colons (`:1`, `:2` and so on). The style to use is outlined in this table:\n\n| Driver | Placeholder Style |\n|---|---|\n| `clickhouse` | Dollar sign |\n| `mysql` | Question mark |\n| `postgres` | Dollar sign |\n| `mssql` | Question mark |\n| `sqlite` | Question mark |\n| `oracle` | Colon |\n| `snowflake` | Question mark |\n| `trino` | Question mark |\n| `gocosmos` | Colon |\n","is_optional":true,"examples":["INSERT INTO footable (foo, bar, baz) VALUES (?, ?, ?);","SELECT * FROM footable WHERE user_id = $1;"]},{"name":"unsafe_dynamic_query","type":"bool","kind":"scalar","description":"Whether to enable xref:configuration:interpolation.adoc#bloblang-queries[interpolation functions] in the query. Great care should be made to ensure your queries are defended against injection attacks.","is_advanced":true,"default":false},{"name":"args_mapping","type":"string","kind":"scalar","description":"An optional xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `query`.","is_optional":true,"bloblang":true,"examples":["root = [ this.cat.meow, this.doc.woofs[0] ]","root = [ meta(\"user.id\") ]"]},{"name":"exec_only","type":"bool","kind":"scalar","description":"Whether the query result should be discarded. When set to `true` the message contents will remain unchanged, which is useful in cases where you are executing inserts, updates, etc. By default this is true for the last query, and previous queries don't change the results. If set to true for any query but the last one, the subsequent `args_mappings` input is overwritten.","is_optional":true},{"name":"queries","type":"object","kind":"array","description":"A list of statements to run in addition to `query`. When specifying multiple statements, they are all executed within a transaction. The output of the processor is always the last query that runs, unless `exec_only` is used.","is_optional":true,"children":[{"name":"query","type":"string","kind":"scalar","description":"The query to execute. The style of placeholder to use depends on the driver, some drivers require question marks (`?`) whereas others expect incrementing dollar signs (`$1`, `$2`, and so on) or colons (`:1`, `:2` and so on). The style to use is outlined in this table:\n\n| Driver | Placeholder Style |\n|---|---|\n| `clickhouse` | Dollar sign |\n| `mysql` | Question mark |\n| `postgres` | Dollar sign |\n| `mssql` | Question mark |\n| `sqlite` | Question mark |\n| `oracle` | Colon |\n| `snowflake` | Question mark |\n| `trino` | Question mark |\n| `gocosmos` | Colon |\n"},{"name":"args_mapping","type":"string","kind":"scalar","description":"An optional xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `query`.","is_optional":true,"bloblang":true,"examples":["root = [ this.cat.meow, this.doc.woofs[0] ]","root = [ meta(\"user.id\") ]"]},{"name":"exec_only","type":"bool","kind":"scalar","description":"Whether the query result should be discarded. When set to `true` the message contents will remain unchanged, which is useful in cases where you are executing inserts, updates, etc. By default this is true for the last query, and previous queries don't change the results. If set to true for any query but the last one, the subsequent `args_mappings` input is overwritten.","is_optional":true}]},{"name":"init_files","type":"string","kind":"array","description":"\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":[["./init/*.sql"],["./foo.sql","./bar.sql"]],"version":"4.10.0"},{"name":"init_statement","type":"string","kind":"scalar","description":"\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":["\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n"],"version":"4.10.0"},{"name":"conn_max_idle_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time.","is_advanced":true,"is_optional":true},{"name":"conn_max_life_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age.","is_advanced":true,"is_optional":true},{"name":"conn_max_idle","type":"int","kind":"scalar","description":"An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release.","is_advanced":true,"is_optional":true,"default":2},{"name":"conn_max_open","type":"int","kind":"scalar","description":"An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited).","is_advanced":true,"is_optional":true}],"linter":"root = match {\n !this.exists(\"queries\") \u0026\u0026 !this.exists(\"query\") =\u003e [ \"either `query` or `queries` is required\" ],\n }"},"version":"3.65.0"},{"name":"sql_select","type":"processor","status":"stable","plugin":true,"summary":"Runs an SQL select query against a database and returns the result as an array of objects, one for each row returned, containing a key for each column queried and its value.","description":"\nIf the query fails to execute then the message will remain unchanged and the error can be caught using xref:configuration:error_handling.adoc[error handling methods].","categories":["Integration"],"examples":[{"title":"Table Query (PostgreSQL)","summary":"\nHere we query a database for columns of footable that share a `user_id`\nwith the message `user.id`. A xref:components:processors/branch.adoc[`branch` processor]\nis used in order to insert the resulting array into the original message at the\npath `foo_rows`:","config":"\npipeline:\n processors:\n - branch:\n processors:\n - sql_select:\n driver: postgres\n dsn: postgres://foouser:foopass@localhost:5432/testdb?sslmode=disable\n table: footable\n columns: [ '*' ]\n where: user_id = ?\n args_mapping: '[ this.user.id ]'\n result_map: 'root.foo_rows = this'\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"driver","type":"string","kind":"scalar","description":"A database \u003c\u003cdrivers, driver\u003e\u003e to use.","options":["mysql","postgres","clickhouse","mssql","sqlite","oracle","snowflake","trino","gocosmos","spanner"],"linter":"\nlet options = {\n \"mysql\": true,\n \"postgres\": true,\n \"clickhouse\": true,\n \"mssql\": true,\n \"sqlite\": true,\n \"oracle\": true,\n \"snowflake\": true,\n \"trino\": true,\n \"gocosmos\": true,\n \"spanner\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"dsn","type":"string","kind":"scalar","description":"A Data Source Name to identify the target database.\n\n==== Drivers\n\n:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community, spanner=community\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n|===\n| Driver | Data Source Name Format\n\n| `clickhouse` \n| https://github.com/ClickHouse/clickhouse-go#dsn[`clickhouse://[username[:password\\]@\\][netloc\\][:port\\]/dbname[?param1=value1\u0026...\u0026paramN=valueN\\]`^] \n\n| `mysql` \n| `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` \n\n| `postgres` \n| `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` \n\n| `mssql` \n| `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` \n\n| `sqlite` \n| `file:/path/to/filename.db[?param\u0026=value1\u0026...]` \n\n| `oracle` \n| `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` \n\n| `snowflake` \n| `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` \n\n| `trino` \n| https://github.com/trinodb/trino-go-client#dsn-data-source-name[`http[s\\]://user[:pass\\]@host[:port\\][?parameters\\]`^] \n\n| `gocosmos` \n| https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage[`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e\\][;Version=\u003ccosmosdb-api-version\u003e\\][;DefaultDb/Db=\u003cdb-name\u003e\\][;AutoId=\u003ctrue/false\u003e\\][;InsecureSkipVerify=\u003ctrue/false\u003e\\]`^] \n\n| `spanner` \n| projects/[PROJECT]/instances/[INSTANCE]/databases/[DATABASE] \n|===\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String[the docs^] for more details. For https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication[key pair authentication^], the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe https://pkg.go.dev/github.com/microsoft/gocosmos[`gocosmos`^] driver is still experimental, but it has support for https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys[hierarchical partition keys^] as well as https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query[cross-partition queries^]. Please refer to the https://github.com/microsoft/gocosmos/blob/main/SQL.md[SQL notes^] for details.","examples":["clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60","foouser:foopassword@tcp(localhost:3306)/foodb","postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable","oracle://foouser:foopass@localhost:1521/service_name"]},{"name":"table","type":"string","kind":"scalar","description":"The table to query.","examples":["foo"]},{"name":"columns","type":"string","kind":"array","description":"A list of columns to query.","examples":[["*"],["foo","bar","baz"]]},{"name":"where","type":"string","kind":"scalar","description":"An optional where clause to add. Placeholder arguments are populated with the `args_mapping` field. Placeholders should always be question marks, and will automatically be converted to dollar syntax when the postgres or clickhouse drivers are used.","is_optional":true,"examples":["meow = ? and woof = ?","user_id = ?"]},{"name":"args_mapping","type":"string","kind":"scalar","description":"An optional xref:guides:bloblang/about.adoc[Bloblang mapping] which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `where`.","is_optional":true,"bloblang":true,"examples":["root = [ this.cat.meow, this.doc.woofs[0] ]","root = [ meta(\"user.id\") ]"]},{"name":"prefix","type":"string","kind":"scalar","description":"An optional prefix to prepend to the query (before SELECT).","is_advanced":true,"is_optional":true},{"name":"suffix","type":"string","kind":"scalar","description":"An optional suffix to append to the select query.","is_advanced":true,"is_optional":true},{"name":"init_files","type":"string","kind":"array","description":"\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":[["./init/*.sql"],["./foo.sql","./bar.sql"]],"version":"4.10.0"},{"name":"init_statement","type":"string","kind":"scalar","description":"\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n","is_advanced":true,"is_optional":true,"examples":["\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n"],"version":"4.10.0"},{"name":"conn_max_idle_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time.","is_advanced":true,"is_optional":true},{"name":"conn_max_life_time","type":"string","kind":"scalar","description":"An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age.","is_advanced":true,"is_optional":true},{"name":"conn_max_idle","type":"int","kind":"scalar","description":"An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release.","is_advanced":true,"is_optional":true,"default":2},{"name":"conn_max_open","type":"int","kind":"scalar","description":"An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited).","is_advanced":true,"is_optional":true}]},"version":"3.59.0"},{"name":"subprocess","type":"processor","status":"stable","plugin":true,"summary":"Executes a command as a subprocess and, for each message, will pipe its contents to the stdin stream of the process followed by a newline.","description":"\n[NOTE]\n====\nThis processor keeps the subprocess alive and requires very specific behavior from the command executed. If you wish to simply execute a command for each message take a look at the xref:components:processors/command.adoc[`command` processor] instead.\n====\n\nThe subprocess must then either return a line over stdout or stderr. If a response is returned over stdout then its contents will replace the message. If a response is instead returned from stderr it will be logged and the message will continue unchanged and will be xref:configuration:error_handling.adoc[marked as failed].\n\nRather than separating data by a newline it's possible to specify alternative \u003c\u003ccodec_send,`codec_send`\u003e\u003e and \u003c\u003ccodec_recv,`codec_recv`\u003e\u003e values, which allow binary messages to be encoded for logical separation.\n\nThe execution environment of the subprocess is the same as the Redpanda Connect instance, including environment variables and the current working directory.\n\nThe field `max_buffer` defines the maximum response size able to be read from the subprocess. This value should be set significantly above the real expected maximum response size.\n\n== Subprocess requirements\n\nIt is required that subprocesses flush their stdout and stderr pipes for each line. Redpanda Connect will attempt to keep the process alive for as long as the pipeline is running. If the process exits early it will be restarted.\n\n== Messages containing line breaks\n\nIf a message contains line breaks each line of the message is piped to the subprocess and flushed, and a response is expected from the subprocess before another line is fed in.","categories":["Integration"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"name","type":"string","kind":"scalar","description":"The command to execute as a subprocess.","examples":["cat","sed","awk"]},{"name":"args","type":"string","kind":"array","description":"A list of arguments to provide the command.","default":[]},{"name":"max_buffer","type":"int","kind":"scalar","description":"The maximum expected response size.","is_advanced":true,"default":65536},{"name":"codec_send","type":"string","kind":"scalar","description":"Determines how messages written to the subprocess are encoded, which allows them to be logically separated.","is_advanced":true,"default":"lines","options":["lines","length_prefixed_uint32_be","netstring"],"version":"3.37.0","linter":"\nlet options = {\n \"lines\": true,\n \"length_prefixed_uint32_be\": true,\n \"netstring\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"codec_recv","type":"string","kind":"scalar","description":"Determines how messages read from the subprocess are decoded, which allows them to be logically separated.","is_advanced":true,"default":"lines","options":["lines","length_prefixed_uint32_be","netstring"],"version":"3.37.0","linter":"\nlet options = {\n \"lines\": true,\n \"length_prefixed_uint32_be\": true,\n \"netstring\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"}]}},{"name":"switch","type":"processor","status":"stable","plugin":true,"summary":"Conditionally processes messages based on their contents.","description":"For each switch case a xref:guides:bloblang/about.adoc[Bloblang query] is checked and, if the result is true (or the check is empty) the child processors are executed on the message.","categories":["Composition"],"footnotes":"\n== Batching\n\nWhen a switch processor executes on a xref:configuration:batching.adoc[batch of messages] they are checked individually and can be matched independently against cases. During processing the messages matched against a case are processed as a batch, although the ordering of messages during case processing cannot be guaranteed to match the order as received.\n\nAt the end of switch processing the resulting batch will follow the same ordering as the batch was received. If any child processors have split or otherwise grouped messages this grouping will be lost as the result of a switch is always a single batch. In order to perform conditional grouping and/or splitting use the xref:components:processors/group_by.adoc[`group_by` processor].","examples":[{"title":"Ignore George","summary":"\nWe have a system where we're counting a metric for all messages that pass through our system. However, occasionally we get messages from George that we don't care about.\n\nFor George's messages we want to instead emit a metric that gauges how angry he is about being ignored and then we drop it.","config":"\npipeline:\n processors:\n - switch:\n - check: this.user.name.first != \"George\"\n processors:\n - metric:\n type: counter\n name: MessagesWeCareAbout\n\n - processors:\n - metric:\n type: gauge\n name: GeorgesAnger\n value: ${! json(\"user.anger\") }\n - mapping: root = deleted()\n"}],"config":{"name":"","type":"object","kind":"array","children":[{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should have the processors of this case executed on it. If left empty the case always passes. If the check mapping throws an error the message will be flagged xref:configuration:error_handling.adoc[as having failed] and will not be tested against any other cases.","default":"","bloblang":true,"examples":["this.type == \"foo\"","this.contents.urls.contains(\"https://benthos.dev/\")"]},{"name":"processors","type":"processor","kind":"array","description":"A list of xref:components:processors/about.adoc[processors] to execute on a message.","default":[]},{"name":"fallthrough","type":"bool","kind":"scalar","description":"Indicates whether, if this case passes for a message, the next case should also be executed.","is_advanced":true,"default":false}]}},{"name":"sync_response","type":"processor","status":"stable","plugin":true,"summary":"Adds the payload in its current state as a synchronous response to the input source, where it is dealt with according to that specific input type.","description":"\nFor most inputs this mechanism is ignored entirely, in which case the sync response is dropped without penalty. It is therefore safe to use this processor even when combining input types that might not have support for sync responses. An example of an input able to utilize this is the `http_server`.\n\nFor more information please read xref:guides:sync_responses.adoc[synchronous responses].","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","default":{}}},{"name":"text_chunker","type":"processor","status":"experimental","plugin":true,"summary":"A processor that allows chunking and splitting text based on some strategy. Usually used for creating vector embeddings of large documents.","description":"A processor allowing splitting text into chunks based on several different strategies.","categories":["AI"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"strategy","type":"string","kind":"scalar","annotated_options":[["markdown","Split text by markdown headers."],["recursive_character","Split text recursively by characters (defined in `separators`)."],["token","Split text by tokens."]],"linter":"\nlet options = {\n \"markdown\": true,\n \"recursive_character\": true,\n \"token\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"chunk_size","type":"int","kind":"scalar","description":"The maximum size of each chunk.","default":512},{"name":"chunk_overlap","type":"int","kind":"scalar","description":"The number of characters to overlap between chunks.","default":100},{"name":"separators","type":"string","kind":"array","description":"A list of strings that should be considered as separators between chunks.","default":["\n\n","\n"," ",""]},{"name":"length_measure","type":"string","kind":"scalar","description":"The method for measuring the length of a string.","default":"runes","annotated_options":[["graphemes","Use unicode graphemes to determine the length of a string."],["runes","Use the number of codepoints to determine the length of a string."],["token","Use the number of tokens (using the `token_encoding` tokenizer) to determine the length of a string."],["utf8","Determine the length of text using the number of utf8 bytes."]],"linter":"\nlet options = {\n \"graphemes\": true,\n \"runes\": true,\n \"token\": true,\n \"utf8\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"token_encoding","type":"string","kind":"scalar","description":"The encoding to use for tokenization.","is_advanced":true,"is_optional":true,"examples":["cl100k_base","r50k_base"]},{"name":"allowed_special","type":"string","kind":"array","description":"A list of special tokens that are allowed in the output.","is_advanced":true,"default":[]},{"name":"disallowed_special","type":"string","kind":"array","description":"A list of special tokens that are disallowed in the output.","is_advanced":true,"default":["all"]},{"name":"include_code_blocks","type":"bool","kind":"scalar","description":"Whether to include code blocks in the output.","default":false},{"name":"keep_reference_links","type":"bool","kind":"scalar","description":"Whether to keep reference links in the output.","default":false}]}},{"name":"try","type":"processor","status":"stable","plugin":true,"summary":"Executes a list of child processors on messages only if no prior processors have failed (or the errors have been cleared).","description":"\nThis processor behaves similarly to the xref:components:processors/for_each.adoc[`for_each`] processor, where a list of child processors are applied to individual messages of a batch. However, if a message has failed any prior processor (before or during the try block) then that message will skip all following processors.\n\nFor example, with the following config:\n\n```yaml\npipeline:\n processors:\n - resource: foo\n - try:\n - resource: bar\n - resource: baz\n - resource: buz\n```\n\nIf the processor `bar` fails for a particular message, that message will skip the processors `baz` and `buz`. Similarly, if `bar` succeeds but `baz` does not then `buz` will be skipped. If the processor `foo` fails for a message then none of `bar`, `baz` or `buz` are executed on that message.\n\nThis processor is useful for when child processors depend on the successful output of previous processors. This processor can be followed with a xref:components:processors/catch.adoc[catch] processor for defining child processors to be applied only to failed messages.\n\nMore information about error handing can be found in xref:configuration:error_handling.adoc[].\n\n== Nest within a catch block\n\nIn some cases it might be useful to nest a try block within a catch block, since the xref:components:processors/catch.adoc[`catch` processor] only clears errors _after_ executing its child processors this means a nested try processor will not execute unless the errors are explicitly cleared beforehand.\n\nThis can be done by inserting an empty catch block before the try block like as follows:\n\n```yaml\npipeline:\n processors:\n - resource: foo\n - catch:\n - log:\n level: ERROR\n message: \"Foo failed due to: ${! error() }\"\n - catch: [] # Clear prior error\n - try:\n - resource: bar\n - resource: baz\n```","categories":["Composition"],"config":{"name":"","type":"processor","kind":"array","default":[]}},{"name":"unarchive","type":"processor","status":"stable","plugin":true,"summary":"Unarchives messages according to the selected archive format into multiple messages within a xref:configuration:batching.adoc[batch].","description":"\nWhen a message is unarchived the new messages replace the original message in the batch. Messages that are selected but fail to unarchive (invalid format) will remain unchanged in the message batch but will be flagged as having failed, allowing you to xref:configuration:error_handling.adoc[error handle them].\n\n== Metadata\n\nThe metadata found on the messages handled by this processor will be copied into the resulting messages. For the unarchive formats that contain file information (tar, zip), a metadata field is also added to each message called `archive_filename` with the extracted filename.\n","categories":["Parsing","Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"format","type":"string","kind":"scalar","description":"The unarchiving format to apply.","annotated_options":[["binary","Extract messages from a https://github.com/redpanda-data/benthos/blob/main/internal/message/message.go#L96[binary blob format^]."],["csv","Attempt to parse the message as a csv file (header required) and for each row in the file expands its contents into a json object in a new message."],["csv:x","Attempt to parse the message as a csv file (header required) and for each row in the file expands its contents into a json object in a new message using a custom delimiter. The custom delimiter must be a single character, e.g. the format \"csv:\\t\" would consume a tab delimited file."],["json_array","Attempt to parse a message as a JSON array, and extract each element into its own message."],["json_documents","Attempt to parse a message as a stream of concatenated JSON documents. Each parsed document is expanded into a new message."],["json_map","Attempt to parse the message as a JSON map and for each element of the map expands its contents into a new message. A metadata field is added to each message called `archive_key` with the relevant key from the top-level map."],["lines","Extract the lines of a message each into their own message."],["tar","Extract messages from a unix standard tape archive."],["zip","Extract messages from a zip file."]]}]}},{"name":"wasm","type":"processor","status":"experimental","plugin":true,"summary":"Executes a function exported by a WASM module for each message.","description":"\nThis processor uses https://github.com/tetratelabs/wazero[Wazero^] to execute a WASM module (with support for WASI), calling a specific function for each message being processed. From within the WASM module it is possible to query and mutate the message being processed via a suite of functions exported to the module.\n\nThis ecosystem is delicate as WASM doesn't have a single clearly defined way to pass strings back and forth between the host and the module. In order to remedy this we're gradually working on introducing libraries and examples for multiple languages which can be found in https://github.com/redpanda-data/benthos/tree/main/public/wasm/README.md[the codebase^].\n\nThese examples, as well as the processor itself, is a work in progress.\n\n== Parallelism\n\nIt's not currently possible to execute a single WASM runtime across parallel threads with this processor. Therefore, in order to support parallel processing this processor implements pooling of module runtimes. Ideally your WASM module shouldn't depend on any global state, but if it does then you need to ensure the processor xref:configuration:processing_pipelines.adoc[is only run on a single thread].\n","categories":["Utility"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"module_path","type":"string","kind":"scalar","description":"The path of the target WASM module to execute."},{"name":"function","type":"string","kind":"scalar","description":"The name of the function exported by the target WASM module to run for each message.","default":"process"}]},"version":"4.11.0"},{"name":"while","type":"processor","status":"stable","plugin":true,"summary":"A processor that checks a xref:guides:bloblang/about.adoc[Bloblang query] against each batch of messages and executes child processors on them for as long as the query resolves to true.","description":"\nThe field `at_least_once`, if true, ensures that the child processors are always executed at least one time (like a do .. while loop.)\n\nThe field `max_loops`, if greater than zero, caps the number of loops for a message batch to this value.\n\nIf following a loop execution the number of messages in a batch is reduced to zero the loop is exited regardless of the condition result. If following a loop execution there are more than 1 message batches the query is checked against the first batch only.\n\nThe conditions of this processor are applied across entire message batches. You can find out more about batching xref:configuration:batching.adoc[in this doc].","categories":["Composition"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"at_least_once","type":"bool","kind":"scalar","description":"Whether to always run the child processors at least one time.","default":false},{"name":"max_loops","type":"int","kind":"scalar","description":"An optional maximum number of loops to execute. Helps protect against accidentally creating infinite loops.","is_advanced":true,"default":0},{"name":"check","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether the while loop should execute again.","default":"","bloblang":true,"examples":["errored()","this.urls.unprocessed.length() \u003e 0"]},{"name":"processors","type":"processor","kind":"array","description":"A list of child processors to execute on each loop."}]}},{"name":"workflow","type":"processor","status":"stable","plugin":true,"summary":"Executes a topology of xref:components:processors/branch.adoc[`branch` processors], performing them in parallel where possible.","description":"\n== Why use a workflow\n\n=== Performance\n\nMost of the time the best way to compose processors is also the simplest, just configure them in series. This is because processors are often CPU bound, low-latency, and you can gain vertical scaling by increasing the number of processor pipeline threads, allowing Redpanda Connect to process xref:configuration:processing_pipelines.adoc[multiple messages in parallel].\n\nHowever, some processors such as xref:components:processors/http.adoc[`http`], xref:components:processors/aws_lambda.adoc[`aws_lambda`] or xref:components:processors/cache.adoc[`cache`] interact with external services and therefore spend most of their time waiting for a response. These processors tend to be high-latency and low CPU activity, which causes messages to process slowly.\n\nWhen a processing pipeline contains multiple network processors that aren't dependent on each other we can benefit from performing these processors in parallel for each individual message, reducing the overall message processing latency.\n\n=== Simplifying processor topology\n\nA workflow is often expressed as a https://en.wikipedia.org/wiki/Directed_acyclic_graph[DAG^] of processing stages, where each stage can result in N possible next stages, until finally the flow ends at an exit node.\n\nFor example, if we had processing stages A, B, C and D, where stage A could result in either stage B or C being next, always followed by D, it might look something like this:\n\n```text\n /--\u003e B --\\\nA --| |--\u003e D\n \\--\u003e C --/\n```\n\nThis flow would be easy to express in a standard Redpanda Connect config, we could simply use a xref:components:processors/switch.adoc[`switch` processor] to route to either B or C depending on a condition on the result of A. However, this method of flow control quickly becomes unfeasible as the DAG gets more complicated, imagine expressing this flow using switch processors:\n\n```text\n /--\u003e B -------------|--\u003e D\n / /\nA --| /--\u003e E --|\n \\--\u003e C --| \\\n \\----------|--\u003e F\n```\n\nAnd imagine doing so knowing that the diagram is subject to change over time. Yikes! Instead, with a workflow we can either trust it to automatically resolve the DAG or express it manually as simply as `order: [ [ A ], [ B, C ], [ E ], [ D, F ] ]`, and the conditional logic for determining if a stage is executed is defined as part of the branch itself.","categories":["Composition"],"footnotes":"\n== Structured metadata\n\nWhen the field `meta_path` is non-empty the workflow processor creates an object describing which workflows were successful, skipped or failed for each message and stores the object within the message at the end.\n\nThe object is of the following form:\n\n```json\n{\n\t\"succeeded\": [ \"foo\" ],\n\t\"skipped\": [ \"bar\" ],\n\t\"failed\": {\n\t\t\"baz\": \"the error message from the branch\"\n\t}\n}\n```\n\nIf a message already has a meta object at the given path when it is processed then the object is used in order to determine which branches have already been performed on the message (or skipped) and can therefore be skipped on this run.\n\nThis is a useful pattern when replaying messages that have failed some branches previously. For example, given the above example object the branches foo and bar would automatically be skipped, and baz would be reattempted.\n\nThe previous meta object will also be preserved in the field `\u003cmeta_path\u003e.previous` when the new meta object is written, preserving a full record of all workflow executions.\n\nIf a field `\u003cmeta_path\u003e.apply` exists in the meta object for a message and is an array then it will be used as an explicit list of stages to apply, all other stages will be skipped.\n\n== Resources\n\nIt's common to configure processors (and other components) xref:configuration:resources.adoc[as resources] in order to keep the pipeline configuration cleaner. With the workflow processor you can include branch processors configured as resources within your workflow either by specifying them by name in the field `order`, if Redpanda Connect doesn't find a branch within the workflow configuration of that name it'll refer to the resources.\n\nAlternatively, if you do not wish to have an explicit ordering, you can add resource names to the field `branch_resources` and they will be included in the workflow with automatic DAG resolution along with any branches configured in the `branches` field.\n\n=== Resource error conditions\n\nThere are two error conditions that could potentially occur when resources included in your workflow are mutated, and if you are planning to mutate resources in your workflow it is important that you understand them.\n\nThe first error case is that a resource in the workflow is removed and not replaced, when this happens the workflow will still be executed but the individual branch will fail. This should only happen if you explicitly delete a branch resource, as any mutation operation will create the new resource before removing the old one.\n\nThe second error case is when automatic DAG resolution is being used and a resource in the workflow is changed in a way that breaks the DAG (circular dependencies, etc). When this happens it is impossible to execute the workflow and therefore the processor will fail, which is possible to capture and handle using xref:configuration:error_handling.adoc[standard error handling patterns].\n\n== Error handling\n\nThe recommended approach to handle failures within a workflow is to query against the \u003c\u003cstructured-metadata, structured metadata\u003e\u003e it provides, as it provides granular information about exactly which branches failed and which ones succeeded and therefore aren't necessary to perform again.\n\nFor example, if our meta object is stored at the path `meta.workflow` and we wanted to check whether a message has failed for any branch we can do that using a xref:guides:bloblang/about.adoc[Bloblang query] like `this.meta.workflow.failed.length() | 0 \u003e 0`, or to check whether a specific branch failed we can use `this.exists(\"meta.workflow.failed.foo\")`.\n\nHowever, if structured metadata is disabled by setting the field `meta_path` to empty then the workflow processor instead adds a general error flag to messages when any executed branch fails. In this case it's possible to handle failures using xref:configuration:error_handling.adoc[standard error handling patterns].\n\n","examples":[{"title":"Automatic Ordering","summary":"\nWhen the field `order` is omitted a best attempt is made to determine a dependency tree between branches based on their request and result mappings. In the following example the branches foo and bar will be executed first in parallel, and afterwards the branch baz will be executed.","config":"\npipeline:\n processors:\n - workflow:\n meta_path: meta.workflow\n branches:\n foo:\n request_map: 'root = \"\"'\n processors:\n - http:\n url: TODO\n result_map: 'root.foo = this'\n\n bar:\n request_map: 'root = this.body'\n processors:\n - aws_lambda:\n function: TODO\n result_map: 'root.bar = this'\n\n baz:\n request_map: |\n root.fooid = this.foo.id\n root.barstuff = this.bar.content\n processors:\n - cache:\n resource: TODO\n operator: set\n key: ${! json(\"fooid\") }\n value: ${! json(\"barstuff\") }\n"},{"title":"Conditional Branches","summary":"\nBranches of a workflow are skipped when the `request_map` assigns `deleted()` to the root. In this example the branch A is executed when the document type is \"foo\", and branch B otherwise. Branch C is executed afterwards and is skipped unless either A or B successfully provided a result at `tmp.result`.","config":"\npipeline:\n processors:\n - workflow:\n branches:\n A:\n request_map: |\n root = if this.document.type != \"foo\" {\n deleted()\n }\n processors:\n - http:\n url: TODO\n result_map: 'root.tmp.result = this'\n\n B:\n request_map: |\n root = if this.document.type == \"foo\" {\n deleted()\n }\n processors:\n - aws_lambda:\n function: TODO\n result_map: 'root.tmp.result = this'\n\n C:\n request_map: |\n root = if this.tmp.result != null {\n deleted()\n }\n processors:\n - http:\n url: TODO_SOMEWHERE_ELSE\n result_map: 'root.tmp.result = this'\n"},{"title":"Resources","summary":"\nThe `order` field can be used in order to refer to \u003c\u003cresources, branch processor resources\u003e\u003e, this can sometimes make your pipeline configuration cleaner, as well as allowing you to reuse branch configurations in order places. It's also possible to mix and match branches configured within the workflow and configured as resources.","config":"\npipeline:\n processors:\n - workflow:\n order: [ [ foo, bar ], [ baz ] ]\n branches:\n bar:\n request_map: 'root = this.body'\n processors:\n - aws_lambda:\n function: TODO\n result_map: 'root.bar = this'\n\nprocessor_resources:\n - label: foo\n branch:\n request_map: 'root = \"\"'\n processors:\n - http:\n url: TODO\n result_map: 'root.foo = this'\n\n - label: baz\n branch:\n request_map: |\n root.fooid = this.foo.id\n root.barstuff = this.bar.content\n processors:\n - cache:\n resource: TODO\n operator: set\n key: ${! json(\"fooid\") }\n value: ${! json(\"barstuff\") }\n"}],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"meta_path","type":"string","kind":"scalar","description":"A xref:configuration:field_paths.adoc[dot path] indicating where to store and reference \u003c\u003cstructured-metadata, structured metadata\u003e\u003e about the workflow execution.","default":"meta.workflow"},{"name":"order","type":"string","kind":"2darray","description":"An explicit declaration of branch ordered tiers, which describes the order in which parallel tiers of branches should be executed. Branches should be identified by the name as they are configured in the field `branches`. It's also possible to specify branch processors configured \u003c\u003cresources, as a resource\u003e\u003e.","default":[],"examples":[[["foo","bar"],["baz"]],[["foo"],["bar"],["baz"]]]},{"name":"branch_resources","type":"string","kind":"array","description":"An optional list of xref:components:processors/branch.adoc[`branch` processor] names that are configured as \u003c\u003cresources\u003e\u003e. These resources will be included in the workflow with any branches configured inline within the \u003c\u003cbranches, `branches`\u003e\u003e field. The order and parallelism in which branches are executed is automatically resolved based on the mappings of each branch. When using resources with an explicit order it is not necessary to list resources in this field.","is_advanced":true,"default":[],"version":"3.38.0"},{"name":"branches","type":"object","kind":"map","description":"An object of named xref:components:processors/branch.adoc[`branch` processors] that make up the workflow. The order and parallelism in which branches are executed can either be made explicit with the field `order`, or if omitted an attempt is made to automatically resolve an ordering based on the mappings of each branch.","default":{},"children":[{"name":"request_map","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] that describes how to create a request payload suitable for the child processors of this branch. If left empty then the branch will begin with an exact copy of the origin message (including metadata).","default":"","bloblang":true,"examples":["root = {\n\t\"id\": this.doc.id,\n\t\"content\": this.doc.body.text\n}","root = if this.type == \"foo\" {\n\tthis.foo.request\n} else {\n\tdeleted()\n}"]},{"name":"processors","type":"processor","kind":"array","description":"A list of processors to apply to mapped requests. When processing message batches the resulting batch must match the size and ordering of the input batch, therefore filtering, grouping should not be performed within these processors."},{"name":"result_map","type":"string","kind":"scalar","description":"A xref:guides:bloblang/about.adoc[Bloblang mapping] that describes how the resulting messages from branched processing should be mapped back into the original payload. If left empty the origin message will remain unchanged (including metadata).","default":"","bloblang":true,"examples":["meta foo_code = metadata(\"code\")\nroot.foo_result = this","meta = metadata()\nroot.bar.body = this.body\nroot.bar.id = this.user.id","root.raw_result = content().string()","root.enrichments.foo = if metadata(\"request_failed\") != null {\n throw(metadata(\"request_failed\"))\n} else {\n this\n}","# Retain only the updated metadata fields which were present in the origin message\nmeta = metadata().filter(v -\u003e @.get(v.key) != null)"]}]}]}},{"name":"xml","type":"processor","status":"beta","plugin":true,"summary":"Parses messages as an XML document, performs a mutation on the data, and then overwrites the previous contents with the new value.","description":"\n== Operators\n\n=== `to_json`\n\nConverts an XML document into a JSON structure, where elements appear as keys of an object according to the following rules:\n\n- If an element contains attributes they are parsed by prefixing a hyphen, `-`, to the attribute label.\n- If the element is a simple element and has attributes, the element value is given the key `#text`.\n- XML comments, directives, and process instructions are ignored.\n- When elements are repeated the resulting JSON value is an array.\n\nFor example, given the following XML:\n\n```xml\n\u003croot\u003e\n \u003ctitle\u003eThis is a title\u003c/title\u003e\n \u003cdescription tone=\"boring\"\u003eThis is a description\u003c/description\u003e\n \u003celements id=\"1\"\u003efoo1\u003c/elements\u003e\n \u003celements id=\"2\"\u003efoo2\u003c/elements\u003e\n \u003celements\u003efoo3\u003c/elements\u003e\n\u003c/root\u003e\n```\n\nThe resulting JSON structure would look like this:\n\n```json\n{\n \"root\":{\n \"title\":\"This is a title\",\n \"description\":{\n \"#text\":\"This is a description\",\n \"-tone\":\"boring\"\n },\n \"elements\":[\n {\"#text\":\"foo1\",\"-id\":\"1\"},\n {\"#text\":\"foo2\",\"-id\":\"2\"},\n \"foo3\"\n ]\n }\n}\n```\n\nWith cast set to true, the resulting JSON structure would look like this:\n\n```json\n{\n \"root\":{\n \"title\":\"This is a title\",\n \"description\":{\n \"#text\":\"This is a description\",\n \"-tone\":\"boring\"\n },\n \"elements\":[\n {\"#text\":\"foo1\",\"-id\":1},\n {\"#text\":\"foo2\",\"-id\":2},\n \"foo3\"\n ]\n }\n}\n```","categories":["Parsing"],"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"operator","type":"string","kind":"scalar","description":"An XML \u003c\u003coperators, operation\u003e\u003e to apply to messages.","default":"","options":["to_json"],"linter":"\nlet options = {\n \"to_json\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"cast","type":"bool","kind":"scalar","description":"Whether to try to cast values that are numbers and booleans to the right type. Default: all values are strings.","default":false}]}}],"rate-limits":[{"name":"local","type":"rate_limit","status":"stable","plugin":true,"summary":"The local rate limit is a simple X every Y type rate limit that can be shared across any number of components within the pipeline but does not support distributed rate limits across multiple running instances of Benthos.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"count","type":"int","kind":"scalar","description":"The maximum number of requests to allow for a given period of time.","default":1000},{"name":"interval","type":"string","kind":"scalar","description":"The time window to limit requests by.","default":"1s"}]}},{"name":"redis","type":"rate_limit","status":"experimental","plugin":true,"summary":"A rate limit implementation using Redis. It works by using a simple token bucket algorithm to limit the number of requests to a given count within a given time period. The rate limit is shared across all instances of Redpanda Connect that use the same Redis instance, which must all have a consistent count and interval.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"The URL of the target Redis server. Database is optional and is supplied as the URL path.","examples":["redis://:6379","redis://localhost:6379","redis://foousername:foopassword@redisplace:6379","redis://:foopassword@redisplace:6379","redis://localhost:6379/1","redis://localhost:6379/1,redis://localhost:6380/1"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"kind","type":"string","kind":"scalar","description":"Specifies a simple, cluster-aware, or failover-aware redis client.","is_advanced":true,"default":"simple","options":["simple","cluster","failover"],"linter":"\nlet options = {\n \"simple\": true,\n \"cluster\": true,\n \"failover\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"master","type":"string","kind":"scalar","description":"Name of the redis master when `kind` is `failover`","is_advanced":true,"default":"","examples":["mymaster"]},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"count","type":"int","kind":"scalar","description":"The maximum number of messages to allow for a given period of time.","default":1000,"linter":"root = if this \u003c= 0 { [ \"count must be larger than zero\" ] }"},{"name":"interval","type":"string","kind":"scalar","description":"The time window to limit requests by.","default":"1s"},{"name":"key","type":"string","kind":"scalar","description":"The key to use for the rate limit."}]},"version":"4.12.0"}],"metrics":[{"name":"aws_cloudwatch","type":"metrics","status":"stable","plugin":true,"summary":"Send metrics to AWS CloudWatch using the PutMetricData endpoint.","description":"\n== Timing metrics\n\nThe smallest timing unit that CloudWatch supports is microseconds, therefore timing metrics are automatically downgraded to microseconds (by dividing delta values by 1000). This conversion will also apply to custom timing metrics produced with a `metric` processor.\n\n== Billing\n\nAWS bills per metric series exported, it is therefore STRONGLY recommended that you reduce the metrics that are exposed with a `mapping` like this:\n\n```yaml\nmetrics:\n mapping: |\n if ![\n \"input_received\",\n \"input_latency\",\n \"output_sent\",\n ].contains(this) { deleted() }\n aws_cloudwatch:\n namespace: Foo\n```","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"namespace","type":"string","kind":"scalar","description":"The namespace used to distinguish metrics from other services.","default":"Benthos"},{"name":"flush_period","type":"string","kind":"scalar","description":"The period of time between PutMetricData requests.","is_advanced":true,"default":"100ms"},{"name":"region","type":"string","kind":"scalar","description":"The AWS region to target.","is_advanced":true,"is_optional":true},{"name":"endpoint","type":"string","kind":"scalar","description":"Allows you to specify a custom endpoint for the AWS API.","is_advanced":true,"is_optional":true},{"name":"credentials","type":"object","kind":"scalar","description":"Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].","is_advanced":true,"is_optional":true,"children":[{"name":"profile","type":"string","kind":"scalar","description":"A profile from `~/.aws/credentials` to use.","is_advanced":true,"is_optional":true},{"name":"id","type":"string","kind":"scalar","description":"The ID of credentials to use.","is_advanced":true,"is_optional":true},{"name":"secret","type":"string","kind":"scalar","description":"The secret for the credentials being used.","is_advanced":true,"is_optional":true,"is_secret":true,"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"token","type":"string","kind":"scalar","description":"The token for the credentials being used, required when using short term credentials.","is_advanced":true,"is_optional":true},{"name":"from_ec2_role","type":"bool","kind":"scalar","description":"Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].","is_advanced":true,"is_optional":true,"version":"4.2.0"},{"name":"role","type":"string","kind":"scalar","description":"A role ARN to assume.","is_advanced":true,"is_optional":true},{"name":"role_external_id","type":"string","kind":"scalar","description":"An external ID to provide when assuming a role.","is_advanced":true,"is_optional":true}]}]},"version":"3.36.0"},{"name":"influxdb","type":"metrics","status":"beta","plugin":true,"summary":"Send metrics to InfluxDB 1.x using the `/write` endpoint.","description":"See https://docs.influxdata.com/influxdb/v1.8/tools/api/#write-http-endpoint for further details on the write API.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"url","type":"string","kind":"scalar","description":"A URL of the format `[https|http|udp]://host:port` to the InfluxDB host.","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"db","type":"string","kind":"scalar","description":"The name of the database to use."},{"name":"tls","type":"object","kind":"scalar","description":"Custom TLS settings can be used to override system defaults.","is_advanced":true,"children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether custom TLS settings are enabled.","is_advanced":true,"default":false},{"name":"skip_cert_verify","type":"bool","kind":"scalar","description":"Whether to skip server side certificate verification.","is_advanced":true,"default":false},{"name":"enable_renegotiation","type":"bool","kind":"scalar","description":"Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`.","is_advanced":true,"default":false,"version":"3.45.0"},{"name":"root_cas","type":"string","kind":"scalar","description":"An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"is_secret":true,"default":"","examples":["-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"root_cas_file","type":"string","kind":"scalar","description":"An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate.","is_advanced":true,"default":"","examples":["./root_cas.pem"]},{"name":"client_certs","type":"object","kind":"array","description":"A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both.","is_advanced":true,"default":[],"examples":[[{"cert":"foo","key":"bar"}],[{"cert_file":"./example.pem","key_file":"./example.key"}]],"children":[{"name":"cert","type":"string","kind":"scalar","description":"A plain text certificate to use.","is_advanced":true,"default":""},{"name":"key","type":"string","kind":"scalar","description":"A plain text certificate key to use.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"cert_file","type":"string","kind":"scalar","description":"The path of a certificate to use.","is_advanced":true,"default":""},{"name":"key_file","type":"string","kind":"scalar","description":"The path of a certificate key to use.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format.\n\nBecause the obsolete pbeWithMD5AndDES-CBC algorithm does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext.\n","is_advanced":true,"is_secret":true,"default":"","examples":["foo","${KEY_PASSWORD}"],"scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]}]},{"name":"username","type":"string","kind":"scalar","description":"A username (when applicable).","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"A password (when applicable).","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"},{"name":"include","type":"object","kind":"scalar","description":"Optional additional metrics to collect, enabling these metrics may have some performance implications as it acquires a global semaphore and does `stoptheworld()`.","is_advanced":true,"children":[{"name":"runtime","type":"string","kind":"scalar","description":"A duration string indicating how often to poll and collect runtime metrics. Leave empty to disable this metric","is_advanced":true,"default":"","examples":["1m"]},{"name":"debug_gc","type":"string","kind":"scalar","description":"A duration string indicating how often to poll and collect GC metrics. Leave empty to disable this metric.","is_advanced":true,"default":"","examples":["1m"]}]},{"name":"interval","type":"string","kind":"scalar","description":"A duration string indicating how often metrics should be flushed.","is_advanced":true,"default":"1m"},{"name":"ping_interval","type":"string","kind":"scalar","description":"A duration string indicating how often to ping the host.","is_advanced":true,"default":"20s"},{"name":"precision","type":"string","kind":"scalar","description":"[ns|us|ms|s] timestamp precision passed to write api.","is_advanced":true,"default":"s"},{"name":"timeout","type":"string","kind":"scalar","description":"How long to wait for response for both ping and writing metrics.","is_advanced":true,"default":"5s"},{"name":"tags","type":"string","kind":"map","description":"Global tags added to each metric.","is_advanced":true,"default":{},"examples":[{"hostname":"localhost","zone":"danger"}]},{"name":"retention_policy","type":"string","kind":"scalar","description":"Sets the retention policy for each write.","is_advanced":true,"is_optional":true},{"name":"write_consistency","type":"string","kind":"scalar","description":"[any|one|quorum|all] sets write consistency when available.","is_advanced":true,"is_optional":true}]},"version":"3.36.0"},{"name":"json_api","type":"metrics","status":"stable","plugin":true,"summary":"Serves metrics as JSON object with the service wide HTTP service at the endpoints `/stats` and `/metrics`.","description":"This metrics type is useful for debugging as it provides a human readable format that you can parse with tools such as `jq`","categories":null,"config":{"name":"","type":"object","kind":"scalar","default":{}}},{"name":"logger","type":"metrics","status":"beta","plugin":true,"summary":"Prints aggregated metrics through the logger.","description":"\nPrints each metric produced by Redpanda Connect as a log event (level `info` by default) during shutdown, and optionally on an interval.\n\nThis metrics type is useful for debugging pipelines when you only have access to the logger output and not the service-wide server. Otherwise it's recommended that you use either the `prometheus` or `json_api`types.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"push_interval","type":"string","kind":"scalar","description":"An optional period of time to continuously print all metrics.","is_optional":true},{"name":"flush_metrics","type":"bool","kind":"scalar","description":"Whether counters and timing metrics should be reset to 0 each time metrics are printed.","default":false}]}},{"name":"none","type":"metrics","status":"stable","plugin":true,"summary":"Disable metrics entirely.","categories":null,"config":{"name":"","type":"object","kind":"scalar","default":{}}},{"name":"prometheus","type":"metrics","status":"stable","plugin":true,"summary":"Host endpoints (`/metrics` and `/stats`) for Prometheus scraping.","categories":null,"footnotes":"\n== Push gateway\n\nThe field `push_url` is optional and when set will trigger a push of metrics to a https://prometheus.io/docs/instrumenting/pushing/[Prometheus Push Gateway^] once Redpanda Connect shuts down. It is also possible to specify a `push_interval` which results in periodic pushes.\n\nThe Push Gateway is useful for when Redpanda Connect instances are short lived. Do not include the \"/metrics/jobs/...\" path in the push URL.\n\nIf the Push Gateway requires HTTP Basic Authentication it can be configured with `push_basic_auth`.","config":{"name":"","type":"object","kind":"scalar","children":[{"name":"use_histogram_timing","type":"bool","kind":"scalar","description":"Whether to export timing metrics as a histogram, if `false` a summary is used instead. When exporting histogram timings the delta values are converted from nanoseconds into seconds in order to better fit within bucket definitions. For more information on histograms and summaries refer to: https://prometheus.io/docs/practices/histograms/.","is_advanced":true,"default":false,"version":"3.63.0"},{"name":"histogram_buckets","type":"float","kind":"array","description":"Timing metrics histogram buckets (in seconds). If left empty defaults to DefBuckets (https://pkg.go.dev/github.com/prometheus/client_golang/prometheus#pkg-variables). Applicable when `use_histogram_timing` is set to `true`.","is_advanced":true,"default":[],"version":"3.63.0"},{"name":"summary_quantiles_objectives","type":"object","kind":"array","description":"A list of timing metrics summary buckets (as quantiles). Applicable when `use_histogram_timing` is set to `false`.","is_advanced":true,"default":[{"error":0.05,"quantile":0.5},{"error":0.01,"quantile":0.9},{"error":0.001,"quantile":0.99}],"examples":[[{"error":0.05,"quantile":0.5},{"error":0.01,"quantile":0.9},{"error":0.001,"quantile":0.99}]],"children":[{"name":"quantile","type":"float","kind":"scalar","description":"Quantile value.","is_advanced":true,"default":0},{"name":"error","type":"float","kind":"scalar","description":"Permissible margin of error for quantile calculations. Precise calculations in a streaming context (without prior knowledge of the full dataset) can be resource-intensive. To balance accuracy with computational efficiency, an error margin is introduced. For instance, if the 90th quantile (`0.9`) is determined to be `100ms` with a 1% error margin (`0.01`), the true value will fall within the `[99ms, 101ms]` range.)","is_advanced":true,"default":0}],"version":"4.23.0"},{"name":"add_process_metrics","type":"bool","kind":"scalar","description":"Whether to export process metrics such as CPU and memory usage in addition to Redpanda Connect metrics.","is_advanced":true,"default":false},{"name":"add_go_metrics","type":"bool","kind":"scalar","description":"Whether to export Go runtime metrics such as GC pauses in addition to Redpanda Connect metrics.","is_advanced":true,"default":false},{"name":"push_url","type":"string","kind":"scalar","description":"An optional \u003c\u003cpush-gateway, Push Gateway URL\u003e\u003e to push metrics to.","is_advanced":true,"is_optional":true,"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"push_interval","type":"string","kind":"scalar","description":"The period of time between each push when sending metrics to a Push Gateway.","is_advanced":true,"is_optional":true},{"name":"push_job_name","type":"string","kind":"scalar","description":"An identifier for push jobs.","is_advanced":true,"default":"benthos_push"},{"name":"push_basic_auth","type":"object","kind":"scalar","description":"The Basic Authentication credentials.","is_advanced":true,"children":[{"name":"username","type":"string","kind":"scalar","description":"The Basic Authentication username.","is_advanced":true,"default":""},{"name":"password","type":"string","kind":"scalar","description":"The Basic Authentication password.","is_advanced":true,"is_secret":true,"default":"","scrubber":"root = if this != null \u0026\u0026 this != \"\" \u0026\u0026 !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"}]},{"name":"file_output_path","type":"string","kind":"scalar","description":"An optional file path to write all prometheus metrics on service shutdown.","is_advanced":true,"default":""}]}},{"name":"statsd","type":"metrics","status":"stable","plugin":true,"summary":"Pushes metrics using the https://github.com/statsd/statsd[StatsD protocol^]. Supported tagging formats are 'none', 'datadog' and 'influxdb'.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"address","type":"string","kind":"scalar","description":"The address to send metrics to."},{"name":"flush_period","type":"string","kind":"scalar","description":"The time interval between metrics flushes.","default":"100ms"},{"name":"tag_format","type":"string","kind":"scalar","description":"Metrics tagging is supported in a variety of formats.","default":"none","options":["none","datadog","influxdb"],"linter":"\nlet options = {\n \"none\": true,\n \"datadog\": true,\n \"influxdb\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"}]}}],"tracers":[{"name":"gcp_cloudtrace","type":"tracer","status":"experimental","plugin":true,"summary":"Send tracing events to a https://cloud.google.com/trace[Google Cloud Trace^].","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"project","type":"string","kind":"scalar","description":"The google project with Cloud Trace API enabled. If this is omitted then the Google Cloud SDK will attempt auto-detect it from the environment."},{"name":"sampling_ratio","type":"float","kind":"scalar","description":"Sets the ratio of traces to sample. Tuning the sampling ratio is recommended for high-volume production workloads.","default":1,"examples":[1]},{"name":"tags","type":"string","kind":"map","description":"A map of tags to add to tracing spans.","is_advanced":true,"default":{}},{"name":"flush_interval","type":"string","kind":"scalar","description":"The period of time between each flush of tracing spans.","is_optional":true}]},"version":"4.2.0"},{"name":"jaeger","type":"tracer","status":"stable","plugin":true,"summary":"Send tracing events to a https://www.jaegertracing.io/[Jaeger^] agent or collector.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"agent_address","type":"string","kind":"scalar","description":"The address of a Jaeger agent to send tracing events to.","default":"","examples":["jaeger-agent:6831"]},{"name":"collector_url","type":"string","kind":"scalar","description":"The URL of a Jaeger collector to send tracing events to. If set, this will override `agent_address`.","default":"","examples":["https://jaeger-collector:14268/api/traces"],"version":"3.38.0"},{"name":"sampler_type","type":"string","kind":"scalar","description":"The sampler type to use.","default":"const","annotated_options":[["const","Sample a percentage of traces. 1 or more means all traces are sampled, 0 means no traces are sampled and anything in between means a percentage of traces are sampled. Tuning the sampling rate is recommended for high-volume production workloads."]],"linter":"\nlet options = {\n \"const\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"},{"name":"sampler_param","type":"float","kind":"scalar","description":"A parameter to use for sampling. This field is unused for some sampling types.","is_advanced":true,"default":1},{"name":"tags","type":"string","kind":"map","description":"A map of tags to add to tracing spans.","is_advanced":true,"default":{}},{"name":"flush_interval","type":"string","kind":"scalar","description":"The period of time between each flush of tracing spans.","is_optional":true}]}},{"name":"none","type":"tracer","status":"stable","plugin":true,"summary":"Do not send tracing events anywhere.","categories":null,"config":{"name":"","type":"object","kind":"scalar","default":{}}},{"name":"open_telemetry_collector","type":"tracer","status":"experimental","plugin":true,"summary":"Send tracing events to an https://opentelemetry.io/docs/collector/[Open Telemetry collector^].","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"http","type":"object","kind":"array","description":"A list of http collectors.","children":[{"name":"address","type":"string","kind":"scalar","description":"The endpoint of a collector to send tracing events to.","is_optional":true,"examples":["localhost:4318"]},{"name":"url","type":"string","kind":"scalar","description":"The URL of a collector to send tracing events to.","is_deprecated":true,"default":"localhost:4318"},{"name":"secure","type":"bool","kind":"scalar","description":"Connect to the collector over HTTPS","default":false}]},{"name":"grpc","type":"object","kind":"array","description":"A list of grpc collectors.","children":[{"name":"address","type":"string","kind":"scalar","description":"The endpoint of a collector to send tracing events to.","is_optional":true,"examples":["localhost:4317"],"scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"url","type":"string","kind":"scalar","description":"The URL of a collector to send tracing events to.","is_deprecated":true,"default":"localhost:4317","scrubber":"\nlet pass = this.parse_url().user.password.or(\"\")\nroot = if $pass != \"\" \u0026\u0026 !$pass.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n}\n"},{"name":"secure","type":"bool","kind":"scalar","description":"Connect to the collector with client transport security","default":false}]},{"name":"tags","type":"string","kind":"map","description":"A map of tags to add to all tracing spans.","is_advanced":true,"default":{}},{"name":"sampling","type":"object","kind":"scalar","description":"Settings for trace sampling. Sampling is recommended for high-volume production workloads.","children":[{"name":"enabled","type":"bool","kind":"scalar","description":"Whether to enable sampling.","default":false},{"name":"ratio","type":"float","kind":"scalar","description":"Sets the ratio of traces to sample.","is_optional":true,"examples":[0.85,0.5]}],"version":"4.25.0"}]}}],"scanners":[{"name":"avro","type":"scanner","status":"stable","plugin":true,"summary":"Consume a stream of Avro OCF datum.","description":"\n== Avro JSON format\n\nThis scanner yields documents formatted as https://avro.apache.org/docs/current/specification/_print/#json-encoding[Avro JSON^] when decoding with Avro schemas. In this format the value of a union is encoded in JSON as follows:\n\n- if its type is `null`, then it is encoded as a JSON `null`;\n- otherwise it is encoded as a JSON object with one name/value pair whose name is the type's name and whose value is the recursively encoded value. For Avro's named types (record, fixed or enum) the user-specified name is used, for other types the type name is used.\n\nFor example, the union schema `[\"null\",\"string\",\"Foo\"]`, where `Foo` is a record name, would encode:\n\n- `null` as `null`;\n- the string `\"a\"` as `{\"string\": \"a\"}`; and\n- a `Foo` instance as `{\"Foo\": {...}}`, where `{...}` indicates the JSON encoding of a `Foo` instance.\n\nHowever, it is possible to instead create documents in https://pkg.go.dev/github.com/linkedin/goavro/v2#NewCodecForStandardJSONFull[standard/raw JSON format^] by setting the field \u003c\u003cavro_raw_json,`avro_raw_json`\u003e\u003e to `true`.\n\nThis scanner also emits the canonical Avro schema as `@avro_schema` metadata, along with the schema's fingerprint available via `@avro_schema_fingerprint`.\n","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"raw_json","type":"bool","kind":"scalar","description":"Whether messages should be decoded into normal JSON (\"json that meets the expectations of regular internet json\") rather than https://avro.apache.org/docs/current/specification/_print/#json-encoding[Avro JSON^]. If `true` the schema returned from the subject should be decoded as https://pkg.go.dev/github.com/linkedin/goavro/v2#NewCodecForStandardJSONFull[standard json^] instead of as https://pkg.go.dev/github.com/linkedin/goavro/v2#NewCodec[avro json^]. There is a https://github.com/linkedin/goavro/blob/5ec5a5ee7ec82e16e6e2b438d610e1cab2588393/union.go#L224-L249[comment in goavro^], the https://github.com/linkedin/goavro[underlining library used for avro serialization^], that explains in more detail the difference between the standard json and avro json.","is_advanced":true,"default":false}]}},{"name":"chunker","type":"scanner","status":"stable","plugin":true,"summary":"Split an input stream into chunks of a given number of bytes.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"size","type":"int","kind":"scalar","description":"The size of each chunk in bytes."}]}},{"name":"csv","type":"scanner","status":"stable","plugin":true,"summary":"Consume comma-separated values row by row, including support for custom delimiters.","description":"\n== Metadata\n\nThis scanner adds the following metadata to each message:\n\n- `csv_row` The index of each row, beginning at 0.\n\n","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"custom_delimiter","type":"string","kind":"scalar","description":"Use a provided custom delimiter instead of the default comma.","is_optional":true},{"name":"parse_header_row","type":"bool","kind":"scalar","description":"Whether to reference the first row as a header row. If set to true the output structure for messages will be an object where field keys are determined by the header row. Otherwise, each message will consist of an array of values from the corresponding CSV row.","default":true},{"name":"lazy_quotes","type":"bool","kind":"scalar","description":"If set to `true`, a quote may appear in an unquoted field and a non-doubled quote may appear in a quoted field.","default":false},{"name":"continue_on_error","type":"bool","kind":"scalar","description":"If a row fails to parse due to any error emit an empty message marked with the error and then continue consuming subsequent rows when possible. This can sometimes be useful in situations where input data contains individual rows which are malformed. However, when a row encounters a parsing error it is impossible to guarantee that following rows are valid, as this indicates that the input data is unreliable and could potentially emit misaligned rows.","default":false}]}},{"name":"decompress","type":"scanner","status":"stable","plugin":true,"summary":"Decompress the stream of bytes according to an algorithm, before feeding it into a child scanner.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"algorithm","type":"string","kind":"scalar","description":"One of `gzip`, `pgzip`, `zlib`, `bzip2`, `flate`, `snappy`, `lz4`, `zstd`."},{"name":"into","type":"scanner","kind":"scalar","description":"The child scanner to feed the decompressed stream into.","default":{"to_the_end":{}}}]}},{"name":"json_documents","type":"scanner","status":"stable","plugin":true,"summary":"Consumes a stream of one or more JSON documents.","categories":null,"config":{"name":"","type":"object","kind":"scalar","default":{}},"version":"4.27.0"},{"name":"lines","type":"scanner","status":"stable","plugin":true,"summary":"Split an input stream into a message per line of data.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"custom_delimiter","type":"string","kind":"scalar","description":"Use a provided custom delimiter for detecting the end of a line rather than a single line break.","is_optional":true},{"name":"max_buffer_size","type":"int","kind":"scalar","description":"Set the maximum buffer size for storing line data, this limits the maximum size that a line can be without causing an error.","default":65536},{"name":"omit_empty","type":"bool","kind":"scalar","description":"Omit empty lines.","default":false}]}},{"name":"re_match","type":"scanner","status":"stable","plugin":true,"summary":"Split an input stream into segments matching against a regular expression.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"pattern","type":"string","kind":"scalar","description":"The pattern to match against.","examples":["(?m)^\\d\\d:\\d\\d:\\d\\d"]},{"name":"max_buffer_size","type":"int","kind":"scalar","description":"Set the maximum buffer size for storing line data, this limits the maximum size that a message can be without causing an error.","default":65536}]}},{"name":"skip_bom","type":"scanner","status":"stable","plugin":true,"summary":"Skip one or more byte order marks for each opened child scanner.","categories":null,"config":{"name":"","type":"object","kind":"scalar","children":[{"name":"into","type":"scanner","kind":"scalar","description":"The child scanner to feed the resulting stream into.","default":{"to_the_end":{}}}]}},{"name":"switch","type":"scanner","status":"stable","plugin":true,"summary":"Select a child scanner dynamically for source data based on factors such as the filename.","description":"This scanner outlines a list of potential child scanner candidates to be chosen, and for each source of data the first candidate to pass will be selected. A candidate without any conditions acts as a catch-all and will pass for every source, it is recommended to always have a catch-all scanner at the end of your list. If a given source of data does not pass a candidate an error is returned and the data is rejected.","categories":null,"examples":[{"title":"Switch based on file name","summary":"In this example a file input chooses a scanner based on the extension of each file","config":"\ninput:\n file:\n paths: [ ./data/* ]\n scanner:\n switch:\n - re_match_name: '\\.avro$'\n scanner: { avro: {} }\n\n - re_match_name: '\\.csv$'\n scanner: { csv: {} }\n\n - re_match_name: '\\.csv.gz$'\n scanner:\n decompress:\n algorithm: gzip\n into:\n csv: {}\n\n - re_match_name: '\\.tar$'\n scanner: { tar: {} }\n\n - re_match_name: '\\.tar.gz$'\n scanner:\n decompress:\n algorithm: gzip\n into:\n tar: {}\n\n - scanner: { to_the_end: {} }\n"}],"config":{"name":"","type":"object","kind":"array","children":[{"name":"re_match_name","type":"string","kind":"scalar","description":"A regular expression to test against the name of each source of data fed into the scanner (filename or equivalent). If this pattern matches the child scanner is selected.","is_optional":true},{"name":"scanner","type":"scanner","kind":"scalar","description":"The scanner to activate if this candidate passes."}]}},{"name":"tar","type":"scanner","status":"stable","plugin":true,"summary":"Consume a tar archive file by file.","description":"\n== Metadata\n\nThis scanner adds the following metadata to each message:\n\n- `tar_name`\n\n","categories":null,"config":{"name":"","type":"object","kind":"scalar","default":{}}},{"name":"to_the_end","type":"scanner","status":"stable","plugin":true,"summary":"Read the input stream all the way until the end and deliver it as a single message.","description":"\n[CAUTION]\n====\nSome sources of data may not have a logical end, therefore caution should be made to exclusively use this scanner when the end of an input stream is clearly defined (and well within memory).\n====\n","categories":null,"config":{"name":"","type":"object","kind":"scalar","default":{}}}],"bloblang-functions":[{"status":"stable","category":"Message Info","name":"batch_index","description":"Returns the index of the mapped message within a batch. This is useful for applying maps only on certain messages of a batch.","params":{},"examples":[{"mapping":"root = if batch_index() \u003e 0 { deleted() }","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"Message Info","name":"batch_size","description":"Returns the size of the message batch.","params":{},"examples":[{"mapping":"root.foo = batch_size()","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"Message Info","name":"content","description":"Returns the full raw contents of the mapping target message as a byte array. When mapping to a JSON field the value should be encoded using the method xref:guides:bloblang/methods.adoc#encode[`encode`], or cast to a string directly using the method xref:guides:bloblang/methods.adoc#string[`string`], otherwise it will be base64 encoded by default.","params":{},"examples":[{"mapping":"root.doc = content().string()","summary":"","results":[["{\"foo\":\"bar\"}","{\"doc\":\"{\\\"foo\\\":\\\"bar\\\"}\"}"]],"skip_testing":false}],"impure":false},{"status":"deprecated","category":"Deprecated","name":"count","description":"The `count` function is a counter starting at 1 which increments after each time it is called. Count takes an argument which is an identifier for the counter, allowing you to specify multiple unique counters in your configuration.","params":{"named":[{"name":"name","description":"An identifier for the counter.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"examples":[{"mapping":"root = this\nroot.id = count(\"bloblang_function_example\")","summary":"","results":[["{\"message\":\"foo\"}","{\"id\":1,\"message\":\"foo\"}"],["{\"message\":\"bar\"}","{\"id\":2,\"message\":\"bar\"}"]],"skip_testing":false}],"impure":true},{"status":"experimental","category":"General","name":"counter","description":"Returns a non-negative integer that increments each time it is resolved, yielding the minimum (`1` by default) as the first value. Each instantiation of `counter` has its own independent count. Once the maximum integer (or `max` argument) is reached the counter resets back to the minimum.","params":{"named":[{"name":"min","description":"The minimum value of the counter, this is the first value that will be yielded. If this parameter is dynamic it will be resolved only once during the lifetime of the mapping.","type":"query expression","no_dynamic":false,"scalars_to_literal":true,"default":1},{"name":"max","description":"The maximum value of the counter, once this value is yielded the counter will reset back to the min. If this parameter is dynamic it will be resolved only once during the lifetime of the mapping.","type":"query expression","no_dynamic":false,"scalars_to_literal":true,"default":9223372036854775807},{"name":"set","description":"An optional mapping that when specified will be executed each time the counter is resolved. When this mapping resolves to a non-negative integer value it will cause the counter to reset to this value and yield it. If this mapping is omitted or doesn't resolve to anything then the counter will increment and yield the value as normal. If this mapping resolves to `null` then the counter is not incremented and the current value is yielded. If this mapping resolves to a deletion then the counter is reset to the `min` value.","type":"query expression","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"examples":[{"mapping":"root.id = counter()","summary":"","results":[["{}","{\"id\":1}"],["{}","{\"id\":2}"]],"skip_testing":false},{"mapping":"\nmap foos {\n root = counter()\n}\n\nroot.meow_id = null.apply(\"foos\")\nroot.woof_id = null.apply(\"foos\")\n","summary":"It's possible to increment a counter multiple times within a single mapping invocation using a map.","results":[["{}","{\"meow_id\":1,\"woof_id\":2}"],["{}","{\"meow_id\":3,\"woof_id\":4}"]],"skip_testing":false},{"mapping":"root.consecutive_doggos = counter(min: 1, set: if !this.sound.lowercase().contains(\"woof\") { 0 })","summary":"By specifying an optional `set` parameter it is possible to dynamically reset the counter based on input data.","results":[["{\"sound\":\"woof woof\"}","{\"consecutive_doggos\":1}"],["{\"sound\":\"woofer wooooo\"}","{\"consecutive_doggos\":2}"],["{\"sound\":\"meow\"}","{\"consecutive_doggos\":0}"],["{\"sound\":\"uuuuh uh uh woof uhhhhhh\"}","{\"consecutive_doggos\":1}"]],"skip_testing":false},{"mapping":"root.things = counter(set: if this.id == null { null })","summary":"The `set` parameter can also be utilized to peek at the counter without mutating it by returning `null`.","results":[["{\"id\":\"a\"}","{\"things\":1}"],["{\"id\":\"b\"}","{\"things\":2}"],["{\"what\":\"just checking\"}","{\"things\":2}"],["{\"id\":\"c\"}","{\"things\":3}"]],"skip_testing":false}],"impure":false},{"status":"stable","category":"General","name":"deleted","description":"A function that returns a result indicating that the mapping target should be deleted. Deleting, also known as dropping, messages will result in them being acknowledged as successfully processed to inputs in a Redpanda Connect pipeline. For more information about error handling patterns read xref:configuration:error_handling.adoc[].","params":{},"examples":[{"mapping":"root = this\nroot.bar = deleted()","summary":"","results":[["{\"bar\":\"bar_value\",\"baz\":\"baz_value\",\"foo\":\"foo value\"}","{\"baz\":\"baz_value\",\"foo\":\"foo value\"}"]],"skip_testing":false},{"mapping":"root.new_nums = this.nums.map_each(num -\u003e if num \u003c 10 { deleted() } else { num - 10 })","summary":"Since the result is a value it can be used to do things like remove elements of an array within `map_each`.","results":[["{\"nums\":[3,11,4,17]}","{\"new_nums\":[1,7]}"]],"skip_testing":false}],"impure":false},{"status":"stable","category":"Environment","name":"env","description":"Returns the value of an environment variable, or `null` if the environment variable does not exist.","params":{"named":[{"name":"name","description":"The name of an environment variable.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"no_cache","description":"Force the variable lookup to occur for each mapping invocation.","type":"bool","no_dynamic":false,"scalars_to_literal":false,"default":false}]},"examples":[{"mapping":"root.thing.key = env(\"key\").or(\"default value\")","summary":"","results":[],"skip_testing":false},{"mapping":"root.thing.key = env(this.thing.key_name)","summary":"","results":[],"skip_testing":false},{"mapping":"root.thing.key = env(name: \"key\", no_cache: true)","summary":"When the name parameter is static this function will only resolve once and yield the same result for each invocation as an optimization, this means that updates to env vars during runtime will not be reflected. You can disable this cache with the optional parameter `no_cache`, which when set to `true` will cause the variable lookup to be performed for each execution of the mapping.","results":[],"skip_testing":false}],"impure":true},{"status":"stable","category":"Message Info","name":"error","description":"If an error has occurred during the processing of a message this function returns the reported cause of the error as a string, otherwise `null`. For more information about error handling patterns read xref:configuration:error_handling.adoc[].","params":{},"examples":[{"mapping":"root.doc.error = error()","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"Message Info","name":"error_source_label","description":"Returns the label of the source component which raised the error during the processing of a message or an empty string if not set. `null` is returned when the error is null or no source component is associated with it. For more information about error handling patterns read xref:configuration:error_handling.adoc[].","params":{},"examples":[{"mapping":"root.doc.error_source_label = error_source_label()","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"Message Info","name":"error_source_name","description":"Returns the name of the source component which raised the error during the processing of a message. `null` is returned when the error is null or no source component is associated with it. For more information about error handling patterns read xref:configuration:error_handling.adoc[].","params":{},"examples":[{"mapping":"root.doc.error_source_name = error_source_name()","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"Message Info","name":"error_source_path","description":"Returns the path of the source component which raised the error during the processing of a message. `null` is returned when the error is null or no source component is associated with it. For more information about error handling patterns read xref:configuration:error_handling.adoc[].","params":{},"examples":[{"mapping":"root.doc.error_source_path = error_source_path()","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"Message Info","name":"errored","description":"Returns a boolean value indicating whether an error has occurred during the processing of a message. For more information about error handling patterns read xref:configuration:error_handling.adoc[].","params":{},"examples":[{"mapping":"root.doc.status = if errored() { 400 } else { 200 }","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"beta","category":"Fake Data Generation","name":"fake","description":"Takes in a string that maps to a https://github.com/go-faker/faker[faker^] function and returns the result from that faker function. Returns an error if the given string doesn't match a supported faker function. Supported functions: `latitude`, `longitude`, `unix_time`, `date`, `time_string`, `month_name`, `year_string`, `day_of_week`, `day_of_month`, `timestamp`, `century`, `timezone`, `time_period`, `email`, `mac_address`, `domain_name`, `url`, `username`, `ipv4`, `ipv6`, `password`, `jwt`, `word`, `sentence`, `paragraph`, `cc_type`, `cc_number`, `currency`, `amount_with_currency`, `title_male`, `title_female`, `first_name`, `first_name_male`, `first_name_female`, `last_name`, `name`, `gender`, `chinese_first_name`, `chinese_last_name`, `chinese_name`, `phone_number`, `toll_free_phone_number`, `e164_phone_number`, `uuid_hyphenated`, `uuid_digit`. Refer to the https://github.com/go-faker/faker[faker^] docs for details on these functions.","params":{"named":[{"name":"function","description":"The name of the function to use to generate the value.","type":"string","no_dynamic":false,"scalars_to_literal":false,"default":""}]},"examples":[{"mapping":"root.time = fake(\"time_string\")","summary":"Use `time_string` to generate a time in the format `00:00:00`:","results":[],"skip_testing":false},{"mapping":"root.email = fake(\"email\")","summary":"Use `email` to generate a string in email address format:","results":[],"skip_testing":false},{"mapping":"root.jwt = fake(\"jwt\")","summary":"Use `jwt` to generate a JWT token:","results":[],"skip_testing":false},{"mapping":"root.uuid = fake(\"uuid_hyphenated\")","summary":"Use `uuid_hyphenated` to generate a hyphenated UUID:","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"Environment","name":"file","description":"Reads a file and returns its contents. Relative paths are resolved from the directory of the process executing the mapping. In order to read files relative to the mapping file use the newer \u003c\u003cfile_rel, `file_rel` function\u003e\u003e","params":{"named":[{"name":"path","description":"The path of the target file.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"no_cache","description":"Force the file to be read for each mapping invocation.","type":"bool","no_dynamic":false,"scalars_to_literal":false,"default":false}]},"examples":[{"mapping":"root.doc = file(env(\"BENTHOS_TEST_BLOBLANG_FILE\")).parse_json()","summary":"","results":[["{}","{\"doc\":{\"foo\":\"bar\"}}"]],"skip_testing":false},{"mapping":"root.doc = file(path: env(\"BENTHOS_TEST_BLOBLANG_FILE\"), no_cache: true).parse_json()","summary":"When the path parameter is static this function will only read the specified file once and yield the same result for each invocation as an optimization, this means that updates to files during runtime will not be reflected. You can disable this cache with the optional parameter `no_cache`, which when set to `true` will cause the file to be read for each execution of the mapping.","results":[["{}","{\"doc\":{\"foo\":\"bar\"}}"]],"skip_testing":false}],"impure":true},{"status":"stable","category":"Environment","name":"file_rel","description":"Reads a file and returns its contents. Relative paths are resolved from the directory of the mapping.","params":{"named":[{"name":"path","description":"The path of the target file.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"no_cache","description":"Force the file to be read for each mapping invocation.","type":"bool","no_dynamic":false,"scalars_to_literal":false,"default":false}]},"examples":[{"mapping":"root.doc = file_rel(env(\"BENTHOS_TEST_BLOBLANG_FILE\")).parse_json()","summary":"","results":[["{}","{\"doc\":{\"foo\":\"bar\"}}"]],"skip_testing":false},{"mapping":"root.doc = file_rel(path: env(\"BENTHOS_TEST_BLOBLANG_FILE\"), no_cache: true).parse_json()","summary":"When the path parameter is static this function will only read the specified file once and yield the same result for each invocation as an optimization, this means that updates to files during runtime will not be reflected. You can disable this cache with the optional parameter `no_cache`, which when set to `true` will cause the file to be read for each execution of the mapping.","results":[["{}","{\"doc\":{\"foo\":\"bar\"}}"]],"skip_testing":false}],"impure":true},{"status":"stable","category":"Environment","name":"hostname","description":"Returns a string matching the hostname of the machine running Benthos.","params":{},"examples":[{"mapping":"root.thing.host = hostname()","summary":"","results":[],"skip_testing":false}],"impure":true},{"status":"stable","category":"Message Info","name":"json","description":"Returns the value of a field within a JSON message located by a [dot path][field_paths] argument. This function always targets the entire source JSON document regardless of the mapping context.","params":{"named":[{"name":"path","description":"An optional [dot path][field_paths] identifying a field to obtain.","type":"string","no_dynamic":false,"scalars_to_literal":false,"default":""}]},"examples":[{"mapping":"root.mapped = json(\"foo.bar\")","summary":"","results":[["{\"foo\":{\"bar\":\"hello world\"}}","{\"mapped\":\"hello world\"}"]],"skip_testing":false},{"mapping":"root.doc = json()","summary":"The path argument is optional and if omitted the entire JSON payload is returned.","results":[["{\"foo\":{\"bar\":\"hello world\"}}","{\"doc\":{\"foo\":{\"bar\":\"hello world\"}}}"]],"skip_testing":false}],"impure":false},{"status":"stable","category":"General","name":"ksuid","description":"Generates a new ksuid each time it is invoked and prints a string representation.","params":{},"examples":[{"mapping":"root.id = ksuid()","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"deprecated","category":"Deprecated","name":"meta","description":"Returns the value of a metadata key from the input message as a string, or `null` if the key does not exist. Since values are extracted from the read-only input message they do NOT reflect changes made from within the map. In order to query metadata mutations made within a mapping use the \u003c\u003croot_meta, `root_meta` function\u003e\u003e. This function supports extracting metadata from other messages of a batch with the `from` method.","params":{"named":[{"name":"key","description":"An optional key of a metadata value to obtain.","type":"string","no_dynamic":false,"scalars_to_literal":false,"default":""}]},"examples":[{"mapping":"root.topic = meta(\"kafka_topic\")","summary":"","results":[],"skip_testing":false},{"mapping":"root.all_metadata = meta()","summary":"The key parameter is optional and if omitted the entire metadata contents are returned as an object.","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"Message Info","name":"metadata","description":"Returns the value of a metadata key from the input message, or `null` if the key does not exist. Since values are extracted from the read-only input message they do NOT reflect changes made from within the map, in order to query metadata mutations made within a mapping use the xref:guides:bloblang/about.adoc#metadata[`@` operator]. This function supports extracting metadata from other messages of a batch with the `from` method.","params":{"named":[{"name":"key","description":"An optional key of a metadata value to obtain.","type":"string","no_dynamic":false,"scalars_to_literal":false,"default":""}]},"examples":[{"mapping":"root.topic = metadata(\"kafka_topic\")","summary":"","results":[],"skip_testing":false},{"mapping":"root.all_metadata = metadata()","summary":"The key parameter is optional and if omitted the entire metadata contents are returned as an object.","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"General","name":"nanoid","description":"Generates a new nanoid each time it is invoked and prints a string representation.","params":{"named":[{"name":"length","description":"An optional length.","type":"integer","no_dynamic":false,"scalars_to_literal":false,"is_optional":true},{"name":"alphabet","description":"An optional custom alphabet to use for generating IDs. When specified the field `length` must also be present.","type":"string","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"examples":[{"mapping":"root.id = nanoid()","summary":"","results":[],"skip_testing":false},{"mapping":"root.id = nanoid(54)","summary":"It is possible to specify an optional length parameter.","results":[],"skip_testing":false},{"mapping":"root.id = nanoid(54, \"abcde\")","summary":"It is also possible to specify an optional custom alphabet after the length parameter.","results":[],"skip_testing":false}],"impure":false},{"status":"hidden","category":"","name":"nothing","params":{},"impure":false},{"status":"stable","category":"Environment","name":"now","description":"Returns the current timestamp as a string in RFC 3339 format with the local timezone. Use the method `ts_format` in order to change the format and timezone.","params":{},"examples":[{"mapping":"root.received_at = now()","summary":"","results":[],"skip_testing":false},{"mapping":"root.received_at = now().ts_format(\"Mon Jan 2 15:04:05 -0700 MST 2006\", \"UTC\")","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"General","name":"pi","description":"Returns the value of the mathematical constant Pi.","params":{},"examples":[{"mapping":"root.radians = this.degrees * (pi() / 180)","summary":"","results":[["{\"degrees\":45}","{\"radians\":0.7853981633974483}"]],"skip_testing":false},{"mapping":"root.degrees = this.radians * (180 / pi())","summary":"","results":[["{\"radians\":0.78540}","{\"degrees\":45.00010522957486}"]],"skip_testing":false}],"impure":false},{"status":"stable","category":"General","name":"random_int","description":"\nGenerates a non-negative pseudo-random 64-bit integer. An optional integer argument can be provided in order to seed the random number generator.\n\nOptional `min` and `max` arguments can be provided in order to only generate numbers within a range. Neither of these parameters can be set via a dynamic expression (i.e. from values taken from mapped data). Instead, for dynamic ranges extract a min and max manually using a modulo operator (`random_int() % a + b`).","params":{"named":[{"name":"seed","description":"A seed to use, if a query is provided it will only be resolved once during the lifetime of the mapping.","type":"query expression","no_dynamic":false,"scalars_to_literal":true,"default":{"Value":0}},{"name":"min","description":"The minimum value the random generated number will have. The default value is 0.","type":"integer","no_dynamic":true,"scalars_to_literal":false,"default":0},{"name":"max","description":"The maximum value the random generated number will have. The default value is 9223372036854775806 (math.MaxInt64 - 1).","type":"integer","no_dynamic":true,"scalars_to_literal":false,"default":9223372036854775806}]},"examples":[{"mapping":"root.first = random_int()\nroot.second = random_int(1)\nroot.third = random_int(max:20)\nroot.fourth = random_int(min:10, max:20)\nroot.fifth = random_int(timestamp_unix_nano(), 5, 20)\nroot.sixth = random_int(seed:timestamp_unix_nano(), max:20)\n","summary":"","results":[],"skip_testing":false},{"mapping":"root.first = random_int(timestamp_unix_nano())","summary":"It is possible to specify a dynamic seed argument, in which case the argument will only be resolved once during the lifetime of the mapping.","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"General","name":"range","description":"The `range` function creates an array of integers following a range between a start, stop and optional step integer argument. If the step argument is omitted then it defaults to 1. A negative step can be provided as long as stop \u003c start.","params":{"named":[{"name":"start","description":"The start value.","type":"integer","no_dynamic":false,"scalars_to_literal":false},{"name":"stop","description":"The stop value.","type":"integer","no_dynamic":false,"scalars_to_literal":false},{"name":"step","description":"The step value.","type":"integer","no_dynamic":false,"scalars_to_literal":false,"default":1}]},"examples":[{"mapping":"root.a = range(0, 10)\nroot.b = range(start: 0, stop: this.max, step: 2) # Using named params\nroot.c = range(0, -this.max, -2)","summary":"","results":[["{\"max\":10}","{\"a\":[0,1,2,3,4,5,6,7,8,9],\"b\":[0,2,4,6,8],\"c\":[0,-2,-4,-6,-8]}"]],"skip_testing":false}],"impure":false},{"status":"deprecated","category":"Deprecated","name":"root_meta","description":"Returns the value of a metadata key from the new message being created as a string, or `null` if the key does not exist. Changes made to metadata during a mapping will be reflected by this function.","params":{"named":[{"name":"key","description":"An optional key of a metadata value to obtain.","type":"string","no_dynamic":false,"scalars_to_literal":false,"default":""}]},"examples":[{"mapping":"root.topic = root_meta(\"kafka_topic\")","summary":"","results":[],"skip_testing":false},{"mapping":"root.all_metadata = root_meta()","summary":"The key parameter is optional and if omitted the entire metadata contents are returned as an object.","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"General","name":"snowflake_id","description":"Generate a new snowflake ID each time it is invoked and prints a string representation. I.e.: 1559229974454472704","params":{"named":[{"name":"node_id","description":"It is possible to specify the node_id.","type":"integer","no_dynamic":false,"scalars_to_literal":false,"default":1}]},"examples":[{"mapping":"root.id = snowflake_id()","summary":"","results":[],"skip_testing":false},{"mapping":"root.id = snowflake_id(2)","summary":"It is possible to specify the node_id.","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"General","name":"throw","description":"Throws an error similar to a regular mapping error. This is useful for abandoning a mapping entirely given certain conditions.","params":{"named":[{"name":"why","description":"A string explanation for why an error was thrown, this will be added to the resulting error message.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"examples":[{"mapping":"root.doc.type = match {\n this.exists(\"header.id\") =\u003e \"foo\"\n this.exists(\"body.data\") =\u003e \"bar\"\n _ =\u003e throw(\"unknown type\")\n}\nroot.doc.contents = (this.body.content | this.thing.body)","summary":"","results":[["{\"header\":{\"id\":\"first\"},\"thing\":{\"body\":\"hello world\"}}","{\"doc\":{\"contents\":\"hello world\",\"type\":\"foo\"}}"],["{\"nothing\":\"matches\"}","Error(\"failed assignment (line 1): unknown type\")"]],"skip_testing":false}],"impure":false},{"status":"stable","category":"Environment","name":"timestamp_unix","description":"Returns the current unix timestamp in seconds.","params":{},"examples":[{"mapping":"root.received_at = timestamp_unix()","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"Environment","name":"timestamp_unix_micro","description":"Returns the current unix timestamp in microseconds.","params":{},"examples":[{"mapping":"root.received_at = timestamp_unix_micro()","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"Environment","name":"timestamp_unix_milli","description":"Returns the current unix timestamp in milliseconds.","params":{},"examples":[{"mapping":"root.received_at = timestamp_unix_milli()","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"Environment","name":"timestamp_unix_nano","description":"Returns the current unix timestamp in nanoseconds.","params":{},"examples":[{"mapping":"root.received_at = timestamp_unix_nano()","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"experimental","category":"Message Info","name":"tracing_id","description":"Provides the message trace id. The returned value will be zeroed if the message does not contain a span.","params":{},"examples":[{"mapping":"meta trace_id = tracing_id()","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"experimental","category":"Message Info","name":"tracing_span","description":"Provides the message tracing span xref:components:tracers/about.adoc[(created via Open Telemetry APIs)] as an object serialized via text map formatting. The returned value will be `null` if the message does not have a span.","params":{},"examples":[{"mapping":"root.headers.traceparent = tracing_span().traceparent","summary":"","results":[["{\"some_stuff\":\"just can't be explained by science\"}","{\"headers\":{\"traceparent\":\"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\"}}"]],"skip_testing":false}],"impure":false},{"status":"experimental","category":"General","name":"ulid","description":"Generate a random ULID.","params":{"named":[{"name":"encoding","description":"The format to encode a ULID into. Valid options are: crockford, hex","type":"string","no_dynamic":false,"scalars_to_literal":false,"default":"crockford"},{"name":"random_source","description":"The source of randomness to use for generating ULIDs. \"secure_random\" is recommended for most use cases. \"fast_random\" can be used if security is not a concern.","type":"string","no_dynamic":false,"scalars_to_literal":false,"default":"secure_random"}]},"examples":[{"mapping":"root.id = ulid()","summary":"Using the defaults of Crockford Base32 encoding and secure random source","results":[],"skip_testing":false},{"mapping":"root.id = ulid(\"hex\")","summary":"ULIDs can be hex-encoded too.","results":[],"skip_testing":false},{"mapping":"root.id = ulid(\"crockford\", \"fast_random\")","summary":"They can be generated using a fast, but unsafe, random source for use cases that are not security-sensitive.","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"General","name":"uuid_v4","description":"Generates a new RFC-4122 UUID each time it is invoked and prints a string representation.","params":{},"examples":[{"mapping":"root.id = uuid_v4()","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"stable","category":"General","name":"uuid_v7","description":"Generates a new time ordered UUID each time it is invoked and prints a string representation.","params":{"named":[{"name":"time","description":"An optional timestamp to use for the time ordered portion of the UUID.","type":"timestamp","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"examples":[{"mapping":"root.id = uuid_v7()","summary":"","results":[],"skip_testing":false},{"mapping":"root.id = uuid_v7(now().ts_sub_iso8601(\"PT1M\"))","summary":"It is also possible to specify the timestamp for the uuid_v7","results":[],"skip_testing":false}],"impure":false},{"status":"hidden","category":"","name":"var","params":{"named":[{"name":"name","description":"The name of the target variable.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"impure":false}],"bloblang-methods":[{"status":"stable","name":"abs","description":"Returns the absolute value of an int64 or float64 number. As a special case, when an integer is provided that is the minimum value it is converted to the maximum value.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"\nroot.outs = this.ins.map_each(ele -\u003e ele.abs())\n","summary":"","results":[["{\"ins\":[9,-18,1.23,-4.56]}","{\"outs\":[9,18,1.23,4.56]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"all","description":"Checks each element of an array against a query and returns true if all elements passed. An error occurs if the target is not an array, or if any element results in the provided query returning a non-boolean result. Returns false if the target array is empty.","params":{"named":[{"name":"test","description":"A test query to apply to each element.","type":"query expression","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.all_over_21 = this.patrons.all(patron -\u003e patron.age \u003e= 21)","summary":"","results":[["{\"patrons\":[{\"id\":\"1\",\"age\":18},{\"id\":\"2\",\"age\":23}]}","{\"all_over_21\":false}"],["{\"patrons\":[{\"id\":\"1\",\"age\":45},{\"id\":\"2\",\"age\":23}]}","{\"all_over_21\":true}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"any","description":"Checks the elements of an array against a query and returns true if any element passes. An error occurs if the target is not an array, or if an element results in the provided query returning a non-boolean result. Returns false if the target array is empty.","params":{"named":[{"name":"test","description":"A test query to apply to each element.","type":"query expression","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.any_over_21 = this.patrons.any(patron -\u003e patron.age \u003e= 21)","summary":"","results":[["{\"patrons\":[{\"id\":\"1\",\"age\":18},{\"id\":\"2\",\"age\":23}]}","{\"any_over_21\":true}"],["{\"patrons\":[{\"id\":\"1\",\"age\":10},{\"id\":\"2\",\"age\":12}]}","{\"any_over_21\":false}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"append","description":"Returns an array with new elements appended to the end.","params":{"variadic":true},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.foo = this.foo.append(\"and\", \"this\")","summary":"","results":[["{\"foo\":[\"bar\",\"baz\"]}","{\"foo\":[\"bar\",\"baz\",\"and\",\"this\"]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"apply","description":"Apply a declared mapping to a target value.","params":{"named":[{"name":"mapping","description":"The mapping to apply.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"examples":[{"mapping":"map thing {\n root.inner = this.first\n}\n\nroot.foo = this.doc.apply(\"thing\")","summary":"","results":[["{\"doc\":{\"first\":\"hello world\"}}","{\"foo\":{\"inner\":\"hello world\"}}"]],"skip_testing":false},{"mapping":"map create_foo {\n root.name = \"a foo\"\n root.purpose = \"to be a foo\"\n}\n\nroot = this\nroot.foo = null.apply(\"create_foo\")","summary":"","results":[["{\"id\":\"1234\"}","{\"foo\":{\"name\":\"a foo\",\"purpose\":\"to be a foo\"},\"id\":\"1234\"}"]],"skip_testing":false}],"impure":false},{"status":"stable","name":"array","params":{},"categories":[{"Category":"Type Coercion","Description":"Return an array containing the target value. If the value is already an array it is unchanged.","Examples":[{"mapping":"root.my_array = this.name.array()","summary":"","results":[["{\"name\":\"foobar bazson\"}","{\"my_array\":[\"foobar bazson\"]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"assign","description":"Merge a source object into an existing destination object. When a collision is found within the merged structures (both a source and destination object contain the same non-object keys) the value in the destination object will be overwritten by that of source object. In order to preserve both values on collision use the \u003c\u003cmerge, `merge`\u003e\u003e method.","params":{"named":[{"name":"with","description":"A value to merge the target value with.","type":"unknown","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root = this.foo.assign(this.bar)","summary":"","results":[["{\"foo\":{\"first_name\":\"fooer\",\"likes\":\"bars\"},\"bar\":{\"second_name\":\"barer\",\"likes\":\"foos\"}}","{\"first_name\":\"fooer\",\"likes\":\"foos\",\"second_name\":\"barer\"}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"bloblang","description":"Executes an argument Bloblang mapping on the target. This method can be used in order to execute dynamic mappings. Imports and functions that interact with the environment, such as `file` and `env`, or that access message information directly, such as `content` or `json`, are not enabled for dynamic Bloblang mappings.","params":{"named":[{"name":"mapping","description":"The mapping to execute.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Parsing","Description":"","Examples":[{"mapping":"root.body = this.body.bloblang(this.mapping)","summary":"","results":[["{\"body\":{\"foo\":\"hello world\"},\"mapping\":\"root.foo = this.foo.uppercase()\"}","{\"body\":{\"foo\":\"HELLO WORLD\"}}"],["{\"body\":{\"foo\":\"hello world 2\"},\"mapping\":\"root.foo = this.foo.capitalize()\"}","{\"body\":{\"foo\":\"Hello World 2\"}}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"bool","params":{"named":[{"name":"default","description":"An optional value to yield if the target cannot be parsed as a boolean.","type":"bool","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"categories":[{"Category":"Type Coercion","Description":"Attempt to parse a value into a boolean. An optional argument can be provided, in which case if the value cannot be parsed the argument will be returned instead. If the value is a number then any non-zero value will resolve to `true`, if the value is a string then any of the following values are considered valid: `1, t, T, TRUE, true, True, 0, f, F, FALSE`.","Examples":[{"mapping":"root.foo = this.thing.bool()\nroot.bar = this.thing.bool(true)","summary":"","results":[],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"bytes","params":{},"categories":[{"Category":"Type Coercion","Description":"Marshal a value into a byte array. If the value is already a byte array it is unchanged.","Examples":[{"mapping":"root.first_byte = this.name.bytes().index(0)","summary":"","results":[["{\"name\":\"foobar bazson\"}","{\"first_byte\":102}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"capitalize","params":{},"categories":[{"Category":"String Manipulation","Description":"Takes a string value and returns a copy with all Unicode letters that begin words mapped to their Unicode title case.","Examples":[{"mapping":"root.title = this.title.capitalize()","summary":"","results":[["{\"title\":\"the foo bar\"}","{\"title\":\"The Foo Bar\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"catch","description":"If the result of a target query fails (due to incorrect types, failed parsing, etc) the argument is returned instead.","params":{"named":[{"name":"fallback","description":"A value to yield, or query to execute, if the target query fails.","type":"query expression","no_dynamic":false,"scalars_to_literal":true}]},"examples":[{"mapping":"root.doc.id = this.thing.id.string().catch(uuid_v4())","summary":"","results":[],"skip_testing":false},{"mapping":"root.url = this.url.parse_url().catch(err -\u003e {\"error\":err,\"input\":this.url})","summary":"The fallback argument can be a mapping, allowing you to capture the error string and yield structured data back.","results":[["{\"url\":\"invalid %\u0026# url\"}","{\"url\":{\"error\":\"field `this.url`: parse \\\"invalid %\u0026\\\": invalid URL escape \\\"%\u0026\\\"\",\"input\":\"invalid %\u0026# url\"}}"]],"skip_testing":false},{"mapping":"root = this.catch(deleted())","summary":"When the input document is not structured attempting to reference structured fields with `this` will result in an error. Therefore, a convenient way to delete non-structured data is with a catch.","results":[["{\"doc\":{\"foo\":\"bar\"}}","{\"doc\":{\"foo\":\"bar\"}}"],["not structured data","\u003cMessage deleted\u003e"]],"skip_testing":false}],"impure":false},{"status":"stable","name":"ceil","description":"Returns the least integer value greater than or equal to a number. If the resulting value fits within a 64-bit integer then that is returned, otherwise a new floating point number is returned.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"root.new_value = this.value.ceil()","summary":"","results":[["{\"value\":5.3}","{\"new_value\":6}"],["{\"value\":-5.9}","{\"new_value\":-5}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"collapse","params":{"named":[{"name":"include_empty","description":"Whether to include empty objects and arrays in the resulting object.","type":"bool","no_dynamic":false,"scalars_to_literal":false,"default":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"Collapse an array or object into an object of key/value pairs for each field, where the key is the full path of the structured field in dot path notation. Empty arrays an objects are ignored by default.","Examples":[{"mapping":"root.result = this.collapse()","summary":"","results":[["{\"foo\":[{\"bar\":\"1\"},{\"bar\":{}},{\"bar\":\"2\"},{\"bar\":[]}]}","{\"result\":{\"foo.0.bar\":\"1\",\"foo.2.bar\":\"2\"}}"]],"skip_testing":false},{"mapping":"root.result = this.collapse(include_empty: true)","summary":"An optional boolean parameter can be set to true in order to include empty objects and arrays.","results":[["{\"foo\":[{\"bar\":\"1\"},{\"bar\":{}},{\"bar\":\"2\"},{\"bar\":[]}]}","{\"result\":{\"foo.0.bar\":\"1\",\"foo.1.bar\":{},\"foo.2.bar\":\"2\",\"foo.3.bar\":[]}}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"compare_argon2","description":"Checks whether a string matches a hashed secret using Argon2.","params":{"named":[{"name":"hashed_secret","description":"The hashed secret to compare with the input. This must be a fully-qualified string which encodes the Argon2 options used to generate the hash.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"String Manipulation","Description":"","Examples":[{"mapping":"root.match = this.secret.compare_argon2(\"$argon2id$v=19$m=4096,t=3,p=1$c2FsdHktbWNzYWx0ZmFjZQ$RMUMwgtS32/mbszd+ke4o4Ej1jFpYiUqY6MHWa69X7Y\")","summary":"","results":[["{\"secret\":\"there-are-many-blobs-in-the-sea\"}","{\"match\":true}"]],"skip_testing":false},{"mapping":"root.match = this.secret.compare_argon2(\"$argon2id$v=19$m=4096,t=3,p=1$c2FsdHktbWNzYWx0ZmFjZQ$RMUMwgtS32/mbszd+ke4o4Ej1jFpYiUqY6MHWa69X7Y\")","summary":"","results":[["{\"secret\":\"will-i-ever-find-love\"}","{\"match\":false}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"compare_bcrypt","description":"Checks whether a string matches a hashed secret using bcrypt.","params":{"named":[{"name":"hashed_secret","description":"The hashed secret value to compare with the input.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"String Manipulation","Description":"","Examples":[{"mapping":"root.match = this.secret.compare_bcrypt(\"$2y$10$Dtnt5NNzVtMCOZONT705tOcS8It6krJX8bEjnDJnwxiFKsz1C.3Ay\")","summary":"","results":[["{\"secret\":\"there-are-many-blobs-in-the-sea\"}","{\"match\":true}"]],"skip_testing":false},{"mapping":"root.match = this.secret.compare_bcrypt(\"$2y$10$Dtnt5NNzVtMCOZONT705tOcS8It6krJX8bEjnDJnwxiFKsz1C.3Ay\")","summary":"","results":[["{\"secret\":\"will-i-ever-find-love\"}","{\"match\":false}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"compress","description":"Compresses a string or byte array value according to a specified algorithm.","params":{"named":[{"name":"algorithm","description":"One of `flate`, `gzip`, `pgzip`, `lz4`, `snappy`, `zlib`, `zstd`.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"level","description":"The level of compression to use. May not be applicable to all algorithms.","type":"integer","no_dynamic":false,"scalars_to_literal":false,"default":-1}]},"categories":[{"Category":"Encoding and Encryption","Description":"","Examples":[{"mapping":"let long_content = range(0, 1000).map_each(content()).join(\" \")\nroot.a_len = $long_content.length()\nroot.b_len = $long_content.compress(\"gzip\").length()\n","summary":"","results":[["hello world this is some content","{\"a_len\":32999,\"b_len\":161}"]],"skip_testing":false},{"mapping":"root.compressed = content().compress(\"lz4\").encode(\"base64\")","summary":"","results":[["hello world I love space","{\"compressed\":\"BCJNGGRwuRgAAIBoZWxsbyB3b3JsZCBJIGxvdmUgc3BhY2UAAAAAGoETLg==\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"concat","description":"Concatenates an array value with one or more argument arrays.","params":{"variadic":true},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.foo = this.foo.concat(this.bar, this.baz)","summary":"","results":[["{\"foo\":[\"a\",\"b\"],\"bar\":[\"c\"],\"baz\":[\"d\",\"e\",\"f\"]}","{\"foo\":[\"a\",\"b\",\"c\",\"d\",\"e\",\"f\"]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"contains","params":{"named":[{"name":"value","description":"A value to test against elements of the target.","type":"unknown","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"Checks whether an array contains an element matching the argument, or an object contains a value matching the argument, and returns a boolean result. Numerical comparisons are made irrespective of the representation type (float versus integer).","Examples":[{"mapping":"root.has_foo = this.thing.contains(\"foo\")","summary":"","results":[["{\"thing\":[\"this\",\"foo\",\"that\"]}","{\"has_foo\":true}"],["{\"thing\":[\"this\",\"bar\",\"that\"]}","{\"has_foo\":false}"]],"skip_testing":false},{"mapping":"root.has_bar = this.thing.contains(20)","summary":"","results":[["{\"thing\":[10.3,20.0,\"huh\",3]}","{\"has_bar\":true}"],["{\"thing\":[2,3,40,67]}","{\"has_bar\":false}"]],"skip_testing":false}]},{"Category":"String Manipulation","Description":"Checks whether a string contains a substring and returns a boolean result.","Examples":[{"mapping":"root.has_foo = this.thing.contains(\"foo\")","summary":"","results":[["{\"thing\":\"this foo that\"}","{\"has_foo\":true}"],["{\"thing\":\"this bar that\"}","{\"has_foo\":false}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"cos","description":"Calculates the cosine of a given angle specified in radians.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"root.new_value = (this.value * (pi() / 180)).cos()","summary":"","results":[["{\"value\":45}","{\"new_value\":0.7071067811865476}"],["{\"value\":0}","{\"new_value\":1}"],["{\"value\":180}","{\"new_value\":-1}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"decode","params":{"named":[{"name":"scheme","description":"The decoding scheme to use.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Encoding and Encryption","Description":"Decodes an encoded string target according to a chosen scheme and returns the result as a byte array. When mapping the result to a JSON field the value should be cast to a string using the method `string`, or encoded using the method `encode`, otherwise it will be base64 encoded by default.\n\nAvailable schemes are: `base64`, `base64url` https://rfc-editor.org/rfc/rfc4648.html[(RFC 4648 with padding characters)], `base64rawurl` https://rfc-editor.org/rfc/rfc4648.html[(RFC 4648 without padding characters)], `hex`, `ascii85`.","Examples":[{"mapping":"root.decoded = this.value.decode(\"hex\").string()","summary":"","results":[["{\"value\":\"68656c6c6f20776f726c64\"}","{\"decoded\":\"hello world\"}"]],"skip_testing":false},{"mapping":"root = this.encoded.decode(\"ascii85\")","summary":"","results":[["{\"encoded\":\"FD,B0+DGm\u003eFDl80Ci\\\"A\u003eF`)8BEckl6F`M\u0026(+Cno\u0026@/\"}","this is totally unstructured data"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"decompress","description":"Decompresses a string or byte array value according to a specified algorithm. The result of decompression ","params":{"named":[{"name":"algorithm","description":"One of `gzip`, `pgzip`, `zlib`, `bzip2`, `flate`, `snappy`, `lz4`, `zstd`.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Encoding and Encryption","Description":"","Examples":[{"mapping":"root = this.compressed.decode(\"base64\").decompress(\"lz4\")","summary":"","results":[["{\"compressed\":\"BCJNGGRwuRgAAIBoZWxsbyB3b3JsZCBJIGxvdmUgc3BhY2UAAAAAGoETLg==\"}","hello world I love space"]],"skip_testing":false},{"mapping":"root.result = this.compressed.decode(\"base64\").decompress(\"lz4\").string()","summary":"Use the `.string()` method in order to coerce the result into a string, this makes it possible to place the data within a JSON document without automatic base64 encoding.","results":[["{\"compressed\":\"BCJNGGRwuRgAAIBoZWxsbyB3b3JsZCBJIGxvdmUgc3BhY2UAAAAAGoETLg==\"}","{\"result\":\"hello world I love space\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"decrypt_aes","params":{"named":[{"name":"scheme","description":"The scheme to use for decryption, one of `ctr`, `gcm`, `ofb`, `cbc`.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"key","description":"A key to decrypt with.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"iv","description":"An initialization vector / nonce.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Encoding and Encryption","Description":"Decrypts an encrypted string or byte array target according to a chosen AES encryption method and returns the result as a byte array. The algorithms require a key and an initialization vector / nonce. Available schemes are: `ctr`, `gcm`, `ofb`, `cbc`.","Examples":[{"mapping":"let key = \"2b7e151628aed2a6abf7158809cf4f3c\".decode(\"hex\")\nlet vector = \"f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff\".decode(\"hex\")\nroot.decrypted = this.value.decode(\"hex\").decrypt_aes(\"ctr\", $key, $vector).string()","summary":"","results":[["{\"value\":\"84e9b31ff7400bdf80be7254\"}","{\"decrypted\":\"hello world!\"}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"diff","description":"Create a diff by comparing the current value with the given one. Wraps the github.com/r3labs/diff/v3 package. See its https://pkg.go.dev/github.com/r3labs/diff/v3[docs^] for more information.","params":{"named":[{"name":"other","description":"The value to compare against.","type":"unknown","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":null}],"impure":false,"version":"4.25.0"},{"status":"stable","name":"encode","params":{"named":[{"name":"scheme","description":"The encoding scheme to use.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Encoding and Encryption","Description":"Encodes a string or byte array target according to a chosen scheme and returns a string result. Available schemes are: `base64`, `base64url` https://rfc-editor.org/rfc/rfc4648.html[(RFC 4648 with padding characters)], `base64rawurl` https://rfc-editor.org/rfc/rfc4648.html[(RFC 4648 without padding characters)], `hex`, `ascii85`.","Examples":[{"mapping":"root.encoded = this.value.encode(\"hex\")","summary":"","results":[["{\"value\":\"hello world\"}","{\"encoded\":\"68656c6c6f20776f726c64\"}"]],"skip_testing":false},{"mapping":"root.encoded = content().encode(\"ascii85\")","summary":"","results":[["this is totally unstructured data","{\"encoded\":\"FD,B0+DGm\u003eFDl80Ci\\\"A\u003eF`)8BEckl6F`M\u0026(+Cno\u0026@/\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"encrypt_aes","params":{"named":[{"name":"scheme","description":"The scheme to use for encryption, one of `ctr`, `gcm`, `ofb`, `cbc`.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"key","description":"A key to encrypt with.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"iv","description":"An initialization vector / nonce.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Encoding and Encryption","Description":"Encrypts a string or byte array target according to a chosen AES encryption method and returns a string result. The algorithms require a key and an initialization vector / nonce. Available schemes are: `ctr`, `gcm`, `ofb`, `cbc`.","Examples":[{"mapping":"let key = \"2b7e151628aed2a6abf7158809cf4f3c\".decode(\"hex\")\nlet vector = \"f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff\".decode(\"hex\")\nroot.encrypted = this.value.encrypt_aes(\"ctr\", $key, $vector).encode(\"hex\")","summary":"","results":[["{\"value\":\"hello world!\"}","{\"encrypted\":\"84e9b31ff7400bdf80be7254\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"enumerated","description":"Converts an array into a new array of objects, where each object has a field index containing the `index` of the element and a field `value` containing the original value of the element.","params":{},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.foo = this.foo.enumerated()","summary":"","results":[["{\"foo\":[\"bar\",\"baz\"]}","{\"foo\":[{\"index\":0,\"value\":\"bar\"},{\"index\":1,\"value\":\"baz\"}]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"escape_html","params":{},"categories":[{"Category":"String Manipulation","Description":"Escapes a string so that special characters like `\u003c` to become `\u0026lt;`. It escapes only five such characters: `\u003c`, `\u003e`, `\u0026`, `'` and `\"` so that it can be safely placed within an HTML entity.","Examples":[{"mapping":"root.escaped = this.value.escape_html()","summary":"","results":[["{\"value\":\"foo \u0026 bar\"}","{\"escaped\":\"foo \u0026amp; bar\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"escape_url_query","params":{},"categories":[{"Category":"String Manipulation","Description":"Escapes a string so that it can be safely placed within a URL query.","Examples":[{"mapping":"root.escaped = this.value.escape_url_query()","summary":"","results":[["{\"value\":\"foo \u0026 bar\"}","{\"escaped\":\"foo+%26+bar\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"exists","description":"Checks that a field, identified via a xref:configuration:field_paths.adoc[dot path], exists in an object.","params":{"named":[{"name":"path","description":"A xref:configuration:field_paths.adoc[dot path] to a field.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"examples":[{"mapping":"root.result = this.foo.exists(\"bar.baz\")","summary":"","results":[["{\"foo\":{\"bar\":{\"baz\":\"yep, I exist\"}}}","{\"result\":true}"],["{\"foo\":{\"bar\":{}}}","{\"result\":false}"],["{\"foo\":{}}","{\"result\":false}"]],"skip_testing":false}],"impure":false},{"status":"stable","name":"explode","params":{"named":[{"name":"path","description":"A xref:configuration:field_paths.adoc[dot path] to a field to explode.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"Explodes an array or object at a xref:configuration:field_paths.adoc[field path].","Examples":[{"mapping":"root = this.explode(\"value\")","summary":"##### On arrays\n\nExploding arrays results in an array containing elements matching the original document, where the target field of each element is an element of the exploded array:","results":[["{\"id\":1,\"value\":[\"foo\",\"bar\",\"baz\"]}","[{\"id\":1,\"value\":\"foo\"},{\"id\":1,\"value\":\"bar\"},{\"id\":1,\"value\":\"baz\"}]"]],"skip_testing":false},{"mapping":"root = this.explode(\"value\")","summary":"##### On objects\n\nExploding objects results in an object where the keys match the target object, and the values match the original document but with the target field replaced by the exploded value:","results":[["{\"id\":1,\"value\":{\"foo\":2,\"bar\":[3,4],\"baz\":{\"bev\":5}}}","{\"bar\":{\"id\":1,\"value\":[3,4]},\"baz\":{\"id\":1,\"value\":{\"bev\":5}},\"foo\":{\"id\":1,\"value\":2}}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"filepath_join","params":{},"categories":[{"Category":"String Manipulation","Description":"Joins an array of path elements into a single file path. The separator depends on the operating system of the machine.","Examples":[{"mapping":"root.path = this.path_elements.filepath_join()","summary":"","results":[["{\"path_elements\":[\"/foo/\",\"bar.txt\"]}","{\"path\":\"/foo/bar.txt\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"filepath_split","params":{},"categories":[{"Category":"String Manipulation","Description":"Splits a file path immediately following the final Separator, separating it into a directory and file name component returned as a two element array of strings. If there is no Separator in the path, the first element will be empty and the second will contain the path. The separator depends on the operating system of the machine.","Examples":[{"mapping":"root.path_sep = this.path.filepath_split()","summary":"","results":[["{\"path\":\"/foo/bar.txt\"}","{\"path_sep\":[\"/foo/\",\"bar.txt\"]}"],["{\"path\":\"baz.txt\"}","{\"path_sep\":[\"\",\"baz.txt\"]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"filter","params":{"named":[{"name":"test","description":"A query to apply to each element, if this query resolves to any value other than a boolean `true` the element will be removed from the result.","type":"query expression","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"Executes a mapping query argument for each element of an array or key/value pair of an object. If the query returns `false` the item is removed from the resulting array or object. The item will also be removed if the query returns any non-boolean value.","Examples":[{"mapping":"root.new_nums = this.nums.filter(num -\u003e num \u003e 10)","summary":"","results":[["{\"nums\":[3,11,4,17]}","{\"new_nums\":[11,17]}"]],"skip_testing":false},{"mapping":"root.new_dict = this.dict.filter(item -\u003e item.value.contains(\"foo\"))","summary":"##### On objects\n\nWhen filtering objects the mapping query argument is provided a context with a field `key` containing the value key, and a field `value` containing the value.","results":[["{\"dict\":{\"first\":\"hello foo\",\"second\":\"world\",\"third\":\"this foo is great\"}}","{\"new_dict\":{\"first\":\"hello foo\",\"third\":\"this foo is great\"}}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"find","description":"Returns the index of the first occurrence of a value in an array. `-1` is returned if there are no matches. Numerical comparisons are made irrespective of the representation type (float versus integer).","params":{"named":[{"name":"value","description":"A value to find.","type":"unknown","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.index = this.find(\"bar\")","summary":"","results":[["[\"foo\", \"bar\", \"baz\"]","{\"index\":1}"]],"skip_testing":false},{"mapping":"root.index = this.things.find(this.goal)","summary":"","results":[["{\"goal\":\"bar\",\"things\":[\"foo\", \"bar\", \"baz\"]}","{\"index\":1}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"find_all","description":"Returns an array containing the indexes of all occurrences of a value in an array. An empty array is returned if there are no matches. Numerical comparisons are made irrespective of the representation type (float versus integer).","params":{"named":[{"name":"value","description":"A value to find.","type":"unknown","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.index = this.find_all(\"bar\")","summary":"","results":[["[\"foo\", \"bar\", \"baz\", \"bar\"]","{\"index\":[1,3]}"]],"skip_testing":false},{"mapping":"root.indexes = this.things.find_all(this.goal)","summary":"","results":[["{\"goal\":\"bar\",\"things\":[\"foo\", \"bar\", \"baz\", \"bar\", \"buz\"]}","{\"indexes\":[1,3]}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"find_all_by","description":"Returns an array containing the indexes of all occurrences of an array where the provided query resolves to a boolean `true`. An empty array is returned if there are no matches. Numerical comparisons are made irrespective of the representation type (float versus integer).","params":{"named":[{"name":"query","description":"A query to execute for each element.","type":"query expression","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.index = this.find_all_by(v -\u003e v != \"bar\")","summary":"","results":[["[\"foo\", \"bar\", \"baz\"]","{\"index\":[0,2]}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"find_by","description":"Returns the index of the first occurrence of an array where the provided query resolves to a boolean `true`. `-1` is returned if there are no matches.","params":{"named":[{"name":"query","description":"A query to execute for each element.","type":"query expression","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.index = this.find_by(v -\u003e v != \"bar\")","summary":"","results":[["[\"foo\", \"bar\", \"baz\"]","{\"index\":0}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"flatten","description":"Iterates an array and any element that is itself an array is removed and has its elements inserted directly in the resulting array.","params":{},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.result = this.flatten()","summary":"","results":[["[\"foo\",[\"bar\",\"baz\"],\"buz\"]","{\"result\":[\"foo\",\"bar\",\"baz\",\"buz\"]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"float32","description":"\nConverts a numerical type into a 32-bit floating point number, this is for advanced use cases where a specific data type is needed for a given component (such as the ClickHouse SQL driver).\n\nIf the value is a string then an attempt will be made to parse it as a 32-bit floating point number. Please refer to the https://pkg.go.dev/strconv#ParseFloat[`strconv.ParseFloat` documentation] for details regarding the supported formats.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"\nroot.out = this.in.float32()\n","summary":"","results":[["{\"in\":\"6.674282313423543523453425345e-11\"}","{\"out\":6.674283e-11}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"float64","description":"\nConverts a numerical type into a 64-bit floating point number, this is for advanced use cases where a specific data type is needed for a given component (such as the ClickHouse SQL driver).\n\nIf the value is a string then an attempt will be made to parse it as a 64-bit floating point number. Please refer to the https://pkg.go.dev/strconv#ParseFloat[`strconv.ParseFloat` documentation] for details regarding the supported formats.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"\nroot.out = this.in.float64()\n","summary":"","results":[["{\"in\":\"6.674282313423543523453425345e-11\"}","{\"out\":6.674282313423544e-11}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"floor","description":"Returns the greatest integer value less than or equal to the target number. If the resulting value fits within a 64-bit integer then that is returned, otherwise a new floating point number is returned.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"root.new_value = this.value.floor()","summary":"","results":[["{\"value\":5.7}","{\"new_value\":5}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"fold","description":"Takes two arguments: an initial value, and a mapping query. For each element of an array the mapping context is an object with two fields `tally` and `value`, where `tally` contains the current accumulated value and `value` is the value of the current element. The mapping must return the result of adding the value to the tally.\n\nThe first argument is the value that `tally` will have on the first call.","params":{"named":[{"name":"initial","description":"The initial value to start the fold with. For example, an empty object `{}`, a zero count `0`, or an empty string `\"\"`.","type":"unknown","no_dynamic":false,"scalars_to_literal":false},{"name":"query","description":"A query to apply for each element. The query is provided an object with two fields; `tally` containing the current tally, and `value` containing the value of the current element. The query should result in a new tally to be passed to the next element query.","type":"query expression","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.sum = this.foo.fold(0, item -\u003e item.tally + item.value)","summary":"","results":[["{\"foo\":[3,8,11]}","{\"sum\":22}"]],"skip_testing":false},{"mapping":"root.result = this.foo.fold(\"\", item -\u003e \"%v%v\".format(item.tally, item.value))","summary":"","results":[["{\"foo\":[\"hello \", \"world\"]}","{\"result\":\"hello world\"}"]],"skip_testing":false},{"mapping":"root.smoothie = this.fruits.fold({}, item -\u003e item.tally.merge(item.value))","summary":"You can use fold to merge an array of objects together:","results":[["{\"fruits\":[{\"apple\":5},{\"banana\":3},{\"orange\":8}]}","{\"smoothie\":{\"apple\":5,\"banana\":3,\"orange\":8}}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"format","params":{"variadic":true},"categories":[{"Category":"String Manipulation","Description":"Use a value string as a format specifier in order to produce a new string, using any number of provided arguments. Please refer to the Go https://pkg.go.dev/fmt[`fmt` package documentation^] for the list of valid format verbs.","Examples":[{"mapping":"root.foo = \"%s(%v): %v\".format(this.name, this.age, this.fingers)","summary":"","results":[["{\"name\":\"lance\",\"age\":37,\"fingers\":13}","{\"foo\":\"lance(37): 13\"}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"format_json","params":{"named":[{"name":"indent","description":"Indentation string. Each element in a JSON object or array will begin on a new, indented line followed by one or more copies of indent according to the indentation nesting.","type":"string","no_dynamic":false,"scalars_to_literal":false,"default":" "},{"name":"no_indent","description":"Disable indentation.","type":"bool","no_dynamic":false,"scalars_to_literal":false,"default":false},{"name":"escape_html","description":"Escape problematic HTML characters.","type":"bool","no_dynamic":false,"scalars_to_literal":false,"default":true}]},"categories":[{"Category":"Parsing","Description":"Serializes a target value into a pretty-printed JSON byte array (with 4 space indentation by default).","Examples":[{"mapping":"root = this.doc.format_json()","summary":"","results":[["{\"doc\":{\"foo\":\"bar\"}}","{\n \"foo\": \"bar\"\n}"]],"skip_testing":false},{"mapping":"root = this.format_json(\" \")","summary":"Pass a string to the `indent` parameter in order to customise the indentation.","results":[["{\"doc\":{\"foo\":\"bar\"}}","{\n \"doc\": {\n \"foo\": \"bar\"\n }\n}"]],"skip_testing":false},{"mapping":"root.doc = this.doc.format_json().string()","summary":"Use the `.string()` method in order to coerce the result into a string.","results":[["{\"doc\":{\"foo\":\"bar\"}}","{\"doc\":\"{\\n \\\"foo\\\": \\\"bar\\\"\\n}\"}"]],"skip_testing":false},{"mapping":"root = this.doc.format_json(no_indent: true)","summary":"Set the `no_indent` parameter to true to disable indentation. The result is equivalent to calling `bytes()`.","results":[["{\"doc\":{\"foo\":\"bar\"}}","{\"foo\":\"bar\"}"]],"skip_testing":false},{"mapping":"root = this.doc.format_json()","summary":"Escapes problematic HTML characters.","results":[["{\"doc\":{\"email\":\"foo\u0026bar@benthos.dev\",\"name\":\"foo\u003ebar\"}}","{\n \"email\": \"foo\\u0026bar@benthos.dev\",\n \"name\": \"foo\\u003ebar\"\n}"]],"skip_testing":false},{"mapping":"root = this.doc.format_json(escape_html: false)","summary":"Set the `escape_html` parameter to false to disable escaping of problematic HTML characters.","results":[["{\"doc\":{\"email\":\"foo\u0026bar@benthos.dev\",\"name\":\"foo\u003ebar\"}}","{\n \"email\": \"foo\u0026bar@benthos.dev\",\n \"name\": \"foo\u003ebar\"\n}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"format_msgpack","description":"Formats data as a https://msgpack.org/[MessagePack^] message in bytes format.","params":{},"categories":[{"Category":"Parsing","Description":"","Examples":[{"mapping":"root = this.format_msgpack().encode(\"hex\")","summary":"","results":[["{\"foo\":\"bar\"}","81a3666f6fa3626172"]],"skip_testing":false},{"mapping":"root.encoded = this.format_msgpack().encode(\"base64\")","summary":"","results":[["{\"foo\":\"bar\"}","{\"encoded\":\"gaNmb2+jYmFy\"}"]],"skip_testing":false}]}],"impure":false},{"status":"deprecated","name":"format_timestamp","description":"Attempts to format a timestamp value as a string according to a specified format, or RFC 3339 by default. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format.\n\nThe output format is defined by showing how the reference time, defined to be Mon Jan 2 15:04:05 -0700 MST 2006, would be displayed if it were the value. For an alternative way to specify formats check out the \u003c\u003cts_strftime, `ts_strftime`\u003e\u003e method.","params":{"named":[{"name":"format","description":"The output format to use.","type":"string","no_dynamic":false,"scalars_to_literal":false,"default":"2006-01-02T15:04:05.999999999Z07:00"},{"name":"tz","description":"An optional timezone to use, otherwise the timezone of the input string is used, or in the case of unix timestamps the local timezone is used.","type":"string","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"categories":[{"Category":"Deprecated","Description":"","Examples":null}],"impure":false},{"status":"deprecated","name":"format_timestamp_strftime","description":"Attempts to format a timestamp value as a string according to a specified strftime-compatible format. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format.","params":{"named":[{"name":"format","description":"The output format to use.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"tz","description":"An optional timezone to use, otherwise the timezone of the input string is used.","type":"string","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"categories":[{"Category":"Deprecated","Description":"","Examples":null}],"impure":false},{"status":"deprecated","name":"format_timestamp_unix","description":"Attempts to format a timestamp value as a unix timestamp. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format. The \u003c\u003cts_parse, `ts_parse`\u003e\u003e method can be used in order to parse different timestamp formats.","params":{},"categories":[{"Category":"Deprecated","Description":"","Examples":null}],"impure":false},{"status":"deprecated","name":"format_timestamp_unix_micro","description":"Attempts to format a timestamp value as a unix timestamp with microsecond precision. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format. The \u003c\u003cts_parse, `ts_parse`\u003e\u003e method can be used in order to parse different timestamp formats.","params":{},"categories":[{"Category":"Deprecated","Description":"","Examples":null}],"impure":false},{"status":"deprecated","name":"format_timestamp_unix_milli","description":"Attempts to format a timestamp value as a unix timestamp with millisecond precision. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format. The \u003c\u003cts_parse, `ts_parse`\u003e\u003e method can be used in order to parse different timestamp formats.","params":{},"categories":[{"Category":"Deprecated","Description":"","Examples":null}],"impure":false},{"status":"deprecated","name":"format_timestamp_unix_nano","description":"Attempts to format a timestamp value as a unix timestamp with nanosecond precision. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format. The \u003c\u003cts_parse, `ts_parse`\u003e\u003e method can be used in order to parse different timestamp formats.","params":{},"categories":[{"Category":"Deprecated","Description":"","Examples":null}],"impure":false},{"status":"stable","name":"format_xml","description":"\nSerializes a target value into an XML byte array.\n","params":{"named":[{"name":"indent","description":"Indentation string. Each element in an XML object or array will begin on a new, indented line followed by one or more copies of indent according to the indentation nesting.","type":"string","no_dynamic":false,"scalars_to_literal":false,"default":" "},{"name":"no_indent","description":"Disable indentation.","type":"bool","no_dynamic":false,"scalars_to_literal":false,"default":false}]},"categories":[{"Category":"Parsing","Description":"","Examples":[{"mapping":"root = this.format_xml()","summary":"Serializes a target value into a pretty-printed XML byte array (with 4 space indentation by default).","results":[["{\"foo\":{\"bar\":{\"baz\":\"foo bar baz\"}}}","\u003cfoo\u003e\n \u003cbar\u003e\n \u003cbaz\u003efoo bar baz\u003c/baz\u003e\n \u003c/bar\u003e\n\u003c/foo\u003e"]],"skip_testing":false},{"mapping":"root = this.format_xml(\" \")","summary":"Pass a string to the `indent` parameter in order to customise the indentation.","results":[["{\"foo\":{\"bar\":{\"baz\":\"foo bar baz\"}}}","\u003cfoo\u003e\n \u003cbar\u003e\n \u003cbaz\u003efoo bar baz\u003c/baz\u003e\n \u003c/bar\u003e\n\u003c/foo\u003e"]],"skip_testing":false},{"mapping":"root.doc = this.format_xml(\"\").string()","summary":"Use the `.string()` method in order to coerce the result into a string.","results":[["{\"foo\":{\"bar\":{\"baz\":\"foo bar baz\"}}}","{\"doc\":\"\u003cfoo\u003e\\n\u003cbar\u003e\\n\u003cbaz\u003efoo bar baz\u003c/baz\u003e\\n\u003c/bar\u003e\\n\u003c/foo\u003e\"}"]],"skip_testing":false},{"mapping":"root = this.format_xml(no_indent: true)","summary":"Set the `no_indent` parameter to true to disable indentation.","results":[["{\"foo\":{\"bar\":{\"baz\":\"foo bar baz\"}}}","\u003cfoo\u003e\u003cbar\u003e\u003cbaz\u003efoo bar baz\u003c/baz\u003e\u003c/bar\u003e\u003c/foo\u003e"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"format_yaml","params":{},"categories":[{"Category":"Parsing","Description":"Serializes a target value into a YAML byte array.","Examples":[{"mapping":"root = this.doc.format_yaml()","summary":"","results":[["{\"doc\":{\"foo\":\"bar\"}}","foo: bar\n"]],"skip_testing":false},{"mapping":"root.doc = this.doc.format_yaml().string()","summary":"Use the `.string()` method in order to coerce the result into a string.","results":[["{\"doc\":{\"foo\":\"bar\"}}","{\"doc\":\"foo: bar\\n\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"from","description":"Modifies a target query such that certain functions are executed from the perspective of another message in the batch. This allows you to mutate events based on the contents of other messages. Functions that support this behavior are `content`, `json` and `meta`.","params":{"named":[{"name":"index","description":"The message index to use as a perspective.","type":"integer","no_dynamic":false,"scalars_to_literal":false}]},"examples":[{"mapping":"root = this\nroot.foo = json(\"foo\").from(1)","summary":"For example, the following map extracts the contents of the JSON field `foo` specifically from message index `1` of a batch, effectively overriding the field `foo` for all messages of a batch to that of message 1:","results":[],"skip_testing":false}],"impure":false},{"status":"stable","name":"from_all","description":"Modifies a target query such that certain functions are executed from the perspective of each message in the batch, and returns the set of results as an array. Functions that support this behavior are `content`, `json` and `meta`.","params":{},"examples":[{"mapping":"root = this\nroot.foo_summed = json(\"foo\").from_all().sum()","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"experimental","name":"geoip_anonymous_ip","description":"Looks up an IP address against a https://www.maxmind.com/en/home[MaxMind database file^] and, if found, returns an object describing the anonymous IP associated with it.","params":{"named":[{"name":"path","description":"A path to an mmdb (maxmind) file.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"GeoIP","Description":"","Examples":null}],"impure":false},{"status":"experimental","name":"geoip_asn","description":"Looks up an IP address against a https://www.maxmind.com/en/home[MaxMind database file^] and, if found, returns an object describing the ASN associated with it.","params":{"named":[{"name":"path","description":"A path to an mmdb (maxmind) file.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"GeoIP","Description":"","Examples":null}],"impure":false},{"status":"experimental","name":"geoip_city","description":"Looks up an IP address against a https://www.maxmind.com/en/home[MaxMind database file^] and, if found, returns an object describing the city associated with it.","params":{"named":[{"name":"path","description":"A path to an mmdb (maxmind) file.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"GeoIP","Description":"","Examples":null}],"impure":false},{"status":"experimental","name":"geoip_connection_type","description":"Looks up an IP address against a https://www.maxmind.com/en/home[MaxMind database file^] and, if found, returns an object describing the connection type associated with it.","params":{"named":[{"name":"path","description":"A path to an mmdb (maxmind) file.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"GeoIP","Description":"","Examples":null}],"impure":false},{"status":"experimental","name":"geoip_country","description":"Looks up an IP address against a https://www.maxmind.com/en/home[MaxMind database file^] and, if found, returns an object describing the country associated with it.","params":{"named":[{"name":"path","description":"A path to an mmdb (maxmind) file.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"GeoIP","Description":"","Examples":null}],"impure":false},{"status":"experimental","name":"geoip_domain","description":"Looks up an IP address against a https://www.maxmind.com/en/home[MaxMind database file^] and, if found, returns an object describing the domain associated with it.","params":{"named":[{"name":"path","description":"A path to an mmdb (maxmind) file.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"GeoIP","Description":"","Examples":null}],"impure":false},{"status":"experimental","name":"geoip_enterprise","description":"Looks up an IP address against a https://www.maxmind.com/en/home[MaxMind database file^] and, if found, returns an object describing the enterprise associated with it.","params":{"named":[{"name":"path","description":"A path to an mmdb (maxmind) file.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"GeoIP","Description":"","Examples":null}],"impure":false},{"status":"experimental","name":"geoip_isp","description":"Looks up an IP address against a https://www.maxmind.com/en/home[MaxMind database file^] and, if found, returns an object describing the ISP associated with it.","params":{"named":[{"name":"path","description":"A path to an mmdb (maxmind) file.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"GeoIP","Description":"","Examples":null}],"impure":false},{"status":"stable","name":"get","description":"Extract a field value, identified via a xref:configuration:field_paths.adoc[dot path], from an object.","params":{"named":[{"name":"path","description":"A xref:configuration:field_paths.adoc[dot path] identifying a field to obtain.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.result = this.foo.get(this.target)","summary":"","results":[["{\"foo\":{\"bar\":\"from bar\",\"baz\":\"from baz\"},\"target\":\"bar\"}","{\"result\":\"from bar\"}"],["{\"foo\":{\"bar\":\"from bar\",\"baz\":\"from baz\"},\"target\":\"baz\"}","{\"result\":\"from baz\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"has_prefix","params":{"named":[{"name":"value","description":"The string to test.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"String Manipulation","Description":"Checks whether a string has a prefix argument and returns a bool.","Examples":[{"mapping":"root.t1 = this.v1.has_prefix(\"foo\")\nroot.t2 = this.v2.has_prefix(\"foo\")","summary":"","results":[["{\"v1\":\"foobar\",\"v2\":\"barfoo\"}","{\"t1\":true,\"t2\":false}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"has_suffix","params":{"named":[{"name":"value","description":"The string to test.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"String Manipulation","Description":"Checks whether a string has a suffix argument and returns a bool.","Examples":[{"mapping":"root.t1 = this.v1.has_suffix(\"foo\")\nroot.t2 = this.v2.has_suffix(\"foo\")","summary":"","results":[["{\"v1\":\"foobar\",\"v2\":\"barfoo\"}","{\"t1\":false,\"t2\":true}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"hash","params":{"named":[{"name":"algorithm","description":"The hasing algorithm to use.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"key","description":"An optional key to use.","type":"string","no_dynamic":false,"scalars_to_literal":false,"is_optional":true},{"name":"polynomial","description":"An optional polynomial key to use when selecting the `crc32` algorithm, otherwise ignored. Options are `IEEE` (default), `Castagnoli` and `Koopman`","type":"string","no_dynamic":false,"scalars_to_literal":false,"default":"IEEE"}]},"categories":[{"Category":"Encoding and Encryption","Description":"\nHashes a string or byte array according to a chosen algorithm and returns the result as a byte array. When mapping the result to a JSON field the value should be cast to a string using the method xref:guides:bloblang/methods.adoc#string[`string`], or encoded using the method xref:guides:bloblang/methods.adoc#encode[`encode`], otherwise it will be base64 encoded by default.\n\nAvailable algorithms are: `hmac_sha1`, `hmac_sha256`, `hmac_sha512`, `md5`, `sha1`, `sha256`, `sha512`, `xxhash64`, `crc32`, `fnv32`.\n\nThe following algorithms require a key, which is specified as a second argument: `hmac_sha1`, `hmac_sha256`, `hmac_sha512`.","Examples":[{"mapping":"root.h1 = this.value.hash(\"sha1\").encode(\"hex\")\nroot.h2 = this.value.hash(\"hmac_sha1\",\"static-key\").encode(\"hex\")","summary":"","results":[["{\"value\":\"hello world\"}","{\"h1\":\"2aae6c35c94fcfb415dbe95f408b9ce91ee846ed\",\"h2\":\"d87e5f068fa08fe90bb95bc7c8344cb809179d76\"}"]],"skip_testing":false},{"mapping":"root.h1 = this.value.hash(algorithm: \"crc32\", polynomial: \"Castagnoli\").encode(\"hex\")\nroot.h2 = this.value.hash(algorithm: \"crc32\", polynomial: \"Koopman\").encode(\"hex\")","summary":"The `crc32` algorithm supports options for the polynomial.","results":[["{\"value\":\"hello world\"}","{\"h1\":\"c99465aa\",\"h2\":\"df373d3c\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"index","description":"Extract an element from an array by an index. The index can be negative, and if so the element will be selected from the end counting backwards starting from -1. E.g. an index of -1 returns the last element, an index of -2 returns the element before the last, and so on.","params":{"named":[{"name":"index","description":"The index to obtain from an array.","type":"integer","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.last_name = this.names.index(-1)","summary":"","results":[["{\"names\":[\"rachel\",\"stevens\"]}","{\"last_name\":\"stevens\"}"]],"skip_testing":false},{"mapping":"root.last_byte = this.name.bytes().index(-1)","summary":"It is also possible to use this method on byte arrays, in which case the selected element will be returned as an integer.","results":[["{\"name\":\"foobar bazson\"}","{\"last_byte\":110}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"index_of","params":{"named":[{"name":"value","description":"A string to search for.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"String Manipulation","Description":"Returns the starting index of the argument substring in a string target, or `-1` if the target doesn't contain the argument.","Examples":[{"mapping":"root.index = this.thing.index_of(\"bar\")","summary":"","results":[["{\"thing\":\"foobar\"}","{\"index\":3}"]],"skip_testing":false},{"mapping":"root.index = content().index_of(\"meow\")","summary":"","results":[["the cat meowed, the dog woofed","{\"index\":8}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"int16","description":"\nConverts a numerical type into a 16-bit signed integer, this is for advanced use cases where a specific data type is needed for a given component (such as the ClickHouse SQL driver).\n\nIf the value is a string then an attempt will be made to parse it as a 16-bit signed integer. If the target value exceeds the capacity of an integer or contains decimal values then this method will throw an error. In order to convert a floating point number containing decimals first use \u003c\u003cround, `.round()`\u003e\u003e on the value. Please refer to the https://pkg.go.dev/strconv#ParseInt[`strconv.ParseInt` documentation] for details regarding the supported formats.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"\nroot.a = this.a.int16()\nroot.b = this.b.round().int16()\nroot.c = this.c.int16()\nroot.d = this.d.int16().catch(0)\n","summary":"","results":[["{\"a\":12,\"b\":12.34,\"c\":\"12\",\"d\":-12}","{\"a\":12,\"b\":12,\"c\":12,\"d\":-12}"]],"skip_testing":false},{"mapping":"\nroot = this.int16()\n","summary":"","results":[["\"0xDE\"","222"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"int32","description":"\nConverts a numerical type into a 32-bit signed integer, this is for advanced use cases where a specific data type is needed for a given component (such as the ClickHouse SQL driver).\n\nIf the value is a string then an attempt will be made to parse it as a 32-bit signed integer. If the target value exceeds the capacity of an integer or contains decimal values then this method will throw an error. In order to convert a floating point number containing decimals first use \u003c\u003cround, `.round()`\u003e\u003e on the value. Please refer to the https://pkg.go.dev/strconv#ParseInt[`strconv.ParseInt` documentation] for details regarding the supported formats.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"\nroot.a = this.a.int32()\nroot.b = this.b.round().int32()\nroot.c = this.c.int32()\nroot.d = this.d.int32().catch(0)\n","summary":"","results":[["{\"a\":12,\"b\":12.34,\"c\":\"12\",\"d\":-12}","{\"a\":12,\"b\":12,\"c\":12,\"d\":-12}"]],"skip_testing":false},{"mapping":"\nroot = this.int32()\n","summary":"","results":[["\"0xDEAD\"","57005"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"int64","description":"\nConverts a numerical type into a 64-bit signed integer, this is for advanced use cases where a specific data type is needed for a given component (such as the ClickHouse SQL driver).\n\nIf the value is a string then an attempt will be made to parse it as a 64-bit signed integer. If the target value exceeds the capacity of an integer or contains decimal values then this method will throw an error. In order to convert a floating point number containing decimals first use \u003c\u003cround, `.round()`\u003e\u003e on the value. Please refer to the https://pkg.go.dev/strconv#ParseInt[`strconv.ParseInt` documentation] for details regarding the supported formats.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"\nroot.a = this.a.int64()\nroot.b = this.b.round().int64()\nroot.c = this.c.int64()\nroot.d = this.d.int64().catch(0)\n","summary":"","results":[["{\"a\":12,\"b\":12.34,\"c\":\"12\",\"d\":-12}","{\"a\":12,\"b\":12,\"c\":12,\"d\":-12}"]],"skip_testing":false},{"mapping":"\nroot = this.int64()\n","summary":"","results":[["\"0xDEADBEEF\"","3735928559"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"int8","description":"\nConverts a numerical type into a 8-bit signed integer, this is for advanced use cases where a specific data type is needed for a given component (such as the ClickHouse SQL driver).\n\nIf the value is a string then an attempt will be made to parse it as a 8-bit signed integer. If the target value exceeds the capacity of an integer or contains decimal values then this method will throw an error. In order to convert a floating point number containing decimals first use \u003c\u003cround, `.round()`\u003e\u003e on the value. Please refer to the https://pkg.go.dev/strconv#ParseInt[`strconv.ParseInt` documentation] for details regarding the supported formats.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"\nroot.a = this.a.int8()\nroot.b = this.b.round().int8()\nroot.c = this.c.int8()\nroot.d = this.d.int8().catch(0)\n","summary":"","results":[["{\"a\":12,\"b\":12.34,\"c\":\"12\",\"d\":-12}","{\"a\":12,\"b\":12,\"c\":12,\"d\":-12}"]],"skip_testing":false},{"mapping":"\nroot = this.int8()\n","summary":"","results":[["\"0xD\"","13"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"join","params":{"named":[{"name":"delimiter","description":"An optional delimiter to add between each string.","type":"string","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"Join an array of strings with an optional delimiter into a single string.","Examples":[{"mapping":"root.joined_words = this.words.join()\nroot.joined_numbers = this.numbers.map_each(this.string()).join(\",\")","summary":"","results":[["{\"words\":[\"hello\",\"world\"],\"numbers\":[3,8,11]}","{\"joined_numbers\":\"3,8,11\",\"joined_words\":\"helloworld\"}"]],"skip_testing":false}]}],"impure":false},{"status":"experimental","name":"json_path","description":"Executes the given JSONPath expression on an object or array and returns the result. The JSONPath expression syntax can be found at https://goessner.net/articles/JsonPath/. For more complex logic, you can use Gval expressions (https://github.com/PaesslerAG/gval).","params":{"named":[{"name":"expression","description":"The JSONPath expression to execute.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.all_names = this.json_path(\"$..name\")","summary":"","results":[["{\"name\":\"alice\",\"foo\":{\"name\":\"bob\"}}","{\"all_names\":[\"alice\",\"bob\"]}"],["{\"thing\":[\"this\",\"bar\",{\"name\":\"alice\"}]}","{\"all_names\":[\"alice\"]}"]],"skip_testing":false},{"mapping":"root.text_objects = this.json_path(\"$.body[?(@.type=='text')]\")","summary":"","results":[["{\"body\":[{\"type\":\"image\",\"id\":\"foo\"},{\"type\":\"text\",\"id\":\"bar\"}]}","{\"text_objects\":[{\"id\":\"bar\",\"type\":\"text\"}]}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"json_schema","description":"Checks a https://json-schema.org/[JSON schema^] against a value and returns the value if it matches or throws and error if it does not.","params":{"named":[{"name":"schema","description":"The schema to check values against.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root = this.json_schema(\"\"\"{\n \"type\":\"object\",\n \"properties\":{\n \"foo\":{\n \"type\":\"string\"\n }\n }\n}\"\"\")","summary":"","results":[["{\"foo\":\"bar\"}","{\"foo\":\"bar\"}"],["{\"foo\":5}","Error(\"failed assignment (line 1): field `this`: foo invalid type. expected: string, given: integer\")"]],"skip_testing":false},{"mapping":"root = this.json_schema(file(env(\"BENTHOS_TEST_BLOBLANG_SCHEMA_FILE\")))","summary":"In order to load a schema from a file use the `file` function.","results":[],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"key_values","description":"Returns the key/value pairs of an object as an array, where each element is an object with a `key` field and a `value` field. The order of the resulting array will be random.","params":{},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.foo_key_values = this.foo.key_values().sort_by(pair -\u003e pair.key)","summary":"","results":[["{\"foo\":{\"bar\":1,\"baz\":2}}","{\"foo_key_values\":[{\"key\":\"bar\",\"value\":1},{\"key\":\"baz\",\"value\":2}]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"keys","description":"Returns the keys of an object as an array.","params":{},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.foo_keys = this.foo.keys()","summary":"","results":[["{\"foo\":{\"bar\":1,\"baz\":2}}","{\"foo_keys\":[\"bar\",\"baz\"]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"length","params":{},"categories":[{"Category":"String Manipulation","Description":"Returns the length of a string.","Examples":[{"mapping":"root.foo_len = this.foo.length()","summary":"","results":[["{\"foo\":\"hello world\"}","{\"foo_len\":11}"]],"skip_testing":false}]},{"Category":"Object \u0026 Array Manipulation","Description":"Returns the length of an array or object (number of keys).","Examples":[{"mapping":"root.foo_len = this.foo.length()","summary":"","results":[["{\"foo\":[\"first\",\"second\"]}","{\"foo_len\":2}"],["{\"foo\":{\"first\":\"bar\",\"second\":\"baz\"}}","{\"foo_len\":2}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"log","description":"Returns the natural logarithm of a number.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"root.new_value = this.value.log().round()","summary":"","results":[["{\"value\":1}","{\"new_value\":0}"],["{\"value\":2.7183}","{\"new_value\":1}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"log10","description":"Returns the decimal logarithm of a number.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"root.new_value = this.value.log10()","summary":"","results":[["{\"value\":100}","{\"new_value\":2}"],["{\"value\":1000}","{\"new_value\":3}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"lowercase","params":{},"categories":[{"Category":"String Manipulation","Description":"Convert a string value into lowercase.","Examples":[{"mapping":"root.foo = this.foo.lowercase()","summary":"","results":[["{\"foo\":\"HELLO WORLD\"}","{\"foo\":\"hello world\"}"]],"skip_testing":false}]}],"impure":false},{"status":"hidden","name":"map","params":{"named":[{"name":"query","description":"A query to execute on the target.","type":"query expression","no_dynamic":false,"scalars_to_literal":false}]},"impure":false},{"status":"stable","name":"map_each","params":{"named":[{"name":"query","description":"A query that will be used to map each element.","type":"query expression","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.new_nums = this.nums.map_each(num -\u003e if num \u003c 10 {\n deleted()\n} else {\n num - 10\n})","summary":"##### On arrays\n\nApply a mapping to each element of an array and replace the element with the result. Within the argument mapping the context is the value of the element being mapped.","results":[["{\"nums\":[3,11,4,17]}","{\"new_nums\":[1,7]}"]],"skip_testing":false},{"mapping":"root.new_dict = this.dict.map_each(item -\u003e item.value.uppercase())","summary":"##### On objects\n\nApply a mapping to each value of an object and replace the value with the result. Within the argument mapping the context is an object with a field `key` containing the value key, and a field `value`.","results":[["{\"dict\":{\"foo\":\"hello\",\"bar\":\"world\"}}","{\"new_dict\":{\"bar\":\"WORLD\",\"foo\":\"HELLO\"}}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"map_each_key","params":{"named":[{"name":"query","description":"A query that will be used to map each key.","type":"query expression","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"Apply a mapping to each key of an object, and replace the key with the result, which must be a string.","Examples":[{"mapping":"root.new_dict = this.dict.map_each_key(key -\u003e key.uppercase())","summary":"","results":[["{\"dict\":{\"keya\":\"hello\",\"keyb\":\"world\"}}","{\"new_dict\":{\"KEYA\":\"hello\",\"KEYB\":\"world\"}}"]],"skip_testing":false},{"mapping":"root = this.map_each_key(key -\u003e if key.contains(\"kafka\") { \"_\" + key })","summary":"","results":[["{\"amqp_key\":\"foo\",\"kafka_key\":\"bar\",\"kafka_topic\":\"baz\"}","{\"_kafka_key\":\"bar\",\"_kafka_topic\":\"baz\",\"amqp_key\":\"foo\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"max","description":"Returns the largest numerical value found within an array. All values must be numerical and the array must not be empty, otherwise an error is returned.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"root.biggest = this.values.max()","summary":"","results":[["{\"values\":[0,3,2.5,7,5]}","{\"biggest\":7}"]],"skip_testing":false},{"mapping":"root.new_value = [0,this.value].max()","summary":"","results":[["{\"value\":-1}","{\"new_value\":0}"],["{\"value\":7}","{\"new_value\":7}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"merge","description":"Merge a source object into an existing destination object. When a collision is found within the merged structures (both a source and destination object contain the same non-object keys) the result will be an array containing both values, where values that are already arrays will be expanded into the resulting array. In order to simply override destination fields on collision use the \u003c\u003cassign, `assign`\u003e\u003e method.","params":{"named":[{"name":"with","description":"A value to merge the target value with.","type":"unknown","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root = this.foo.merge(this.bar)","summary":"","results":[["{\"foo\":{\"first_name\":\"fooer\",\"likes\":\"bars\"},\"bar\":{\"second_name\":\"barer\",\"likes\":\"foos\"}}","{\"first_name\":\"fooer\",\"likes\":[\"bars\",\"foos\"],\"second_name\":\"barer\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"min","description":"Returns the smallest numerical value found within an array. All values must be numerical and the array must not be empty, otherwise an error is returned.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"root.smallest = this.values.min()","summary":"","results":[["{\"values\":[0,3,-2.5,7,5]}","{\"smallest\":-2.5}"]],"skip_testing":false},{"mapping":"root.new_value = [10,this.value].min()","summary":"","results":[["{\"value\":2}","{\"new_value\":2}"],["{\"value\":23}","{\"new_value\":10}"]],"skip_testing":false}]}],"impure":false},{"status":"hidden","name":"not","params":{},"impure":false},{"status":"stable","name":"not_empty","params":{},"categories":[{"Category":"Type Coercion","Description":"Ensures that the given string, array or object value is not empty, and if so returns it, otherwise an error is returned.","Examples":[{"mapping":"root.a = this.a.not_empty()","summary":"","results":[["{\"a\":\"foo\"}","{\"a\":\"foo\"}"],["{\"a\":\"\"}","Error(\"failed assignment (line 1): field `this.a`: string value is empty\")"],["{\"a\":[\"foo\",\"bar\"]}","{\"a\":[\"foo\",\"bar\"]}"],["{\"a\":[]}","Error(\"failed assignment (line 1): field `this.a`: array value is empty\")"],["{\"a\":{\"b\":\"foo\",\"c\":\"bar\"}}","{\"a\":{\"b\":\"foo\",\"c\":\"bar\"}}"],["{\"a\":{}}","Error(\"failed assignment (line 1): field `this.a`: object value is empty\")"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"not_null","params":{},"categories":[{"Category":"Type Coercion","Description":"Ensures that the given value is not `null`, and if so returns it, otherwise an error is returned.","Examples":[{"mapping":"root.a = this.a.not_null()","summary":"","results":[["{\"a\":\"foobar\",\"b\":\"barbaz\"}","{\"a\":\"foobar\"}"],["{\"b\":\"barbaz\"}","Error(\"failed assignment (line 1): field `this.a`: value is null\")"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"number","params":{"named":[{"name":"default","description":"An optional value to yield if the target cannot be parsed as a number.","type":"float","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"categories":[{"Category":"Type Coercion","Description":"Attempt to parse a value into a number. An optional argument can be provided, in which case if the value cannot be parsed into a number the argument will be returned instead.","Examples":[{"mapping":"root.foo = this.thing.number() + 10\nroot.bar = this.thing.number(5) * 10","summary":"","results":[],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"or","description":"If the result of the target query fails or resolves to `null`, returns the argument instead. This is an explicit method alternative to the coalesce pipe operator `|`.","params":{"named":[{"name":"fallback","description":"A value to yield, or query to execute, if the target query fails or resolves to `null`.","type":"query expression","no_dynamic":false,"scalars_to_literal":true}]},"examples":[{"mapping":"root.doc.id = this.thing.id.or(uuid_v4())","summary":"","results":[],"skip_testing":false}],"impure":false},{"status":"stable","name":"parse_csv","params":{"named":[{"name":"parse_header_row","description":"Whether to reference the first row as a header row. If set to true the output structure for messages will be an object where field keys are determined by the header row. Otherwise, the output will be an array of row arrays.","type":"bool","no_dynamic":false,"scalars_to_literal":false,"default":true},{"name":"delimiter","description":"The delimiter to use for splitting values in each record. It must be a single character.","type":"string","no_dynamic":false,"scalars_to_literal":false,"default":","},{"name":"lazy_quotes","description":"If set to `true`, a quote may appear in an unquoted field and a non-doubled quote may appear in a quoted field.","type":"bool","no_dynamic":false,"scalars_to_literal":false,"default":false}]},"categories":[{"Category":"Parsing","Description":"Attempts to parse a string into an array of objects by following the CSV format described in RFC 4180.","Examples":[{"mapping":"root.orders = this.orders.parse_csv()","summary":"Parses CSV data with a header row","results":[["{\"orders\":\"foo,bar\\nfoo 1,bar 1\\nfoo 2,bar 2\"}","{\"orders\":[{\"bar\":\"bar 1\",\"foo\":\"foo 1\"},{\"bar\":\"bar 2\",\"foo\":\"foo 2\"}]}"]],"skip_testing":false},{"mapping":"root.orders = this.orders.parse_csv(false)","summary":"Parses CSV data without a header row","results":[["{\"orders\":\"foo 1,bar 1\\nfoo 2,bar 2\"}","{\"orders\":[[\"foo 1\",\"bar 1\"],[\"foo 2\",\"bar 2\"]]}"]],"skip_testing":false},{"mapping":"root.orders = this.orders.parse_csv(delimiter:\".\")","summary":"Parses CSV data delimited by dots","results":[["{\"orders\":\"foo.bar\\nfoo 1.bar 1\\nfoo 2.bar 2\"}","{\"orders\":[{\"bar\":\"bar 1\",\"foo\":\"foo 1\"},{\"bar\":\"bar 2\",\"foo\":\"foo 2\"}]}"]],"skip_testing":false},{"mapping":"root.orders = this.orders.parse_csv(lazy_quotes:true)","summary":"Parses CSV data containing a quote in an unquoted field","results":[["{\"orders\":\"foo,bar\\nfoo 1,bar 1\\nfoo\\\" \\\"2,bar\\\" \\\"2\"}","{\"orders\":[{\"bar\":\"bar 1\",\"foo\":\"foo 1\"},{\"bar\":\"bar\\\" \\\"2\",\"foo\":\"foo\\\" \\\"2\"}]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"parse_duration","description":"Attempts to parse a string as a duration and returns an integer of nanoseconds. A duration string is a possibly signed sequence of decimal numbers, each with an optional fraction and a unit suffix, such as \"300ms\", \"-1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"Β΅s\"), \"ms\", \"s\", \"m\", \"h\".","params":{},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":[{"mapping":"root.delay_for_ns = this.delay_for.parse_duration()","summary":"","results":[["{\"delay_for\":\"50us\"}","{\"delay_for_ns\":50000}"]],"skip_testing":false},{"mapping":"root.delay_for_s = this.delay_for.parse_duration() / 1000000000","summary":"","results":[["{\"delay_for\":\"2h\"}","{\"delay_for_s\":7200}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"parse_duration_iso8601","description":"Attempts to parse a string using ISO-8601 rules as a duration and returns an integer of nanoseconds. A duration string is represented by the format \"P[n]Y[n]M[n]DT[n]H[n]M[n]S\" or \"P[n]W\". In these representations, the \"[n]\" is replaced by the value for each of the date and time elements that follow the \"[n]\". For example, \"P3Y6M4DT12H30M5S\" represents a duration of \"three years, six months, four days, twelve hours, thirty minutes, and five seconds\". The last field of the format allows fractions with one decimal place, so \"P3.5S\" will return 3500000000ns. Any additional decimals will be truncated.","params":{},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":[{"mapping":"root.delay_for_ns = this.delay_for.parse_duration_iso8601()","summary":"Arbitrary ISO-8601 duration string to nanoseconds:","results":[["{\"delay_for\":\"P3Y6M4DT12H30M5S\"}","{\"delay_for_ns\":110839937000000000}"]],"skip_testing":false},{"mapping":"root.delay_for_s = this.delay_for.parse_duration_iso8601() / 1000000000","summary":"Two hours ISO-8601 duration string to seconds:","results":[["{\"delay_for\":\"PT2H\"}","{\"delay_for_s\":7200}"]],"skip_testing":false},{"mapping":"root.delay_for_s = this.delay_for.parse_duration_iso8601() / 1000000000","summary":"Two and a half seconds ISO-8601 duration string to seconds:","results":[["{\"delay_for\":\"PT2.5S\"}","{\"delay_for_s\":2.5}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"parse_form_url_encoded","description":"Attempts to parse a url-encoded query string (from an x-www-form-urlencoded request body) and returns a structured result.","params":{},"categories":[{"Category":"Parsing","Description":"","Examples":[{"mapping":"root.values = this.body.parse_form_url_encoded()","summary":"","results":[["{\"body\":\"noise=meow\u0026animal=cat\u0026fur=orange\u0026fur=fluffy\"}","{\"values\":{\"animal\":\"cat\",\"fur\":[\"orange\",\"fluffy\"],\"noise\":\"meow\"}}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"parse_json","params":{"named":[{"name":"use_number","description":"An optional flag that when set makes parsing numbers as json.Number instead of the default float64.","type":"bool","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"categories":[{"Category":"Parsing","Description":"Attempts to parse a string as a JSON document and returns the result.","Examples":[{"mapping":"root.doc = this.doc.parse_json()","summary":"","results":[["{\"doc\":\"{\\\"foo\\\":\\\"bar\\\"}\"}","{\"doc\":{\"foo\":\"bar\"}}"]],"skip_testing":false},{"mapping":"root.doc = this.doc.parse_json(use_number: true)","summary":"","results":[["{\"doc\":\"{\\\"foo\\\":\\\"11380878173205700000000000000000000000000000000\\\"}\"}","{\"doc\":{\"foo\":\"11380878173205700000000000000000000000000000000\"}}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"parse_jwt_es256","description":"Parses a claims object from a JWT string encoded with ES256. This method does not validate JWT claims.","params":{"named":[{"name":"signing_secret","description":"The ES256 secret that was used for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.claims = this.signed.parse_jwt_es256(\"\"\"-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEGtLqIBePHmIhQcf0JLgc+F/4W/oI\ndp0Gta53G35VerNDgUUXmp78J2kfh4qLdh0XtmOMI587tCaqjvDAXfs//w==\n-----END PUBLIC KEY-----\"\"\")","summary":"","results":[["{\"signed\":\"eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE1MTYyMzkwMjIsIm1vb2QiOiJEaXNkYWluZnVsIiwic3ViIjoiMTIzNDU2Nzg5MCJ9.GIRajP9JJbpTlqSCdNEz4qpQkRvzX4Q51YnTwVyxLDM9tKjR_a8ggHWn9CWj7KG0x8J56OWtmUxn112SRTZVhQ\"}","{\"claims\":{\"iat\":1516239022,\"mood\":\"Disdainful\",\"sub\":\"1234567890\"}}"]],"skip_testing":false}]}],"impure":false,"version":"v4.20.0"},{"status":"stable","name":"parse_jwt_es384","description":"Parses a claims object from a JWT string encoded with ES384. This method does not validate JWT claims.","params":{"named":[{"name":"signing_secret","description":"The ES384 secret that was used for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.claims = this.signed.parse_jwt_es384(\"\"\"-----BEGIN PUBLIC KEY-----\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAERoz74/B6SwmLhs8X7CWhnrWyRrB13AuU\n8OYeqy0qHRu9JWNw8NIavqpTmu6XPT4xcFanYjq8FbeuM11eq06C52mNmS4LLwzA\n2imlFEgn85bvJoC3bnkuq4mQjwt9VxdH\n-----END PUBLIC KEY-----\"\"\")","summary":"","results":[["{\"signed\":\"eyJhbGciOiJFUzM4NCIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE1MTYyMzkwMjIsIm1vb2QiOiJEaXNkYWluZnVsIiwic3ViIjoiMTIzNDU2Nzg5MCJ9.H2HBSlrvQBaov2tdreGonbBexxtQB-xzaPL4-tNQZ6TVh7VH8VBcSwcWHYa1lBAHmdsKOFcB2Wk0SB7QWeGT3ptSgr-_EhDMaZ8bA5spgdpq5DsKfaKHrd7DbbQlmxNq\"}","{\"claims\":{\"iat\":1516239022,\"mood\":\"Disdainful\",\"sub\":\"1234567890\"}}"]],"skip_testing":false}]}],"impure":false,"version":"v4.20.0"},{"status":"stable","name":"parse_jwt_es512","description":"Parses a claims object from a JWT string encoded with ES512. This method does not validate JWT claims.","params":{"named":[{"name":"signing_secret","description":"The ES512 secret that was used for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.claims = this.signed.parse_jwt_es512(\"\"\"-----BEGIN PUBLIC KEY-----\nMIGbMBAGByqGSM49AgEGBSuBBAAjA4GGAAQAkHLdts9P56fFkyhpYQ31M/Stwt3w\nvpaxhlfudxnXgTO1IP4RQRgryRxZ19EUzhvWDcG3GQIckoNMY5PelsnCGnIBT2Xh\n9NQkjWF5K6xS4upFsbGSAwQ+GIyyk5IPJ2LHgOyMSCVh5gRZXV3CZLzXujx/umC9\nUeYyTt05zRRWuD+p5bY=\n-----END PUBLIC KEY-----\"\"\")","summary":"","results":[["{\"signed\":\"eyJhbGciOiJFUzUxMiIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE1MTYyMzkwMjIsIm1vb2QiOiJEaXNkYWluZnVsIiwic3ViIjoiMTIzNDU2Nzg5MCJ9.ACrpLuU7TKpAnncDCpN9m85nkL55MJ45NFOBl6-nEXmNT1eIxWjiP4pwWVbFH9et_BgN14119jbL_KqEJInPYc9nAXC6dDLq0aBU-dalvNl4-O5YWpP43-Y-TBGAsWnbMTrchILJ4-AEiICe73Ck5yWPleKg9c3LtkEFWfGs7BoPRguZ\"}","{\"claims\":{\"iat\":1516239022,\"mood\":\"Disdainful\",\"sub\":\"1234567890\"}}"]],"skip_testing":false}]}],"impure":false,"version":"v4.20.0"},{"status":"stable","name":"parse_jwt_hs256","description":"Parses a claims object from a JWT string encoded with HS256. This method does not validate JWT claims.","params":{"named":[{"name":"signing_secret","description":"The HS256 secret that was used for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.claims = this.signed.parse_jwt_hs256(\"\"\"dont-tell-anyone\"\"\")","summary":"","results":[["{\"signed\":\"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE1MTYyMzkwMjIsIm1vb2QiOiJEaXNkYWluZnVsIiwic3ViIjoiMTIzNDU2Nzg5MCJ9.YwXOM8v3gHVWcQRRRQc_zDlhmLnM62fwhFYGpiA0J1A\"}","{\"claims\":{\"iat\":1516239022,\"mood\":\"Disdainful\",\"sub\":\"1234567890\"}}"]],"skip_testing":false}]}],"impure":false,"version":"v4.12.0"},{"status":"stable","name":"parse_jwt_hs384","description":"Parses a claims object from a JWT string encoded with HS384. This method does not validate JWT claims.","params":{"named":[{"name":"signing_secret","description":"The HS384 secret that was used for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.claims = this.signed.parse_jwt_hs384(\"\"\"dont-tell-anyone\"\"\")","summary":"","results":[["{\"signed\":\"eyJhbGciOiJIUzM4NCIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE1MTYyMzkwMjIsIm1vb2QiOiJEaXNkYWluZnVsIiwic3ViIjoiMTIzNDU2Nzg5MCJ9.2Y8rf_ijwN4t8hOGGViON_GrirLkCQVbCOuax6EoZ3nluX0tCGezcJxbctlIfsQ2\"}","{\"claims\":{\"iat\":1516239022,\"mood\":\"Disdainful\",\"sub\":\"1234567890\"}}"]],"skip_testing":false}]}],"impure":false,"version":"v4.12.0"},{"status":"stable","name":"parse_jwt_hs512","description":"Parses a claims object from a JWT string encoded with HS512. This method does not validate JWT claims.","params":{"named":[{"name":"signing_secret","description":"The HS512 secret that was used for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.claims = this.signed.parse_jwt_hs512(\"\"\"dont-tell-anyone\"\"\")","summary":"","results":[["{\"signed\":\"eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE1MTYyMzkwMjIsIm1vb2QiOiJEaXNkYWluZnVsIiwic3ViIjoiMTIzNDU2Nzg5MCJ9.utRb0urG6LGGyranZJVo5Dk0Fns1QNcSUYPN0TObQ-YzsGGB8jrxHwM5NAJccjJZzKectEUqmmKCaETZvuX4Fg\"}","{\"claims\":{\"iat\":1516239022,\"mood\":\"Disdainful\",\"sub\":\"1234567890\"}}"]],"skip_testing":false}]}],"impure":false,"version":"v4.12.0"},{"status":"stable","name":"parse_jwt_rs256","description":"Parses a claims object from a JWT string encoded with RS256. This method does not validate JWT claims.","params":{"named":[{"name":"signing_secret","description":"The RS256 secret that was used for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.claims = this.signed.parse_jwt_rs256(\"\"\"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAs/ibN8r68pLMR6gRzg4S\n8v8l6Q7yi8qURjkEbcNeM1rkokC7xh0I4JVTwxYSVv/JIW8qJdyspl5NIfuAVi32\nWfKvSAs+NIs+DMsNPYw3yuQals4AX8hith1YDvYpr8SD44jxhz/DR9lYKZFGhXGB\n+7NqQ7vpTWp3BceLYocazWJgusZt7CgecIq57ycM5hjM93BvlrUJ8nQ1a46wfL/8\nCy4P0et70hzZrsjjN41KFhKY0iUwlyU41yEiDHvHDDsTMBxAZosWjSREGfJL6Mfp\nXOInTHs/Gg6DZMkbxjQu6L06EdJ+Q/NwglJdAXM7Zo9rNELqRig6DdvG5JesdMsO\n+QIDAQAB\n-----END PUBLIC KEY-----\"\"\")","summary":"","results":[["{\"signed\":\"eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE1MTYyMzkwMjIsIm1vb2QiOiJEaXNkYWluZnVsIiwic3ViIjoiMTIzNDU2Nzg5MCJ9.b0lH3jEupZZ4zoaly4Y_GCvu94HH6UKdKY96zfGNsIkPZpQLHIkZ7jMWlLlNOAd8qXlsBGP_i8H2qCKI4zlWJBGyPZgxXDzNRPVrTDfFpn4t4nBcA1WK2-ntXP3ehQxsaHcQU8Z_nsogId7Pme5iJRnoHWEnWtbwz5DLSXL3ZZNnRdrHM9MdI7QSDz9mojKDCaMpGN9sG7Xl-tGdBp1XzXuUOzG8S03mtZ1IgVR1uiBL2N6oohHIAunk8DIAmNWI-zgycTgzUGU7mvPkKH43qO8Ua1-13tCUBKKa8VxcotZ67Mxm1QAvBGoDnTKwWMwghLzs6d6WViXQg6eWlJcpBA\"}","{\"claims\":{\"iat\":1516239022,\"mood\":\"Disdainful\",\"sub\":\"1234567890\"}}"]],"skip_testing":false}]}],"impure":false,"version":"v4.20.0"},{"status":"stable","name":"parse_jwt_rs384","description":"Parses a claims object from a JWT string encoded with RS384. This method does not validate JWT claims.","params":{"named":[{"name":"signing_secret","description":"The RS384 secret that was used for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.claims = this.signed.parse_jwt_rs384(\"\"\"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAs/ibN8r68pLMR6gRzg4S\n8v8l6Q7yi8qURjkEbcNeM1rkokC7xh0I4JVTwxYSVv/JIW8qJdyspl5NIfuAVi32\nWfKvSAs+NIs+DMsNPYw3yuQals4AX8hith1YDvYpr8SD44jxhz/DR9lYKZFGhXGB\n+7NqQ7vpTWp3BceLYocazWJgusZt7CgecIq57ycM5hjM93BvlrUJ8nQ1a46wfL/8\nCy4P0et70hzZrsjjN41KFhKY0iUwlyU41yEiDHvHDDsTMBxAZosWjSREGfJL6Mfp\nXOInTHs/Gg6DZMkbxjQu6L06EdJ+Q/NwglJdAXM7Zo9rNELqRig6DdvG5JesdMsO\n+QIDAQAB\n-----END PUBLIC KEY-----\"\"\")","summary":"","results":[["{\"signed\":\"eyJhbGciOiJSUzM4NCIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE1MTYyMzkwMjIsIm1vb2QiOiJEaXNkYWluZnVsIiwic3ViIjoiMTIzNDU2Nzg5MCJ9.orcXYBcjVE5DU7mvq4KKWFfNdXR4nEY_xupzWoETRpYmQZIozlZnM_nHxEk2dySvpXlAzVm7kgOPK2RFtGlOVaNRIa3x-pMMr-bhZTno4L8Hl4sYxOks3bWtjK7wql4uqUbqThSJB12psAXw2-S-I_FMngOPGIn4jDT9b802ottJSvTpXcy0-eKTjrV2PSkRRu-EYJh0CJZW55MNhqlt6kCGhAXfbhNazN3ASX-dmpd_JixyBKphrngr_zRA-FCn_Xf3QQDA-5INopb4Yp5QiJ7UxVqQEKI80X_JvJqz9WE1qiAw8pq5-xTen1t7zTP-HT1NbbD3kltcNa3G8acmNg\"}","{\"claims\":{\"iat\":1516239022,\"mood\":\"Disdainful\",\"sub\":\"1234567890\"}}"]],"skip_testing":false}]}],"impure":false,"version":"v4.20.0"},{"status":"stable","name":"parse_jwt_rs512","description":"Parses a claims object from a JWT string encoded with RS512. This method does not validate JWT claims.","params":{"named":[{"name":"signing_secret","description":"The RS512 secret that was used for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.claims = this.signed.parse_jwt_rs512(\"\"\"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAs/ibN8r68pLMR6gRzg4S\n8v8l6Q7yi8qURjkEbcNeM1rkokC7xh0I4JVTwxYSVv/JIW8qJdyspl5NIfuAVi32\nWfKvSAs+NIs+DMsNPYw3yuQals4AX8hith1YDvYpr8SD44jxhz/DR9lYKZFGhXGB\n+7NqQ7vpTWp3BceLYocazWJgusZt7CgecIq57ycM5hjM93BvlrUJ8nQ1a46wfL/8\nCy4P0et70hzZrsjjN41KFhKY0iUwlyU41yEiDHvHDDsTMBxAZosWjSREGfJL6Mfp\nXOInTHs/Gg6DZMkbxjQu6L06EdJ+Q/NwglJdAXM7Zo9rNELqRig6DdvG5JesdMsO\n+QIDAQAB\n-----END PUBLIC KEY-----\"\"\")","summary":"","results":[["{\"signed\":\"eyJhbGciOiJSUzUxMiIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE1MTYyMzkwMjIsIm1vb2QiOiJEaXNkYWluZnVsIiwic3ViIjoiMTIzNDU2Nzg5MCJ9.rsMp_X5HMrUqKnZJIxo27aAoscovRA6SSQYR9rq7pifIj0YHXxMyNyOBDGnvVALHKTi25VUGHpfNUW0VVMmae0A4t_ObNU6hVZHguWvetKZZq4FZpW1lgWHCMqgPGwT5_uOqwYCH6r8tJuZT3pqXeL0CY4putb1AN2w6CVp620nh3l8d3XWb4jaifycd_4CEVCqHuWDmohfug4VhmoVKlIXZkYoAQowgHlozATDssBSWdYtv107Wd2AzEoiXPu6e3pflsuXULlyqQnS4ELEKPYThFLafh1NqvZDPddqozcPZ-iODBW-xf3A4DYDdivnMYLrh73AZOGHexxu8ay6nDA\"}","{\"claims\":{\"iat\":1516239022,\"mood\":\"Disdainful\",\"sub\":\"1234567890\"}}"]],"skip_testing":false}]}],"impure":false,"version":"v4.20.0"},{"status":"stable","name":"parse_msgpack","description":"Parses a https://msgpack.org/[MessagePack^] message into a structured document.","params":{},"categories":[{"Category":"Parsing","Description":"","Examples":[{"mapping":"root = content().decode(\"hex\").parse_msgpack()","summary":"","results":[["81a3666f6fa3626172","{\"foo\":\"bar\"}"]],"skip_testing":false},{"mapping":"root = this.encoded.decode(\"base64\").parse_msgpack()","summary":"","results":[["{\"encoded\":\"gaNmb2+jYmFy\"}","{\"foo\":\"bar\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"parse_parquet","description":"Decodes a https://parquet.apache.org/docs/[Parquet file^] into an array of objects, one for each row within the file.","params":{"named":[{"name":"byte_array_as_string","description":"Deprecated: This parameter is no longer used.","type":"bool","no_dynamic":false,"scalars_to_literal":false,"default":false}]},"categories":[{"Category":"Parsing","Description":"","Examples":[{"mapping":"root = content().parse_parquet()","summary":"","results":[],"skip_testing":false}]}],"impure":false},{"status":"deprecated","name":"parse_timestamp","description":"Attempts to parse a string as a timestamp following a specified format and outputs a timestamp, which can then be fed into methods such as \u003c\u003cts_format, `ts_format`\u003e\u003e.\n\nThe input format is defined by showing how the reference time, defined to be Mon Jan 2 15:04:05 -0700 MST 2006, would be displayed if it were the value. For an alternative way to specify formats check out the \u003c\u003cts_strptime, `ts_strptime`\u003e\u003e method.","params":{"named":[{"name":"format","description":"The format of the target string.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Deprecated","Description":"","Examples":null}],"impure":false},{"status":"deprecated","name":"parse_timestamp_strptime","description":"Attempts to parse a string as a timestamp following a specified strptime-compatible format and outputs a timestamp, which can then be fed into \u003c\u003cts_format, `ts_format`\u003e\u003e.","params":{"named":[{"name":"format","description":"The format of the target string.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Deprecated","Description":"","Examples":null}],"impure":false},{"status":"stable","name":"parse_url","description":"Attempts to parse a URL from a string value, returning a structured result that describes the various facets of the URL. The fields returned within the structured result roughly follow https://pkg.go.dev/net/url#URL, and may be expanded in future in order to present more information.","params":{},"categories":[{"Category":"Parsing","Description":"","Examples":[{"mapping":"root.foo_url = this.foo_url.parse_url()","summary":"","results":[["{\"foo_url\":\"https://docs.redpanda.com/redpanda-connect/guides/bloblang/about/\"}","{\"foo_url\":{\"fragment\":\"\",\"host\":\"docs.redpanda.com\",\"opaque\":\"\",\"path\":\"/redpanda-connect/guides/bloblang/about/\",\"raw_fragment\":\"\",\"raw_path\":\"\",\"raw_query\":\"\",\"scheme\":\"https\"}}"]],"skip_testing":false},{"mapping":"root.username = this.url.parse_url().user.name | \"unknown\"","summary":"","results":[["{\"url\":\"amqp://foo:bar@127.0.0.1:5672/\"}","{\"username\":\"foo\"}"],["{\"url\":\"redis://localhost:6379\"}","{\"username\":\"unknown\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"parse_xml","description":"\nAttempts to parse a string as an XML document and returns a structured result, where elements appear as keys of an object according to the following rules:\n\n- If an element contains attributes they are parsed by prefixing a hyphen, `-`, to the attribute label.\n- If the element is a simple element and has attributes, the element value is given the key `#text`.\n- XML comments, directives, and process instructions are ignored.\n- When elements are repeated the resulting JSON value is an array.\n- If cast is true, try to cast values to numbers and booleans instead of returning strings.\n","params":{"named":[{"name":"cast","description":"whether to try to cast values that are numbers and booleans to the right type.","type":"bool","no_dynamic":false,"scalars_to_literal":false,"is_optional":true,"default":false}]},"categories":[{"Category":"Parsing","Description":"","Examples":[{"mapping":"root.doc = this.doc.parse_xml()","summary":"","results":[["{\"doc\":\"\u003croot\u003e\u003ctitle\u003eThis is a title\u003c/title\u003e\u003ccontent\u003eThis is some content\u003c/content\u003e\u003c/root\u003e\"}","{\"doc\":{\"root\":{\"content\":\"This is some content\",\"title\":\"This is a title\"}}}"]],"skip_testing":false},{"mapping":"root.doc = this.doc.parse_xml(cast: false)","summary":"","results":[["{\"doc\":\"\u003croot\u003e\u003ctitle\u003eThis is a title\u003c/title\u003e\u003cnumber id=99\u003e123\u003c/number\u003e\u003cbool\u003eTrue\u003c/bool\u003e\u003c/root\u003e\"}","{\"doc\":{\"root\":{\"bool\":\"True\",\"number\":{\"#text\":\"123\",\"-id\":\"99\"},\"title\":\"This is a title\"}}}"]],"skip_testing":false},{"mapping":"root.doc = this.doc.parse_xml(cast: true)","summary":"","results":[["{\"doc\":\"\u003croot\u003e\u003ctitle\u003eThis is a title\u003c/title\u003e\u003cnumber id=99\u003e123\u003c/number\u003e\u003cbool\u003eTrue\u003c/bool\u003e\u003c/root\u003e\"}","{\"doc\":{\"root\":{\"bool\":true,\"number\":{\"#text\":123,\"-id\":99},\"title\":\"This is a title\"}}}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"parse_yaml","params":{},"categories":[{"Category":"Parsing","Description":"Attempts to parse a string as a single YAML document and returns the result.","Examples":[{"mapping":"root.doc = this.doc.parse_yaml()","summary":"","results":[["{\"doc\":\"foo: bar\"}","{\"doc\":{\"foo\":\"bar\"}}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"patch","description":"Create a diff by comparing the current value with the given one. Wraps the github.com/r3labs/diff/v3 package. See its https://pkg.go.dev/github.com/r3labs/diff/v3[docs^] for more information.","params":{"named":[{"name":"changelog","description":"The changelog to apply.","type":"unknown","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":null}],"impure":false,"version":"4.25.0"},{"status":"stable","name":"pow","description":"Returns the number raised to the specified exponent.","params":{"named":[{"name":"exponent","description":"The exponent you want to raise to the power of.","type":"float","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"root.new_value = this.value * 10.pow(-2)","summary":"","results":[["{\"value\":2}","{\"new_value\":0.02}"]],"skip_testing":false},{"mapping":"root.new_value = this.value.pow(-2)","summary":"","results":[["{\"value\":2}","{\"new_value\":0.25}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"quote","params":{},"categories":[{"Category":"String Manipulation","Description":"Quotes a target string using escape sequences (`\\t`, `\\n`, `\\xFF`, `\\u0100`) for control characters and non-printable characters.","Examples":[{"mapping":"root.quoted = this.thing.quote()","summary":"","results":[["{\"thing\":\"foo\\nbar\"}","{\"quoted\":\"\\\"foo\\\\nbar\\\"\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"re_find_all","params":{"named":[{"name":"pattern","description":"The pattern to match against.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Regular Expressions","Description":"Returns an array containing all successive matches of a regular expression in a string.","Examples":[{"mapping":"root.matches = this.value.re_find_all(\"a.\")","summary":"","results":[["{\"value\":\"paranormal\"}","{\"matches\":[\"ar\",\"an\",\"al\"]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"re_find_all_object","params":{"named":[{"name":"pattern","description":"The pattern to match against.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Regular Expressions","Description":"Returns an array of objects containing all matches of the regular expression and the matches of its subexpressions. The key of each match value is the name of the group when specified, otherwise it is the index of the matching group, starting with the expression as a whole at 0.","Examples":[{"mapping":"root.matches = this.value.re_find_all_object(\"a(?P\u003cfoo\u003ex*)b\")","summary":"","results":[["{\"value\":\"-axxb-ab-\"}","{\"matches\":[{\"0\":\"axxb\",\"foo\":\"xx\"},{\"0\":\"ab\",\"foo\":\"\"}]}"]],"skip_testing":false},{"mapping":"root.matches = this.value.re_find_all_object(\"(?m)(?P\u003ckey\u003e\\\\w+):\\\\s+(?P\u003cvalue\u003e\\\\w+)$\")","summary":"","results":[["{\"value\":\"option1: value1\\noption2: value2\\noption3: value3\"}","{\"matches\":[{\"0\":\"option1: value1\",\"key\":\"option1\",\"value\":\"value1\"},{\"0\":\"option2: value2\",\"key\":\"option2\",\"value\":\"value2\"},{\"0\":\"option3: value3\",\"key\":\"option3\",\"value\":\"value3\"}]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"re_find_all_submatch","params":{"named":[{"name":"pattern","description":"The pattern to match against.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Regular Expressions","Description":"Returns an array of arrays containing all successive matches of the regular expression in a string and the matches, if any, of its subexpressions.","Examples":[{"mapping":"root.matches = this.value.re_find_all_submatch(\"a(x*)b\")","summary":"","results":[["{\"value\":\"-axxb-ab-\"}","{\"matches\":[[\"axxb\",\"xx\"],[\"ab\",\"\"]]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"re_find_object","params":{"named":[{"name":"pattern","description":"The pattern to match against.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Regular Expressions","Description":"Returns an object containing the first match of the regular expression and the matches of its subexpressions. The key of each match value is the name of the group when specified, otherwise it is the index of the matching group, starting with the expression as a whole at 0.","Examples":[{"mapping":"root.matches = this.value.re_find_object(\"a(?P\u003cfoo\u003ex*)b\")","summary":"","results":[["{\"value\":\"-axxb-ab-\"}","{\"matches\":{\"0\":\"axxb\",\"foo\":\"xx\"}}"]],"skip_testing":false},{"mapping":"root.matches = this.value.re_find_object(\"(?P\u003ckey\u003e\\\\w+):\\\\s+(?P\u003cvalue\u003e\\\\w+)\")","summary":"","results":[["{\"value\":\"option1: value1\"}","{\"matches\":{\"0\":\"option1: value1\",\"key\":\"option1\",\"value\":\"value1\"}}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"re_match","params":{"named":[{"name":"pattern","description":"The pattern to match against.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Regular Expressions","Description":"Checks whether a regular expression matches against any part of a string and returns a boolean.","Examples":[{"mapping":"root.matches = this.value.re_match(\"[0-9]\")","summary":"","results":[["{\"value\":\"there are 10 puppies\"}","{\"matches\":true}"],["{\"value\":\"there are ten puppies\"}","{\"matches\":false}"]],"skip_testing":false}]}],"impure":false},{"status":"hidden","name":"re_replace","params":{"named":[{"name":"pattern","description":"The pattern to match against.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"value","description":"The value to replace with.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"impure":false},{"status":"stable","name":"re_replace_all","params":{"named":[{"name":"pattern","description":"The pattern to match against.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"value","description":"The value to replace with.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Regular Expressions","Description":"Replaces all occurrences of the argument regular expression in a string with a value. Inside the value $ signs are interpreted as submatch expansions, e.g. `$1` represents the text of the first submatch.","Examples":[{"mapping":"root.new_value = this.value.re_replace_all(\"ADD ([0-9]+)\",\"+($1)\")","summary":"","results":[["{\"value\":\"foo ADD 70\"}","{\"new_value\":\"foo +(70)\"}"]],"skip_testing":false}]}],"impure":false},{"status":"hidden","name":"replace","params":{"named":[{"name":"old","description":"A string to match against.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"new","description":"A string to replace with.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"impure":false},{"status":"stable","name":"replace_all","params":{"named":[{"name":"old","description":"A string to match against.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"new","description":"A string to replace with.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"String Manipulation","Description":"Replaces all occurrences of the first argument in a target string with the second argument.","Examples":[{"mapping":"root.new_value = this.value.replace_all(\"foo\",\"dog\")","summary":"","results":[["{\"value\":\"The foo ate my homework\"}","{\"new_value\":\"The dog ate my homework\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"replace_all_many","params":{"named":[{"name":"values","description":"An array of values, each even value will be replaced with the following odd value.","type":"array","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"String Manipulation","Description":"For each pair of strings in an argument array, replaces all occurrences of the first item of the pair with the second. This is a more compact way of chaining a series of `replace_all` methods.","Examples":[{"mapping":"root.new_value = this.value.replace_all_many([\n \"\u003cb\u003e\", \"\u0026lt;b\u0026gt;\",\n \"\u003c/b\u003e\", \"\u0026lt;/b\u0026gt;\",\n \"\u003ci\u003e\", \"\u0026lt;i\u0026gt;\",\n \"\u003c/i\u003e\", \"\u0026lt;/i\u0026gt;\",\n])","summary":"","results":[["{\"value\":\"\u003ci\u003eHello\u003c/i\u003e \u003cb\u003eWorld\u003c/b\u003e\"}","{\"new_value\":\"\u0026lt;i\u0026gt;Hello\u0026lt;/i\u0026gt; \u0026lt;b\u0026gt;World\u0026lt;/b\u0026gt;\"}"]],"skip_testing":false}]}],"impure":false},{"status":"hidden","name":"replace_many","params":{"named":[{"name":"values","description":"An array of values, each even value will be replaced with the following odd value.","type":"array","no_dynamic":false,"scalars_to_literal":false}]},"impure":false},{"status":"stable","name":"reverse","params":{},"categories":[{"Category":"String Manipulation","Description":"Returns the target string in reverse order.","Examples":[{"mapping":"root.reversed = this.thing.reverse()","summary":"","results":[["{\"thing\":\"backwards\"}","{\"reversed\":\"sdrawkcab\"}"]],"skip_testing":false},{"mapping":"root = content().reverse()","summary":"","results":[["{\"thing\":\"backwards\"}","}\"sdrawkcab\":\"gniht\"{"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"round","description":"Rounds numbers to the nearest integer, rounding half away from zero. If the resulting value fits within a 64-bit integer then that is returned, otherwise a new floating point number is returned.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"root.new_value = this.value.round()","summary":"","results":[["{\"value\":5.3}","{\"new_value\":5}"],["{\"value\":5.9}","{\"new_value\":6}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"sign_jwt_es256","description":"Hash and sign an object representing JSON Web Token (JWT) claims using ES256.","params":{"named":[{"name":"signing_secret","description":"The secret to use for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.signed = this.claims.sign_jwt_es256(\"\"\"-----BEGIN EC PRIVATE KEY-----\n... signature data ...\n-----END EC PRIVATE KEY-----\"\"\")","summary":"","results":[["{\"claims\":{\"sub\":\"user123\"}}","{\"signed\":\"eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE1MTYyMzkwMjIsIm1vb2QiOiJEaXNkYWluZnVsIiwic3ViIjoiMTIzNDU2Nzg5MCJ9.-8LrOdkEiv_44ADWW08lpbq41ZmHCel58NMORPq1q4Dyw0zFhqDVLrRoSvCvuyyvgXAFb9IHfR-9MlJ_2ShA9A\"}"]],"skip_testing":true}]}],"impure":false,"version":"v4.20.0"},{"status":"stable","name":"sign_jwt_es384","description":"Hash and sign an object representing JSON Web Token (JWT) claims using ES384.","params":{"named":[{"name":"signing_secret","description":"The secret to use for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.signed = this.claims.sign_jwt_es384(\"\"\"-----BEGIN EC PRIVATE KEY-----\n... signature data ...\n-----END EC PRIVATE KEY-----\"\"\")","summary":"","results":[["{\"claims\":{\"sub\":\"user123\"}}","{\"signed\":\"eyJhbGciOiJFUzM4NCIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJ1c2VyMTIzIn0.8FmTKH08dl7dyxrNu0rmvhegiIBCy-O9cddGco2e9lpZtgv5mS5qHgPkgBC5eRw1d7SRJsHwHZeehzdqT5Ba7aZJIhz9ds0sn37YQ60L7jT0j2gxCzccrt4kECHnUnLw\"}"]],"skip_testing":true}]}],"impure":false,"version":"v4.20.0"},{"status":"stable","name":"sign_jwt_es512","description":"Hash and sign an object representing JSON Web Token (JWT) claims using ES512.","params":{"named":[{"name":"signing_secret","description":"The secret to use for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.signed = this.claims.sign_jwt_es512(\"\"\"-----BEGIN EC PRIVATE KEY-----\n... signature data ...\n-----END EC PRIVATE KEY-----\"\"\")","summary":"","results":[["{\"claims\":{\"sub\":\"user123\"}}","{\"signed\":\"eyJhbGciOiJFUzUxMiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJ1c2VyMTIzIn0.AQbEWymoRZxDJEJtKSFFG2k2VbDCTYSuBwAZyMqexCspr3If8aERTVGif8HXG3S7TzMBCCzxkcKr3eIU441l3DlpAMNfQbkcOlBqMvNBn-CX481WyKf3K5rFHQ-6wRonz05aIsWAxCDvAozI_9J0OWllxdQ2MBAuTPbPJ38OqXsYkCQs\"}"]],"skip_testing":true}]}],"impure":false,"version":"v4.20.0"},{"status":"stable","name":"sign_jwt_hs256","description":"Hash and sign an object representing JSON Web Token (JWT) claims using HS256.","params":{"named":[{"name":"signing_secret","description":"The secret to use for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.signed = this.claims.sign_jwt_hs256(\"\"\"dont-tell-anyone\"\"\")","summary":"","results":[["{\"claims\":{\"sub\":\"user123\"}}","{\"signed\":\"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJ1c2VyMTIzIn0.hUl-nngPMY_3h9vveWJUPsCcO5PeL6k9hWLnMYeFbFQ\"}"]],"skip_testing":true}]}],"impure":false,"version":"v4.12.0"},{"status":"stable","name":"sign_jwt_hs384","description":"Hash and sign an object representing JSON Web Token (JWT) claims using HS384.","params":{"named":[{"name":"signing_secret","description":"The secret to use for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.signed = this.claims.sign_jwt_hs384(\"\"\"dont-tell-anyone\"\"\")","summary":"","results":[["{\"claims\":{\"sub\":\"user123\"}}","{\"signed\":\"eyJhbGciOiJIUzM4NCIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJ1c2VyMTIzIn0.zGYLr83aToon1efUNq-hw7XgT20lPvZb8sYei8x6S6mpHwb433SJdXJXx0Oio8AZ\"}"]],"skip_testing":true}]}],"impure":false,"version":"v4.12.0"},{"status":"stable","name":"sign_jwt_hs512","description":"Hash and sign an object representing JSON Web Token (JWT) claims using HS512.","params":{"named":[{"name":"signing_secret","description":"The secret to use for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.signed = this.claims.sign_jwt_hs512(\"\"\"dont-tell-anyone\"\"\")","summary":"","results":[["{\"claims\":{\"sub\":\"user123\"}}","{\"signed\":\"eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJ1c2VyMTIzIn0.zBNR9o_6EDwXXKkpKLNJhG26j8Dc-mV-YahBwmEdCrmiWt5les8I9rgmNlWIowpq6Yxs4kLNAdFhqoRz3NXT3w\"}"]],"skip_testing":true}]}],"impure":false,"version":"v4.12.0"},{"status":"stable","name":"sign_jwt_rs256","description":"Hash and sign an object representing JSON Web Token (JWT) claims using RS256.","params":{"named":[{"name":"signing_secret","description":"The secret to use for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.signed = this.claims.sign_jwt_rs256(\"\"\"-----BEGIN RSA PRIVATE KEY-----\n... signature data ...\n-----END RSA PRIVATE KEY-----\"\"\")","summary":"","results":[["{\"claims\":{\"sub\":\"user123\"}}","{\"signed\":\"eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE1MTYyMzkwMjIsIm1vb2QiOiJEaXNkYWluZnVsIiwic3ViIjoiMTIzNDU2Nzg5MCJ9.b0lH3jEupZZ4zoaly4Y_GCvu94HH6UKdKY96zfGNsIkPZpQLHIkZ7jMWlLlNOAd8qXlsBGP_i8H2qCKI4zlWJBGyPZgxXDzNRPVrTDfFpn4t4nBcA1WK2-ntXP3ehQxsaHcQU8Z_nsogId7Pme5iJRnoHWEnWtbwz5DLSXL3ZZNnRdrHM9MdI7QSDz9mojKDCaMpGN9sG7Xl-tGdBp1XzXuUOzG8S03mtZ1IgVR1uiBL2N6oohHIAunk8DIAmNWI-zgycTgzUGU7mvPkKH43qO8Ua1-13tCUBKKa8VxcotZ67Mxm1QAvBGoDnTKwWMwghLzs6d6WViXQg6eWlJcpBA\"}"]],"skip_testing":true}]}],"impure":false,"version":"v4.18.0"},{"status":"stable","name":"sign_jwt_rs384","description":"Hash and sign an object representing JSON Web Token (JWT) claims using RS384.","params":{"named":[{"name":"signing_secret","description":"The secret to use for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.signed = this.claims.sign_jwt_rs384(\"\"\"-----BEGIN RSA PRIVATE KEY-----\n... signature data ...\n-----END RSA PRIVATE KEY-----\"\"\")","summary":"","results":[["{\"claims\":{\"sub\":\"user123\"}}","{\"signed\":\"eyJhbGciOiJSUzM4NCIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE1MTYyMzkwMjIsIm1vb2QiOiJEaXNkYWluZnVsIiwic3ViIjoiMTIzNDU2Nzg5MCJ9.orcXYBcjVE5DU7mvq4KKWFfNdXR4nEY_xupzWoETRpYmQZIozlZnM_nHxEk2dySvpXlAzVm7kgOPK2RFtGlOVaNRIa3x-pMMr-bhZTno4L8Hl4sYxOks3bWtjK7wql4uqUbqThSJB12psAXw2-S-I_FMngOPGIn4jDT9b802ottJSvTpXcy0-eKTjrV2PSkRRu-EYJh0CJZW55MNhqlt6kCGhAXfbhNazN3ASX-dmpd_JixyBKphrngr_zRA-FCn_Xf3QQDA-5INopb4Yp5QiJ7UxVqQEKI80X_JvJqz9WE1qiAw8pq5-xTen1t7zTP-HT1NbbD3kltcNa3G8acmNg\"}"]],"skip_testing":true}]}],"impure":false,"version":"v4.18.0"},{"status":"stable","name":"sign_jwt_rs512","description":"Hash and sign an object representing JSON Web Token (JWT) claims using RS512.","params":{"named":[{"name":"signing_secret","description":"The secret to use for signing the token.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"JSON Web Tokens","Description":"","Examples":[{"mapping":"root.signed = this.claims.sign_jwt_rs512(\"\"\"-----BEGIN RSA PRIVATE KEY-----\n... signature data ...\n-----END RSA PRIVATE KEY-----\"\"\")","summary":"","results":[["{\"claims\":{\"sub\":\"user123\"}}","{\"signed\":\"eyJhbGciOiJSUzUxMiIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE1MTYyMzkwMjIsIm1vb2QiOiJEaXNkYWluZnVsIiwic3ViIjoiMTIzNDU2Nzg5MCJ9.rsMp_X5HMrUqKnZJIxo27aAoscovRA6SSQYR9rq7pifIj0YHXxMyNyOBDGnvVALHKTi25VUGHpfNUW0VVMmae0A4t_ObNU6hVZHguWvetKZZq4FZpW1lgWHCMqgPGwT5_uOqwYCH6r8tJuZT3pqXeL0CY4putb1AN2w6CVp620nh3l8d3XWb4jaifycd_4CEVCqHuWDmohfug4VhmoVKlIXZkYoAQowgHlozATDssBSWdYtv107Wd2AzEoiXPu6e3pflsuXULlyqQnS4ELEKPYThFLafh1NqvZDPddqozcPZ-iODBW-xf3A4DYDdivnMYLrh73AZOGHexxu8ay6nDA\"}"]],"skip_testing":true}]}],"impure":false,"version":"v4.18.0"},{"status":"stable","name":"sin","description":"Calculates the sine of a given angle specified in radians.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"root.new_value = (this.value * (pi() / 180)).sin()","summary":"","results":[["{\"value\":45}","{\"new_value\":0.7071067811865475}"],["{\"value\":0}","{\"new_value\":0}"],["{\"value\":90}","{\"new_value\":1}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"slice","params":{"named":[{"name":"low","description":"The low bound, which is the first element of the selection, or if negative selects from the end.","type":"integer","no_dynamic":false,"scalars_to_literal":false},{"name":"high","description":"An optional high bound.","type":"integer","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"categories":[{"Category":"String Manipulation","Description":"Extract a slice from a string by specifying two indices, a low and high bound, which selects a half-open range that includes the first character, but excludes the last one. If the second index is omitted then it defaults to the length of the input sequence.","Examples":[{"mapping":"root.beginning = this.value.slice(0, 2)\nroot.end = this.value.slice(4)","summary":"","results":[["{\"value\":\"foo bar\"}","{\"beginning\":\"fo\",\"end\":\"bar\"}"]],"skip_testing":false},{"mapping":"root.last_chunk = this.value.slice(-4)\nroot.the_rest = this.value.slice(0, -4)","summary":"A negative low index can be used, indicating an offset from the end of the sequence. If the low index is greater than the length of the sequence then an empty result is returned.","results":[["{\"value\":\"foo bar\"}","{\"last_chunk\":\" bar\",\"the_rest\":\"foo\"}"]],"skip_testing":false}]},{"Category":"Object \u0026 Array Manipulation","Description":"Extract a slice from an array by specifying two indices, a low and high bound, which selects a half-open range that includes the first element, but excludes the last one. If the second index is omitted then it defaults to the length of the input sequence.","Examples":[{"mapping":"root.beginning = this.value.slice(0, 2)\nroot.end = this.value.slice(4)","summary":"","results":[["{\"value\":[\"foo\",\"bar\",\"baz\",\"buz\",\"bev\"]}","{\"beginning\":[\"foo\",\"bar\"],\"end\":[\"bev\"]}"]],"skip_testing":false},{"mapping":"root.last_chunk = this.value.slice(-2)\nroot.the_rest = this.value.slice(0, -2)","summary":"A negative low index can be used, indicating an offset from the end of the sequence. If the low index is greater than the length of the sequence then an empty result is returned.","results":[["{\"value\":[\"foo\",\"bar\",\"baz\",\"buz\",\"bev\"]}","{\"last_chunk\":[\"buz\",\"bev\"],\"the_rest\":[\"foo\",\"bar\",\"baz\"]}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"slug","description":"Creates a \"slug\" from a given string. Wraps the github.com/gosimple/slug package. See its https://pkg.go.dev/github.com/gosimple/slug[docs^] for more information.","params":{"named":[{"name":"lang","type":"string","no_dynamic":false,"scalars_to_literal":false,"is_optional":true,"default":"en"}]},"categories":[{"Category":"String Manipulation","Description":"","Examples":[{"mapping":"root.slug = this.value.slug()","summary":"Creates a slug from an English string","results":[["{\"value\":\"Gopher \u0026 Benthos\"}","{\"slug\":\"gopher-and-benthos\"}"]],"skip_testing":false},{"mapping":"root.slug = this.value.slug(\"fr\")","summary":"Creates a slug from a French string","results":[["{\"value\":\"Gaufre \u0026 Poisson d'Eau Profonde\"}","{\"slug\":\"gaufre-et-poisson-deau-profonde\"}"]],"skip_testing":false}]}],"impure":false,"version":"4.2.0"},{"status":"stable","name":"sort","params":{"named":[{"name":"compare","description":"An optional query that should explicitly compare elements `left` and `right` and provide a boolean result.","type":"query expression","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"Attempts to sort the values of an array in increasing order. The type of all values must match in order for the ordering to succeed. Supports string and number values.","Examples":[{"mapping":"root.sorted = this.foo.sort()","summary":"","results":[["{\"foo\":[\"bbb\",\"ccc\",\"aaa\"]}","{\"sorted\":[\"aaa\",\"bbb\",\"ccc\"]}"]],"skip_testing":false},{"mapping":"root.sorted = this.foo.sort(item -\u003e item.left.v \u003c item.right.v)","summary":"It's also possible to specify a mapping argument, which is provided an object context with fields `left` and `right`, the mapping must return a boolean indicating whether the `left` value is less than `right`. This allows you to sort arrays containing non-string or non-number values.","results":[["{\"foo\":[{\"id\":\"foo\",\"v\":\"bbb\"},{\"id\":\"bar\",\"v\":\"ccc\"},{\"id\":\"baz\",\"v\":\"aaa\"}]}","{\"sorted\":[{\"id\":\"baz\",\"v\":\"aaa\"},{\"id\":\"foo\",\"v\":\"bbb\"},{\"id\":\"bar\",\"v\":\"ccc\"}]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"sort_by","params":{"named":[{"name":"query","description":"A query to apply to each element that yields a value used for sorting.","type":"query expression","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"Attempts to sort the elements of an array, in increasing order, by a value emitted by an argument query applied to each element. The type of all values must match in order for the ordering to succeed. Supports string and number values.","Examples":[{"mapping":"root.sorted = this.foo.sort_by(ele -\u003e ele.id)","summary":"","results":[["{\"foo\":[{\"id\":\"bbb\",\"message\":\"bar\"},{\"id\":\"aaa\",\"message\":\"foo\"},{\"id\":\"ccc\",\"message\":\"baz\"}]}","{\"sorted\":[{\"id\":\"aaa\",\"message\":\"foo\"},{\"id\":\"bbb\",\"message\":\"bar\"},{\"id\":\"ccc\",\"message\":\"baz\"}]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"split","params":{"named":[{"name":"delimiter","description":"The delimiter to split with.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"String Manipulation","Description":"Split a string value into an array of strings by splitting it on a string separator.","Examples":[{"mapping":"root.new_value = this.value.split(\",\")","summary":"","results":[["{\"value\":\"foo,bar,baz\"}","{\"new_value\":[\"foo\",\"bar\",\"baz\"]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"squash","description":"Squashes an array of objects into a single object, where key collisions result in the values being merged (following similar rules as the `.merge()` method)","params":{},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.locations = this.locations.map_each(loc -\u003e {loc.state: [loc.name]}).squash()","summary":"","results":[["{\"locations\":[{\"name\":\"Seattle\",\"state\":\"WA\"},{\"name\":\"New York\",\"state\":\"NY\"},{\"name\":\"Bellevue\",\"state\":\"WA\"},{\"name\":\"Olympia\",\"state\":\"WA\"}]}","{\"locations\":{\"NY\":[\"New York\"],\"WA\":[\"Seattle\",\"Bellevue\",\"Olympia\"]}}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"string","params":{},"categories":[{"Category":"Type Coercion","Description":"Marshal a value into a string. If the value is already a string it is unchanged.","Examples":[{"mapping":"root.nested_json = this.string()","summary":"","results":[["{\"foo\":\"bar\"}","{\"nested_json\":\"{\\\"foo\\\":\\\"bar\\\"}\"}"]],"skip_testing":false},{"mapping":"root.id = this.id.string()","summary":"","results":[["{\"id\":228930314431312345}","{\"id\":\"228930314431312345\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"strip_html","description":"Attempts to remove all HTML tags from a target string.","params":{"named":[{"name":"preserve","description":"An optional array of element types to preserve in the output.","type":"unknown","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"categories":[{"Category":"String Manipulation","Description":"","Examples":[{"mapping":"root.stripped = this.value.strip_html()","summary":"","results":[["{\"value\":\"\u003cp\u003ethe plain \u003cstrong\u003eold text\u003c/strong\u003e\u003c/p\u003e\"}","{\"stripped\":\"the plain old text\"}"]],"skip_testing":false},{"mapping":"root.stripped = this.value.strip_html([\"article\"])","summary":"It's also possible to provide an explicit list of element types to preserve in the output.","results":[["{\"value\":\"\u003carticle\u003e\u003cp\u003ethe plain \u003cstrong\u003eold text\u003c/strong\u003e\u003c/p\u003e\u003c/article\u003e\"}","{\"stripped\":\"\u003carticle\u003ethe plain old text\u003c/article\u003e\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"sum","params":{},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"Sum the numerical values of an array.","Examples":[{"mapping":"root.sum = this.foo.sum()","summary":"","results":[["{\"foo\":[3,8,4]}","{\"sum\":15}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"tan","description":"Calculates the tangent of a given angle specified in radians.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"root.new_value = \"%f\".format((this.value * (pi() / 180)).tan())","summary":"","results":[["{\"value\":0}","{\"new_value\":\"0.000000\"}"],["{\"value\":45}","{\"new_value\":\"1.000000\"}"],["{\"value\":180}","{\"new_value\":\"-0.000000\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"timestamp","params":{"named":[{"name":"default","description":"An optional value to yield if the target cannot be parsed as a timestamp.","type":"timestamp","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"categories":[{"Category":"Type Coercion","Description":"Attempt to parse a value into a timestamp. An optional argument can be provided, in which case if the value cannot be parsed into a timestamp the argument will be returned instead.","Examples":[{"mapping":"root.foo = this.ts.timestamp()\nroot.bar = this.none.timestamp(1234567890.timestamp())","summary":"","results":[],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"trim","params":{"named":[{"name":"cutset","description":"An optional string of characters to trim from the target value.","type":"string","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"categories":[{"Category":"String Manipulation","Description":"Remove all leading and trailing characters from a string that are contained within an argument cutset. If no arguments are provided then whitespace is removed.","Examples":[{"mapping":"root.title = this.title.trim(\"!?\")\nroot.description = this.description.trim()","summary":"","results":[["{\"description\":\" something happened and its amazing! \",\"title\":\"!!!watch out!?\"}","{\"description\":\"something happened and its amazing!\",\"title\":\"watch out\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"trim_prefix","params":{"named":[{"name":"prefix","description":"The leading prefix substring to trim from the string.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"String Manipulation","Description":"Remove the provided leading prefix substring from a string. If the string does not have the prefix substring, it is returned unchanged.","Examples":[{"mapping":"root.name = this.name.trim_prefix(\"foobar_\")\nroot.description = this.description.trim_prefix(\"foobar_\")","summary":"","results":[["{\"description\":\"unchanged\",\"name\":\"foobar_blobton\"}","{\"description\":\"unchanged\",\"name\":\"blobton\"}"]],"skip_testing":false}]}],"impure":false,"version":"4.12.0"},{"status":"stable","name":"trim_suffix","params":{"named":[{"name":"suffix","description":"The trailing suffix substring to trim from the string.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"String Manipulation","Description":"Remove the provided trailing suffix substring from a string. If the string does not have the suffix substring, it is returned unchanged.","Examples":[{"mapping":"root.name = this.name.trim_suffix(\"_foobar\")\nroot.description = this.description.trim_suffix(\"_foobar\")","summary":"","results":[["{\"description\":\"unchanged\",\"name\":\"blobton_foobar\"}","{\"description\":\"unchanged\",\"name\":\"blobton\"}"]],"skip_testing":false}]}],"impure":false,"version":"4.12.0"},{"status":"beta","name":"ts_add_iso8601","description":"Parse parameter string as ISO 8601 period and add it to value with high precision for units larger than an hour.","params":{"named":[{"name":"duration","description":"Duration in ISO 8601 format","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":null}],"impure":false},{"status":"beta","name":"ts_format","description":"Attempts to format a timestamp value as a string according to a specified format, or RFC 3339 by default. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format.\n\nThe output format is defined by showing how the reference time, defined to be Mon Jan 2 15:04:05 -0700 MST 2006, would be displayed if it were the value. For an alternative way to specify formats check out the \u003c\u003cts_strftime, `ts_strftime`\u003e\u003e method.","params":{"named":[{"name":"format","description":"The output format to use.","type":"string","no_dynamic":false,"scalars_to_literal":false,"default":"2006-01-02T15:04:05.999999999Z07:00"},{"name":"tz","description":"An optional timezone to use, otherwise the timezone of the input string is used, or in the case of unix timestamps the local timezone is used.","type":"string","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":[{"mapping":"root.something_at = (this.created_at + 300).ts_format()","summary":"","results":[],"skip_testing":false},{"mapping":"root.something_at = (this.created_at + 300).ts_format(\"2006-Jan-02 15:04:05\")","summary":"An optional string argument can be used in order to specify the output format of the timestamp. The format is defined by showing how the reference time, defined to be Mon Jan 2 15:04:05 -0700 MST 2006, would be displayed if it were the value.","results":[],"skip_testing":false},{"mapping":"root.something_at = this.created_at.ts_format(format: \"2006-Jan-02 15:04:05\", tz: \"UTC\")","summary":"A second optional string argument can also be used in order to specify a timezone, otherwise the timezone of the input string is used, or in the case of unix timestamps the local timezone is used.","results":[["{\"created_at\":1597405526}","{\"something_at\":\"2020-Aug-14 11:45:26\"}"],["{\"created_at\":\"2020-08-14T11:50:26.371Z\"}","{\"something_at\":\"2020-Aug-14 11:50:26\"}"]],"skip_testing":false},{"mapping":"root.something_at = this.created_at.ts_format(\"2006-Jan-02 15:04:05.999999\", \"UTC\")","summary":"And `ts_format` supports up to nanosecond precision with floating point timestamp values.","results":[["{\"created_at\":1597405526.123456}","{\"something_at\":\"2020-Aug-14 11:45:26.123456\"}"],["{\"created_at\":\"2020-08-14T11:50:26.371Z\"}","{\"something_at\":\"2020-Aug-14 11:50:26.371\"}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"ts_parse","description":"Attempts to parse a string as a timestamp following a specified format and outputs a timestamp, which can then be fed into methods such as \u003c\u003cts_format, `ts_format`\u003e\u003e.\n\nThe input format is defined by showing how the reference time, defined to be Mon Jan 2 15:04:05 -0700 MST 2006, would be displayed if it were the value. For an alternative way to specify formats check out the \u003c\u003cts_strptime, `ts_strptime`\u003e\u003e method.","params":{"named":[{"name":"format","description":"The format of the target string.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":[{"mapping":"root.doc.timestamp = this.doc.timestamp.ts_parse(\"2006-Jan-02\")","summary":"","results":[["{\"doc\":{\"timestamp\":\"2020-Aug-14\"}}","{\"doc\":{\"timestamp\":\"2020-08-14T00:00:00Z\"}}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"ts_round","description":"Returns the result of rounding a timestamp to the nearest multiple of the argument duration (nanoseconds). The rounding behavior for halfway values is to round up. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format. The \u003c\u003cts_parse, `ts_parse`\u003e\u003e method can be used in order to parse different timestamp formats.","params":{"named":[{"name":"duration","description":"A duration measured in nanoseconds to round by.","type":"integer","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":[{"mapping":"root.created_at_hour = this.created_at.ts_round(\"1h\".parse_duration())","summary":"Use the method `parse_duration` to convert a duration string into an integer argument.","results":[["{\"created_at\":\"2020-08-14T05:54:23Z\"}","{\"created_at_hour\":\"2020-08-14T06:00:00Z\"}"]],"skip_testing":false}]}],"impure":false,"version":"4.2.0"},{"status":"beta","name":"ts_strftime","description":"Attempts to format a timestamp value as a string according to a specified strftime-compatible format. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format.","params":{"named":[{"name":"format","description":"The output format to use.","type":"string","no_dynamic":false,"scalars_to_literal":false},{"name":"tz","description":"An optional timezone to use, otherwise the timezone of the input string is used.","type":"string","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":[{"mapping":"root.something_at = (this.created_at + 300).ts_strftime(\"%Y-%b-%d %H:%M:%S\")","summary":"The format consists of zero or more conversion specifiers and ordinary characters (except `%`). All ordinary characters are copied to the output string without modification. Each conversion specification begins with `%` character followed by the character that determines the behavior of the specifier. Please refer to https://linux.die.net/man/3/strftime[man 3 strftime] for the list of format specifiers.","results":[],"skip_testing":false},{"mapping":"root.something_at = this.created_at.ts_strftime(\"%Y-%b-%d %H:%M:%S\", \"UTC\")","summary":"A second optional string argument can also be used in order to specify a timezone, otherwise the timezone of the input string is used, or in the case of unix timestamps the local timezone is used.","results":[["{\"created_at\":1597405526}","{\"something_at\":\"2020-Aug-14 11:45:26\"}"],["{\"created_at\":\"2020-08-14T11:50:26.371Z\"}","{\"something_at\":\"2020-Aug-14 11:50:26\"}"]],"skip_testing":false},{"mapping":"root.something_at = this.created_at.ts_strftime(\"%Y-%b-%d %H:%M:%S.%f\", \"UTC\")","summary":"As an extension provided by the underlying formatting library, https://github.com/itchyny/timefmt-go[itchyny/timefmt-go], the `%f` directive is supported for zero-padded microseconds, which originates from Python. Note that E and O modifier characters are not supported.","results":[["{\"created_at\":1597405526}","{\"something_at\":\"2020-Aug-14 11:45:26.000000\"}"],["{\"created_at\":\"2020-08-14T11:50:26.371Z\"}","{\"something_at\":\"2020-Aug-14 11:50:26.371000\"}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"ts_strptime","description":"Attempts to parse a string as a timestamp following a specified strptime-compatible format and outputs a timestamp, which can then be fed into \u003c\u003cts_format, `ts_format`\u003e\u003e.","params":{"named":[{"name":"format","description":"The format of the target string.","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":[{"mapping":"root.doc.timestamp = this.doc.timestamp.ts_strptime(\"%Y-%b-%d\")","summary":"The format consists of zero or more conversion specifiers and ordinary characters (except `%`). All ordinary characters are copied to the output string without modification. Each conversion specification begins with a `%` character followed by the character that determines the behavior of the specifier. Please refer to https://linux.die.net/man/3/strptime[man 3 strptime] for the list of format specifiers.","results":[["{\"doc\":{\"timestamp\":\"2020-Aug-14\"}}","{\"doc\":{\"timestamp\":\"2020-08-14T00:00:00Z\"}}"]],"skip_testing":false},{"mapping":"root.doc.timestamp = this.doc.timestamp.ts_strptime(\"%Y-%b-%d %H:%M:%S.%f\")","summary":"As an extension provided by the underlying formatting library, https://github.com/itchyny/timefmt-go[itchyny/timefmt-go], the `%f` directive is supported for zero-padded microseconds, which originates from Python. Note that E and O modifier characters are not supported.","results":[["{\"doc\":{\"timestamp\":\"2020-Aug-14 11:50:26.371000\"}}","{\"doc\":{\"timestamp\":\"2020-08-14T11:50:26.371Z\"}}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"ts_sub","description":"Returns the difference in nanoseconds between the target timestamp (t1) and the timestamp provided as a parameter (t2). The \u003c\u003cts_parse, `ts_parse`\u003e\u003e method can be used in order to parse different timestamp formats.","params":{"named":[{"name":"t2","description":"The second timestamp to be subtracted from the method target.","type":"timestamp","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":[{"mapping":"root.between = this.started_at.ts_sub(\"2020-08-14T05:54:23Z\").abs()","summary":"Use the `.abs()` method in order to calculate an absolute duration between two timestamps.","results":[["{\"started_at\":\"2020-08-13T05:54:23Z\"}","{\"between\":86400000000000}"]],"skip_testing":false}]}],"impure":false,"version":"4.23.0"},{"status":"beta","name":"ts_sub_iso8601","description":"Parse parameter string as ISO 8601 period and subtract it from value with high precision for units larger than an hour.","params":{"named":[{"name":"duration","description":"Duration in ISO 8601 format","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":null}],"impure":false},{"status":"beta","name":"ts_tz","description":"Returns the result of converting a timestamp to a specified timezone. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format. The \u003c\u003cts_parse, `ts_parse`\u003e\u003e method can be used in order to parse different timestamp formats.","params":{"named":[{"name":"tz","description":"The timezone to change to. If set to \"UTC\" then the timezone will be UTC. If set to \"Local\" then the local timezone will be used. Otherwise, the argument is taken to be a location name corresponding to a file in the IANA Time Zone database, such as \"America/New_York\".","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":[{"mapping":"root.created_at_utc = this.created_at.ts_tz(\"UTC\")","summary":"","results":[["{\"created_at\":\"2021-02-03T17:05:06+01:00\"}","{\"created_at_utc\":\"2021-02-03T16:05:06Z\"}"]],"skip_testing":false}]}],"impure":false,"version":"4.3.0"},{"status":"beta","name":"ts_unix","description":"Attempts to format a timestamp value as a unix timestamp. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format. The \u003c\u003cts_parse, `ts_parse`\u003e\u003e method can be used in order to parse different timestamp formats.","params":{},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":[{"mapping":"root.created_at_unix = this.created_at.ts_unix()","summary":"","results":[["{\"created_at\":\"2009-11-10T23:00:00Z\"}","{\"created_at_unix\":1257894000}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"ts_unix_micro","description":"Attempts to format a timestamp value as a unix timestamp with microsecond precision. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format. The \u003c\u003cts_parse, `ts_parse`\u003e\u003e method can be used in order to parse different timestamp formats.","params":{},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":[{"mapping":"root.created_at_unix = this.created_at.ts_unix_micro()","summary":"","results":[["{\"created_at\":\"2009-11-10T23:00:00Z\"}","{\"created_at_unix\":1257894000000000}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"ts_unix_milli","description":"Attempts to format a timestamp value as a unix timestamp with millisecond precision. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format. The \u003c\u003cts_parse, `ts_parse`\u003e\u003e method can be used in order to parse different timestamp formats.","params":{},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":[{"mapping":"root.created_at_unix = this.created_at.ts_unix_milli()","summary":"","results":[["{\"created_at\":\"2009-11-10T23:00:00Z\"}","{\"created_at_unix\":1257894000000}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"ts_unix_nano","description":"Attempts to format a timestamp value as a unix timestamp with nanosecond precision. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format. The \u003c\u003cts_parse, `ts_parse`\u003e\u003e method can be used in order to parse different timestamp formats.","params":{},"categories":[{"Category":"Timestamp Manipulation","Description":"","Examples":[{"mapping":"root.created_at_unix = this.created_at.ts_unix_nano()","summary":"","results":[["{\"created_at\":\"2009-11-10T23:00:00Z\"}","{\"created_at_unix\":1257894000000000000}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"type","params":{},"categories":[{"Category":"Type Coercion","Description":"Returns the type of a value as a string, providing one of the following values: `string`, `bytes`, `number`, `bool`, `timestamp`, `array`, `object` or `null`.","Examples":[{"mapping":"root.bar_type = this.bar.type()\nroot.foo_type = this.foo.type()","summary":"","results":[["{\"bar\":10,\"foo\":\"is a string\"}","{\"bar_type\":\"number\",\"foo_type\":\"string\"}"]],"skip_testing":false},{"mapping":"root.type = this.type()","summary":"","results":[["\"foobar\"","{\"type\":\"string\"}"],["666","{\"type\":\"number\"}"],["false","{\"type\":\"bool\"}"],["[\"foo\", \"bar\"]","{\"type\":\"array\"}"],["{\"foo\": \"bar\"}","{\"type\":\"object\"}"],["null","{\"type\":\"null\"}"]],"skip_testing":false},{"mapping":"root.type = content().type()","summary":"","results":[["foobar","{\"type\":\"bytes\"}"]],"skip_testing":false},{"mapping":"root.type = this.ts_parse(\"2006-01-02\").type()","summary":"","results":[["\"2022-06-06\"","{\"type\":\"timestamp\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"uint16","description":"\nConverts a numerical type into a 16-bit unsigned integer, this is for advanced use cases where a specific data type is needed for a given component (such as the ClickHouse SQL driver).\n\nIf the value is a string then an attempt will be made to parse it as a 16-bit unsigned integer. If the target value exceeds the capacity of an integer or contains decimal values then this method will throw an error. In order to convert a floating point number containing decimals first use \u003c\u003cround, `.round()`\u003e\u003e on the value. Please refer to the https://pkg.go.dev/strconv#ParseInt[`strconv.ParseInt` documentation] for details regarding the supported formats.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"\nroot.a = this.a.uint16()\nroot.b = this.b.round().uint16()\nroot.c = this.c.uint16()\nroot.d = this.d.uint16().catch(0)\n","summary":"","results":[["{\"a\":12,\"b\":12.34,\"c\":\"12\",\"d\":-12}","{\"a\":12,\"b\":12,\"c\":12,\"d\":0}"]],"skip_testing":false},{"mapping":"\nroot = this.uint16()\n","summary":"","results":[["\"0xDE\"","222"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"uint32","description":"\nConverts a numerical type into a 32-bit unsigned integer, this is for advanced use cases where a specific data type is needed for a given component (such as the ClickHouse SQL driver).\n\nIf the value is a string then an attempt will be made to parse it as a 32-bit unsigned integer. If the target value exceeds the capacity of an integer or contains decimal values then this method will throw an error. In order to convert a floating point number containing decimals first use \u003c\u003cround, `.round()`\u003e\u003e on the value. Please refer to the https://pkg.go.dev/strconv#ParseInt[`strconv.ParseInt` documentation] for details regarding the supported formats.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"\nroot.a = this.a.uint32()\nroot.b = this.b.round().uint32()\nroot.c = this.c.uint32()\nroot.d = this.d.uint32().catch(0)\n","summary":"","results":[["{\"a\":12,\"b\":12.34,\"c\":\"12\",\"d\":-12}","{\"a\":12,\"b\":12,\"c\":12,\"d\":0}"]],"skip_testing":false},{"mapping":"\nroot = this.uint32()\n","summary":"","results":[["\"0xDEAD\"","57005"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"uint64","description":"\nConverts a numerical type into a 64-bit unsigned integer, this is for advanced use cases where a specific data type is needed for a given component (such as the ClickHouse SQL driver).\n\nIf the value is a string then an attempt will be made to parse it as a 64-bit unsigned integer. If the target value exceeds the capacity of an integer or contains decimal values then this method will throw an error. In order to convert a floating point number containing decimals first use \u003c\u003cround, `.round()`\u003e\u003e on the value. Please refer to the https://pkg.go.dev/strconv#ParseInt[`strconv.ParseInt` documentation] for details regarding the supported formats.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"\nroot.a = this.a.uint64()\nroot.b = this.b.round().uint64()\nroot.c = this.c.uint64()\nroot.d = this.d.uint64().catch(0)\n","summary":"","results":[["{\"a\":12,\"b\":12.34,\"c\":\"12\",\"d\":-12}","{\"a\":12,\"b\":12,\"c\":12,\"d\":0}"]],"skip_testing":false},{"mapping":"\nroot = this.uint64()\n","summary":"","results":[["\"0xDEADBEEF\"","3735928559"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"uint8","description":"\nConverts a numerical type into a 8-bit unsigned integer, this is for advanced use cases where a specific data type is needed for a given component (such as the ClickHouse SQL driver).\n\nIf the value is a string then an attempt will be made to parse it as a 8-bit unsigned integer. If the target value exceeds the capacity of an integer or contains decimal values then this method will throw an error. In order to convert a floating point number containing decimals first use \u003c\u003cround, `.round()`\u003e\u003e on the value. Please refer to the https://pkg.go.dev/strconv#ParseInt[`strconv.ParseInt` documentation] for details regarding the supported formats.","params":{},"categories":[{"Category":"Number Manipulation","Description":"","Examples":[{"mapping":"\nroot.a = this.a.uint8()\nroot.b = this.b.round().uint8()\nroot.c = this.c.uint8()\nroot.d = this.d.uint8().catch(0)\n","summary":"","results":[["{\"a\":12,\"b\":12.34,\"c\":\"12\",\"d\":-12}","{\"a\":12,\"b\":12,\"c\":12,\"d\":0}"]],"skip_testing":false},{"mapping":"\nroot = this.uint8()\n","summary":"","results":[["\"0xD\"","13"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"unescape_html","params":{},"categories":[{"Category":"String Manipulation","Description":"Unescapes a string so that entities like `\u0026lt;` become `\u003c`. It unescapes a larger range of entities than `escape_html` escapes. For example, `\u0026aacute;` unescapes to `Γ‘`, as does `\u0026#225;` and `\u0026xE1;`.","Examples":[{"mapping":"root.unescaped = this.value.unescape_html()","summary":"","results":[["{\"value\":\"foo \u0026amp; bar\"}","{\"unescaped\":\"foo \u0026 bar\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"unescape_url_query","params":{},"categories":[{"Category":"String Manipulation","Description":"Expands escape sequences from a URL query string.","Examples":[{"mapping":"root.unescaped = this.value.unescape_url_query()","summary":"","results":[["{\"value\":\"foo+%26+bar\"}","{\"unescaped\":\"foo \u0026 bar\"}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"unicode_segments","description":"Splits text into segments from a given string based on the unicode text segmentation rules.","params":{"named":[{"name":"segmentation_type","type":"string","no_dynamic":false,"scalars_to_literal":false}]},"categories":[{"Category":"String Manipulation","Description":"","Examples":[{"mapping":"root.sentences = this.value.unicode_segments(\"sentence\")","summary":"Splits a string into different sentences","results":[["{\"value\":\"This is sentence 1.0. And this is sentence two.\"}","{\"sentences\":[\"This is sentence 1.0. \",\"And this is sentence two.\"]}"]],"skip_testing":false},{"mapping":"root.graphemes = this.value.unicode_segments(\"grapheme\")","summary":"Splits a string into different graphemes","results":[["{\"value\":\"πŸ•β€πŸ¦Ί 🫠\"}","{\"graphemes\":[\"πŸ•β€πŸ¦Ί\",\" \",\"🫠\"]}"]],"skip_testing":false},{"mapping":"root.words = this.value.unicode_segments(\"word\")","summary":"Splits text into words","results":[["{\"value\":\"Hello, world!\"}","{\"words\":[\"Hello\",\",\",\" \",\"world\",\"!\"]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"unique","params":{"named":[{"name":"emit","description":"An optional query that can be used in order to yield a value for each element to determine uniqueness.","type":"query expression","no_dynamic":false,"scalars_to_literal":false,"is_optional":true}]},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"Attempts to remove duplicate values from an array. The array may contain a combination of different value types, but numbers and strings are checked separately (`\"5\"` is a different element to `5`).","Examples":[{"mapping":"root.uniques = this.foo.unique()","summary":"","results":[["{\"foo\":[\"a\",\"b\",\"a\",\"c\"]}","{\"uniques\":[\"a\",\"b\",\"c\"]}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"unquote","params":{},"categories":[{"Category":"String Manipulation","Description":"Unquotes a target string, expanding any escape sequences (`\\t`, `\\n`, `\\xFF`, `\\u0100`) for control characters and non-printable characters.","Examples":[{"mapping":"root.unquoted = this.thing.unquote()","summary":"","results":[["{\"thing\":\"\\\"foo\\\\nbar\\\"\"}","{\"unquoted\":\"foo\\nbar\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"uppercase","params":{},"categories":[{"Category":"String Manipulation","Description":"Convert a string value into uppercase.","Examples":[{"mapping":"root.foo = this.foo.uppercase()","summary":"","results":[["{\"foo\":\"hello world\"}","{\"foo\":\"HELLO WORLD\"}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"values","params":{},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"Returns the values of an object as an array. The order of the resulting array will be random.","Examples":[{"mapping":"root.foo_vals = this.foo.values().sort()","summary":"","results":[["{\"foo\":{\"bar\":1,\"baz\":2}}","{\"foo_vals\":[1,2]}"]],"skip_testing":false}]}],"impure":false},{"status":"beta","name":"vector","description":"Creates a vector from a given array of floating point numbers.\n\nThis vector can be inserted into various SQL databases if they have support for embeddings vectors (for example `pgvector`).","params":{},"categories":[{"Category":"SQL","Description":"","Examples":[{"mapping":"root.embeddings = [1.2, 0.6, 0.9].vector()","summary":"Create a vector from an array literal","results":[],"skip_testing":false},{"mapping":"root.embedding_vector = this.embedding_array.vector()","summary":"Create a vector from an array","results":[],"skip_testing":false}]}],"impure":false,"version":"4.33.0"},{"status":"stable","name":"with","description":"Returns an object where all but one or more xref:configuration:field_paths.adoc[field path] arguments are removed. Each path specifies a specific field to be retained from the input object, allowing for nested fields.\n\nIf a key within a nested path does not exist then it is ignored.","params":{"variadic":true},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root = this.with(\"inner.a\",\"inner.c\",\"d\")","summary":"","results":[["{\"inner\":{\"a\":\"first\",\"b\":\"second\",\"c\":\"third\"},\"d\":\"fourth\",\"e\":\"fifth\"}","{\"d\":\"fourth\",\"inner\":{\"a\":\"first\",\"c\":\"third\"}}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"without","params":{"variadic":true},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"Returns an object where one or more xref:configuration:field_paths.adoc[field path] arguments are removed. Each path specifies a specific field to be deleted from the input object, allowing for nested fields.\n\nIf a key within a nested path does not exist or is not an object then it is not removed.","Examples":[{"mapping":"root = this.without(\"inner.a\",\"inner.c\",\"d\")","summary":"","results":[["{\"inner\":{\"a\":\"first\",\"b\":\"second\",\"c\":\"third\"},\"d\":\"fourth\",\"e\":\"fifth\"}","{\"e\":\"fifth\",\"inner\":{\"b\":\"second\"}}"]],"skip_testing":false}]}],"impure":false},{"status":"stable","name":"zip","description":"Zip an array value with one or more argument arrays. Each array must match in length.","params":{"variadic":true},"categories":[{"Category":"Object \u0026 Array Manipulation","Description":"","Examples":[{"mapping":"root.foo = this.foo.zip(this.bar, this.baz)","summary":"","results":[["{\"foo\":[\"a\",\"b\",\"c\"],\"bar\":[1,2,3],\"baz\":[4,5,6]}","{\"foo\":[[\"a\",1,4],[\"b\",2,5],[\"c\",3,6]]}"]],"skip_testing":false}]}],"impure":false}]} diff --git a/__tests__/tools/property-extractor/test_known_values.py b/__tests__/tools/property-extractor/test_known_values.py index 60a2ff9b..1e2dd6bd 100644 --- a/__tests__/tools/property-extractor/test_known_values.py +++ b/__tests__/tools/property-extractor/test_known_values.py @@ -380,7 +380,7 @@ def test_secret_string_property(self): class SizeLiteralTest(unittest.TestCase): - """Test size literal conversion (e.g., 20_GiB)""" + """Test size literal conversion (for example, 20_GiB)""" def test_gib_size_literal(self): """Test conversion of _GiB size literal""" diff --git a/admin/redpanda-admin-api.yaml b/admin/redpanda-admin-api.yaml index 33f69910..2c41268a 100644 --- a/admin/redpanda-admin-api.yaml +++ b/admin/redpanda-admin-api.yaml @@ -776,7 +776,7 @@ components: title: has_more_requests type: boolean sampledInFlightRequests: - description: A sample (e.g., the 5 latest) of the currently in-flight requests + description: A sample (for example, the 5 latest) of the currently in-flight requests items: $ref: "#/components/schemas/redpanda.core.admin.v2.InFlightRequests.Request" title: sampled_in_flight_requests @@ -788,7 +788,7 @@ components: properties: apiKey: description: |- - API key for the request type (e.g., produce/fetch/metadata/etc) + API key for the request type (for example, produce/fetch/metadata/etc) https://kafka.apache.org/0101/protocol.html#protocol_api_keys format: int32 title: api_key @@ -923,7 +923,7 @@ components: tlsInfo: $ref: "#/components/schemas/redpanda.core.admin.v2.TLSInfo" description: |- - Information about the TLS state of the connection (e.g., whether TLS + Information about the TLS state of the connection (for example, whether TLS encryption is used for this connection) title: tls_info totalRequestStatistics: diff --git a/bin/doc-tools-mcp.js b/bin/doc-tools-mcp.js new file mode 100755 index 00000000..d99ea59a --- /dev/null +++ b/bin/doc-tools-mcp.js @@ -0,0 +1,701 @@ +#!/usr/bin/env node + +/** + * MCP Server for Redpanda Documentation Tools + * + * This server exposes domain-specific documentation tools to Claude Code + * via the Model Context Protocol. + * + * Features: + * - Context-aware: Works from any repository based on cwd + * - Antora intelligence: Understands component/module structure + * - Automation: Run doc-tools generate commands + * - Auto-discovery: Prompts loaded automatically from mcp/prompts/ + * - Validation: Startup checks ensure all resources are accessible + * - Telemetry: Usage tracking for adoption metrics + */ + +const fs = require('fs'); +const path = require('path'); +const { Server } = require('@modelcontextprotocol/sdk/server/index.js'); +const { StdioServerTransport } = require('@modelcontextprotocol/sdk/server/stdio.js'); +const { + CallToolRequestSchema, + ListToolsRequestSchema, + ListPromptsRequestSchema, + GetPromptRequestSchema, + ListResourcesRequestSchema, + ReadResourceRequestSchema, +} = require('@modelcontextprotocol/sdk/types.js'); +const { findRepoRoot, executeTool } = require('./mcp-tools'); +const { initializeJobQueue, getJob, listJobs } = require('./mcp-tools/job-queue'); + +// New modules for improved architecture +const { + PromptCache, + loadAllPrompts, + watchPrompts, + buildPromptWithArguments, + promptsToMcpFormat +} = require('./mcp-tools/prompt-discovery'); +const { + validatePromptArguments, + validateMcpConfiguration +} = require('./mcp-tools/mcp-validation'); +const { + UsageStats, + createPeriodicReporter, + createShutdownHandler +} = require('./mcp-tools/telemetry'); + +// Get version from package.json +const packageJson = require('../package.json'); + +// Base directory +const baseDir = path.join(__dirname, '..'); + +// Initialize prompt cache and usage stats +const promptCache = new PromptCache(); +const usageStats = new UsageStats(); + +// Create the MCP server +const server = new Server( + { + name: 'redpanda-doc-tools-assistant', + version: packageJson.version, + }, + { + capabilities: { + tools: {}, + prompts: {}, + resources: {}, + }, + } +); + +// Tool definitions - Writer-friendly documentation tools +const tools = [ + { + name: 'get_antora_structure', + description: 'Get information about the Antora documentation structure in the current repository, including components, modules, and available directories. Use this to understand the docs organization.', + inputSchema: { + type: 'object', + properties: {}, + required: [] + } + }, + { + name: 'get_redpanda_version', + description: 'Get the latest Redpanda version information including version number, Docker tag, and release notes URL. Writers use this to find out what version to document.', + inputSchema: { + type: 'object', + properties: { + beta: { + type: 'boolean', + description: 'Whether to get the latest beta/RC version instead of stable (optional, defaults to false)' + } + }, + required: [] + } + }, + { + name: 'get_console_version', + description: 'Get the latest Redpanda Console version information including version number, Docker tag, and release notes URL.', + inputSchema: { + type: 'object', + properties: {}, + required: [] + } + }, + { + name: 'generate_property_docs', + description: 'Generate Redpanda configuration property documentation for a specific version. Creates JSON and optionally AsciiDoc partials with all configuration properties. Writers use this when updating docs for a new Redpanda release. Can run in background with progress streaming.', + inputSchema: { + type: 'object', + properties: { + tag: { + type: 'string', + description: 'Git tag for released content (for example "25.3.1", "v25.3.1", or "latest"). Auto-prepends "v" if not present. Use tags for GA or beta releases.' + }, + branch: { + type: 'string', + description: 'Branch name for in-progress content (for example "dev", "main"). Use branches for documentation under development.' + }, + generate_partials: { + type: 'boolean', + description: 'Whether to generate AsciiDoc partials (cluster-properties.adoc, topic-properties.adoc, etc.). Default: false (only generates JSON)' + }, + background: { + type: 'boolean', + description: 'Run as background job with progress updates. Returns job ID immediately instead of waiting. Default: false (run synchronously)' + } + }, + required: [] + } + }, + { + name: 'generate_metrics_docs', + description: 'Generate Redpanda metrics documentation for a specific version. Creates the public metrics reference page. Writers use this when updating metrics docs for a new release. Can run in background with progress streaming.', + inputSchema: { + type: 'object', + properties: { + tag: { + type: 'string', + description: 'Git tag for released content (for example "25.3.1" or "v25.3.1"). Auto-prepends "v" if not present. Use tags for GA or beta releases.' + }, + branch: { + type: 'string', + description: 'Branch name for in-progress content (for example "dev", "main"). Use branches for documentation under development.' + }, + background: { + type: 'boolean', + description: 'Run as background job with progress updates. Returns job ID immediately instead of waiting. Default: false (run synchronously)' + } + }, + required: [] + } + }, + { + name: 'generate_rpk_docs', + description: 'Generate RPK command-line documentation for a specific version. Creates AsciiDoc files for all rpk commands. Writers use this when updating CLI docs for a new release. Can run in background with progress streaming.', + inputSchema: { + type: 'object', + properties: { + tag: { + type: 'string', + description: 'Git tag for released content (for example "25.3.1" or "v25.3.1"). Auto-prepends "v" if not present. Use tags for GA or beta releases.' + }, + branch: { + type: 'string', + description: 'Branch name for in-progress content (for example "dev", "main"). Use branches for documentation under development.' + }, + background: { + type: 'boolean', + description: 'Run as background job with progress updates. Returns job ID immediately instead of waiting. Default: false (run synchronously)' + } + }, + required: [] + } + }, + { + name: 'generate_rpcn_connector_docs', + description: 'Generate Redpanda Connect connector documentation. Creates component documentation for all connectors. Writers use this when updating connector reference docs.', + inputSchema: { + type: 'object', + properties: { + fetch_connectors: { + type: 'boolean', + description: 'Fetch latest connector data using rpk (optional, defaults to false)' + }, + draft_missing: { + type: 'boolean', + description: 'Generate full-doc drafts for connectors missing in output (optional, defaults to false)' + }, + update_whats_new: { + type: 'boolean', + description: 'Update whats-new.adoc with new section from diff JSON (optional, defaults to false)' + }, + include_bloblang: { + type: 'boolean', + description: 'Include Bloblang functions and methods in generation (optional, defaults to false)' + }, + data_dir: { + type: 'string', + description: 'Directory where versioned connect JSON files live (optional)' + }, + old_data: { + type: 'string', + description: 'Optional override for old data file (for diff)' + }, + csv: { + type: 'string', + description: 'Path to connector metadata CSV file (optional)' + }, + overrides: { + type: 'string', + description: 'Path to optional JSON file with overrides' + } + }, + required: [] + } + }, + { + name: 'generate_helm_docs', + description: 'Generate Helm chart documentation. Creates AsciiDoc documentation for Helm charts from local directories or GitHub repositories. Writers use this when updating Helm chart reference docs.', + inputSchema: { + type: 'object', + properties: { + chart_dir: { + type: 'string', + description: 'Chart directory (contains Chart.yaml) or root containing multiple charts, or a GitHub URL (optional, defaults to Redpanda operator charts)' + }, + tag: { + type: 'string', + description: 'Git tag for released content when using GitHub URL (for example "25.1.2" or "v25.1.2"). Auto-prepends "v" if not present. For redpanda-operator repository, also auto-prepends "operator/".' + }, + branch: { + type: 'string', + description: 'Branch name for in-progress content when using GitHub URL (for example "dev", "main").' + }, + readme: { + type: 'string', + description: 'Relative README.md path inside each chart dir (optional, defaults to "README.md")' + }, + output_dir: { + type: 'string', + description: 'Where to write all generated AsciiDoc files (optional, defaults to "modules/reference/pages")' + }, + output_suffix: { + type: 'string', + description: 'Suffix to append to each chart name including extension (optional, defaults to "-helm-spec.adoc")' + } + }, + required: [] + } + }, + { + name: 'generate_cloud_regions', + description: 'Generate cloud regions table documentation. Creates a Markdown or AsciiDoc table of cloud regions and tiers from GitHub YAML data. Writers use this when updating cloud region documentation.', + inputSchema: { + type: 'object', + properties: { + output: { + type: 'string', + description: 'Output file path relative to repo root (optional, defaults to "cloud-controlplane/x-topics/cloud-regions.md")' + }, + format: { + type: 'string', + description: 'Output format: "md" (Markdown) or "adoc" (AsciiDoc) (optional, defaults to "md")', + enum: ['md', 'adoc'] + }, + owner: { + type: 'string', + description: 'GitHub repository owner (optional, defaults to "redpanda-data")' + }, + repo: { + type: 'string', + description: 'GitHub repository name (optional, defaults to "cloudv2-infra")' + }, + path: { + type: 'string', + description: 'Path to YAML file in repository (optional)' + }, + ref: { + type: 'string', + description: 'Git reference - branch, tag, or commit SHA (optional, defaults to "integration")' + }, + template: { + type: 'string', + description: 'Path to custom Handlebars template relative to repo root (optional)' + }, + dry_run: { + type: 'boolean', + description: 'Print output to stdout instead of writing file (optional, defaults to false)' + } + }, + required: [] + } + }, + { + name: 'generate_crd_docs', + description: 'Generate Kubernetes CRD (Custom Resource Definition) documentation. Creates AsciiDoc documentation for Kubernetes operator CRDs. Writers use this when updating operator reference docs.', + inputSchema: { + type: 'object', + properties: { + tag: { + type: 'string', + description: 'Operator release tag (for example "operator/v25.1.2", "25.1.2", or "v25.1.2"). Auto-prepends "operator/" for redpanda-operator repository.' + }, + branch: { + type: 'string', + description: 'Branch name for in-progress content (for example "dev", "main").' + }, + source_path: { + type: 'string', + description: 'CRD Go types directory or GitHub URL (optional, defaults to Redpanda operator repo)' + }, + depth: { + type: 'number', + description: 'How many levels deep to generate (optional, defaults to 10)' + }, + templates_dir: { + type: 'string', + description: 'Asciidoctor templates directory (optional)' + }, + output: { + type: 'string', + description: 'Where to write the generated AsciiDoc file (optional, defaults to "modules/reference/pages/k-crd.adoc")' + } + }, + required: [] + } + }, + { + name: 'generate_bundle_openapi', + description: 'Bundle Redpanda OpenAPI fragments. Creates complete OpenAPI 3.1 documents for admin and/or connect APIs by bundling fragments from the Redpanda repository. Writers use this when updating API reference docs.', + inputSchema: { + type: 'object', + properties: { + tag: { + type: 'string', + description: 'Git tag for released content (for example "v24.3.2" or "24.3.2"). Use tags for GA or beta releases.' + }, + branch: { + type: 'string', + description: 'Branch name for in-progress content (for example "dev", "main"). Use branches for documentation under development.' + }, + repo: { + type: 'string', + description: 'Repository URL (optional, defaults to Redpanda repo)' + }, + surface: { + type: 'string', + description: 'Which API surface(s) to bundle (optional, defaults to "both")', + enum: ['admin', 'connect', 'both'] + }, + out_admin: { + type: 'string', + description: 'Output path for admin API (optional, defaults to "admin/redpanda-admin-api.yaml")' + }, + out_connect: { + type: 'string', + description: 'Output path for connect API (optional, defaults to "connect/redpanda-connect-api.yaml")' + }, + admin_major: { + type: 'string', + description: 'Admin API major version (optional, defaults to "v2.0.0")' + }, + use_admin_major_version: { + type: 'boolean', + description: 'Use admin major version for info.version instead of git tag (optional, defaults to false)' + }, + quiet: { + type: 'boolean', + description: 'Suppress logs (optional, defaults to false)' + } + }, + required: [] + } + }, + { + name: 'review_generated_docs', + description: 'Review recently generated documentation for quality issues. Checks for missing descriptions, invalid formatting, DRY violations, and other quality problems. Uses the quality criteria from the property-docs-guide and rpcn-connector-docs-guide prompts. Can generate a formatted markdown report for easy review.', + inputSchema: { + type: 'object', + properties: { + doc_type: { + type: 'string', + description: 'Type of documentation to review', + enum: ['properties', 'metrics', 'rpk', 'rpcn_connectors'] + }, + version: { + type: 'string', + description: 'Version of the docs to review (required for properties, metrics, rpk; for example "25.3.1" or "v25.3.1")' + }, + generate_report: { + type: 'boolean', + description: 'Generate a formatted markdown report file for easy review (default: true)' + } + }, + required: ['doc_type'] + } + }, + { + name: 'run_doc_tools_command', + description: 'Advanced: Run a raw doc-tools command. Only use this if none of the specific tools above fit your needs. Requires knowledge of doc-tools CLI syntax.', + inputSchema: { + type: 'object', + properties: { + command: { + type: 'string', + description: 'The doc-tools command to run (without "npx doc-tools" prefix)' + } + }, + required: ['command'] + } + }, + { + name: 'get_job_status', + description: 'Get the status and progress of a background job. Use this to check on long-running documentation generation tasks.', + inputSchema: { + type: 'object', + properties: { + job_id: { + type: 'string', + description: 'Job ID returned when the job was created' + } + }, + required: ['job_id'] + } + }, + { + name: 'list_jobs', + description: 'List all background jobs with optional filtering. Use this to see recent documentation generation jobs and their status.', + inputSchema: { + type: 'object', + properties: { + status: { + type: 'string', + description: 'Filter by job status (optional)', + enum: ['pending', 'running', 'completed', 'failed'] + }, + tool: { + type: 'string', + description: 'Filter by tool name (optional)' + } + }, + required: [] + } + } +]; + +// Resource definitions - Team standards and guidelines +const resources = [ + { + uri: 'redpanda://style-guide', + name: 'Redpanda Documentation Style Guide', + description: 'Complete style guide based on Google Developer Documentation Style Guide with Redpanda-specific guidelines, voice/tone standards, and formatting preferences. Includes references to official glossary sources.', + mimeType: 'text/markdown', + version: '1.0.0', + lastUpdated: '2025-12-11' + } +]; + +// Resource file mappings +const resourceMap = { + 'redpanda://style-guide': { file: 'style-guide.md', mimeType: 'text/markdown' } +}; + +/** + * Load resource content from team-standards directory + * @param {string} uri - Resource URI (such as 'redpanda://style-guide') + * @returns {Object} Resource content + */ +function getResourceContent(uri) { + const resource = resourceMap[uri]; + if (!resource) { + throw new Error(`Unknown resource: ${uri}`); + } + + const resourcePath = path.join(baseDir, 'mcp', 'team-standards', resource.file); + try { + const content = fs.readFileSync(resourcePath, 'utf8'); + return { + contents: [{ + uri, + mimeType: resource.mimeType, + text: content + }] + }; + } catch (err) { + console.error(`Error loading resource ${uri}: ${err.message}`); + throw new Error(`Resource not found: ${uri}`); + } +} + +// Handle list tools request +server.setRequestHandler(ListToolsRequestSchema, async () => { + return { tools }; +}); + +// Handle list prompts request +server.setRequestHandler(ListPromptsRequestSchema, async () => { + const allPrompts = promptCache.getAll(); + return { prompts: promptsToMcpFormat(allPrompts) }; +}); + +// Handle list resources request +server.setRequestHandler(ListResourcesRequestSchema, async () => { + return { resources }; +}); + +// Handle read resource request +server.setRequestHandler(ReadResourceRequestSchema, async (request) => { + const { uri } = request.params; + + // Record usage + usageStats.recordResource(uri); + + return getResourceContent(uri); +}); + +// Handle get prompt request +server.setRequestHandler(GetPromptRequestSchema, async (request) => { + const { name, arguments: args } = request.params; + + // Record usage + usageStats.recordPrompt(name); + + // Get prompt from cache + const prompt = promptCache.get(name); + if (!prompt) { + throw new Error(`Unknown prompt: ${name}`); + } + + // Validate arguments if schema exists + if (prompt.arguments && prompt.arguments.length > 0) { + try { + validatePromptArguments(name, args, prompt.arguments); + } catch (err) { + return { + messages: [{ + role: 'user', + content: { + type: 'text', + text: `Error: ${err.message}` + } + }] + }; + } + } + + // Build prompt with arguments + const promptText = buildPromptWithArguments(prompt, args); + + return { + messages: [{ + role: 'user', + content: { + type: 'text', + text: promptText + } + }] + }; +}); + +// Handle tool execution +server.setRequestHandler(CallToolRequestSchema, async (request) => { + const { name, arguments: args } = request.params; + + // Record usage + usageStats.recordTool(name); + + // Handle job management tools + if (name === 'get_job_status') { + const job = getJob(args.job_id); + if (!job) { + return { + content: [ + { + type: 'text', + text: JSON.stringify({ + success: false, + error: `Job not found: ${args.job_id}`, + suggestion: 'Check the job ID or use list_jobs to see available jobs' + }, null, 2) + } + ] + }; + } + + return { + content: [ + { + type: 'text', + text: JSON.stringify({ + success: true, + job + }, null, 2) + } + ] + }; + } + + if (name === 'list_jobs') { + const jobs = listJobs(args || {}); + return { + content: [ + { + type: 'text', + text: JSON.stringify({ + success: true, + jobs, + total: jobs.length + }, null, 2) + } + ] + }; + } + + // Handle regular tools + const result = executeTool(name, args || {}); + + return { + content: [ + { + type: 'text', + text: JSON.stringify(result, null, 2) + } + ] + }; +}); + +// Start the server +async function main() { + // Load and validate prompts + console.error('Loading prompts...'); + const prompts = loadAllPrompts(baseDir, promptCache); + console.error(`Loaded ${prompts.length} prompts`); + + // Validate configuration at startup + console.error('Validating MCP configuration...'); + const validation = validateMcpConfiguration({ + resources, + resourceMap, + prompts, + baseDir + }); + + if (!validation.valid) { + console.error('\nMCP Configuration validation FAILED:'); + validation.errors.forEach(err => console.error(` ${err}`)); + console.error('\nServer cannot start with invalid configuration.'); + process.exit(1); + } + + if (validation.warnings.length > 0) { + console.error('\nWarnings:'); + validation.warnings.forEach(warn => console.error(` ⚠ ${warn}`)); + } + + console.error('Configuration valid'); + + // Enable file watching in dev mode + if (process.env.MCP_DEV_MODE === 'true') { + watchPrompts(baseDir, promptCache, (reloadedPrompts) => { + console.error(`Prompts reloaded: ${reloadedPrompts.length} available`); + }); + } + + // Initialize usage tracking + createShutdownHandler(usageStats, baseDir); + + // Periodic reporting (every hour) + if (process.env.MCP_TELEMETRY_REPORTING === 'true') { + createPeriodicReporter(usageStats, 3600000); + } + + // Connect MCP server + const transport = new StdioServerTransport(); + await server.connect(transport); + + // Initialize job queue with server instance for progress notifications + initializeJobQueue(server); + + // Log to stderr so it doesn't interfere with MCP protocol on stdout + const repoInfo = findRepoRoot(); + console.error('Redpanda Doc Tools MCP Server running'); + console.error(`Server version: ${packageJson.version}`); + console.error(`Working directory: ${process.cwd()}`); + console.error(`Repository root: ${repoInfo.root} (${repoInfo.detected ? repoInfo.type : 'not detected'})`); + console.error('Background job queue: enabled'); + console.error('Command timeout: 10 minutes'); + console.error('Auto-discovery: enabled'); + console.error('Startup validation: enabled'); + console.error('Usage telemetry: enabled'); +} + +main().catch((error) => { + console.error('Server error:', error); + process.exit(1); +}); diff --git a/bin/doc-tools.js b/bin/doc-tools.js index f883355f..39e7a1ec 100755 --- a/bin/doc-tools.js +++ b/bin/doc-tools.js @@ -57,7 +57,7 @@ function fail(msg) { * * Attempts to execute the tool with a version flag to verify its presence. If the tool is missing or fails to run, the process exits with an error message and optional installation hint. * - * @param {string} cmd - The name of the tool to check (e.g., 'docker', 'helm-docs'). + * @param {string} cmd - The name of the tool to check (for example, 'docker', 'helm-docs'). * @param {object} [opts] - Optional settings. * @param {string} [opts.versionFlag='--version'] - The flag used to test the tool's execution. * @param {string} [opts.help] - An optional hint or installation instruction shown on failure. @@ -240,7 +240,7 @@ function verifyPropertyDependencies() { requirePython(); // Check for Node.js (required for Handlebars templates) - requireCmd('node', 'https://nodejs.org/en/download/ or use your package manager (e.g., brew install node)'); + requireCmd('node', 'https://nodejs.org/en/download/ or use your package manager (for example, brew install node)'); requireCmd('npm', 'Usually installed with Node.js'); // Check for C++ compiler @@ -342,6 +342,32 @@ programCli .version(pkg.version); // Top-level commands. + +/** + * install-test-dependencies + * + * @description + * Installs all packages and dependencies required for documentation testing workflows. + * This includes Redpanda Docker images, Python virtual environments for property extraction, + * and other test dependencies. + * + * @why + * Setting up a documentation environment requires multiple dependencies across different + * package managers (npm, pip, Docker). This command automates the entire setup process. + * + * @example + * # Set up a new documentation environment + * npx doc-tools install-test-dependencies + * + * # Use in CI/CD before running doc tests + * - run: npx doc-tools install-test-dependencies + * - run: npm test + * + * @requirements + * - Node.js and npm + * - Python 3.9 or higher + * - Docker (for some dependencies) + */ programCli .command('install-test-dependencies') .description('Install packages for doc test workflows') @@ -351,6 +377,41 @@ programCli process.exit(result.status); }); +/** + * get-redpanda-version + * + * @description + * Fetches the latest Redpanda version from GitHub releases. Can retrieve either stable + * releases or beta/RC versions. Returns the version in format "v25.3.1" which can be + * used directly with other doc-tools commands. + * + * @why + * Documentation must reference the correct current version. This command ensures version + * numbers are accurate and can be used in CI/CD pipelines or before generating + * version-specific documentation. The version is fetched from GitHub releases, which is + * the source of truth for Redpanda releases. + * + * @example + * # Get latest stable version + * npx doc-tools get-redpanda-version + * # Output: v25.3.1 + * + * # Get latest beta/RC version + * npx doc-tools get-redpanda-version --beta + * # Output: v26.1.1-rc1 + * + * # Auto-detect from antora.yml prerelease flag + * cd docs-site + * npx doc-tools get-redpanda-version --from-antora + * + * # Use in CI/CD or scripts + * VERSION=$(npx doc-tools get-redpanda-version) + * npx doc-tools generate property-docs --tag $VERSION + * + * @requirements + * - Internet connection to access GitHub API + * - GitHub API rate limits apply (60 requests/hour unauthenticated) + */ programCli .command('get-redpanda-version') .description('Print the latest Redpanda version') @@ -365,6 +426,40 @@ programCli } }); +/** + * get-console-version + * + * @description + * Fetches the latest Redpanda Console version from GitHub releases. Can retrieve either + * stable releases or beta versions. Returns the version in format "v2.7.2" which can be + * used for documentation references and Docker image tags. + * + * @why + * Console is released separately from Redpanda core. This command keeps Console + * documentation in sync with releases and provides the correct version for Docker + * Compose files and deployment documentation. + * + * @example + * # Get latest stable Console version + * npx doc-tools get-console-version + * # Output: v2.7.2 + * + * # Get latest beta version + * npx doc-tools get-console-version --beta + * # Output: v2.8.0-beta1 + * + * # Auto-detect from antora.yml prerelease flag + * cd docs-site + * npx doc-tools get-console-version --from-antora + * + * # Use in Docker Compose documentation + * CONSOLE_VERSION=$(npx doc-tools get-console-version) + * echo "image: redpandadata/console:$CONSOLE_VERSION" + * + * @requirements + * - Internet connection to access GitHub API + * - GitHub API rate limits apply (60 requests/hour unauthenticated) + */ programCli .command('get-console-version') .description('Print the latest Console version') @@ -379,6 +474,40 @@ programCli } }); +/** + * link-readme + * + * @description + * Creates a symbolic link from a project's README.adoc file into the Antora documentation + * structure. This allows project README files to be included in the documentation site + * without duplication. The command creates the necessary directory structure and establishes + * a symlink in docs/modules//pages/ that points to the project's README.adoc. + * + * @why + * Documentation repositories often contain multiple sub-projects (like labs or examples) + * that have their own README files. Rather than manually copying these files into the + * Antora structure (which creates maintenance burden), symlinks keep the content in one + * place while making it available to Antora. Changes to the project README automatically + * appear in the docs site. + * + * @example + * # Link a lab project README into documentation + * npx doc-tools link-readme \\ + * --subdir labs/docker-compose \\ + * --target docker-compose-lab.adoc + * + * # Link multiple lab READMEs + * npx doc-tools link-readme -s labs/kubernetes -t k8s-lab.adoc + * npx doc-tools link-readme -s labs/terraform -t terraform-lab.adoc + * + * # The symlink structure created: + * # docs/modules/labs/pages/docker-compose-lab.adoc -> ../../../../labs/docker-compose/README.adoc + * + * @requirements + * - Must run from repository root + * - Target project must have README.adoc file + * - Operating system must support symbolic links + */ programCli .command('link-readme') .description('Symlink a README.adoc into docs/modules//pages/') @@ -419,6 +548,49 @@ programCli } }); +/** + * fetch + * + * @description + * Downloads specific files or directories from GitHub repositories without cloning the entire + * repository. Uses the GitHub API to fetch content and saves it to a local directory. Useful + * for grabbing examples, configuration files, or documentation snippets from other repositories. + * Supports both individual files and entire directories. + * + * @why + * Documentation often needs to reference or include files from other repositories (examples, + * configuration templates, code samples). Cloning entire repositories is inefficient when you + * only need specific files. This command provides targeted fetching, saving bandwidth and time. + * It's particularly useful in CI/CD pipelines where you need specific assets without full clones. + * + * @example + * # Fetch a specific configuration file + * npx doc-tools fetch \\ + * --owner redpanda-data \\ + * --repo redpanda \\ + * --remote-path docker/docker-compose.yml \\ + * --save-dir examples/ + * + * # Fetch an entire directory of examples + * npx doc-tools fetch \\ + * -o redpanda-data \\ + * -r connect-examples \\ + * -p pipelines/mongodb \\ + * -d docs/modules/examples/attachments/ + * + * # Fetch with custom filename + * npx doc-tools fetch \\ + * -o redpanda-data \\ + * -r helm-charts \\ + * -p charts/redpanda/values.yaml \\ + * -d examples/ \\ + * --filename redpanda-values-example.yaml + * + * @requirements + * - Internet connection to access GitHub API + * - GitHub API rate limits apply (60 requests/hour unauthenticated, 5000 with token) + * - For private repositories: GitHub token with repo permissions + */ programCli .command('fetch') .description('Fetch a file or directory from GitHub and save it locally') @@ -443,6 +615,83 @@ programCli } }); +/** + * setup-mcp + * + * @description + * Configures the Redpanda Docs MCP (Model Context Protocol) server for Claude Code or + * Claude Desktop. Automatically detects the installed application, updates the appropriate + * configuration file, and enables Claude to use doc-tools commands through natural conversation. + * Supports both production (npm package) and local development modes. + * + * @why + * Manual MCP configuration requires editing JSON configuration files in the correct location + * with the correct schema. This command handles all setup automatically, including path + * detection, configuration merging, and validation. It enables AI-assisted documentation + * workflows where writers can use natural language to run doc-tools commands. + * + * @example + * # Auto-detect and configure for Claude Code or Desktop + * npx doc-tools setup-mcp + * + * # Configure for local development (run from this repository) + * cd /path/to/docs-extensions-and-macros + * npx doc-tools setup-mcp --local + * + * # Force update existing configuration + * npx doc-tools setup-mcp --force + * + * # Target specific application + * npx doc-tools setup-mcp --target code + * npx doc-tools setup-mcp --target desktop + * + * # Check current configuration status + * npx doc-tools setup-mcp --status + * + * # After setup, restart Claude Code and use natural language + * "What's the latest Redpanda version?" + * "Generate property docs for v25.3.1" + * + * @requirements + * - Claude Code or Claude Desktop must be installed + * - For --local mode: must run from docs-extensions-and-macros repository + * - After setup: restart Claude Code/Desktop to load the MCP server + */ +programCli + .command('setup-mcp') + .description('Configure the Redpanda Docs MCP server for Claude Code/Desktop') + .option('--force', 'Force update even if already configured', false) + .option('--target ', 'Target application: auto, code, or desktop', 'auto') + .option('--local', 'Use local development mode (requires running from this repo)', false) + .option('--status', 'Show current MCP server configuration status', false) + .action(async (options) => { + try { + const { setupMCP, showStatus, printNextSteps } = require('../cli-utils/setup-mcp.js'); + + if (options.status) { + showStatus(); + return; + } + + const result = await setupMCP({ + force: options.force, + target: options.target, + local: options.local + }); + + if (result.success) { + printNextSteps(result); + process.exit(0); + } else { + console.error(`❌ Setup failed: ${result.error}`); + process.exit(1); + } + } catch (err) { + console.error(`❌ ${err.message}`); + process.exit(1); + } + }); + // Create an "automation" subcommand group. const automation = new Command('generate').description('Run docs automations'); @@ -465,7 +714,7 @@ const commonOptions = { * process; if the script exits with a non-zero status, this function will terminate * the Node.js process with that status code. * - * @param {string} mode - Operation mode passed to the script (e.g., "generate" or "clean"). + * @param {string} mode - Operation mode passed to the script (for example, "generate" or "clean"). * @param {string} tag - Release tag or version to generate docs for. * @param {Object} options - Runtime options. * @param {string} options.dockerRepo - Docker repository used by the script. @@ -592,7 +841,7 @@ function generatePropertyComparisonReport(oldTag, newTag, outputDir) { * provided temporary directories. On missing inputs or if the diff subprocess * fails to spawn, the process exits with a non-zero status. * - * @param {string} kind - Logical category for the diff (e.g., "metrics" or "rpk"); used in the output path. + * @param {string} kind - Logical category for the diff (for example, "metrics" or "rpk"); used in the output path. * @param {string} oldTag - Identifier for the "old" version (used in the output path). * @param {string} newTag - Identifier for the "new" version (used in the output path). * @param {string} oldTempDir - Path to the existing temporary directory containing the old output; must exist. @@ -662,10 +911,52 @@ function diffDirs(kind, oldTag, newTag, oldTempDir, newTempDir) { } } +/** + * generate metrics-docs + * + * @description + * Generates comprehensive metrics reference documentation by running Redpanda in Docker and + * scraping the `/public_metrics` Prometheus endpoint. Starts a Redpanda cluster with the + * specified version, waits for it to be ready, collects all exposed metrics, parses the + * Prometheus format, and generates categorized AsciiDoc documentation. Optionally compares + * metrics between versions to identify new, removed, or changed metrics. + * + * @why + * Redpanda exposes hundreds of metrics for monitoring and observability. Manual documentation + * of metrics is error-prone and becomes outdated as new metrics are added or existing ones + * change. This automation ensures metrics documentation accurately reflects what Redpanda + * actually exports at each version. Running Redpanda in Docker and scraping metrics directly + * is the only reliable way to capture the complete and accurate metrics set. + * + * @example + * # Basic: Generate metrics docs for a specific version + * npx doc-tools generate metrics-docs --tag v25.3.1 + * + * # Compare metrics between versions to see what changed + * npx doc-tools generate metrics-docs \\ + * --tag v25.3.1 \\ + * --diff v25.2.1 + * + * # Use custom Docker repository + * npx doc-tools generate metrics-docs \\ + * --tag v25.3.1 \\ + * --docker-repo docker.redpanda.com/redpandadata/redpanda + * + * # Full workflow: document new release + * VERSION=$(npx doc-tools get-redpanda-version) + * npx doc-tools generate metrics-docs --tag $VERSION + * + * @requirements + * - Docker must be installed and running + * - Port 9644 must be available (Redpanda metrics endpoint) + * - Sufficient disk space for Docker image + * - Internet connection to pull Docker images + */ automation .command('metrics-docs') - .description('Generate JSON and AsciiDoc documentation for Redpanda metrics') - .requiredOption('-t, --tag ', 'Redpanda version to use when starting Redpanda in Docker') + .description('Generate JSON and AsciiDoc documentation for Redpanda metrics. Defaults to branch "dev" if neither --tag nor --branch is specified.') + .option('-t, --tag ', 'Git tag for released content (GA/beta)') + .option('-b, --branch ', 'Branch name for in-progress content') .option( '--docker-repo ', 'Docker repository to use when starting Redpanda in Docker', @@ -685,7 +976,14 @@ automation .action((options) => { verifyMetricsDependencies(); - const newTag = options.tag; + // Validate that tag and branch are mutually exclusive + if (options.tag && options.branch) { + console.error('❌ Error: Cannot specify both --tag and --branch'); + process.exit(1); + } + + // Default to 'dev' branch if neither specified + const newTag = options.tag || options.branch || 'dev'; const oldTag = options.diff; if (oldTag) { @@ -706,6 +1004,48 @@ automation process.exit(0); }); +/** + * generate rpcn-connector-docs + * + * @description + * Generates complete reference documentation for Redpanda Connect (formerly Benthos) connectors, + * processors, and components. Clones the Redpanda Connect repository, parses component templates + * and configuration schemas embedded in Go code, reads connector metadata from CSV, and generates + * AsciiDoc documentation for each component. Supports diffing changes between versions and + * automatically updating what's new documentation. Can also generate Bloblang function documentation. + * + * @why + * Redpanda Connect has hundreds of connectors (inputs, outputs, processors) with complex + * configuration schemas. Each component's documentation lives in its Go source code as struct + * tags and comments. Manual documentation is impossible to maintain. This automation extracts + * documentation directly from code, ensuring accuracy and completeness. The diff capability + * automatically identifies new connectors and changed configurations for release notes. + * + * @example + * # Basic: Generate all connector docs + * npx doc-tools generate rpcn-connector-docs + * + * # Generate docs and automatically update what's new page + * npx doc-tools generate rpcn-connector-docs --update-whats-new + * + * # Include Bloblang function documentation + * npx doc-tools generate rpcn-connector-docs --include-bloblang + * + * # Generate with custom metadata CSV + * npx doc-tools generate rpcn-connector-docs \\ + * --csv custom/connector-metadata.csv + * + * # Full workflow with diff and what's new update + * npx doc-tools generate rpcn-connector-docs \\ + * --update-whats-new \\ + * --include-bloblang + * + * @requirements + * - Git to clone Redpanda Connect repository + * - Internet connection to clone repository + * - Node.js for parsing and generation + * - Sufficient disk space for repository clone (~500MB) + */ automation .command('rpcn-connector-docs') .description('Generate RPCN connector docs and diff changes since the last version') @@ -720,7 +1060,7 @@ automation .option('--template-fields ', 'Fields section partial template', path.resolve(__dirname, '../tools/redpanda-connect/templates/fields-partials.hbs')) .option('--template-examples ', 'Examples section partial template', path.resolve(__dirname, '../tools/redpanda-connect/templates/examples-partials.hbs')) .option('--template-bloblang ', 'Custom Handlebars template for bloblang function/method partials') - .option('--overrides ', 'Optional JSON file with overrides') + .option('--overrides ', 'Optional JSON file with overrides', 'docs-data/overrides.json') .option('--include-bloblang', 'Include Bloblang functions and methods in generation') .action(async (options) => { requireTool('rpk', { @@ -1477,16 +1817,72 @@ automation process.exit(0); }); +/** + * generate property-docs + * + * @description + * Generates comprehensive reference documentation for Redpanda cluster and topic configuration + * properties. Clones the Redpanda repository at a specified version, runs a Python extractor + * to parse C++ configuration code, and outputs JSON data files with all property metadata + * (descriptions, types, defaults, constraints). Optionally generates consolidated AsciiDoc + * partials for direct inclusion in documentation sites. + * + * @why + * Property definitions in the C++ source code are the single source of truth for Redpanda + * configuration. Manual documentation becomes outdated quickly. This automation ensures docs + * stay perfectly in sync with implementation by extracting properties directly from code, + * including type information, default values, and constraints that would be error-prone to + * maintain manually. + * + * @example + * # Basic: Extract properties to JSON only (default) + * npx doc-tools generate property-docs --tag v25.3.1 + * + * # Generate AsciiDoc partials for documentation site + * npx doc-tools generate property-docs --tag v25.3.1 --generate-partials + * + * # Include Cloud support tags (requires GitHub token) + * export GITHUB_TOKEN=ghp_xxx + * npx doc-tools generate property-docs \\ + * --tag v25.3.1 \\ + * --generate-partials \\ + * --cloud-support + * + * # Compare properties between versions + * npx doc-tools generate property-docs \\ + * --tag v25.3.1 \\ + * --diff v25.2.1 + * + * # Use custom output directory + * npx doc-tools generate property-docs \\ + * --tag v25.3.1 \\ + * --output-dir docs/modules/reference + * + * # Full workflow: document new release + * VERSION=$(npx doc-tools get-redpanda-version) + * npx doc-tools generate property-docs \\ + * --tag $VERSION \\ + * --generate-partials \\ + * --cloud-support + * + * @requirements + * - Python 3.9 or higher + * - Git + * - Internet connection to clone Redpanda repository + * - For --cloud-support: GitHub token with repo permissions (GITHUB_TOKEN env var) + * - For --cloud-support: Python packages pyyaml and requests + */ automation .command('property-docs') .description( 'Generate JSON and consolidated AsciiDoc partials for Redpanda configuration properties. ' + 'By default, only extracts properties to JSON. Use --generate-partials to create consolidated ' + - 'AsciiDoc partials (including deprecated properties).' + 'AsciiDoc partials (including deprecated properties). Defaults to branch "dev" if neither --tag nor --branch is specified.' ) - .option('--tag ', 'Git tag or branch to extract from', 'dev') - .option('--diff ', 'Also diff autogenerated properties from to ') - .option('--overrides ', 'Optional JSON file with property description overrides') + .option('-t, --tag ', 'Git tag for released content (GA/beta)') + .option('-b, --branch ', 'Branch name for in-progress content') + .option('--diff ', 'Also diff autogenerated properties from to current tag/branch') + .option('--overrides ', 'Optional JSON file with property description overrides', 'docs-data/property-overrides.json') .option('--output-dir ', 'Where to write all generated files', 'modules/reference') .option('--cloud-support', 'Add AsciiDoc tags to generated property docs to indicate which ones are supported in Redpanda Cloud. This data is fetched from the cloudv2 repository so requires a GitHub token with repo permissions. Set the token as an environment variable using GITHUB_TOKEN, GH_TOKEN, or REDPANDA_GITHUB_TOKEN') .option('--template-property ', 'Custom Handlebars template for individual property sections') @@ -1499,12 +1895,22 @@ automation .action((options) => { verifyPropertyDependencies(); + // Validate that tag and branch are mutually exclusive + if (options.tag && options.branch) { + console.error('❌ Error: Cannot specify both --tag and --branch'); + process.exit(1); + } + + // Default to 'dev' branch if neither specified + const newTag = options.tag || options.branch || 'dev'; + // Validate cloud support dependencies if requested if (options.cloudSupport) { console.log('πŸ” Validating cloud support dependencies...'); - const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN || process.env.REDPANDA_GITHUB_TOKEN; + const { getGitHubToken } = require('../cli-utils/github-token'); + const token = getGitHubToken(); if (!token) { - console.error('❌ Cloud support requires GITHUB_TOKEN, GH_TOKEN, or REDPANDA_GITHUB_TOKEN environment variable'); + console.error('❌ Cloud support requires a GitHub token'); console.error(' Set up GitHub token:'); console.error(' 1. Go to https://github.com/settings/tokens'); console.error(' 2. Generate token with "repo" scope'); @@ -1520,8 +1926,6 @@ automation console.log(' Required packages: pyyaml, requests'); console.log('βœ… GitHub token validated'); } - - const newTag = options.tag; let oldTag = options.diff; const overridesPath = options.overrides; const outputDir = options.outputDir; @@ -1611,10 +2015,51 @@ automation process.exit(0); }); +/** + * generate rpk-docs + * + * @description + * Generates comprehensive CLI reference documentation for RPK (Redpanda Keeper), the official + * Redpanda command-line tool. Starts Redpanda in Docker (RPK is bundled with Redpanda), executes + * `rpk --help` for all commands and subcommands recursively, parses the help output, and generates + * structured AsciiDoc documentation for each command with usage, flags, and descriptions. + * Optionally compares RPK commands between versions to identify new or changed commands. + * + * @why + * RPK has dozens of commands and subcommands with complex flags and options. The built-in help + * text is the source of truth for RPK's CLI interface. Manual documentation becomes outdated as + * RPK evolves. This automation extracts documentation directly from RPK's help output, ensuring + * accuracy. Running RPK from Docker guarantees the exact version being documented, and diffing + * between versions automatically highlights CLI changes for release notes. + * + * @example + * # Basic: Generate RPK docs for a specific version + * npx doc-tools generate rpk-docs --tag v25.3.1 + * + * # Compare RPK commands between versions + * npx doc-tools generate rpk-docs \\ + * --tag v25.3.1 \\ + * --diff v25.2.1 + * + * # Use custom Docker repository + * npx doc-tools generate rpk-docs \\ + * --tag v25.3.1 \\ + * --docker-repo docker.redpanda.com/redpandadata/redpanda + * + * # Full workflow: document new release + * VERSION=$(npx doc-tools get-redpanda-version) + * npx doc-tools generate rpk-docs --tag $VERSION + * + * @requirements + * - Docker must be installed and running + * - Sufficient disk space for Docker image + * - Internet connection to pull Docker images + */ automation .command('rpk-docs') - .description('Generate AsciiDoc documentation for rpk CLI commands') - .requiredOption('-t, --tag ', 'Redpanda version to use when starting Redpanda in Docker') + .description('Generate AsciiDoc documentation for rpk CLI commands. Defaults to branch "dev" if neither --tag nor --branch is specified.') + .option('-t, --tag ', 'Git tag for released content (GA/beta)') + .option('-b, --branch ', 'Branch name for in-progress content') .option( '--docker-repo ', 'Docker repository to use when starting Redpanda in Docker', @@ -1634,7 +2079,14 @@ automation .action((options) => { verifyMetricsDependencies(); - const newTag = options.tag; + // Validate that tag and branch are mutually exclusive + if (options.tag && options.branch) { + console.error('❌ Error: Cannot specify both --tag and --branch'); + process.exit(1); + } + + // Default to 'dev' branch if neither specified + const newTag = options.tag || options.branch || 'dev'; const oldTag = options.diff; if (oldTag) { @@ -1655,17 +2107,59 @@ automation process.exit(0); }); +/** + * generate helm-spec + * + * @description + * Generates Helm chart reference documentation by parsing values.yaml files and README.md + * documentation from Helm chart repositories. Supports both local chart directories and + * GitHub URLs. Extracts all configuration options with their types, defaults, and descriptions, + * and generates comprehensive AsciiDoc documentation. Can process single charts or entire + * chart repositories with multiple charts. + * + * @why + * Helm charts have complex configuration with hundreds of values. The values.yaml file and + * chart README contain the configuration options, but they're not in a documentation-friendly + * format. This automation parses the YAML structure and README documentation to generate + * comprehensive reference documentation. Supporting both local and GitHub sources allows + * documenting charts from any source without manual cloning. + * + * @example + * # Generate docs from GitHub repository + * npx doc-tools generate helm-spec \\ + * --chart-dir https://github.com/redpanda-data/helm-charts \\ + * --tag v5.9.0 \\ + * --output-dir modules/deploy/pages + * + * # Generate docs from local chart directory + * npx doc-tools generate helm-spec \\ + * --chart-dir ./charts/redpanda \\ + * --output-dir docs/modules/deploy/pages + * + * # Use custom README and output suffix + * npx doc-tools generate helm-spec \\ + * --chart-dir https://github.com/redpanda-data/helm-charts \\ + * --tag v5.9.0 \\ + * --readme docs/README.md \\ + * --output-suffix -values.adoc + * + * @requirements + * - For GitHub URLs: Git and internet connection + * - For local charts: Chart directory must contain Chart.yaml + * - README.md file in chart directory (optional but recommended) + */ automation .command('helm-spec') .description( - `Generate AsciiDoc documentation for one or more Helm charts (supports local dirs or GitHub URLs)` + `Generate AsciiDoc documentation for one or more Helm charts (supports local dirs or GitHub URLs). When using GitHub URLs, requires either --tag or --branch to be specified.` ) .option( '--chart-dir ', 'Chart directory (contains Chart.yaml) or a root containing multiple charts, or a GitHub URL', 'https://github.com/redpanda-data/redpanda-operator/charts' ) - .requiredOption('-t, --tag ', 'Branch or tag to clone when using a GitHub URL for the chart-dir') + .option('-t, --tag ', 'Git tag for released content when using GitHub URL (auto-prepends "operator/" for redpanda-operator repository)') + .option('-b, --branch ', 'Branch name for in-progress content when using GitHub URL') .option('--readme ', 'Relative README.md path inside each chart dir', 'README.md') .option('--output-dir ', 'Where to write all generated AsciiDoc files', 'modules/reference/pages') .option('--output-suffix ', 'Suffix to append to each chart name (including extension)', '-helm-spec.adoc') @@ -1677,10 +2171,24 @@ automation let tmpClone = null; if (/^https?:\/\/github\.com\//.test(root)) { - if (!opts.tag) { - console.error('❌ When using a GitHub URL you must pass --tag'); + // Validate tag/branch for GitHub URLs + if (!opts.tag && !opts.branch) { + console.error('❌ When using a GitHub URL you must pass either --tag or --branch'); + process.exit(1); + } + if (opts.tag && opts.branch) { + console.error('❌ Cannot specify both --tag and --branch'); process.exit(1); } + + let gitRef = opts.tag || opts.branch; + + // Normalize tag: add 'v' prefix if not present for tags + if (opts.tag && !gitRef.startsWith('v')) { + gitRef = `v${gitRef}`; + console.log(`ℹ️ Auto-prepending "v" to tag: ${gitRef}`); + } + const u = new URL(root); const parts = u.pathname.replace(/\.git$/, '').split('/').filter(Boolean); if (parts.length < 2) { @@ -1689,24 +2197,42 @@ automation } const [owner, repo, ...sub] = parts; const repoUrl = `https://${u.host}/${owner}/${repo}.git`; - const ref = opts.tag; - console.log(`⏳ Verifying ${repoUrl}@${ref}…`); + // Auto-prepend "operator/" for tags in redpanda-operator repository + if (opts.tag && owner === 'redpanda-data' && repo === 'redpanda-operator') { + if (!gitRef.startsWith('operator/')) { + gitRef = `operator/${gitRef}`; + console.log(`ℹ️ Auto-prepending "operator/" to tag: ${gitRef}`); + } + } + + console.log(`⏳ Verifying ${repoUrl}@${gitRef}…`); const ok = spawnSync( 'git', - ['ls-remote', '--exit-code', repoUrl, `refs/heads/${ref}`, `refs/tags/${ref}`], + ['ls-remote', '--exit-code', repoUrl, `refs/heads/${gitRef}`, `refs/tags/${gitRef}`], { stdio: 'ignore' } ).status === 0; if (!ok) { - console.error(`❌ ${ref} not found on ${repoUrl}`); + console.error(`❌ ${gitRef} not found on ${repoUrl}`); process.exit(1); } + const { getAuthenticatedGitHubUrl, hasGitHubToken } = require('../cli-utils/github-token'); + tmpClone = fs.mkdtempSync(path.join(os.tmpdir(), 'helm-')); - console.log(`⏳ Cloning ${repoUrl}@${ref} β†’ ${tmpClone}`); + + // Use token if available for GitHub repos + let cloneUrl = repoUrl; + if (hasGitHubToken() && repoUrl.includes('github.com')) { + cloneUrl = getAuthenticatedGitHubUrl(repoUrl); + console.log(`⏳ Cloning ${repoUrl}@${gitRef} β†’ ${tmpClone} (authenticated)`); + } else { + console.log(`⏳ Cloning ${repoUrl}@${gitRef} β†’ ${tmpClone}`); + } + if ( - spawnSync('git', ['clone', '--depth', '1', '--branch', ref, repoUrl, tmpClone], { + spawnSync('git', ['clone', '--depth', '1', '--branch', gitRef, cloneUrl, tmpClone], { stdio: 'inherit', }).status !== 0 ) { @@ -1766,6 +2292,8 @@ automation const xrefRe = /https:\/\/docs\.redpanda\.com[^\s\]\[\)"]+(?:\[[^\]]*\])?/g; doc = doc .replace(/(\[\d+\])\]\./g, '$1\\].') + .replace(/(\[\d+\])\]\]/g, '$1\\]\\]') + .replace(/^=== +(https?:\/\/[^\[]*)\[([^\]]*)\]/gm, '=== link:++$1++[$2]') .replace(/^== # (.*)$/gm, '= $1') .replace(/^== description: (.*)$/gm, ':description: $1') .replace(xrefRe, (match) => { @@ -1799,6 +2327,50 @@ automation /** * Generate Markdown table of cloud regions and tiers from master-data.yaml */ +/** + * generate cloud-regions + * + * @description + * Generates a formatted table of Redpanda Cloud regions, tiers, and availability information + * by fetching data from the private cloudv2-infra repository. Reads a YAML configuration file + * that contains master data for cloud infrastructure, parses region and tier information, and + * generates either Markdown or AsciiDoc tables for documentation. Supports custom templates + * and dry-run mode for previewing output. + * + * @why + * Cloud region data changes frequently as new regions are added and tier availability evolves. + * The cloudv2-infra repository contains the source of truth for cloud infrastructure. Manual + * documentation becomes outdated quickly. This automation fetches the latest data directly from + * the infrastructure repository, ensuring documentation always reflects current cloud offerings. + * Weekly or triggered updates keep docs in sync with cloud expansion. + * + * @example + * # Basic: Generate Markdown table + * export GITHUB_TOKEN=ghp_xxx + * npx doc-tools generate cloud-regions + * + * # Generate AsciiDoc format + * export GITHUB_TOKEN=ghp_xxx + * npx doc-tools generate cloud-regions --format adoc + * + * # Preview without writing file + * export GITHUB_TOKEN=ghp_xxx + * npx doc-tools generate cloud-regions --dry-run + * + * # Use custom output file + * export GITHUB_TOKEN=ghp_xxx + * npx doc-tools generate cloud-regions \\ + * --output custom/path/regions.md + * + * # Use different branch for testing + * export GITHUB_TOKEN=ghp_xxx + * npx doc-tools generate cloud-regions --ref staging + * + * @requirements + * - GitHub token with access to redpanda-data/cloudv2-infra repository + * - Token must be set via GITHUB_TOKEN, GH_TOKEN, or REDPANDA_GITHUB_TOKEN environment variable + * - Internet connection to access GitHub API + */ automation .command('cloud-regions') .description('Generate Markdown table of cloud regions and tiers from GitHub YAML file') @@ -1812,11 +2384,12 @@ automation .option('--dry-run', 'Print output to stdout instead of writing file') .action(async (options) => { const { generateCloudRegions } = require('../tools/cloud-regions/generate-cloud-regions.js'); + const { getGitHubToken } = require('../cli-utils/github-token'); try { - const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN || process.env.REDPANDA_GITHUB_TOKEN; + const token = getGitHubToken(); if (!token) { - throw new Error('GITHUB_TOKEN, GH_TOKEN, or REDPANDA_GITHUB_TOKEN environment variable is required to fetch from private cloudv2-infra repo.'); + throw new Error('GitHub token is required to fetch from private cloudv2-infra repo. Set GITHUB_TOKEN, GH_TOKEN, or REDPANDA_GITHUB_TOKEN.'); } const fmt = (options.format || 'md').toLowerCase(); let templatePath = undefined; @@ -1853,10 +2426,60 @@ automation } }); +/** + * generate crd-spec + * + * @description + * Generates Kubernetes Custom Resource Definition (CRD) reference documentation by parsing + * Go type definitions from the Redpanda Operator repository. Uses the crd-ref-docs tool to + * extract API field definitions, types, descriptions, and validation rules from Go struct tags + * and comments, then generates comprehensive AsciiDoc documentation. Supports both local Go + * source directories and GitHub URLs for operator versions. + * + * When to use --tag vs --branch: + * - Use --tag for released content (GA or beta releases). Tags reference specific release points. + * - Use --branch for in-progress content (unreleased features). Branches track ongoing development. + * + * @why + * Kubernetes CRDs define complex APIs for deploying and managing Redpanda. The API schema + * is defined in Go code with hundreds of fields across nested structures. Manual documentation + * is error-prone and becomes outdated as the API evolves. This automation uses specialized + * tooling (crd-ref-docs) to extract API documentation directly from Go source code, ensuring + * accuracy and completeness. It captures field types, validation rules, and descriptions that + * are essential for users configuring Redpanda in Kubernetes. + * + * @example + * # Generate CRD docs for specific operator tag + * npx doc-tools generate crd-spec --tag operator/v2.2.6-25.3.1 + * + * # Version without prefix (auto-prepends operator/) + * npx doc-tools generate crd-spec --tag v25.1.2 + * + * # Generate from release branch + * npx doc-tools generate crd-spec --branch release/v2.2.x + * + * # Generate from main branch + * npx doc-tools generate crd-spec --branch main + * + * # Generate from any custom branch + * npx doc-tools generate crd-spec --branch dev + * + * # Use custom templates and output location + * npx doc-tools generate crd-spec \\ + * --tag operator/v2.2.6-25.3.1 \\ + * --templates-dir custom/templates \\ + * --output modules/reference/pages/operator-crd.adoc + * + * @requirements + * - For GitHub URLs: Git and internet connection + * - crd-ref-docs tool (automatically installed if missing) + * - Go toolchain for running crd-ref-docs + */ automation .command('crd-spec') - .description('Generate Asciidoc documentation for Kubernetes CRD references') - .requiredOption('-t, --tag ', 'Operator release tag or branch, such as operator/v25.1.2') + .description('Generate Asciidoc documentation for Kubernetes CRD references. Requires either --tag or --branch to be specified.') + .option('-t, --tag ', 'Operator release tag for GA/beta content (for example operator/v2.2.6-25.3.1 or v25.1.2). Auto-prepends "operator/" if not present.') + .option('-b, --branch ', 'Branch name for in-progress content (for example release/v2.2.x, main, dev)') .option( '-s, --source-path ', 'CRD Go types dir or GitHub URL', @@ -1868,15 +2491,35 @@ automation .action(async (opts) => { verifyCrdDependencies(); - // Fetch upstream config + // Validate that either --tag or --branch is provided (but not both) + if (!opts.tag && !opts.branch) { + console.error('❌ Error: Either --tag or --branch must be specified'); + process.exit(1); + } + if (opts.tag && opts.branch) { + console.error('❌ Error: Cannot specify both --tag and --branch'); + process.exit(1); + } + + // Determine the git ref to use + let configRef; + if (opts.branch) { + // Branch - use as-is + configRef = opts.branch; + } else { + // Tag - auto-prepend operator/ if needed + configRef = opts.tag.startsWith('operator/') ? opts.tag : `operator/${opts.tag}`; + } + const configTmp = fs.mkdtempSync(path.join(os.tmpdir(), 'crd-config-')); - console.log(`⏳ Fetching crd-ref-docs-config.yaml from redpanda-operator@main…`); + console.log(`⏳ Fetching crd-ref-docs-config.yaml from redpanda-operator@${configRef}…`); await fetchFromGithub( 'redpanda-data', 'redpanda-operator', 'operator/crd-ref-docs-config.yaml', configTmp, - 'crd-ref-docs-config.yaml' + 'crd-ref-docs-config.yaml', + configRef ); const configPath = path.join(configTmp, 'crd-ref-docs-config.yaml'); @@ -1892,7 +2535,7 @@ automation console.warn('⚠️ Not inside redpanda-data/docs; skipping branch suggestion.'); } else { try { - docsBranch = await determineDocsBranch(opts.tag); + docsBranch = await determineDocsBranch(configRef); console.log(`βœ… Detected docs repo; you should commit to branch '${docsBranch}'.`); } catch (err) { console.error(`❌ Unable to determine docs branch: ${err.message}`); @@ -1919,19 +2562,30 @@ automation const [owner, repo, ...subpathParts] = parts; const repoUrl = `https://${u.host}/${owner}/${repo}`; const subpath = subpathParts.join('/'); - console.log(`⏳ Verifying "${opts.tag}" in ${repoUrl}…`); + console.log(`⏳ Verifying "${configRef}" in ${repoUrl}…`); const ok = - spawnSync('git', ['ls-remote', '--exit-code', repoUrl, `refs/tags/${opts.tag}`, `refs/heads/${opts.tag}`], { + spawnSync('git', ['ls-remote', '--exit-code', repoUrl, `refs/tags/${configRef}`, `refs/heads/${configRef}`], { stdio: 'ignore', }).status === 0; if (!ok) { - console.error(`❌ Tag or branch "${opts.tag}" not found on ${repoUrl}`); + console.error(`❌ Tag or branch "${configRef}" not found on ${repoUrl}`); process.exit(1); } + const { getAuthenticatedGitHubUrl, hasGitHubToken } = require('../cli-utils/github-token'); + tmpSrc = fs.mkdtempSync(path.join(os.tmpdir(), 'crd-src-')); - console.log(`⏳ Cloning ${repoUrl}@${opts.tag} β†’ ${tmpSrc}`); + + // Use token if available for GitHub repos + let cloneUrl = repoUrl; + if (hasGitHubToken() && repoUrl.includes('github.com')) { + cloneUrl = getAuthenticatedGitHubUrl(repoUrl); + console.log(`⏳ Cloning ${repoUrl}@${configRef} β†’ ${tmpSrc} (authenticated)`); + } else { + console.log(`⏳ Cloning ${repoUrl}@${configRef} β†’ ${tmpSrc}`); + } + if ( - spawnSync('git', ['clone', '--depth', '1', '--branch', opts.tag, repoUrl, tmpSrc], { + spawnSync('git', ['clone', '--depth', '1', '--branch', configRef, cloneUrl, tmpSrc], { stdio: 'inherit', }).status !== 0 ) { @@ -2005,10 +2659,65 @@ automation } }); +/** + * generate bundle-openapi + * + * @description + * Bundles Redpanda's OpenAPI specification fragments into complete, usable OpenAPI 3.1 documents + * for both Admin API and Connect API. Clones the Redpanda repository at a specified version, + * collects OpenAPI fragments that are distributed throughout the codebase (alongside endpoint + * implementations), uses Buf and Redocly CLI to bundle and validate the specifications, and + * generates separate complete OpenAPI files for each API surface. The resulting specifications + * can be used for API documentation, client SDK generation, or API testing tools. + * + * @why + * Redpanda's API documentation is defined as OpenAPI fragments alongside the C++ implementation + * code. This keeps API docs close to code and ensures they stay in sync, but it means the + * specification is fragmented across hundreds of files. Users need complete OpenAPI specifications + * for tooling (Swagger UI, Postman, client generators). This automation collects all fragments, + * bundles them into valid OpenAPI 3.1 documents, and validates the result. It's the only way + * to produce accurate, complete API specifications that match a specific Redpanda version. + * + * @example + * # Bundle both Admin and Connect APIs + * npx doc-tools generate bundle-openapi \\ + * --tag v25.3.1 \\ + * --surface both + * + * # Bundle only Admin API + * npx doc-tools generate bundle-openapi \\ + * --tag v25.3.1 \\ + * --surface admin + * + * # Use custom output paths + * npx doc-tools generate bundle-openapi \\ + * --tag v25.3.1 \\ + * --surface both \\ + * --out-admin api/admin-api.yaml \\ + * --out-connect api/connect-api.yaml + * + * # Use major version for Admin API version field + * npx doc-tools generate bundle-openapi \\ + * --tag v25.3.1 \\ + * --surface admin \\ + * --use-admin-major-version + * + * # Full workflow: generate API specs for new release + * VERSION=$(npx doc-tools get-redpanda-version) + * npx doc-tools generate bundle-openapi --tag $VERSION --surface both + * + * @requirements + * - Git to clone Redpanda repository + * - Buf tool (automatically installed via npm) + * - Redocly CLI or vacuum for OpenAPI bundling (automatically detected) + * - Internet connection to clone repository + * - Sufficient disk space for repository clone (~2GB) + */ automation .command('bundle-openapi') - .description('Bundle Redpanda OpenAPI fragments for admin and connect APIs into complete OpenAPI 3.1 documents') - .requiredOption('-t, --tag ', 'Branch or tag to clone from the repository (for example v24.3.2 or 24.3.2 or dev)') + .description('Bundle Redpanda OpenAPI fragments for admin and connect APIs into complete OpenAPI 3.1 documents. Requires either --tag or --branch to be specified.') + .option('-t, --tag ', 'Git tag for released content (for example, v24.3.2 or 24.3.2)') + .option('-b, --branch ', 'Branch name for in-progress content (for example, dev, main)') .option('--repo ', 'Repository URL', 'https://github.com/redpanda-data/redpanda.git') .addOption(new Option('-s, --surface ', 'Which API surface(s) to bundle').choices(['admin', 'connect', 'both']).makeOptionMandatory()) .option('--out-admin ', 'Output path for admin API', 'admin/redpanda-admin-api.yaml') @@ -2017,6 +2726,18 @@ automation .option('--use-admin-major-version', 'Use admin major version for info.version instead of git tag', false) .option('--quiet', 'Suppress logs', false) .action(async (options) => { + // Validate that either tag or branch is provided (but not both) + if (!options.tag && !options.branch) { + console.error('❌ Error: Either --tag or --branch must be specified'); + process.exit(1); + } + if (options.tag && options.branch) { + console.error('❌ Error: Cannot specify both --tag and --branch'); + process.exit(1); + } + + // Determine the git ref to use + const gitRef = options.tag || options.branch; // Verify dependencies requireCmd('git', 'Install Git: https://git-scm.com/downloads'); requireCmd('buf', 'buf should be automatically available after npm install'); @@ -2032,7 +2753,7 @@ automation try { const { bundleOpenAPI } = require('../tools/bundle-openapi.js'); await bundleOpenAPI({ - tag: options.tag, + tag: gitRef, repo: options.repo, surface: options.surface, outAdmin: options.outAdmin, @@ -2047,5 +2768,249 @@ automation } }); +/** + * Validate MCP configuration + * + * Validates that all prompts and resources are properly configured and accessible. + * Checks for: + * - Valid frontmatter in all prompt files + * - Accessible resource files + * - Proper metadata and descriptions + * - No naming conflicts + * + * @example + * npx doc-tools validate-mcp + */ +programCli + .command('validate-mcp') + .description('Validate MCP server configuration (prompts, resources, metadata)') + .action(() => { + const { + PromptCache, + loadAllPrompts + } = require('./mcp-tools/prompt-discovery'); + const { + validateMcpConfiguration, + formatValidationResults + } = require('./mcp-tools/mcp-validation'); + + const baseDir = findRepoRoot(); + const promptCache = new PromptCache(); + + // Resources configuration + const resources = [ + { + uri: 'redpanda://style-guide', + name: 'Redpanda Documentation Style Guide', + description: 'Complete style guide based on Google Developer Documentation Style Guide with Redpanda-specific guidelines', + mimeType: 'text/markdown', + version: '1.0.0', + lastUpdated: '2025-12-11' + } + ]; + + const resourceMap = { + 'redpanda://style-guide': { file: 'style-guide.md', mimeType: 'text/markdown' } + }; + + try { + // Load prompts + console.log('Loading prompts...'); + const prompts = loadAllPrompts(baseDir, promptCache); + console.log(`Found ${prompts.length} prompts`); + + // Validate configuration + console.log('\nValidating configuration...'); + const validation = validateMcpConfiguration({ + resources, + resourceMap, + prompts, + baseDir + }); + + // Format and display results + const output = formatValidationResults(validation, { resources, prompts }); + console.log('\n' + output); + + if (!validation.valid) { + process.exit(1); + } + } catch (err) { + console.error(`❌ Validation failed: ${err.message}`); + process.exit(1); + } + }); + +/** + * Preview a prompt with arguments + * + * Loads a prompt and shows how it will appear when used with specified arguments. + * Useful for testing prompts before using them in Claude Code. + * + * @example + * npx doc-tools preview-prompt review-for-style --content "Sample content" + * npx doc-tools preview-prompt write-new-guide --topic "Deploy Redpanda" + */ +programCli + .command('preview-prompt') + .description('Preview a prompt with arguments to see the final output') + .argument('', 'Name of the prompt to preview') + .option('--content ', 'Content argument (for review/check prompts)') + .option('--topic ', 'Topic argument (for write prompts)') + .option('--audience ', 'Audience argument (for write prompts)') + .action((promptName, options) => { + const { + PromptCache, + loadAllPrompts, + buildPromptWithArguments + } = require('./mcp-tools/prompt-discovery'); + + const baseDir = findRepoRoot(); + const promptCache = new PromptCache(); + + try { + // Load prompts + loadAllPrompts(baseDir, promptCache); + + // Get the requested prompt + const prompt = promptCache.get(promptName); + if (!prompt) { + console.error(`❌ Prompt not found: ${promptName}`); + console.error(`\nAvailable prompts: ${promptCache.getNames().join(', ')}`); + process.exit(1); + } + + // Build arguments object from options + const args = {}; + if (options.content) args.content = options.content; + if (options.topic) args.topic = options.topic; + if (options.audience) args.audience = options.audience; + + // Build final prompt text + const promptText = buildPromptWithArguments(prompt, args); + + // Display preview + console.log('='.repeat(70)); + console.log(`PROMPT PREVIEW: ${promptName}`); + console.log('='.repeat(70)); + console.log(`Description: ${prompt.description}`); + console.log(`Version: ${prompt.version}`); + if (prompt.arguments.length > 0) { + console.log(`Arguments: ${prompt.arguments.map(a => a.name).join(', ')}`); + } + console.log('='.repeat(70)); + console.log('\n' + promptText); + console.log('\n' + '='.repeat(70)); + } catch (err) { + console.error(`❌ Preview failed: ${err.message}`); + process.exit(1); + } + }); + +/** + * Show MCP server version and statistics + * + * Displays version information for the MCP server, including: + * - Server version + * - Available prompts and their versions + * - Available resources and their versions + * - Usage statistics (if available) + * + * @example + * npx doc-tools mcp-version + * npx doc-tools mcp-version --stats # Show usage statistics if available + */ +programCli + .command('mcp-version') + .description('Show MCP server version and configuration information') + .option('--stats', 'Show usage statistics if available', false) + .action((options) => { + const packageJson = require('../package.json'); + const { + PromptCache, + loadAllPrompts + } = require('./mcp-tools/prompt-discovery'); + + const baseDir = findRepoRoot(); + const promptCache = new PromptCache(); + + try { + // Load prompts + const prompts = loadAllPrompts(baseDir, promptCache); + + // Resources configuration + const resources = [ + { + uri: 'redpanda://style-guide', + name: 'Redpanda Documentation Style Guide', + version: '1.0.0', + lastUpdated: '2025-12-11' + } + ]; + + // Display version information + console.log('Redpanda Doc Tools MCP Server'); + console.log('='.repeat(60)); + console.log(`Server version: ${packageJson.version}`); + console.log(`Base directory: ${baseDir}`); + console.log(''); + + // Prompts + console.log(`Prompts (${prompts.length} available):`); + prompts.forEach(prompt => { + const args = prompt.arguments.length > 0 + ? ` [${prompt.arguments.map(a => a.name).join(', ')}]` + : ''; + console.log(` - ${prompt.name} (v${prompt.version})${args}`); + console.log(` ${prompt.description}`); + }); + console.log(''); + + // Resources + console.log(`Resources (${resources.length} available):`); + resources.forEach(resource => { + console.log(` - ${resource.uri}`); + console.log(` ${resource.name} (v${resource.version})`); + console.log(` Last updated: ${resource.lastUpdated}`); + }); + console.log(''); + + // Check for usage statistics + if (options.stats) { + const statsPath = path.join(baseDir, 'mcp-usage-stats.json'); + if (fs.existsSync(statsPath)) { + console.log('Usage Statistics:'); + console.log('='.repeat(60)); + const stats = JSON.parse(fs.readFileSync(statsPath, 'utf8')); + console.log(`Exported at: ${stats.exportedAt}`); + console.log(`Total prompt calls: ${stats.totalPromptCalls}`); + console.log(`Total resource calls: ${stats.totalResourceCalls}`); + console.log(`Total tool calls: ${stats.totalToolCalls}`); + + if (stats.totalPromptCalls > 0) { + console.log('\nMost used prompts:'); + Object.entries(stats.prompts) + .sort((a, b) => b[1] - a[1]) + .slice(0, 5) + .forEach(([name, count]) => { + console.log(` ${name}: ${count} calls`); + }); + } + } else { + console.log('No usage statistics available yet.'); + console.log('Statistics are exported when the MCP server shuts down.'); + } + } + + console.log(''); + console.log('For more information, see:'); + console.log(' mcp/WRITER_EXTENSION_GUIDE.adoc'); + console.log(' mcp/AI_CONSISTENCY_ARCHITECTURE.adoc'); + } catch (err) { + console.error(`❌ Failed to retrieve version information: ${err.message}`); + process.exit(1); + } + }); + programCli.addCommand(automation); programCli.parse(process.argv); diff --git a/bin/mcp-tools/antora.js b/bin/mcp-tools/antora.js new file mode 100644 index 00000000..22ff95e1 --- /dev/null +++ b/bin/mcp-tools/antora.js @@ -0,0 +1,124 @@ +/** + * MCP Tools - Antora Structure + */ + +const fs = require('fs'); +const path = require('path'); +const yaml = require('js-yaml'); +const { MAX_RECURSION_DEPTH, DEFAULT_SKIP_DIRS, PLAYBOOK_NAMES } = require('./utils'); + +/** + * Get Antora structure information for the current repository + * @param {string|{root: string, detected: boolean, type: string|null}} repoRoot - Repository root path or info object + * @param {string[]} [skipDirs=DEFAULT_SKIP_DIRS] - Directories to skip during search + * @returns {Object} Antora structure information + */ +function getAntoraStructure(repoRoot, skipDirs = DEFAULT_SKIP_DIRS) { + const rootPath = typeof repoRoot === 'string' ? repoRoot : repoRoot.root; + const repoInfo = typeof repoRoot === 'object' ? repoRoot : { root: repoRoot, detected: true, type: null }; + + const playbookPath = PLAYBOOK_NAMES + .map(name => path.join(rootPath, name)) + .find(p => fs.existsSync(p)); + + let playbookContent = null; + if (playbookPath) { + try { + playbookContent = yaml.load(fs.readFileSync(playbookPath, 'utf8')); + } catch (err) { + console.error(`Warning: Failed to parse playbook at ${playbookPath}: ${err.message}`); + } + } + + const antoraYmls = []; + const findAntoraYmls = (dir, depth = 0, visited = new Set()) => { + if (depth > MAX_RECURSION_DEPTH || !fs.existsSync(dir)) return; + + try { + const realPath = fs.realpathSync(dir); + if (visited.has(realPath)) return; + visited.add(realPath); + + const entries = fs.readdirSync(dir, { withFileTypes: true }); + for (const entry of entries) { + if (skipDirs.includes(entry.name)) continue; + + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + findAntoraYmls(fullPath, depth + 1, visited); + } else if (entry.name === 'antora.yml') { + antoraYmls.push(fullPath); + } + } + } catch (err) { + console.error(`Warning: Failed to read directory ${dir}: ${err.message}`); + } + }; + findAntoraYmls(rootPath); + + const components = antoraYmls.map(ymlPath => { + try { + const content = yaml.load(fs.readFileSync(ymlPath, 'utf8')); + const componentDir = path.dirname(ymlPath); + const modulesDir = path.join(componentDir, 'modules'); + const modules = fs.existsSync(modulesDir) + ? fs.readdirSync(modulesDir).filter(m => { + const stat = fs.statSync(path.join(modulesDir, m)); + return stat.isDirectory(); + }) + : []; + + return { + name: content.name, + version: content.version, + title: content.title, + path: componentDir, + modules: modules.map(moduleName => { + const modulePath = path.join(modulesDir, moduleName); + return { + name: moduleName, + path: modulePath, + pages: fs.existsSync(path.join(modulePath, 'pages')), + partials: fs.existsSync(path.join(modulePath, 'partials')), + examples: fs.existsSync(path.join(modulePath, 'examples')), + attachments: fs.existsSync(path.join(modulePath, 'attachments')), + images: fs.existsSync(path.join(modulePath, 'images')), + }; + }), + }; + } catch (err) { + return { error: `Failed to parse ${ymlPath}: ${err.message}` }; + } + }); + + return { + repoRoot: rootPath, + repoInfo, + playbook: playbookContent, + playbookPath, + components, + hasDocTools: (() => { + // Check if we're in the source repo (docs-extensions-and-macros) + if (fs.existsSync(path.join(rootPath, 'bin', 'doc-tools.js'))) { + return true; + } + + // Check if doc-tools is available via npx or as installed dependency + try { + const { execSync } = require('child_process'); + execSync('npx doc-tools --version', { + stdio: 'ignore', + timeout: 5000, + cwd: rootPath + }); + return true; + } catch { + return false; + } + })() + }; +} + +module.exports = { + getAntoraStructure +}; diff --git a/bin/mcp-tools/cloud-regions.js b/bin/mcp-tools/cloud-regions.js new file mode 100644 index 00000000..b935e1f4 --- /dev/null +++ b/bin/mcp-tools/cloud-regions.js @@ -0,0 +1,127 @@ +/** + * MCP Tools - Cloud Regions Table Generation + */ + +const { spawnSync } = require('child_process'); +const { findRepoRoot, getDocToolsCommand, MAX_EXEC_BUFFER_SIZE, DEFAULT_COMMAND_TIMEOUT } = require('./utils'); +const { getAntoraStructure } = require('./antora'); + +/** + * Generate cloud regions table documentation + * @param {Object} args - Arguments + * @param {string} [args.output] - Output file path (relative to repo root) + * @param {string} [args.format] - Output format: 'md' or 'adoc' + * @param {string} [args.owner] - GitHub repository owner + * @param {string} [args.repo] - GitHub repository name + * @param {string} [args.path] - Path to YAML file in repository + * @param {string} [args.ref] - Git reference (branch, tag, or commit SHA) + * @param {string} [args.template] - Path to custom Handlebars template + * @param {boolean} [args.dry_run] - Print output to stdout instead of writing file + * @returns {Object} Generation results + */ +function generateCloudRegions(args = {}) { + const repoRoot = findRepoRoot(); + const structure = getAntoraStructure(repoRoot); + + if (!structure.hasDocTools) { + return { + success: false, + error: 'doc-tools not found in this repository', + suggestion: 'Navigate to the docs-extensions-and-macros repository' + }; + } + + try { + // Get doc-tools command (handles both local and installed) + const docTools = getDocToolsCommand(repoRoot); + + // Build command arguments array + const baseArgs = ['generate', 'cloud-regions']; + + if (args.output) { + baseArgs.push('--output'); + baseArgs.push(args.output); + } + + if (args.format) { + baseArgs.push('--format'); + baseArgs.push(args.format); + } + + if (args.owner) { + baseArgs.push('--owner'); + baseArgs.push(args.owner); + } + + if (args.repo) { + baseArgs.push('--repo'); + baseArgs.push(args.repo); + } + + if (args.path) { + baseArgs.push('--path'); + baseArgs.push(args.path); + } + + if (args.ref) { + baseArgs.push('--ref'); + baseArgs.push(args.ref); + } + + if (args.template) { + baseArgs.push('--template'); + baseArgs.push(args.template); + } + + if (args.dry_run) { + baseArgs.push('--dry-run'); + } + + const result = spawnSync(docTools.program, docTools.getArgs(baseArgs), { + cwd: repoRoot.root, + encoding: 'utf8', + stdio: 'pipe', + maxBuffer: MAX_EXEC_BUFFER_SIZE, + timeout: DEFAULT_COMMAND_TIMEOUT + }); + + // Check for spawn errors + if (result.error) { + throw new Error(`Failed to execute command: ${result.error.message}`); + } + + // Check for non-zero exit codes + if (result.status !== 0) { + const errorMsg = result.stderr || `Command failed with exit code ${result.status}`; + throw new Error(errorMsg); + } + + const output = result.stdout; + + // Parse output to extract information + const regionCountMatch = output.match(/(\d+) regions?/i); + + return { + success: true, + format: args.format || 'md', + ref: args.ref || 'integration', + regions_documented: regionCountMatch ? parseInt(regionCountMatch[1]) : null, + files_generated: args.dry_run ? [] : [args.output || 'cloud-controlplane/x-topics/cloud-regions.md'], + output: output.trim(), + summary: `Generated cloud regions table from GitHub YAML` + }; + } catch (err) { + return { + success: false, + error: err.message, + stdout: err.stdout || '', + stderr: err.stderr || '', + exitCode: err.status, + suggestion: 'Check that you have access to the GitHub repository and the YAML file exists' + }; + } +} + +module.exports = { + generateCloudRegions +}; diff --git a/bin/mcp-tools/crd-docs.js b/bin/mcp-tools/crd-docs.js new file mode 100644 index 00000000..c8c635c2 --- /dev/null +++ b/bin/mcp-tools/crd-docs.js @@ -0,0 +1,149 @@ +/** + * MCP Tools - Kubernetes CRD Documentation Generation + */ + +const { spawnSync } = require('child_process'); +const { findRepoRoot, getDocToolsCommand, MAX_EXEC_BUFFER_SIZE, normalizeVersion, DEFAULT_COMMAND_TIMEOUT } = require('./utils'); +const { getAntoraStructure } = require('./antora'); + +/** + * Generate Kubernetes CRD documentation + * + * Use tags for released content (GA or beta), branches for in-progress content. + * + * @param {Object} args - Arguments + * @param {string} [args.tag] - Operator release tag for GA/beta content (for example, operator/v2.2.6-25.3.1 or v25.1.2). Auto-prepends "operator/" if not present. + * @param {string} [args.branch] - Branch name for in-progress content (for example, release/v2.2.x, main, dev) + * @param {string} [args.source_path] - CRD Go types dir or GitHub URL + * @param {number} [args.depth] - How many levels deep to generate + * @param {string} [args.templates_dir] - Asciidoctor templates directory + * @param {string} [args.output] - Where to write the generated AsciiDoc file + * @returns {Object} Generation results + */ +function generateCrdDocs(args) { + const repoRoot = findRepoRoot(); + const structure = getAntoraStructure(repoRoot); + + if (!structure.hasDocTools) { + return { + success: false, + error: 'doc-tools not found in this repository', + suggestion: 'Navigate to the docs-extensions-and-macros repository' + }; + } + + // Validate that either tag or branch is provided (but not both) + if (!args.tag && !args.branch) { + return { + success: false, + error: 'Either tag or branch is required', + suggestion: 'Provide --tag "operator/v25.1.2" or --branch "main"' + }; + } + + if (args.tag && args.branch) { + return { + success: false, + error: 'Cannot specify both tag and branch', + suggestion: 'Use either --tag or --branch, not both' + }; + } + + try { + // Get the appropriate doc-tools command (local or installed) + const docTools = getDocToolsCommand(repoRoot); + + // Build command arguments array (no shell interpolation) + const cmdArgs = ['generate', 'crd-spec']; + + // Add tag or branch flag + if (args.tag) { + cmdArgs.push('--tag'); + cmdArgs.push(args.tag); + } else { + cmdArgs.push('--branch'); + cmdArgs.push(args.branch); + } + + if (args.source_path) { + cmdArgs.push('--source-path'); + cmdArgs.push(args.source_path); + } + + if (args.depth) { + cmdArgs.push('--depth'); + cmdArgs.push(String(args.depth)); + } + + if (args.templates_dir) { + cmdArgs.push('--templates-dir'); + cmdArgs.push(args.templates_dir); + } + + if (args.output) { + cmdArgs.push('--output'); + cmdArgs.push(args.output); + } + + const result = spawnSync(docTools.program, docTools.getArgs(cmdArgs), { + cwd: repoRoot.root, + encoding: 'utf8', + stdio: 'pipe', + maxBuffer: MAX_EXEC_BUFFER_SIZE, + timeout: DEFAULT_COMMAND_TIMEOUT + }); + + // Check for spawn errors + if (result.error) { + const err = new Error(`Failed to execute command: ${result.error.message}`); + err.stdout = result.stdout || ''; + err.stderr = result.stderr || ''; + err.status = result.status; + throw err; + } + + // Check for non-zero exit codes + if (result.status !== 0) { + const errorMsg = result.stderr || `Command failed with exit code ${result.status}`; + const err = new Error(errorMsg); + err.stdout = result.stdout || ''; + err.stderr = result.stderr || ''; + err.status = result.status; + throw err; + } + + const output = result.stdout; + + const ref = args.tag || args.branch; + const refType = args.tag ? 'tag' : 'branch'; + + return { + success: true, + [refType]: ref, + files_generated: [args.output || 'modules/reference/pages/k-crd.adoc'], + output: output.trim(), + summary: `Generated CRD documentation for operator ${refType} ${ref}` + }; + } catch (err) { + // Check if the error is due to --branch not being supported (old doc-tools version) + const isOldVersionError = err.stderr && + err.stderr.includes('required option') && + err.stderr.includes('--tag') && + args.branch; + + return { + success: false, + error: err.message, + stdout: err.stdout || '', + stderr: err.stderr || '', + exitCode: err.status, + suggestion: isOldVersionError + ? 'Your doc-tools version doesn\'t support --branch. Update with: npm install (in docs-extensions-and-macros repo)' + : 'Check that the operator tag exists in the repository' + }; + } +} + +module.exports = { + generateCrdDocs +}; diff --git a/bin/mcp-tools/frontmatter.js b/bin/mcp-tools/frontmatter.js new file mode 100644 index 00000000..6ac459b5 --- /dev/null +++ b/bin/mcp-tools/frontmatter.js @@ -0,0 +1,138 @@ +/** + * Frontmatter Parser for MCP Prompts + * + * Parses YAML frontmatter from markdown files to extract metadata. + * Supports validation against JSON schemas. + */ + +const yaml = require('js-yaml'); + +/** + * Parse frontmatter from markdown content + * @param {string} content - Markdown content with optional frontmatter + * @returns {{ metadata: Object, content: string }} Parsed frontmatter and remaining content + */ +function parseFrontmatter(content) { + const frontmatterRegex = /^---\s*\n([\s\S]*?)\n---\s*\n([\s\S]*)$/; + const match = content.match(frontmatterRegex); + + if (!match) { + // No frontmatter found - return empty metadata and full content + return { + metadata: {}, + content: content + }; + } + + try { + const metadata = yaml.load(match[1]) || {}; + const remainingContent = match[2]; + + return { + metadata, + content: remainingContent + }; + } catch (err) { + throw new Error(`Failed to parse YAML frontmatter: ${err.message}`); + } +} + +/** + * JSON Schema for prompt frontmatter + */ +const promptFrontmatterSchema = { + type: 'object', + properties: { + description: { + type: 'string', + minLength: 10, + description: 'Description of what this prompt does' + }, + version: { + type: 'string', + pattern: '^\\d+\\.\\d+\\.\\d+$', + description: 'Semantic version (for example, 1.0.0)' + }, + arguments: { + type: 'array', + description: 'Arguments this prompt accepts', + items: { + type: 'object', + required: ['name', 'description', 'required'], + properties: { + name: { + type: 'string', + pattern: '^[a-z_][a-z0-9_]*$', + description: 'Argument name (lowercase, underscores allowed)' + }, + description: { + type: 'string', + minLength: 5, + description: 'What this argument is for' + }, + required: { + type: 'boolean', + description: 'Whether this argument is required' + } + }, + additionalProperties: false + } + }, + argumentFormat: { + type: 'string', + enum: ['content-append', 'structured'], + description: 'How to format arguments when building the prompt' + } + }, + additionalProperties: false +}; + +/** + * Validate frontmatter against schema + * @param {Object} metadata - Parsed frontmatter metadata + * @param {string} filename - File name (for error messages) + * @param {Object} schema - JSON schema to validate against + * @throws {Error} If validation fails + */ +function validateFrontmatter(metadata, filename, schema = promptFrontmatterSchema) { + const Ajv = require('ajv'); + const ajv = new Ajv({ allErrors: true }); + + const valid = ajv.validate(schema, metadata); + + if (!valid) { + const errors = ajv.errors.map(err => { + const path = err.instancePath || 'root'; + return ` - ${path}: ${err.message}`; + }).join('\n'); + + throw new Error(`Invalid frontmatter in ${filename}:\n${errors}`); + } +} + +/** + * Parse and validate prompt file + * @param {string} content - File content + * @param {string} filename - File name + * @returns {{ metadata: Object, content: string }} Validated metadata and content + */ +function parsePromptFile(content, filename) { + const { metadata, content: promptContent } = parseFrontmatter(content); + + // If metadata exists, validate it + if (Object.keys(metadata).length > 0) { + validateFrontmatter(metadata, filename); + } + + return { + metadata, + content: promptContent + }; +} + +module.exports = { + parseFrontmatter, + parsePromptFile, + validateFrontmatter, + promptFrontmatterSchema +}; diff --git a/bin/mcp-tools/helm-docs.js b/bin/mcp-tools/helm-docs.js new file mode 100644 index 00000000..f3b0b446 --- /dev/null +++ b/bin/mcp-tools/helm-docs.js @@ -0,0 +1,148 @@ +/** + * MCP Tools - Helm Chart Documentation Generation + */ + +const { spawnSync } = require('child_process'); +const { findRepoRoot, getDocToolsCommand, MAX_EXEC_BUFFER_SIZE, DEFAULT_COMMAND_TIMEOUT } = require('./utils'); +const { getAntoraStructure } = require('./antora'); + +/** + * Generate Helm chart documentation + * + * Use tags for released content (GA or beta), branches for in-progress content. + * When using GitHub URLs, requires either --tag or --branch to be specified. + * Auto-prepends "operator/" for tags when using redpanda-operator repository. + * + * @param {Object} args - Arguments + * @param {string} [args.chart_dir] - Chart directory, root, or GitHub URL + * @param {string} [args.tag] - Git tag for released content when using GitHub URL (auto-prepends "operator/" for redpanda-operator repository) + * @param {string} [args.branch] - Branch name for in-progress content when using GitHub URL + * @param {string} [args.readme] - Relative README.md path inside each chart dir + * @param {string} [args.output_dir] - Where to write generated AsciiDoc files + * @param {string} [args.output_suffix] - Suffix to append to each chart name + * @returns {Object} Generation results + */ +function generateHelmDocs(args = {}) { + const repoRoot = findRepoRoot(); + const structure = getAntoraStructure(repoRoot); + + if (!structure.hasDocTools) { + return { + success: false, + error: 'doc-tools not found in this repository', + suggestion: 'Navigate to the docs-extensions-and-macros repository' + }; + } + + // Validate that tag and branch are mutually exclusive + if (args.tag && args.branch) { + return { + success: false, + error: 'Cannot specify both tag and branch', + suggestion: 'Use either --tag or --branch, not both' + }; + } + + try { + // Normalize tag: add 'v' prefix if not present + let normalizedTag = null; + if (args.tag) { + normalizedTag = args.tag; + if (!normalizedTag.startsWith('v')) { + normalizedTag = `v${normalizedTag}`; + } + } + + // Get doc-tools command (handles both local and installed) + const docTools = getDocToolsCommand(repoRoot); + + // Build command arguments array + const baseArgs = ['generate', 'helm-spec']; + + if (args.chart_dir) { + baseArgs.push('--chart-dir'); + baseArgs.push(args.chart_dir); + } + + if (normalizedTag) { + baseArgs.push('--tag'); + baseArgs.push(normalizedTag); + } else if (args.branch) { + baseArgs.push('--branch'); + baseArgs.push(args.branch); + } + + if (args.readme) { + baseArgs.push('--readme'); + baseArgs.push(args.readme); + } + + if (args.output_dir) { + baseArgs.push('--output-dir'); + baseArgs.push(args.output_dir); + } + + if (args.output_suffix) { + baseArgs.push('--output-suffix'); + baseArgs.push(args.output_suffix); + } + + const result = spawnSync(docTools.program, docTools.getArgs(baseArgs), { + cwd: repoRoot.root, + encoding: 'utf8', + stdio: 'pipe', + maxBuffer: MAX_EXEC_BUFFER_SIZE, + timeout: DEFAULT_COMMAND_TIMEOUT + }); + + // Check for spawn errors + if (result.error) { + const err = new Error(`Failed to execute command: ${result.error.message}`); + err.stdout = result.stdout || ''; + err.stderr = result.stderr || ''; + err.status = result.status; + throw err; + } + + // Check for non-zero exit codes + if (result.status !== 0) { + const errorMsg = result.stderr || `Command failed with exit code ${result.status}`; + const err = new Error(errorMsg); + err.stdout = result.stdout || ''; + err.stderr = result.stderr || ''; + err.status = result.status; + throw err; + } + + const output = result.stdout; + + // Parse output to extract information + const chartCountMatch = output.match(/(\d+) charts?/i); + + const refType = normalizedTag ? 'tag' : args.branch ? 'branch' : null; + const gitRef = normalizedTag || args.branch; + + return { + success: true, + ...(refType && { [refType]: gitRef }), + chart_dir: args.chart_dir || 'default', + charts_documented: chartCountMatch ? parseInt(chartCountMatch[1]) : null, + files_generated: [args.output_dir || 'modules/reference/pages'], + output: output.trim(), + summary: `Generated Helm chart documentation${gitRef ? ` for ${refType} ${gitRef}` : ''}` + }; + } catch (err) { + return { + success: false, + error: err.message, + stdout: err.stdout || '', + stderr: err.stderr || '', + exitCode: err.status, + suggestion: 'Check that the chart directory or GitHub URL is valid and accessible' + }; + } +} + +module.exports = { + generateHelmDocs +}; diff --git a/bin/mcp-tools/index.js b/bin/mcp-tools/index.js new file mode 100644 index 00000000..e35f1ea7 --- /dev/null +++ b/bin/mcp-tools/index.js @@ -0,0 +1,241 @@ +/** + * MCP Tools - Main Exports + * + * This module exports all MCP tools in a modular structure. + */ + +// Utilities +const { findRepoRoot, executeCommand, normalizeVersion, formatDate } = require('./utils'); + +// Antora +const { getAntoraStructure } = require('./antora'); + +// Versions +const { getRedpandaVersion, getConsoleVersion } = require('./versions'); + +// Documentation generation tools +const { generatePropertyDocs } = require('./property-docs'); +const { generateMetricsDocs } = require('./metrics-docs'); +const { generateRpkDocs } = require('./rpk-docs'); +const { generateRpConnectDocs } = require('./rpcn-docs'); +const { generateHelmDocs } = require('./helm-docs'); +const { generateCloudRegions } = require('./cloud-regions'); +const { generateCrdDocs } = require('./crd-docs'); +const { generateBundleOpenApi } = require('./openapi'); + +// Review tools +const { reviewGeneratedDocs, generateReviewReport } = require('./review'); + +// Job queue +const { initializeJobQueue, createJob, getJob, listJobs, cleanupOldJobs } = require('./job-queue'); + +/** + * Execute a tool and return results + * @param {string} toolName - Name of the tool to execute + * @param {Object} args - Arguments for the tool + * @returns {Object} Tool execution results + */ +function executeTool(toolName, args = {}) { + const repoRoot = findRepoRoot(); + + try { + switch (toolName) { + case 'get_antora_structure': + return getAntoraStructure(repoRoot); + + case 'get_redpanda_version': + return getRedpandaVersion(args); + + case 'get_console_version': + return getConsoleVersion(); + + case 'generate_property_docs': + return generatePropertyDocs(args); + + case 'generate_metrics_docs': + return generateMetricsDocs(args); + + case 'generate_rpk_docs': + return generateRpkDocs(args); + + case 'generate_rpcn_connector_docs': + return generateRpConnectDocs(args); + + case 'generate_helm_docs': + return generateHelmDocs(args); + + case 'generate_cloud_regions': + return generateCloudRegions(args); + + case 'generate_crd_docs': + return generateCrdDocs(args); + + case 'generate_bundle_openapi': + return generateBundleOpenApi(args); + + case 'review_generated_docs': + return reviewGeneratedDocs(args); + + case 'run_doc_tools_command': { + // Validate and execute raw doc-tools command + if (!args || typeof args !== 'object') { + return { + success: false, + error: 'Invalid arguments: expected an object' + }; + } + + const validation = validateDocToolsCommand(args.command); + if (!validation.valid) { + return { + success: false, + error: validation.error + }; + } + + try { + // Get doc-tools command (handles both local and installed) + const { getDocToolsCommand } = require('./utils'); + const docTools = getDocToolsCommand(repoRoot); + + // Parse command into argument array (no shell interpolation) + // Since validation already rejects shell metacharacters, we can safely split by spaces + // Handle quoted strings for paths with spaces + const cmdArgs = parseCommandArgs(args.command); + + // Build full args using the appropriate command + const fullArgs = docTools.getArgs(cmdArgs); + + const output = executeCommand(docTools.program, fullArgs, { + cwd: repoRoot.root + }); + + return { + success: true, + output: output.trim(), + command: `${docTools.program} ${fullArgs.join(' ')}` + }; + } catch (err) { + return { + success: false, + error: err.message, + stdout: err.stdout || '', + stderr: err.stderr || '' + }; + } + } + + default: + return { + success: false, + error: `Unknown tool: ${toolName}`, + suggestion: 'Check the tool name and try again' + }; + } + } catch (err) { + return { + success: false, + error: err.message, + suggestion: 'An unexpected error occurred while executing the tool' + }; + } +} + +/** + * Parse command string into array of arguments + * Handles quoted strings for paths with spaces + * @param {string} command - The command string to parse + * @returns {string[]} Array of arguments + */ +function parseCommandArgs(command) { + const args = []; + let current = ''; + let inQuotes = false; + let quoteChar = ''; + + for (let i = 0; i < command.length; i++) { + const char = command[i]; + + if ((char === '"' || char === "'") && !inQuotes) { + // Start of quoted string + inQuotes = true; + quoteChar = char; + } else if (char === quoteChar && inQuotes) { + // End of quoted string + inQuotes = false; + quoteChar = ''; + } else if (char === ' ' && !inQuotes) { + // Space outside quotes - end of argument + if (current) { + args.push(current); + current = ''; + } + } else { + // Regular character + current += char; + } + } + + // Add final argument if present + if (current) { + args.push(current); + } + + return args; +} + +/** + * Validate doc-tools command to prevent command injection + * @param {string} command - The command string to validate + * @returns {{ valid: boolean, error?: string }} Validation result + */ +function validateDocToolsCommand(command) { + if (!command || typeof command !== 'string') { + return { valid: false, error: 'Command must be a non-empty string' }; + } + + const dangerousChars = /[;|&$`<>(){}[\]!*?~]/; + if (dangerousChars.test(command)) { + return { + valid: false, + error: 'Invalid command: shell metacharacters not allowed. Use simple doc-tools commands only.' + }; + } + + if (command.includes('..') || command.includes('~')) { + return { + valid: false, + error: 'Invalid command: path traversal sequences not allowed' + }; + } + + return { valid: true }; +} + +module.exports = { + // Core functions + findRepoRoot, + executeTool, + + // Individual tool exports (for testing or direct use) + getAntoraStructure, + getRedpandaVersion, + getConsoleVersion, + generatePropertyDocs, + generateMetricsDocs, + generateRpkDocs, + generateRpConnectDocs, + generateHelmDocs, + generateCloudRegions, + generateCrdDocs, + generateBundleOpenApi, + reviewGeneratedDocs, + generateReviewReport, + + // Job queue + initializeJobQueue, + createJob, + getJob, + listJobs, + cleanupOldJobs +}; diff --git a/bin/mcp-tools/job-queue.js b/bin/mcp-tools/job-queue.js new file mode 100644 index 00000000..5873d002 --- /dev/null +++ b/bin/mcp-tools/job-queue.js @@ -0,0 +1,468 @@ +/** + * MCP Tools - Background Job Queue + * + * Manages long-running documentation generation jobs with progress tracking + * and streaming updates via MCP notifications. + */ + +const { spawn } = require('child_process'); +const { randomUUID } = require('crypto'); +const { DEFAULT_COMMAND_TIMEOUT } = require('./utils'); + +// Job status enum +const JobStatus = { + PENDING: 'pending', + RUNNING: 'running', + COMPLETED: 'completed', + FAILED: 'failed' +}; + +// In-memory job storage +const jobs = new Map(); + +// Reference to MCP server for sending notifications +let mcpServer = null; + +/** + * Initialize the job queue with MCP server reference + * @param {Object} server - MCP server instance + */ +function initializeJobQueue(server) { + mcpServer = server; +} + +/** + * Create a new background job + * @param {string} toolName - Name of the tool + * @param {string|Array} command - Command to execute (array format preferred for security) + * @param {Object} options - Execution options + * @returns {string} Job ID + */ +function createJob(toolName, command, options = {}) { + const jobId = randomUUID(); + + const job = { + id: jobId, + tool: toolName, + command, + status: JobStatus.PENDING, + createdAt: new Date().toISOString(), + startedAt: null, + completedAt: null, + progress: 0, + progressMessage: 'Job queued', + output: '', + error: null, + result: null + }; + + jobs.set(jobId, job); + + // Start job execution immediately (can be changed to queue-based if needed) + executeJob(jobId, command, options); + + return jobId; +} + +/** + * Parse a command string into executable and arguments + * Handles basic quoted arguments safely + */ +function parseCommand(command) { + const args = []; + let current = ''; + let inQuotes = false; + let quoteChar = ''; + + for (let i = 0; i < command.length; i++) { + const char = command[i]; + + if (!inQuotes && (char === '"' || char === "'")) { + inQuotes = true; + quoteChar = char; + } else if (inQuotes && char === quoteChar) { + inQuotes = false; + quoteChar = ''; + } else if (!inQuotes && char === ' ') { + if (current.trim()) { + args.push(current.trim()); + current = ''; + } + } else { + current += char; + } + } + + if (current.trim()) { + args.push(current.trim()); + } + + return args; +} + +/** + * Validate that an executable is safe to run + * Whitelist known safe executables + */ +function isValidExecutable(executable) { + const allowedExecutables = [ + 'npx', + 'node', + 'npm', + 'yarn', + 'doc-tools', + 'helm-docs', + 'crd-ref-docs', + 'git', + 'make', + 'docker', + 'timeout' + ]; + + // Allow absolute paths to known tools in common locations + const allowedPaths = [ + '/usr/bin/', + '/usr/local/bin/', + '/bin/', + '/opt/homebrew/bin/' + ]; + + // Check if it's a whitelisted executable + if (allowedExecutables.includes(executable)) { + return true; + } + + // Check if it's an absolute path to a whitelisted location + if (executable.startsWith('/')) { + return allowedPaths.some(path => + executable.startsWith(path) && + allowedExecutables.some(exe => executable.endsWith(`/${exe}`) || executable.endsWith(`/${exe}.exe`)) + ); + } + + return false; +} + +/** + * Execute a job in the background + * @param {string} jobId - Job ID + * @param {string|Array} command - Command to execute (string will be parsed, array is preferred) + * @param {Object} options - Execution options + */ +async function executeJob(jobId, command, options = {}) { + const job = jobs.get(jobId); + if (!job) return; + + try { + job.status = JobStatus.RUNNING; + job.startedAt = new Date().toISOString(); + updateJobProgress(jobId, 10, 'Starting execution...'); + + const cwd = options.cwd || process.cwd(); + const timeout = options.timeout || DEFAULT_COMMAND_TIMEOUT; + + let executable, args; + + if (Array.isArray(command)) { + // Preferred: pre-parsed array [executable, ...args] + [executable, ...args] = command; + } else if (typeof command === 'string') { + // Legacy string command - use safer parsing + // Basic parsing that handles simple quoted arguments + const parsedArgs = parseCommand(command); + [executable, ...args] = parsedArgs; + } else { + throw new Error('Command must be a string or array'); + } + + // Validate executable to prevent injection + if (!isValidExecutable(executable)) { + throw new Error(`Invalid executable: ${executable}`); + } + + const childProcess = spawn(executable, args, { + cwd, + shell: false, // Explicitly disable shell to prevent injection + stdio: ['ignore', 'pipe', 'pipe'] + }); + + let stdout = ''; + let stderr = ''; + let timedOut = false; + + // Set up timeout + const timeoutHandle = setTimeout(() => { + timedOut = true; + childProcess.kill('SIGTERM'); + job.error = `Job timed out after ${timeout}ms`; + job.status = JobStatus.FAILED; + job.completedAt = new Date().toISOString(); + job.result = { + success: false, + error: job.error, + stdout: stdout.trim(), + stderr: stderr.trim() + }; + updateJobProgress(jobId, 100, 'Job timed out'); + }, timeout); + + // Capture stdout + childProcess.stdout.on('data', (data) => { + const chunk = data.toString(); + stdout += chunk; + job.output = stdout; + + // Parse progress from output if available + parseProgressFromOutput(jobId, chunk); + }); + + // Capture stderr + childProcess.stderr.on('data', (data) => { + stderr += data.toString(); + }); + + // Handle completion + childProcess.on('close', (code) => { + clearTimeout(timeoutHandle); + + // If job already timed out, don't overwrite the timeout error + if (timedOut) { + return; + } + + job.completedAt = new Date().toISOString(); + + if (code === 0) { + job.status = JobStatus.COMPLETED; + job.progress = 100; + job.progressMessage = 'Completed successfully'; + job.result = { + success: true, + output: stdout.trim(), + command + }; + updateJobProgress(jobId, 100, 'Completed successfully'); + } else { + job.status = JobStatus.FAILED; + job.error = stderr || `Command exited with code ${code}`; + job.result = { + success: false, + error: job.error, + stdout: stdout.trim(), + stderr: stderr.trim(), + exitCode: code + }; + updateJobProgress(jobId, 100, `Failed with exit code ${code}`); + } + }); + + // Handle errors + childProcess.on('error', (err) => { + clearTimeout(timeoutHandle); + + // If job already timed out, don't overwrite the timeout error + if (timedOut) { + return; + } + + job.status = JobStatus.FAILED; + job.error = err.message; + job.completedAt = new Date().toISOString(); + job.result = { + success: false, + error: err.message + }; + updateJobProgress(jobId, 100, `Error: ${err.message}`); + }); + } catch (err) { + // Catch synchronous errors (validation failures, etc.) + // Record them on the job instead of throwing + job.status = JobStatus.FAILED; + job.error = err.message; + job.completedAt = new Date().toISOString(); + job.result = { + success: false, + error: err.message + }; + updateJobProgress(jobId, 100, `Error: ${err.message}`); + } +} + +/** + * Parse progress information from command output + * @param {string} jobId - Job ID + * @param {string} output - Output chunk + */ +function parseProgressFromOutput(jobId, output) { + const job = jobs.get(jobId); + if (!job) return; + + // Look for common progress patterns + const patterns = [ + // Percentage: "Progress: 45%", "45%", "[45%]" + /(?:progress[:\s]*)?(\d+)%/i, + // Step indicators: "Step 3/5", "3 of 5" + /(?:step\s+)?(\d+)\s*(?:\/|of)\s*(\d+)/i, + // Processing indicators: "Processing file 3 of 10" + /processing.*?(\d+)\s*of\s*(\d+)/i, + // Cloning/downloading indicators + /(?:cloning|downloading|fetching)/i, + // Building indicators + /(?:building|compiling|generating)/i, + // Analyzing indicators + /(?:analyzing|parsing|extracting)/i + ]; + + for (const pattern of patterns) { + const match = output.match(pattern); + if (match) { + if (match.length === 2) { + // Percentage match + const percentage = parseInt(match[1]); + if (percentage >= 0 && percentage <= 100) { + updateJobProgress(jobId, percentage, output.trim().split('\n').pop()); + return; + } + } else if (match.length === 3) { + // Step match (for example, "3/5") + const current = parseInt(match[1]); + const total = parseInt(match[2]); + const percentage = Math.round((current / total) * 100); + updateJobProgress(jobId, percentage, output.trim().split('\n').pop()); + return; + } + } + } + + // If we find action keywords but no percentage, estimate progress based on job runtime + const actionKeywords = ['cloning', 'downloading', 'fetching', 'building', 'compiling', 'generating', 'analyzing', 'parsing', 'extracting']; + const hasAction = actionKeywords.some(keyword => output.toLowerCase().includes(keyword)); + + if (hasAction && job.progress < 90) { + // Gradually increase progress for long-running jobs + const elapsed = new Date() - new Date(job.startedAt); + const estimatedTotal = DEFAULT_COMMAND_TIMEOUT; + const estimatedProgress = Math.min(90, Math.round((elapsed / estimatedTotal) * 100)); + + if (estimatedProgress > job.progress) { + updateJobProgress(jobId, estimatedProgress, output.trim().split('\n').pop()); + } + } +} + +/** + * Update job progress and send notification + * @param {string} jobId - Job ID + * @param {number} progress - Progress percentage (0-100) + * @param {string} message - Progress message + */ +function updateJobProgress(jobId, progress, message) { + const job = jobs.get(jobId); + if (!job) return; + + job.progress = Math.min(100, Math.max(0, progress)); + job.progressMessage = message; + + // Send MCP notification if server is initialized + if (mcpServer) { + try { + mcpServer.notification({ + method: 'notifications/progress', + params: { + progressToken: jobId, + progress: job.progress, + total: 100, + message: message + } + }); + } catch (err) { + // Ignore notification errors - they shouldn't stop the job + console.error(`Failed to send progress notification: ${err.message}`); + } + } +} + +/** + * Get job status + * @param {string} jobId - Job ID + * @returns {Object|null} Job status or null if not found + */ +function getJob(jobId) { + const job = jobs.get(jobId); + if (!job) return null; + + return { + id: job.id, + tool: job.tool, + status: job.status, + progress: job.progress, + progressMessage: job.progressMessage, + createdAt: job.createdAt, + startedAt: job.startedAt, + completedAt: job.completedAt, + error: job.error, + result: job.result + }; +} + +/** + * Get all jobs + * @param {Object} filter - Optional filter + * @returns {Array} List of jobs + */ +function listJobs(filter = {}) { + const jobList = Array.from(jobs.values()); + + return jobList + .filter(job => { + if (filter.status && job.status !== filter.status) return false; + if (filter.tool && job.tool !== filter.tool) return false; + return true; + }) + .map(job => ({ + id: job.id, + tool: job.tool, + status: job.status, + progress: job.progress, + progressMessage: job.progressMessage, + createdAt: job.createdAt, + startedAt: job.startedAt, + completedAt: job.completedAt + })) + .sort((a, b) => new Date(b.createdAt) - new Date(a.createdAt)); +} + +/** + * Clean up old completed/failed jobs + * @param {number} maxAge - Maximum age in milliseconds (default: 1 hour) + */ +function cleanupOldJobs(maxAge = 60 * 60 * 1000) { + const now = Date.now(); + let removed = 0; + + for (const [jobId, job] of jobs.entries()) { + if (job.status === JobStatus.COMPLETED || job.status === JobStatus.FAILED) { + const jobTime = new Date(job.completedAt || job.createdAt).getTime(); + if (now - jobTime > maxAge) { + jobs.delete(jobId); + removed++; + } + } + } + + return removed; +} + +// Clean up old jobs every 10 minutes +setInterval(() => { + cleanupOldJobs(); +}, 10 * 60 * 1000); + +module.exports = { + JobStatus, + initializeJobQueue, + createJob, + getJob, + listJobs, + cleanupOldJobs +}; diff --git a/bin/mcp-tools/mcp-validation.js b/bin/mcp-tools/mcp-validation.js new file mode 100644 index 00000000..76d9a7c8 --- /dev/null +++ b/bin/mcp-tools/mcp-validation.js @@ -0,0 +1,266 @@ +/** + * MCP Configuration Validation + * + * Validates prompts, resources, and overall MCP server configuration. + * Used at startup and by the validate-mcp CLI command. + */ + +const fs = require('fs'); +const path = require('path'); +const { parsePromptFile } = require('./frontmatter'); + +/** + * Validate that a prompt name is safe (no path traversal) + * @param {string} name - Prompt name to validate + * @throws {Error} If name is invalid + */ +function validatePromptName(name) { + if (!name || typeof name !== 'string') { + throw new Error('Prompt name must be a non-empty string'); + } + + // Only allow alphanumeric, hyphens, and underscores + if (!/^[a-z0-9_-]+$/i.test(name)) { + throw new Error( + `Invalid prompt name: ${name}. Use only letters, numbers, hyphens, and underscores.` + ); + } + + return name; +} + +/** + * Validate that required arguments are provided + * @param {string} promptName - Name of the prompt + * @param {Object} providedArgs - Arguments provided by user + * @param {Array} schema - Argument schema from prompt metadata + * @throws {Error} If validation fails + */ +function validatePromptArguments(promptName, providedArgs, schema) { + if (!schema || schema.length === 0) { + return; // No validation needed + } + + const errors = []; + + // Check required arguments + schema + .filter(arg => arg.required) + .forEach(arg => { + if (!providedArgs || !providedArgs[arg.name]) { + errors.push(`Missing required argument: ${arg.name}`); + } + }); + + if (errors.length > 0) { + throw new Error( + `Invalid arguments for prompt "${promptName}":\n${errors.join('\n')}` + ); + } +} + +/** + * Validate all resources are accessible + * @param {Array} resources - Resource definitions + * @param {Object} resourceMap - Resource file mappings + * @param {string} baseDir - Base directory for resources + * @returns {{ errors: string[], warnings: string[] }} + */ +function validateResources(resources, resourceMap, baseDir) { + const errors = []; + const warnings = []; + + for (const resource of resources) { + // Check mapping exists + const mapping = resourceMap[resource.uri]; + if (!mapping) { + errors.push(`Resource ${resource.uri} has no file mapping in resourceMap`); + continue; + } + + // Check file exists + const filePath = path.join(baseDir, 'mcp', 'team-standards', mapping.file); + if (!fs.existsSync(filePath)) { + errors.push(`Resource file missing: ${mapping.file} for ${resource.uri}`); + continue; + } + + // Check file is readable + try { + fs.readFileSync(filePath, 'utf8'); + } catch (err) { + errors.push(`Resource file not readable: ${mapping.file} - ${err.message}`); + continue; + } + + // Validate resource metadata + if (!resource.name || resource.name.length < 3) { + warnings.push(`Resource ${resource.uri} has a very short name`); + } + + if (!resource.description || resource.description.length < 10) { + warnings.push(`Resource ${resource.uri} has a very short description`); + } + + // Check for versioning + if (!resource.version) { + warnings.push(`Resource ${resource.uri} missing version metadata`); + } + + if (!resource.lastUpdated) { + warnings.push(`Resource ${resource.uri} missing lastUpdated metadata`); + } + } + + return { errors, warnings }; +} + +/** + * Validate all prompts are loadable + * @param {Array} prompts - Discovered prompts + * @returns {{ errors: string[], warnings: string[] }} + */ +function validatePrompts(prompts) { + const errors = []; + const warnings = []; + + for (const prompt of prompts) { + // Check basic metadata + if (!prompt.description || prompt.description.length < 10) { + warnings.push(`Prompt "${prompt.name}" has a very short description`); + } + + if (!prompt.content || prompt.content.trim().length < 100) { + warnings.push(`Prompt "${prompt.name}" has very little content (< 100 chars)`); + } + + // Check for version + if (!prompt.version) { + warnings.push(`Prompt "${prompt.name}" missing version metadata`); + } + + // Validate argument format if specified + if (prompt.argumentFormat) { + const validFormats = ['content-append', 'structured']; + if (!validFormats.includes(prompt.argumentFormat)) { + errors.push( + `Prompt "${prompt.name}" has invalid argumentFormat: ${prompt.argumentFormat}` + ); + } + } + + // Check for argument schema consistency + if (prompt.arguments && prompt.arguments.length > 0) { + if (!prompt.argumentFormat) { + warnings.push( + `Prompt "${prompt.name}" has arguments but no argumentFormat specified` + ); + } + + // Check for duplicate argument names + const argNames = new Set(); + for (const arg of prompt.arguments) { + if (argNames.has(arg.name)) { + errors.push(`Prompt "${prompt.name}" has duplicate argument: ${arg.name}`); + } + argNames.add(arg.name); + } + } + } + + return { errors, warnings }; +} + +/** + * Validate entire MCP configuration + * @param {Object} config - Configuration object + * @param {Array} config.resources - Resources + * @param {Object} config.resourceMap - Resource mappings + * @param {Array} config.prompts - Prompts + * @param {string} config.baseDir - Base directory + * @returns {{ valid: boolean, errors: string[], warnings: string[] }} + */ +function validateMcpConfiguration(config) { + const allErrors = []; + const allWarnings = []; + + // Validate resources + const resourceValidation = validateResources( + config.resources, + config.resourceMap, + config.baseDir + ); + allErrors.push(...resourceValidation.errors); + allWarnings.push(...resourceValidation.warnings); + + // Validate prompts + const promptValidation = validatePrompts(config.prompts); + allErrors.push(...promptValidation.errors); + allWarnings.push(...promptValidation.warnings); + + // Check for name collisions + const promptNames = new Set(); + for (const prompt of config.prompts) { + if (promptNames.has(prompt.name)) { + allErrors.push(`Duplicate prompt name: ${prompt.name}`); + } + promptNames.add(prompt.name); + } + + return { + valid: allErrors.length === 0, + errors: allErrors, + warnings: allWarnings + }; +} + +/** + * Format validation results for display + * @param {{ valid: boolean, errors: string[], warnings: string[] }} results + * @returns {string} Formatted output + */ +function formatValidationResults(results, config) { + const lines = []; + + lines.push('MCP Configuration Validation'); + lines.push('='.repeat(60)); + lines.push(''); + + // Summary + lines.push(`Prompts found: ${config.prompts.length}`); + lines.push(`Resources found: ${config.resources.length}`); + lines.push(''); + + // Warnings + if (results.warnings.length > 0) { + lines.push('Warnings:'); + results.warnings.forEach(w => lines.push(` ⚠ ${w}`)); + lines.push(''); + } + + // Errors + if (results.errors.length > 0) { + lines.push('Errors:'); + results.errors.forEach(e => lines.push(` βœ— ${e}`)); + lines.push(''); + } + + // Result + if (results.valid) { + lines.push('βœ“ Validation passed'); + } else { + lines.push('βœ— Validation failed'); + lines.push(` ${results.errors.length} error(s) must be fixed`); + } + + return lines.join('\n'); +} + +module.exports = { + validatePromptName, + validatePromptArguments, + validateResources, + validatePrompts, + validateMcpConfiguration, + formatValidationResults +}; diff --git a/bin/mcp-tools/metrics-docs.js b/bin/mcp-tools/metrics-docs.js new file mode 100644 index 00000000..134f0c48 --- /dev/null +++ b/bin/mcp-tools/metrics-docs.js @@ -0,0 +1,142 @@ +/** + * MCP Tools - Metrics Documentation Generation + */ + +const { spawnSync } = require('child_process'); +const { findRepoRoot, getDocToolsCommand, MAX_EXEC_BUFFER_SIZE, DEFAULT_COMMAND_TIMEOUT } = require('./utils'); +const { getAntoraStructure } = require('./antora'); +const { createJob } = require('./job-queue'); + +/** + * Generate Redpanda metrics documentation + * + * Use tags for released content (GA or beta), branches for in-progress content. + * Defaults to branch "dev" if neither tag nor branch is provided. + * + * @param {Object} args - Arguments + * @param {string} [args.tag] - Git tag for released content (for example, "v25.3.1") + * @param {string} [args.branch] - Branch name for in-progress content (for example, "dev", "main") + * @param {boolean} [args.background] - Run as background job + * @returns {Object} Generation results + */ +function generateMetricsDocs(args) { + const repoRoot = findRepoRoot(); + const structure = getAntoraStructure(repoRoot); + + if (!structure.hasDocTools) { + return { + success: false, + error: 'doc-tools not found in this repository', + suggestion: 'Navigate to the docs-extensions-and-macros repository' + }; + } + + // Validate that tag and branch are mutually exclusive + if (args.tag && args.branch) { + return { + success: false, + error: 'Cannot specify both tag and branch', + suggestion: 'Use either --tag or --branch, not both' + }; + } + + // Default to 'dev' branch if neither provided + const gitRef = args.tag || args.branch || 'dev'; + const refType = args.tag ? 'tag' : 'branch'; + + // Normalize version (add 'v' prefix if tag and not present) + let version = gitRef; + if (args.tag && !version.startsWith('v')) { + version = `v${version}`; + } + + // Validate version string to prevent command injection + const versionRegex = /^[0-9A-Za-z._\/-]+$/; + if (!versionRegex.test(version)) { + return { + success: false, + error: 'Invalid version format', + suggestion: 'Version must contain only alphanumeric characters, dots, underscores, slashes, and hyphens (for example, "v25.3.1", "v25.3.1-rc1", "dev", "main")' + }; + } + + // Get doc-tools command (handles both local and installed) + const docTools = getDocToolsCommand(repoRoot); + + // Build command arguments + const baseArgs = ['generate', 'metrics-docs']; + + if (args.tag) { + baseArgs.push('--tag'); + baseArgs.push(version); + } else { + baseArgs.push('--branch'); + baseArgs.push(version); + } + + // If background mode, create job and return immediately + if (args.background) { + const cmdArgs = [docTools.program, ...docTools.getArgs(baseArgs)]; + const jobId = createJob('generate_metrics_docs', cmdArgs, { + cwd: repoRoot.root + }); + + return { + success: true, + background: true, + job_id: jobId, + message: `Metrics docs generation started in background. Use get_job_status with job_id: ${jobId} to check progress.`, + [refType]: version + }; + } + + // Otherwise run synchronously + try { + const result = spawnSync(docTools.program, docTools.getArgs(baseArgs), { + cwd: repoRoot.root, + encoding: 'utf8', + stdio: 'pipe', + maxBuffer: MAX_EXEC_BUFFER_SIZE, + timeout: DEFAULT_COMMAND_TIMEOUT + }); + + // Check for spawn errors + if (result.error) { + throw new Error(`Failed to execute command: ${result.error.message}`); + } + + // Check for non-zero exit codes + if (result.status !== 0) { + const errorMsg = result.stderr || `Command failed with exit code ${result.status}`; + throw new Error(errorMsg); + } + + const output = result.stdout; + + const metricsCountMatch = output.match(/(\d+) metrics/i); + + return { + success: true, + [refType]: version, + files_generated: [ + 'modules/reference/pages/public-metrics-reference.adoc' + ], + metrics_count: metricsCountMatch ? parseInt(metricsCountMatch[1]) : null, + output: output.trim(), + summary: `Generated metrics documentation for Redpanda ${refType} ${version}` + }; + } catch (err) { + return { + success: false, + error: err.message, + stdout: err.stdout || '', + stderr: err.stderr || '', + exitCode: err.status, + suggestion: 'Check that the version exists in the Redpanda repository' + }; + } +} + +module.exports = { + generateMetricsDocs +}; diff --git a/bin/mcp-tools/openapi.js b/bin/mcp-tools/openapi.js new file mode 100644 index 00000000..a10c06a5 --- /dev/null +++ b/bin/mcp-tools/openapi.js @@ -0,0 +1,174 @@ +/** + * MCP Tools - OpenAPI Bundle Generation + */ + +const { spawnSync } = require('child_process'); +const { findRepoRoot, MAX_EXEC_BUFFER_SIZE, normalizeVersion, DEFAULT_COMMAND_TIMEOUT } = require('./utils'); +const { getAntoraStructure } = require('./antora'); + +/** + * Generate bundled OpenAPI documentation + * + * Use tags for released content (GA or beta), branches for in-progress content. + * Requires either --tag or --branch to be specified. + * + * @param {Object} args - Arguments + * @param {string} [args.tag] - Git tag for released content (for example, "v24.3.2" or "24.3.2") + * @param {string} [args.branch] - Branch name for in-progress content (for example, "dev", "main") + * @param {string} [args.repo] - Repository URL + * @param {string} [args.surface] - Which API surface(s) to bundle: 'admin', 'connect', or 'both' + * @param {string} [args.out_admin] - Output path for admin API + * @param {string} [args.out_connect] - Output path for connect API + * @param {string} [args.admin_major] - Admin API major version + * @param {boolean} [args.use_admin_major_version] - Use admin major version for info.version + * @param {boolean} [args.quiet] - Suppress logs + * @returns {Object} Generation results + */ +function generateBundleOpenApi(args) { + const repoRoot = findRepoRoot(); + const structure = getAntoraStructure(repoRoot); + + if (!structure.hasDocTools) { + return { + success: false, + error: 'doc-tools not found in this repository', + suggestion: 'Navigate to the docs-extensions-and-macros repository' + }; + } + + // Validate that either tag or branch is provided (but not both) + if (!args.tag && !args.branch) { + return { + success: false, + error: 'Either tag or branch is required', + suggestion: 'Provide --tag "v24.3.2" or --branch "dev"' + }; + } + + if (args.tag && args.branch) { + return { + success: false, + error: 'Cannot specify both tag and branch', + suggestion: 'Use either --tag or --branch, not both' + }; + } + + try { + const gitRef = args.tag || args.branch; + const refType = args.tag ? 'tag' : 'branch'; + + // Normalize version (add 'v' prefix if tag and not present and not branch-like names) + let version = gitRef; + if (args.tag && !version.startsWith('v') && version !== 'dev' && version !== 'main') { + version = `v${version}`; + } + + // Build command arguments array (no shell interpolation) + const cmdArgs = ['doc-tools', 'generate', 'bundle-openapi']; + + // Add flags only when present, each as separate array entries + if (args.tag) { + cmdArgs.push('--tag'); + cmdArgs.push(version); + } else { + cmdArgs.push('--branch'); + cmdArgs.push(version); + } + + if (args.repo) { + cmdArgs.push('--repo'); + cmdArgs.push(args.repo); + } + + if (args.surface) { + cmdArgs.push('--surface'); + cmdArgs.push(args.surface); + } + + if (args.out_admin) { + cmdArgs.push('--out-admin'); + cmdArgs.push(args.out_admin); + } + + if (args.out_connect) { + cmdArgs.push('--out-connect'); + cmdArgs.push(args.out_connect); + } + + if (args.admin_major) { + cmdArgs.push('--admin-major'); + cmdArgs.push(args.admin_major); + } + + if (args.use_admin_major_version) { + cmdArgs.push('--use-admin-major-version'); + } + + if (args.quiet) { + cmdArgs.push('--quiet'); + } + + const result = spawnSync('npx', cmdArgs, { + cwd: repoRoot.root, + encoding: 'utf8', + stdio: 'pipe', + maxBuffer: MAX_EXEC_BUFFER_SIZE, + timeout: DEFAULT_COMMAND_TIMEOUT + }); + + // Check for spawn errors + if (result.error) { + const err = new Error(`Failed to execute command: ${result.error.message}`); + err.stdout = result.stdout || ''; + err.stderr = result.stderr || ''; + err.status = result.status; + throw err; + } + + // Check for non-zero exit codes + if (result.status !== 0) { + const errorMsg = result.stderr || `Command failed with exit code ${result.status}`; + const err = new Error(errorMsg); + err.stdout = result.stdout || ''; + err.stderr = result.stderr || ''; + err.status = result.status; + throw err; + } + + const output = result.stdout; + + // Determine which files were generated based on surface + const filesGenerated = []; + const surface = args.surface || 'both'; + + if (surface === 'admin' || surface === 'both') { + filesGenerated.push(args.out_admin || 'admin/redpanda-admin-api.yaml'); + } + + if (surface === 'connect' || surface === 'both') { + filesGenerated.push(args.out_connect || 'connect/redpanda-connect-api.yaml'); + } + + return { + success: true, + [refType]: version, + surface, + files_generated: filesGenerated, + output: output.trim(), + summary: `Bundled OpenAPI documentation for ${surface} API(s) at ${refType} ${version}` + }; + } catch (err) { + return { + success: false, + error: err.message, + stdout: err.stdout || '', + stderr: err.stderr || '', + exitCode: err.status, + suggestion: 'Check that the tag exists in the Redpanda repository' + }; + } +} + +module.exports = { + generateBundleOpenApi +}; diff --git a/bin/mcp-tools/prompt-discovery.js b/bin/mcp-tools/prompt-discovery.js new file mode 100644 index 00000000..9098da2f --- /dev/null +++ b/bin/mcp-tools/prompt-discovery.js @@ -0,0 +1,283 @@ +/** + * Prompt Discovery and Caching + * + * Automatically discovers prompts from the mcp/prompts directory. + * Caches prompt content and metadata for performance. + * Supports file watching in development mode. + */ + +const fs = require('fs'); +const path = require('path'); +const { parsePromptFile, validateFrontmatter } = require('./frontmatter'); +const { validatePromptName } = require('./mcp-validation'); + +/** + * In-memory prompt cache + */ +class PromptCache { + constructor() { + this.prompts = new Map(); // name -> { metadata, content, filePath } + this.watchers = []; + } + + /** + * Set prompt in cache + * @param {string} name - Prompt name + * @param {Object} data - Prompt data + */ + set(name, data) { + this.prompts.set(name, data); + } + + /** + * Get prompt from cache + * @param {string} name - Prompt name + * @returns {Object|null} Prompt data or null + */ + get(name) { + return this.prompts.get(name) || null; + } + + /** + * Get all prompts + * @returns {Array} Array of prompt objects + */ + getAll() { + return Array.from(this.prompts.values()); + } + + /** + * Get all prompt names + * @returns {Array} Array of prompt names + */ + getNames() { + return Array.from(this.prompts.keys()); + } + + /** + * Check if prompt exists + * @param {string} name - Prompt name + * @returns {boolean} + */ + has(name) { + return this.prompts.has(name); + } + + /** + * Clear all prompts + */ + clear() { + this.prompts.clear(); + } + + /** + * Stop all file watchers + */ + stopWatching() { + this.watchers.forEach(watcher => watcher.close()); + this.watchers = []; + } + + /** + * Add a file watcher + * @param {FSWatcher} watcher - File system watcher + */ + addWatcher(watcher) { + this.watchers.push(watcher); + } +} + +/** + * Default argument formatters for different types + */ +const argumentFormatters = { + 'content-append': (args, schema) => { + if (!args) return ''; + + let result = '\n\n---\n\n'; + + // If there's a 'content' argument, add it specially + if (args.content) { + result += `**Content to review:**\n\n${args.content}`; + return result; + } + + // Otherwise, format all arguments + Object.entries(args).forEach(([key, value]) => { + const argDef = schema?.find(a => a.name === key); + const label = argDef?.name || key; + result += `**${label.charAt(0).toUpperCase() + label.slice(1)}:** ${value}\n`; + }); + + return result; + }, + + 'structured': (args, schema) => { + if (!args) return ''; + + let result = '\n\n---\n\n'; + Object.entries(args).forEach(([key, value]) => { + result += `**${key}:** ${value}\n`; + }); + + return result; + } +}; + +/** + * Discover all prompts in the prompts directory + * @param {string} promptsDir - Path to prompts directory + * @returns {Array} Array of discovered prompts + */ +function discoverPrompts(promptsDir) { + if (!fs.existsSync(promptsDir)) { + throw new Error(`Prompts directory not found: ${promptsDir}`); + } + + const files = fs.readdirSync(promptsDir).filter(f => f.endsWith('.md')); + const prompts = []; + + for (const file of files) { + try { + const name = path.basename(file, '.md'); + validatePromptName(name); // Ensure safe name + + const filePath = path.join(promptsDir, file); + const fileContent = fs.readFileSync(filePath, 'utf8'); + + const { metadata, content } = parsePromptFile(fileContent, file); + + // Build prompt object + const prompt = { + name, + description: metadata.description || `Prompt: ${name}`, + version: metadata.version || '1.0.0', + arguments: metadata.arguments || [], + argumentFormat: metadata.argumentFormat || 'content-append', + content, + filePath, + _rawMetadata: metadata + }; + + prompts.push(prompt); + } catch (err) { + console.error(`Error loading prompt ${file}: ${err.message}`); + // Continue loading other prompts + } + } + + return prompts; +} + +/** + * Load all prompts into cache + * @param {string} baseDir - Base directory (repo root) + * @param {PromptCache} cache - Prompt cache instance + * @returns {Array} Loaded prompts + */ +function loadAllPrompts(baseDir, cache) { + const promptsDir = path.join(baseDir, 'mcp', 'prompts'); + const prompts = discoverPrompts(promptsDir); + + // Clear and reload cache + cache.clear(); + + prompts.forEach(prompt => { + cache.set(prompt.name, prompt); + }); + + return prompts; +} + +/** + * Watch prompts directory for changes (development mode) + * @param {string} baseDir - Base directory + * @param {PromptCache} cache - Prompt cache instance + * @param {Function} onChange - Callback when prompts change + */ +function watchPrompts(baseDir, cache, onChange) { + const promptsDir = path.join(baseDir, 'mcp', 'prompts'); + + if (!fs.existsSync(promptsDir)) { + console.error(`Cannot watch prompts directory (not found): ${promptsDir}`); + return; + } + + const watcher = fs.watch(promptsDir, (eventType, filename) => { + if (!filename || !filename.endsWith('.md')) { + return; + } + + console.error(`Prompt file changed: ${filename} (${eventType})`); + console.error('Reloading all prompts...'); + + try { + const prompts = loadAllPrompts(baseDir, cache); + console.error(`Reloaded ${prompts.length} prompts`); + + if (onChange) { + onChange(prompts); + } + } catch (err) { + console.error(`Error reloading prompts: ${err.message}`); + } + }); + + cache.addWatcher(watcher); + console.error('File watching enabled for prompts (dev mode)'); +} + +/** + * Build prompt text with arguments + * @param {Object} prompt - Prompt object from cache + * @param {Object} args - Arguments provided by user + * @returns {string} Complete prompt text + */ +function buildPromptWithArguments(prompt, args) { + let promptText = prompt.content; + + if (!args || Object.keys(args).length === 0) { + return promptText; + } + + // Use the formatter specified by the prompt + const formatter = argumentFormatters[prompt.argumentFormat]; + if (!formatter) { + console.error( + `Unknown argument format: ${prompt.argumentFormat} for prompt ${prompt.name}` + ); + return promptText; + } + + const formattedArgs = formatter(args, prompt.arguments); + promptText += formattedArgs; + + return promptText; +} + +/** + * Convert prompts to MCP protocol format + * @param {Array} prompts - Discovered prompts + * @returns {Array} Prompts in MCP format + */ +function promptsToMcpFormat(prompts) { + return prompts.map(prompt => ({ + name: prompt.name, + description: prompt.description, + arguments: prompt.arguments.map(arg => ({ + name: arg.name, + description: arg.description, + required: arg.required + })) + })); +} + +module.exports = { + PromptCache, + discoverPrompts, + loadAllPrompts, + watchPrompts, + buildPromptWithArguments, + promptsToMcpFormat, + argumentFormatters +}; diff --git a/bin/mcp-tools/property-docs.js b/bin/mcp-tools/property-docs.js new file mode 100644 index 00000000..51636d18 --- /dev/null +++ b/bin/mcp-tools/property-docs.js @@ -0,0 +1,153 @@ +/** + * MCP Tools - Property Documentation Generation + */ + +const { spawnSync } = require('child_process'); +const { findRepoRoot, getDocToolsCommand, MAX_EXEC_BUFFER_SIZE, DEFAULT_COMMAND_TIMEOUT } = require('./utils'); +const { getAntoraStructure } = require('./antora'); +const { createJob } = require('./job-queue'); + +/** + * Generate Redpanda property documentation + * + * Use tags for released content (GA or beta), branches for in-progress content. + * Defaults to branch "dev" if neither tag nor branch is provided. + * + * @param {Object} args - Arguments + * @param {string} [args.tag] - Git tag for released content (for example, "v25.3.1") + * @param {string} [args.branch] - Branch name for in-progress content (for example, "dev", "main") + * @param {boolean} [args.generate_partials] - Whether to generate AsciiDoc partials + * @param {boolean} [args.background] - Run as background job + * @returns {Object} Generation results + */ +function generatePropertyDocs(args) { + const repoRoot = findRepoRoot(); + const structure = getAntoraStructure(repoRoot); + + if (!structure.hasDocTools) { + return { + success: false, + error: 'doc-tools not found in this repository', + suggestion: 'Navigate to the docs-extensions-and-macros repository' + }; + } + + // Validate that tag and branch are mutually exclusive + if (args.tag && args.branch) { + return { + success: false, + error: 'Cannot specify both tag and branch', + suggestion: 'Use either --tag or --branch, not both' + }; + } + + // Default to 'dev' branch if neither provided + const gitRef = args.tag || args.branch || 'dev'; + const refType = args.tag ? 'tag' : 'branch'; + + // Normalize version (add 'v' prefix if tag and not present and not "latest") + let version = gitRef; + if (args.tag && version !== 'latest' && !version.startsWith('v')) { + version = `v${version}`; + } + + // Get doc-tools command (handles both local and installed) + const docTools = getDocToolsCommand(repoRoot); + + // Build command arguments + const baseArgs = ['generate', 'property-docs']; + + if (args.tag) { + baseArgs.push('--tag'); + baseArgs.push(version); + } else { + baseArgs.push('--branch'); + baseArgs.push(version); + } + + if (args.generate_partials) { + baseArgs.push('--generate-partials'); + } + + // If background mode, create job and return immediately + if (args.background) { + const cmdArgs = [docTools.program, ...docTools.getArgs(baseArgs)]; + const jobId = createJob('generate_property_docs', cmdArgs, { + cwd: repoRoot.root + }); + + return { + success: true, + background: true, + job_id: jobId, + message: `Property docs generation started in background. Use get_job_status with job_id: ${jobId} to check progress.`, + [refType]: version + }; + } + + // Otherwise run synchronously + try { + const result = spawnSync(docTools.program, docTools.getArgs(baseArgs), { + cwd: repoRoot.root, + encoding: 'utf8', + stdio: 'pipe', + maxBuffer: MAX_EXEC_BUFFER_SIZE, + timeout: DEFAULT_COMMAND_TIMEOUT + }); + + // Check for spawn errors + if (result.error) { + const err = new Error(`Failed to execute command: ${result.error.message}`); + err.stdout = result.stdout || ''; + err.stderr = result.stderr || ''; + err.status = result.status; + throw err; + } + + // Check for non-zero exit codes + if (result.status !== 0) { + const errorMsg = result.stderr || `Command failed with exit code ${result.status}`; + const err = new Error(errorMsg); + err.stdout = result.stdout || ''; + err.stderr = result.stderr || ''; + err.status = result.status; + throw err; + } + + const output = result.stdout; + + // Parse output to extract useful information + const propertyCountMatch = output.match(/(\d+) properties/i); + const filesGenerated = []; + + if (args.generate_partials) { + filesGenerated.push('modules/reference/partials/cluster-properties.adoc'); + filesGenerated.push('modules/reference/partials/broker-properties.adoc'); + filesGenerated.push('modules/reference/partials/tunable-properties.adoc'); + filesGenerated.push('modules/reference/partials/topic-properties.adoc'); + } + filesGenerated.push('modules/reference/partials/properties.json'); + + return { + success: true, + [refType]: version, + files_generated: filesGenerated, + property_count: propertyCountMatch ? parseInt(propertyCountMatch[1]) : null, + output: output.trim(), + summary: `Generated property documentation for Redpanda ${refType} ${version}` + }; + } catch (err) { + return { + success: false, + error: err.message, + stdout: err.stdout || '', + stderr: err.stderr || '', + exitCode: err.status, + suggestion: 'Check that the version exists in the Redpanda repository' + }; + } +} + +module.exports = { + generatePropertyDocs +}; diff --git a/bin/mcp-tools/review.js b/bin/mcp-tools/review.js new file mode 100644 index 00000000..f9ddf0a4 --- /dev/null +++ b/bin/mcp-tools/review.js @@ -0,0 +1,718 @@ +/** + * MCP Tools - Documentation Review + */ + +const fs = require('fs'); +const path = require('path'); +const { findRepoRoot, formatDate } = require('./utils'); + +/** + * Sanitize version string to prevent path traversal + * @param {string} version - Version string to sanitize + * @returns {string} Sanitized version string + * @throws {Error} If version contains invalid characters + */ +function sanitizeVersion(version) { + if (!version || typeof version !== 'string') { + throw new Error('Version must be a non-empty string'); + } + + // Reject path separators + if (version.includes('/') || version.includes('\\')) { + throw new Error('Version cannot contain path separators (/ or \\)'); + } + + // Reject path traversal sequences + if (version.includes('..')) { + throw new Error('Version cannot contain path traversal sequences (..)'); + } + + // Reject null bytes + if (version.includes('\0')) { + throw new Error('Version cannot contain null bytes'); + } + + // Whitelist: only allow alphanumeric, dots, dashes, and underscores + const sanitized = version.replace(/[^a-zA-Z0-9._-]/g, '_'); + + // Ensure the sanitized version is not empty and not too long + if (sanitized.length === 0) { + throw new Error('Version contains only invalid characters'); + } + + if (sanitized.length > 100) { + throw new Error('Version string is too long (max 100 characters)'); + } + + return sanitized; +} + +/** + * Generate an AsciiDoc report from review results + * @param {Object} results - Review results + * @param {string} outputPath - Path to save the report + * @returns {string} Generated report content + */ +function generateReviewReport(results, outputPath) { + const { doc_type, version, quality_score, issues, suggestions, files_analyzed } = results; + + const errorIssues = issues.filter(i => i.severity === 'error'); + const warningIssues = issues.filter(i => i.severity === 'warning'); + const infoIssues = issues.filter(i => i.severity === 'info'); + + let report = `= Documentation Review Report\n\n`; + report += `[cols="1,3"]\n`; + report += `|===\n`; + report += `| Type | ${doc_type}\n`; + report += `| Version | ${version || 'N/A'}\n`; + report += `| Date | ${formatDate()}\n`; + report += `| Files Analyzed | ${files_analyzed}\n`; + report += `|===\n\n`; + + report += `== Quality Score: ${quality_score}/100\n\n`; + + // Score interpretation + if (quality_score >= 90) { + report += `[NOTE]\n====\n*Excellent* - Documentation quality is very high.\n====\n\n`; + } else if (quality_score >= 75) { + report += `[WARNING]\n====\n*Good* - Documentation quality is acceptable but has room for improvement.\n====\n\n`; + } else if (quality_score >= 50) { + report += `[WARNING]\n====\n*Fair* - Documentation needs improvement in several areas.\n====\n\n`; + } else { + report += `[CAUTION]\n====\n*Poor* - Documentation requires significant improvements.\n====\n\n`; + } + + // Scoring breakdown with detailed calculation + report += `=== Scoring Breakdown\n\n`; + report += `*How the score is calculated:*\n\n`; + + let runningScore = 100; + report += `. Starting score: *${runningScore}*\n`; + + if (errorIssues.length > 0) { + const errorDeduction = errorIssues.reduce((sum, issue) => { + if (issue.issue.includes('enterprise license') || issue.issue.includes('cloud-specific')) return sum + 5; + if (issue.issue.includes('invalid xref')) return sum + 3; + return sum + 3; + }, 0); + runningScore -= errorDeduction; + report += `. Errors: ${errorIssues.length} issues Γ— 3-5 points = -*${errorDeduction}* (now ${runningScore})\n`; + } + + const missingDescCount = warningIssues.filter(i => i.issue === 'Missing description').length; + const shortDescCount = infoIssues.filter(i => i.issue.includes('Very short description')).length; + const exampleCount = infoIssues.filter(i => i.issue.includes('would benefit from an example')).length; + const otherWarningCount = warningIssues.length - missingDescCount; + + if (missingDescCount > 0) { + const deduction = Math.min(20, missingDescCount * 2); + runningScore -= deduction; + report += `. Missing descriptions: ${missingDescCount} issues Γ— 2 points = -*${deduction}* (capped at 20, now ${runningScore})\n`; + } + + if (shortDescCount > 0) { + const deduction = Math.min(10, shortDescCount); + runningScore -= deduction; + report += `. Short descriptions: ${shortDescCount} issues Γ— 1 point = -*${deduction}* (capped at 10, now ${runningScore})\n`; + } + + if (exampleCount > 0) { + const deduction = Math.min(5, Math.floor(exampleCount / 5)); + runningScore -= deduction; + report += `. Missing examples: ${exampleCount} complex properties = -*${deduction}* (1 point per 5 properties, capped at 5, now ${runningScore})\n`; + } + + if (otherWarningCount > 0) { + const deduction = Math.min(otherWarningCount * 2, 10); + runningScore -= deduction; + report += `. Other warnings: ${otherWarningCount} issues Γ— 1-2 points = -*${deduction}* (now ${runningScore})\n`; + } + + report += `\n*Final Score: ${quality_score}/100*\n\n`; + + // Summary + report += `== Summary\n\n`; + report += `* *Total Issues:* ${issues.length}\n`; + report += `** Errors: ${errorIssues.length}\n`; + report += `** Warnings: ${warningIssues.length}\n`; + report += `** Info: ${infoIssues.length}\n\n`; + + // General suggestions + if (suggestions.length > 0) { + report += `=== Key Findings\n\n`; + suggestions.forEach(s => { + report += `* ${s}\n`; + }); + report += `\n`; + } + + // Errors (highest priority) + if (errorIssues.length > 0) { + report += `== Errors (high priority)\n\n`; + report += `These issues violate documentation standards and should be fixed immediately.\n\n`; + + errorIssues.forEach((issue, idx) => { + report += `=== ${idx + 1}. ${issue.property || issue.path || 'General'}\n\n`; + report += `*Issue:* ${issue.issue}\n\n`; + if (issue.suggestion) { + report += `*Fix:* ${issue.suggestion}\n\n`; + } + report += `*File:* \`${issue.file}\`\n\n`; + + // Add specific instructions based on issue type + if (issue.issue.includes('enterprise license')) { + report += `*Action:*\n\n`; + report += `. Open \`docs-data/property-overrides.json\`\n`; + report += `. Find property \`${issue.property}\`\n`; + report += `. Remove the \`include::reference:partial$enterprise-licensed-property.adoc[]\` from the description\n`; + report += `. Regenerate docs\n\n`; + } else if (issue.issue.includes('cloud-specific conditional')) { + report += `*Action:*\n\n`; + report += `. Open \`docs-data/property-overrides.json\`\n`; + report += `. Find property \`${issue.property}\`\n`; + report += `. Remove the \`ifdef::env-cloud\` blocks from the description\n`; + report += `. Cloud-specific info will appear in metadata automatically\n`; + report += `. Regenerate docs\n\n`; + } else if (issue.issue.includes('invalid xref')) { + report += `*Action:*\n\n`; + report += `. Open \`docs-data/property-overrides.json\`\n`; + report += `. Find property \`${issue.property}\`\n`; + report += `. Update xref links to use full Antora resource IDs\n`; + report += `. Example: \`xref:reference:properties/cluster-properties.adoc[Link]\`\n`; + report += `. Regenerate docs\n\n`; + } else if (issue.issue.includes('Invalid $ref')) { + report += `*Action:*\n\n`; + report += `. Open \`docs-data/overrides.json\`\n`; + report += `. Find the reference at \`${issue.path}\`\n`; + report += `. Either add the missing definition or fix the reference\n`; + report += `. Regenerate docs\n\n`; + } + }); + } + + // Warnings + if (warningIssues.length > 0) { + report += `== Warnings\n\n`; + report += `These issues should be addressed to improve documentation quality.\n\n`; + + // Group warnings by issue type + const warningsByType = {}; + warningIssues.forEach(issue => { + const issueType = issue.issue.split(':')[0] || issue.issue; + if (!warningsByType[issueType]) { + warningsByType[issueType] = []; + } + warningsByType[issueType].push(issue); + }); + + Object.entries(warningsByType).forEach(([type, typeIssues]) => { + report += `=== ${type} (${typeIssues.length})\n\n`; + + if (type === 'Missing description') { + report += `*Fix:* Add descriptions to these properties in \`docs-data/property-overrides.json\`\n\n`; + report += `*Properties needing descriptions:*\n\n`; + typeIssues.forEach(issue => { + report += `* \`${issue.property}\`\n`; + }); + report += `\n`; + } else { + typeIssues.forEach(issue => { + report += `* *${issue.property || issue.path}*: ${issue.issue}\n`; + }); + report += `\n`; + } + }); + } + + // Info items + if (infoIssues.length > 0) { + report += `== Info\n\n`; + report += `These are suggestions for enhancement.\n\n`; + + // Group by issue type + const infoByType = {}; + infoIssues.forEach(issue => { + const issueType = issue.issue.split('(')[0].trim(); + if (!infoByType[issueType]) { + infoByType[issueType] = []; + } + infoByType[issueType].push(issue); + }); + + Object.entries(infoByType).forEach(([type, typeIssues]) => { + report += `=== ${type}\n\n`; + typeIssues.forEach(issue => { + report += `* *${issue.property || issue.path}*: ${issue.issue}\n`; + }); + report += `\n`; + }); + } + + // Next steps + report += `== Next Steps\n\n`; + if (errorIssues.length > 0) { + report += `. *Fix errors first* - Address the ${errorIssues.length} error(s) above\n`; + } + if (warningIssues.length > 0) { + report += `${errorIssues.length > 0 ? '. ' : '. '}*Review warnings* - Prioritize the ${warningIssues.length} warning(s)\n`; + } + const step = errorIssues.length > 0 && warningIssues.length > 0 ? 3 : errorIssues.length > 0 || warningIssues.length > 0 ? 2 : 1; + report += `${step > 1 ? '. ' : '. '}*Regenerate documentation* - After making changes, regenerate the docs\n`; + report += `. *Review again* - Run the review tool again to verify fixes\n\n`; + + // Write report + fs.writeFileSync(outputPath, report, 'utf8'); + + return report; +} + +/** + * Review generated documentation for quality issues + * @param {Object} args - Arguments + * @param {string} args.doc_type - Type of docs to review (properties, metrics, rpk, rpcn_connectors) + * @param {string} args.version - Version of the docs to review (for properties, metrics, rpk) + * @param {boolean} [args.generate_report] - Whether to generate a markdown report file + * @returns {Object} Review results with issues and suggestions + */ +function reviewGeneratedDocs(args) { + const repoRoot = findRepoRoot(); + const { doc_type, version, generate_report } = args; + + if (!doc_type) { + return { + success: false, + error: 'doc_type is required', + suggestion: 'Provide one of: properties, metrics, rpk, rpcn_connectors' + }; + } + + const issues = []; + const suggestions = []; + let filesAnalyzed = 0; + let qualityScore = 100; + + try { + switch (doc_type) { + case 'properties': { + if (!version) { + return { + success: false, + error: 'version is required for property docs review' + }; + } + + // Sanitize version to prevent path traversal + let sanitizedVersion; + try { + sanitizedVersion = sanitizeVersion(version); + } catch (err) { + return { + success: false, + error: `Invalid version: ${err.message}` + }; + } + + // Normalize version + let normalizedVersion = sanitizedVersion; + if (!normalizedVersion.startsWith('v') && normalizedVersion !== 'latest') { + normalizedVersion = `v${normalizedVersion}`; + } + + // Check for generated JSON file + const jsonPath = path.join(repoRoot.root, 'modules', 'reference', 'attachments', `redpanda-properties-${normalizedVersion}.json`); + if (!fs.existsSync(jsonPath)) { + return { + success: false, + error: `Properties JSON not found at ${jsonPath}`, + suggestion: 'Generate property docs first using generate_property_docs tool' + }; + } + + filesAnalyzed++; + + // Read and parse the properties JSON + const propertiesData = JSON.parse(fs.readFileSync(jsonPath, 'utf8')); + const allProperties = Object.values(propertiesData.properties || {}); + + // Properties that typically benefit from examples + const shouldHaveExample = (prop) => { + const name = prop.name.toLowerCase(); + // Properties with specific formats, complex values, or commonly misconfigured + return name.includes('pattern') || + name.includes('regex') || + name.includes('format') || + name.includes('template') || + name.includes('config') || + name.includes('override') || + name.includes('mapping') || + name.includes('filter') || + name.includes('selector') || + (prop.type && prop.type.includes('array')) || + (prop.type && prop.type.includes('object')); + }; + + // Check for missing or short descriptions + let missingDescriptions = 0; + let shortDescriptions = 0; + let emptyDefaults = 0; + let missingExamples = 0; + + allProperties.forEach(prop => { + if (!prop.description || prop.description.trim() === '') { + missingDescriptions++; + if (!prop.is_deprecated) { + issues.push({ + severity: 'warning', + file: jsonPath, + property: prop.name, + issue: 'Missing description' + }); + } + } else if (prop.description.length < 20 && !prop.is_deprecated) { + shortDescriptions++; + issues.push({ + severity: 'info', + file: jsonPath, + property: prop.name, + issue: `Very short description (${prop.description.length} chars): "${prop.description}"` + }); + } + + if ((!prop.default || (typeof prop.default === 'string' && prop.default.trim() === '')) && !prop.is_deprecated && prop.config_scope !== 'broker') { + emptyDefaults++; + } + + // Track properties that should have examples + if (shouldHaveExample(prop) && !prop.is_deprecated) { + missingExamples++; + } + }); + + // Check ALL properties for missing examples + const propertiesNeedingExamples = []; + const overridesPath = path.join(repoRoot.root, 'docs-data', 'property-overrides.json'); + const overrides = fs.existsSync(overridesPath) ? JSON.parse(fs.readFileSync(overridesPath, 'utf8')) : { properties: {} }; + + allProperties.forEach(prop => { + if (shouldHaveExample(prop) && !prop.is_deprecated) { + const override = overrides.properties && overrides.properties[prop.name]; + if (!override || !override.example) { + propertiesNeedingExamples.push(prop.name); + } + } + }); + + // Read property overrides to check for quality issues + if (fs.existsSync(overridesPath)) { + filesAnalyzed++; + + if (overrides.properties) { + Object.entries(overrides.properties).forEach(([propName, override]) => { + let propData = allProperties.find(p => p.name === propName); + + // Check for enterprise license includes (should not be in descriptions) + if (override.description && override.description.includes('include::reference:partial$enterprise-licensed-property.adoc')) { + issues.push({ + severity: 'error', + file: overridesPath, + property: propName, + issue: 'Description contains enterprise license include (should be in metadata only)', + suggestion: 'Remove the include statement from the description' + }); + qualityScore -= 5; + } + + // Check for cloud-specific conditional blocks + if (override.description && (override.description.includes('ifdef::env-cloud') || override.description.includes('ifndef::env-cloud'))) { + issues.push({ + severity: 'error', + file: overridesPath, + property: propName, + issue: 'Description contains cloud-specific conditional blocks', + suggestion: 'Remove cloud conditionals - this info belongs in metadata' + }); + qualityScore -= 5; + } + + // Check for deprecated properties with descriptions (should not have overrides) + if (!propData) propData = allProperties.find(p => p.name === propName); + if (propData && propData.is_deprecated && override.description) { + issues.push({ + severity: 'warning', + file: overridesPath, + property: propName, + issue: 'Override exists for deprecated property', + suggestion: 'Remove override for deprecated properties' + }); + qualityScore -= 2; + } + + // Check for invalid xref links (not using full Antora resource IDs) + if (override.description) { + const invalidXrefPattern = /xref:\.\/|xref:(?![\w-]+:)/g; + const invalidXrefs = override.description.match(invalidXrefPattern); + if (invalidXrefs) { + issues.push({ + severity: 'error', + file: overridesPath, + property: propName, + issue: 'Description contains invalid xref links (not using full Antora resource IDs)', + suggestion: 'Use full resource IDs like xref:reference:path/to/doc.adoc[Link]' + }); + qualityScore -= 3; + } + } + + // Check for duplicate links in related_topics + if (override.related_topics && Array.isArray(override.related_topics)) { + const uniqueLinks = new Set(override.related_topics); + if (uniqueLinks.size < override.related_topics.length) { + issues.push({ + severity: 'warning', + file: overridesPath, + property: propName, + issue: 'Duplicate links in related_topics', + suggestion: 'Remove duplicate links' + }); + qualityScore -= 1; + } + } + }); + } + } + + // Add summary suggestions + if (missingDescriptions > 0) { + suggestions.push(`${missingDescriptions} properties have missing descriptions`); + qualityScore -= Math.min(20, missingDescriptions * 2); + } + if (shortDescriptions > 0) { + suggestions.push(`${shortDescriptions} properties have very short descriptions (< 20 chars)`); + qualityScore -= Math.min(10, shortDescriptions); + } + if (emptyDefaults > 0) { + suggestions.push(`${emptyDefaults} non-deprecated properties have no default value listed`); + } + if (propertiesNeedingExamples.length > 0) { + suggestions.push(`${propertiesNeedingExamples.length} complex properties would benefit from examples`); + // Add info-level issues for properties that should have examples + propertiesNeedingExamples.forEach(propName => { + issues.push({ + severity: 'info', + file: overridesPath, + property: propName, + issue: 'Complex property would benefit from an example', + suggestion: 'Add an example array to the property override showing typical usage' + }); + }); + qualityScore -= Math.min(5, Math.floor(propertiesNeedingExamples.length / 5)); + } + + break; + } + + case 'rpcn_connectors': { + // Read overrides.json + const overridesPath = path.join(repoRoot.root, 'docs-data', 'overrides.json'); + if (!fs.existsSync(overridesPath)) { + return { + success: false, + error: 'overrides.json not found', + suggestion: 'Generate RPCN connector docs first using generate_rpcn_connector_docs tool' + }; + } + + filesAnalyzed++; + const overrides = JSON.parse(fs.readFileSync(overridesPath, 'utf8')); + + // Validate $ref references + const definitions = overrides.definitions || {}; + const allRefs = new Set(); + const invalidRefs = []; + + const findRefs = (obj, path = '') => { + if (typeof obj !== 'object' || obj === null) return; + + if (obj.$ref) { + allRefs.add(obj.$ref); + // Check if ref is valid + const refPath = obj.$ref.replace('#/definitions/', ''); + if (!definitions[refPath]) { + invalidRefs.push({ + ref: obj.$ref, + path + }); + } + } + + for (const [key, value] of Object.entries(obj)) { + if (key !== '$ref') { + findRefs(value, path ? `${path}.${key}` : key); + } + } + }; + + ['inputs', 'outputs', 'processors', 'caches'].forEach(section => { + if (overrides[section]) { + findRefs(overrides[section], section); + } + }); + + invalidRefs.forEach(({ ref, path }) => { + issues.push({ + severity: 'error', + file: overridesPath, + path, + issue: `Invalid $ref: ${ref}`, + suggestion: 'Ensure the reference exists in the definitions section' + }); + qualityScore -= 5; + }); + + // Check for duplicate descriptions (DRY violations) + const descriptions = new Map(); + const checkDuplicates = (obj, path = '') => { + if (typeof obj !== 'object' || obj === null) return; + + if (obj.description && !obj.$ref && typeof obj.description === 'string' && obj.description.length > 30) { + const key = obj.description.trim().toLowerCase(); + if (descriptions.has(key)) { + descriptions.get(key).push(path); + } else { + descriptions.set(key, [path]); + } + } + + for (const [key, value] of Object.entries(obj)) { + checkDuplicates(value, path ? `${path}.${key}` : key); + } + }; + + ['inputs', 'outputs', 'processors', 'caches'].forEach(section => { + if (overrides[section]) { + checkDuplicates(overrides[section], section); + } + }); + + const duplicates = Array.from(descriptions.entries()).filter(([_, paths]) => paths.length > 1); + duplicates.forEach(([desc, paths]) => { + suggestions.push(`Duplicate description found at: ${paths.join(', ')}. Consider creating a definition and using $ref`); + qualityScore -= 3; + }); + + if (invalidRefs.length === 0 && duplicates.length === 0) { + suggestions.push('All $ref references are valid and DRY principles are maintained'); + } + + break; + } + + case 'metrics': + case 'rpk': { + // For metrics and RPK, we just check if the files exist + if (!version) { + return { + success: false, + error: 'version is required for metrics/rpk docs review' + }; + } + + // Sanitize version to prevent path traversal + let sanitizedVersion; + try { + sanitizedVersion = sanitizeVersion(version); + } catch (err) { + return { + success: false, + error: `Invalid version: ${err.message}` + }; + } + + let filePath; + if (doc_type === 'metrics') { + filePath = path.join(repoRoot.root, 'modules', 'reference', 'pages', 'public-metrics-reference.adoc'); + } else { + // RPK files are version-specific + const normalizedVersion = sanitizedVersion.startsWith('v') ? sanitizedVersion : `v${sanitizedVersion}`; + const rpkDir = path.join(repoRoot.root, 'autogenerated', normalizedVersion, 'rpk'); + if (!fs.existsSync(rpkDir)) { + return { + success: false, + error: `RPK docs directory not found at ${rpkDir}`, + suggestion: 'Generate RPK docs first using generate_rpk_docs tool' + }; + } + filePath = rpkDir; + } + + if (!fs.existsSync(filePath)) { + return { + success: false, + error: `Generated docs not found at ${filePath}`, + suggestion: `Generate ${doc_type} docs first using generate_${doc_type}_docs tool` + }; + } + + filesAnalyzed++; + suggestions.push(`${doc_type} documentation generated successfully`); + suggestions.push('Manual review recommended for technical accuracy'); + + break; + } + + default: + return { + success: false, + error: `Unknown doc_type: ${doc_type}`, + suggestion: 'Use one of: properties, metrics, rpk, rpcn_connectors' + }; + } + + // Ensure quality score doesn't go below 0 + qualityScore = Math.max(0, qualityScore); + + const results = { + success: true, + doc_type, + version: version || 'N/A', + files_analyzed: filesAnalyzed, + issues, + quality_score: qualityScore, + suggestions, + summary: `Reviewed ${doc_type} documentation. Quality score: ${qualityScore}/100. Found ${issues.length} issues.` + }; + + // Generate AsciiDoc report if requested + if (generate_report) { + // Sanitize version for filename if provided + let safeVersion = ''; + if (version) { + try { + safeVersion = `-${sanitizeVersion(version)}`; + } catch (err) { + // If version sanitization fails, skip it in filename + safeVersion = ''; + } + } + const reportFilename = `review-${doc_type}${safeVersion}-${formatDate()}.adoc`; + const reportPath = path.join(repoRoot.root, reportFilename); + generateReviewReport(results, reportPath); + results.report_path = reportPath; + results.report_generated = true; + } + + return results; + + } catch (err) { + return { + success: false, + error: err.message, + suggestion: 'Check that the documentation has been generated and files exist' + }; + } +} + +module.exports = { + generateReviewReport, + reviewGeneratedDocs +}; diff --git a/bin/mcp-tools/rpcn-docs.js b/bin/mcp-tools/rpcn-docs.js new file mode 100644 index 00000000..cbd32a7b --- /dev/null +++ b/bin/mcp-tools/rpcn-docs.js @@ -0,0 +1,104 @@ +/** + * MCP Tools - Redpanda Connect Documentation Generation + */ + +const { execSync, spawnSync } = require('child_process'); +const { findRepoRoot, MAX_EXEC_BUFFER_SIZE, DEFAULT_COMMAND_TIMEOUT } = require('./utils'); +const { getAntoraStructure } = require('./antora'); + +/** + * Generate Redpanda Connect connector documentation + * @param {Object} args - Arguments + * @returns {Object} Generation results + */ +function generateRpConnectDocs(args = {}) { + const repoRoot = findRepoRoot(); + const structure = getAntoraStructure(repoRoot); + + if (!structure.hasDocTools) { + return { + success: false, + error: 'doc-tools not found in this repository', + suggestion: 'Navigate to the docs-extensions-and-macros repository' + }; + } + + try { + // Build command arguments array (no shell interpolation) + const cmdArgs = ['doc-tools', 'generate', 'rpcn-connector-docs']; + + // Add flags only when present, each as separate array entries + if (args.fetch_connectors) cmdArgs.push('--fetch-connectors'); + if (args.draft_missing) cmdArgs.push('--draft-missing'); + if (args.update_whats_new) cmdArgs.push('--update-whats-new'); + if (args.include_bloblang) cmdArgs.push('--include-bloblang'); + if (args.data_dir) { + cmdArgs.push('--data-dir'); + cmdArgs.push(args.data_dir); + } + if (args.old_data) { + cmdArgs.push('--old-data'); + cmdArgs.push(args.old_data); + } + if (args.csv) { + cmdArgs.push('--csv'); + cmdArgs.push(args.csv); + } + if (args.overrides) { + cmdArgs.push('--overrides'); + cmdArgs.push(args.overrides); + } + + const result = spawnSync('npx', cmdArgs, { + cwd: repoRoot.root, + encoding: 'utf8', + stdio: 'pipe', + maxBuffer: MAX_EXEC_BUFFER_SIZE, + timeout: DEFAULT_COMMAND_TIMEOUT + }); + + // Check for spawn errors + if (result.error) { + const err = new Error(`Failed to execute command: ${result.error.message}`); + err.stdout = result.stdout || ''; + err.stderr = result.stderr || ''; + err.status = result.status; + throw err; + } + + // Check for non-zero exit codes + if (result.status !== 0) { + const errorMsg = result.stderr || `Command failed with exit code ${result.status}`; + const err = new Error(errorMsg); + err.stdout = result.stdout || ''; + err.stderr = result.stderr || ''; + err.status = result.status; + throw err; + } + + const output = result.stdout; + + const connectorsMatch = output.match(/(\d+) connectors/i); + + return { + success: true, + connectors_documented: connectorsMatch ? parseInt(connectorsMatch[1]) : null, + files_generated: ['modules/reference/pages/redpanda-connect/components/'], + output: output.trim(), + summary: 'Generated Redpanda Connect connector documentation' + }; + } catch (err) { + return { + success: false, + error: err.message, + stdout: err.stdout || '', + stderr: err.stderr || '', + exitCode: err.status, + suggestion: 'Check that you have network access to fetch connector data if using --fetch-connectors' + }; + } +} + +module.exports = { + generateRpConnectDocs +}; diff --git a/bin/mcp-tools/rpk-docs.js b/bin/mcp-tools/rpk-docs.js new file mode 100644 index 00000000..6386e44e --- /dev/null +++ b/bin/mcp-tools/rpk-docs.js @@ -0,0 +1,137 @@ +/** + * MCP Tools - RPK Documentation Generation + */ + +const { spawnSync } = require('child_process'); +const { findRepoRoot, getDocToolsCommand, MAX_EXEC_BUFFER_SIZE, DEFAULT_COMMAND_TIMEOUT } = require('./utils'); +const { getAntoraStructure } = require('./antora'); +const { createJob } = require('./job-queue'); + +/** + * Generate RPK command documentation + * + * Use tags for released content (GA or beta), branches for in-progress content. + * Defaults to branch "dev" if neither tag nor branch is provided. + * + * @param {Object} args - Arguments + * @param {string} [args.tag] - Git tag for released content (for example, "v25.3.1") + * @param {string} [args.branch] - Branch name for in-progress content (for example, "dev", "main") + * @param {boolean} [args.background] - Run as background job + * @returns {Object} Generation results + */ +function generateRpkDocs(args) { + const repoRoot = findRepoRoot(); + const structure = getAntoraStructure(repoRoot); + + if (!structure.hasDocTools) { + return { + success: false, + error: 'doc-tools not found in this repository', + suggestion: 'Navigate to the docs-extensions-and-macros repository' + }; + } + + // Validate that tag and branch are mutually exclusive + if (args.tag && args.branch) { + return { + success: false, + error: 'Cannot specify both tag and branch', + suggestion: 'Use either --tag or --branch, not both' + }; + } + + // Default to 'dev' branch if neither provided + const gitRef = args.tag || args.branch || 'dev'; + const refType = args.tag ? 'tag' : 'branch'; + + // Normalize version (add 'v' prefix if not present and not a branch name like 'dev' or 'main') + let version = gitRef; + if (args.tag && !version.startsWith('v')) { + version = `v${version}`; + } + + // Get doc-tools command (handles both local and installed) + const docTools = getDocToolsCommand(repoRoot); + + // Build command arguments array + const baseArgs = ['generate', 'rpk-docs']; + + if (args.tag) { + baseArgs.push('--tag'); + baseArgs.push(version); + } else { + baseArgs.push('--branch'); + baseArgs.push(version); + } + + // If background mode, create job and return immediately + if (args.background) { + const cmdArgs = [docTools.program, ...docTools.getArgs(baseArgs)]; + const jobId = createJob('generate_rpk_docs', cmdArgs, { + cwd: repoRoot.root + }); + + return { + success: true, + background: true, + job_id: jobId, + message: `RPK docs generation started in background. Use get_job_status with job_id: ${jobId} to check progress.`, + [refType]: version + }; + } + + // Otherwise run synchronously + try { + const result = spawnSync(docTools.program, docTools.getArgs(baseArgs), { + cwd: repoRoot.root, + encoding: 'utf8', + stdio: 'pipe', + maxBuffer: MAX_EXEC_BUFFER_SIZE, + timeout: DEFAULT_COMMAND_TIMEOUT + }); + + // Check for spawn errors + if (result.error) { + const err = new Error(`Failed to execute command: ${result.error.message}`); + err.stdout = result.stdout || ''; + err.stderr = result.stderr || ''; + err.status = result.status; + throw err; + } + + // Check for non-zero exit codes + if (result.status !== 0) { + const errorMsg = result.stderr || `Command failed with exit code ${result.status}`; + const err = new Error(errorMsg); + err.stdout = result.stdout || ''; + err.stderr = result.stderr || ''; + err.status = result.status; + throw err; + } + + const output = result.stdout; + const commandCountMatch = output.match(/(\d+) commands/i); + + return { + success: true, + [refType]: version, + files_generated: [`autogenerated/${version}/rpk/*.adoc`], + commands_documented: commandCountMatch ? parseInt(commandCountMatch[1]) : null, + output: output.trim(), + summary: `Generated RPK documentation for Redpanda ${refType} ${version}` + }; + } catch (err) { + return { + success: false, + error: err.message, + stdout: err.stdout || '', + stderr: err.stderr || '', + exitCode: err.status, + suggestion: 'Check that the version exists in the Redpanda repository' + }; + } +} + +module.exports = { + generateRpkDocs +}; diff --git a/bin/mcp-tools/telemetry.js b/bin/mcp-tools/telemetry.js new file mode 100644 index 00000000..1dd07015 --- /dev/null +++ b/bin/mcp-tools/telemetry.js @@ -0,0 +1,211 @@ +/** + * Usage Telemetry and Statistics + * + * Tracks usage of prompts, resources, and tools to provide insights + * into adoption and identify unused features. + */ + +const fs = require('fs'); +const path = require('path'); + +/** + * Usage statistics tracker + */ +class UsageStats { + constructor() { + this.prompts = new Map(); // name count + this.resources = new Map(); // uri count + this.tools = new Map(); // name count + this.startTime = new Date(); + this.lastReportTime = new Date(); + } + + /** + * Record prompt usage + * @param {string} name - Prompt name + */ + recordPrompt(name) { + const count = this.prompts.get(name) || 0; + this.prompts.set(name, count + 1); + } + + /** + * Record resource usage + * @param {string} uri - Resource URI + */ + recordResource(uri) { + const count = this.resources.get(uri) || 0; + this.resources.set(uri, count + 1); + } + + /** + * Record tool usage + * @param {string} name - Tool name + */ + recordTool(name) { + const count = this.tools.get(name) || 0; + this.tools.set(name, count + 1); + } + + /** + * Get stats summary + * @returns {Object} Statistics summary + */ + getSummary() { + const now = new Date(); + const uptime = Math.floor((now - this.startTime) / 1000); // seconds + const timeSinceLastReport = Math.floor((now - this.lastReportTime) / 1000); + + return { + uptime, + timeSinceLastReport, + promptCount: this.prompts.size, + resourceCount: this.resources.size, + toolCount: this.tools.size, + totalPromptCalls: Array.from(this.prompts.values()).reduce((a, b) => a + b, 0), + totalResourceCalls: Array.from(this.resources.values()).reduce((a, b) => a + b, 0), + totalToolCalls: Array.from(this.tools.values()).reduce((a, b) => a + b, 0), + prompts: Object.fromEntries(this.prompts), + resources: Object.fromEntries(this.resources), + tools: Object.fromEntries(this.tools) + }; + } + + /** + * Get top N most used items + * @param {Map} map - Map to get top from + * @param {number} n - Number of items + * @returns {Array} Top N items as [name, count] pairs + */ + getTopN(map, n = 5) { + return Array.from(map.entries()) + .sort((a, b) => b[1] - a[1]) + .slice(0, n); + } + + /** + * Format stats for display + * @returns {string} Formatted stats + */ + format() { + const summary = this.getSummary(); + const lines = []; + + lines.push('MCP Server Usage Statistics'); + lines.push('='.repeat(60)); + lines.push(''); + + // Uptime + const hours = Math.floor(summary.uptime / 3600); + const minutes = Math.floor((summary.uptime % 3600) / 60); + lines.push(`Uptime: ${hours}h ${minutes}m`); + lines.push(''); + + // Totals + lines.push(`Total calls:`); + lines.push(` Prompts: ${summary.totalPromptCalls}`); + lines.push(` Resources: ${summary.totalResourceCalls}`); + lines.push(` Tools: ${summary.totalToolCalls}`); + lines.push(''); + + // Top prompts + if (this.prompts.size > 0) { + lines.push('Most used prompts:'); + const topPrompts = this.getTopN(this.prompts, 5); + topPrompts.forEach(([name, count]) => { + lines.push(` ${name}: ${count} calls`); + }); + lines.push(''); + } + + // Top resources + if (this.resources.size > 0) { + lines.push('Most used resources:'); + const topResources = this.getTopN(this.resources, 5); + topResources.forEach(([uri, count]) => { + lines.push(` ${uri}: ${count} calls`); + }); + lines.push(''); + } + + // Top tools + if (this.tools.size > 0) { + lines.push('Most used tools:'); + const topTools = this.getTopN(this.tools, 5); + topTools.forEach(([name, count]) => { + lines.push(` ${name}: ${count} calls`); + }); + lines.push(''); + } + + return lines.join('\n'); + } + + /** + * Export stats to JSON file + * @param {string} filepath - Path to export to + */ + exportToFile(filepath) { + const summary = this.getSummary(); + summary.exportedAt = new Date().toISOString(); + + fs.writeFileSync(filepath, JSON.stringify(summary, null, 2), 'utf8'); + } + + /** + * Reset stats (for periodic reporting) + */ + reset() { + this.lastReportTime = new Date(); + // Don't reset the maps - keep cumulative stats + } +} + +/** + * Create a periodic reporter + * @param {UsageStats} stats - Stats instance + * @param {number} intervalMs - Interval in milliseconds + * @returns {NodeJS.Timeout} Interval handle + */ +function createPeriodicReporter(stats, intervalMs = 3600000) { + return setInterval(() => { + const output = stats.format(); + console.error('\n' + output); + stats.reset(); + }, intervalMs); +} + +/** + * Create a shutdown handler to export stats + * @param {UsageStats} stats - Stats instance + * @param {string} baseDir - Base directory for export + */ +function createShutdownHandler(stats, baseDir) { + const handler = () => { + const exportPath = path.join(baseDir, 'mcp-usage-stats.json'); + try { + stats.exportToFile(exportPath); + console.error(`\nUsage stats exported to: ${exportPath}`); + } catch (err) { + console.error(`Failed to export usage stats: ${err.message}`); + } + }; + + process.on('SIGINT', () => { + handler(); + process.exit(0); + }); + + process.on('SIGTERM', () => { + handler(); + process.exit(0); + }); + + return handler; +} + +module.exports = { + UsageStats, + createPeriodicReporter, + createShutdownHandler +}; diff --git a/bin/mcp-tools/utils.js b/bin/mcp-tools/utils.js new file mode 100644 index 00000000..d1bc37fe --- /dev/null +++ b/bin/mcp-tools/utils.js @@ -0,0 +1,131 @@ +/** + * MCP Tools - Shared Utilities + */ + +const fs = require('fs'); +const path = require('path'); +const { spawnSync } = require('child_process'); + +// Constants +const MAX_RECURSION_DEPTH = 3; +const MAX_EXEC_BUFFER_SIZE = 50 * 1024 * 1024; // 50MB +const DEFAULT_COMMAND_TIMEOUT = 10 * 60 * 1000; // 10 minutes +const DEFAULT_SKIP_DIRS = ['node_modules', '.git', 'venv', '__pycache__', '.pytest_cache']; +const PLAYBOOK_NAMES = [ + 'local-antora-playbook.yml', + 'antora-playbook.yml', + 'docs-playbook.yml' +]; + +/** + * Find the repository root from current working directory + * @param {string} [start=process.cwd()] - Starting directory for search + * @returns {{ root: string, detected: boolean, type: string|null }} Repository information + */ +function findRepoRoot(start = process.cwd()) { + let dir = start; + while (dir !== path.parse(dir).root) { + if (fs.existsSync(path.join(dir, '.git'))) { + return { root: dir, detected: true, type: 'git' }; + } + if (fs.existsSync(path.join(dir, 'package.json'))) { + return { root: dir, detected: true, type: 'npm' }; + } + dir = path.dirname(dir); + } + return { root: start, detected: false, type: null }; +} + +/** + * Get the doc-tools command and arguments + * Handles both development mode (source repo) and installed mode (npm package) + * @param {Object} repoRoot - Repository root information from findRepoRoot() + * @returns {{ program: string, getArgs: function }} Command configuration + */ +function getDocToolsCommand(repoRoot) { + // Check if we're in the source repository (development mode) + const localDocTools = path.join(repoRoot.root, 'bin', 'doc-tools.js'); + if (fs.existsSync(localDocTools)) { + return { + program: 'node', + getArgs: (cmdArgs) => [localDocTools, ...cmdArgs] + }; + } + + // Otherwise use npx, which will find the installed package in node_modules + return { + program: 'npx', + getArgs: (cmdArgs) => ['doc-tools', ...cmdArgs] + }; +} + +/** + * Execute a command safely using spawnSync (no shell) + * @param {string} program - Program to execute (for example, 'npx') + * @param {string[]} args - Array of arguments (for example, ['doc-tools', 'generate', 'property-docs']) + * @param {Object} options - Execution options + * @returns {string} Command output (stdout) + * @throws {Error} Error with stdout, stderr, and status properties on failure + */ +function executeCommand(program, args, options = {}) { + const defaultOptions = { + encoding: 'utf8', + maxBuffer: MAX_EXEC_BUFFER_SIZE, + timeout: DEFAULT_COMMAND_TIMEOUT, + stdio: 'pipe' + }; + + const result = spawnSync(program, args, { ...defaultOptions, ...options }); + + // Check for spawn errors + if (result.error) { + const err = new Error(`Failed to execute command: ${result.error.message}`); + err.stdout = result.stdout || ''; + err.stderr = result.stderr || ''; + err.status = result.status; + throw err; + } + + // Check for non-zero exit codes + if (result.status !== 0) { + const errorMsg = result.stderr || `Command failed with exit code ${result.status}`; + const err = new Error(errorMsg); + err.stdout = result.stdout || ''; + err.stderr = result.stderr || ''; + err.status = result.status; + throw err; + } + + return result.stdout; +} + +/** + * Normalize version string (remove 'v' prefix if present) + * @param {string} version - Version string + * @returns {string} Normalized version + */ +function normalizeVersion(version) { + return version.startsWith('v') ? version.substring(1) : version; +} + +/** + * Format date as YYYY-MM-DD + * @param {Date} date - Date to format + * @returns {string} Formatted date string + */ +function formatDate(date = new Date()) { + return date.toISOString().split('T')[0]; +} + +module.exports = { + MAX_RECURSION_DEPTH, + MAX_EXEC_BUFFER_SIZE, + DEFAULT_COMMAND_TIMEOUT, + DEFAULT_SKIP_DIRS, + PLAYBOOK_NAMES, + findRepoRoot, + getDocToolsCommand, + executeCommand, + normalizeVersion, + formatDate +}; diff --git a/bin/mcp-tools/versions.js b/bin/mcp-tools/versions.js new file mode 100644 index 00000000..340af834 --- /dev/null +++ b/bin/mcp-tools/versions.js @@ -0,0 +1,136 @@ +/** + * MCP Tools - Version Information + */ + +const { spawnSync } = require('child_process'); +const { findRepoRoot, getDocToolsCommand } = require('./utils'); + +/** + * Get the latest Redpanda version information + * @param {Object} args - Arguments + * @param {boolean} [args.beta] - Whether to get beta/RC version + * @returns {Object} Version information + */ +function getRedpandaVersion(args = {}) { + try { + // Get doc-tools command (handles both local and installed) + const repoRoot = findRepoRoot(); + const docTools = getDocToolsCommand(repoRoot); + + // Build command arguments array + const baseArgs = ['get-redpanda-version']; + if (args.beta) { + baseArgs.push('--beta'); + } + + const result = spawnSync(docTools.program, docTools.getArgs(baseArgs), { + encoding: 'utf8', + stdio: 'pipe', + timeout: 30000 + }); + + // Check for errors + if (result.error) { + throw new Error(`Failed to execute command: ${result.error.message}`); + } + + if (result.status !== 0) { + const errorMsg = result.stderr || `Command failed with exit code ${result.status}`; + throw new Error(errorMsg); + } + + const output = result.stdout; + + // Parse the output (format: REDPANDA_VERSION=vX.Y.Z\nREDPANDA_DOCKER_REPO=redpanda) + const lines = output.trim().split('\n'); + const versionLine = lines.find(l => l.startsWith('REDPANDA_VERSION=')); + const dockerLine = lines.find(l => l.startsWith('REDPANDA_DOCKER_REPO=')); + + if (!versionLine) { + return { + success: false, + error: 'Failed to parse version from output' + }; + } + + const version = versionLine.split('=')[1]; + const dockerRepo = dockerLine ? dockerLine.split('=')[1] : 'redpanda'; + + return { + success: true, + version, + docker_tag: `docker.redpanda.com/redpandadata/${dockerRepo}:${version}`, + is_beta: args.beta || false, + notes_url: `https://github.com/redpanda-data/redpanda/releases/tag/${version}` + }; + } catch (err) { + return { + success: false, + error: err.message, + suggestion: 'Make sure you have network access to fetch version information from GitHub' + }; + } +} + +/** + * Get the latest Redpanda Console version information + * @returns {Object} Version information + */ +function getConsoleVersion() { + try { + // Get doc-tools command (handles both local and installed) + const repoRoot = findRepoRoot(); + const docTools = getDocToolsCommand(repoRoot); + + const result = spawnSync(docTools.program, docTools.getArgs(['get-console-version']), { + encoding: 'utf8', + stdio: 'pipe', + timeout: 30000 + }); + + // Check for errors + if (result.error) { + throw new Error(`Failed to execute command: ${result.error.message}`); + } + + if (result.status !== 0) { + const errorMsg = result.stderr || `Command failed with exit code ${result.status}`; + throw new Error(errorMsg); + } + + const output = result.stdout; + + // Parse the output (format: CONSOLE_VERSION=vX.Y.Z\nCONSOLE_DOCKER_REPO=console) + const lines = output.trim().split('\n'); + const versionLine = lines.find(l => l.startsWith('CONSOLE_VERSION=')); + const dockerLine = lines.find(l => l.startsWith('CONSOLE_DOCKER_REPO=')); + + if (!versionLine) { + return { + success: false, + error: 'Failed to parse version from output' + }; + } + + const version = versionLine.split('=')[1]; + const dockerRepo = dockerLine ? dockerLine.split('=')[1] : 'console'; + + return { + success: true, + version, + docker_tag: `docker.redpanda.com/redpandadata/${dockerRepo}:${version}`, + notes_url: `https://github.com/redpanda-data/console/releases/tag/${version}` + }; + } catch (err) { + return { + success: false, + error: err.message, + suggestion: 'Make sure you have network access to fetch version information from GitHub' + }; + } +} + +module.exports = { + getRedpandaVersion, + getConsoleVersion +}; diff --git a/cli-utils/convert-doc-links.js b/cli-utils/convert-doc-links.js index 7c297aa7..5d0d95dd 100644 --- a/cli-utils/convert-doc-links.js +++ b/cli-utils/convert-doc-links.js @@ -3,7 +3,7 @@ const { URL } = require('url'); /** * Converts a docs.redpanda.com URL, optionally suffixed with a label in brackets, into an Antora xref resource ID string. * - * If the input includes a label in square brackets (e.g., `[Label]`), the label is preserved and appended to the resulting xref. + * If the input includes a label in square brackets (for example, `[Label]`), the label is preserved and appended to the resulting xref. * * @param {string} input - A docs.redpanda.com URL, optionally followed by a label in square brackets. * @returns {string} The corresponding Antora xref resource ID, with the label preserved if present. diff --git a/cli-utils/github-token.js b/cli-utils/github-token.js new file mode 100644 index 00000000..fb55c83c --- /dev/null +++ b/cli-utils/github-token.js @@ -0,0 +1,58 @@ +/** + * GitHub Token Utility + * + * Provides a consistent way to retrieve GitHub tokens from environment variables. + * Supports multiple common token variable names with priority order. + */ + +/** + * Get GitHub token from environment variables + * Checks multiple common variable names in priority order: + * 1. REDPANDA_GITHUB_TOKEN - Custom Redpanda token + * 2. GITHUB_TOKEN - GitHub Actions default + * 3. GH_TOKEN - GitHub CLI default + * + * @returns {string|null} GitHub token or null if not found + */ +function getGitHubToken() { + return process.env.REDPANDA_GITHUB_TOKEN || + process.env.GITHUB_TOKEN || + process.env.GH_TOKEN || + null; +} + +/** + * Get an authenticated GitHub URL by injecting the token + * @param {string} url - The GitHub HTTPS URL (for example, https://github.com/owner/repo.git) + * @returns {string} Authenticated URL with token, or original URL if no token available + */ +function getAuthenticatedGitHubUrl(url) { + const token = getGitHubToken(); + + if (!token || !url.includes('github.com')) { + return url; + } + + try { + const urlObj = new URL(url); + urlObj.username = token; + return urlObj.toString(); + } catch (err) { + // If URL parsing fails, return original + return url; + } +} + +/** + * Check if a GitHub token is available + * @returns {boolean} True if a token is available + */ +function hasGitHubToken() { + return getGitHubToken() !== null; +} + +module.exports = { + getGitHubToken, + getAuthenticatedGitHubUrl, + hasGitHubToken +}; diff --git a/cli-utils/self-managed-docs-branch.js b/cli-utils/self-managed-docs-branch.js index aa2aaddd..d1bb82ce 100644 --- a/cli-utils/self-managed-docs-branch.js +++ b/cli-utils/self-managed-docs-branch.js @@ -5,7 +5,7 @@ const { spawnSync } = require('child_process'); /** * Retrieves the current Self-Managed documentation version from the remote antora.yml file. * - * @returns {Promise} Resolves with the version string (e.g., "25.1"). + * @returns {Promise} Resolves with the version string (for example, "25.1"). * * @throws {Error} If the antora.yml file cannot be fetched, parsed, or if the version field is missing. */ @@ -39,7 +39,7 @@ function fetchRemoteAntoraVersion() { * * Normalizes the input tag, extracts the major.minor version, and applies version-specific logic to select the correct branch. Verifies that the chosen branch exists in the remote repository. * - * @param {string} operatorTag - The operator tag to evaluate (e.g., "operator/v25.1.2" or "v25.1.2"). + * @param {string} operatorTag - The operator tag to evaluate (for example, "operator/v25.1.2" or "v25.1.2"). * @returns {Promise} The name of the documentation branch to use. * * @throws {Error} If the tag cannot be parsed or if the determined branch does not exist in the remote repository. diff --git a/cli-utils/setup-mcp.js b/cli-utils/setup-mcp.js new file mode 100644 index 00000000..fa9ab749 --- /dev/null +++ b/cli-utils/setup-mcp.js @@ -0,0 +1,313 @@ +/** + * Cross-platform MCP Server Setup Utility + * + * Configures the Redpanda Docs MCP server for Claude Code across + * macOS, Linux, and Windows platforms using the 'claude mcp add' command. + */ + +const fs = require('fs'); +const path = require('path'); +const os = require('os'); +const chalk = require('chalk'); +const { execSync } = require('child_process'); + +/** + * Detect the operating system + */ +function detectOS() { + const platform = os.platform(); + switch (platform) { + case 'darwin': + return 'macOS'; + case 'linux': + return 'Linux'; + case 'win32': + return 'Windows'; + default: + return platform; + } +} + +/** + * Get Claude Code config file path + * Claude Code uses ~/.claude.json for configuration + */ +function getConfigPath() { + return path.join(os.homedir(), '.claude.json'); +} + +/** + * Check if config file exists + */ +function configExists() { + return fs.existsSync(getConfigPath()); +} + +/** + * Read and parse JSON config file + */ +function readConfig(configPath) { + try { + const content = fs.readFileSync(configPath, 'utf8'); + return JSON.parse(content); + } catch (err) { + throw new Error(`Failed to read config: ${err.message}`); + } +} + +/** + * Check if MCP server is already configured + */ +function isMCPServerConfigured(config, serverName = 'redpanda-doc-tools-assistant') { + return config.mcpServers && config.mcpServers[serverName]; +} + +/** + * Get the absolute path to the MCP server script + */ +function getMCPServerPath() { + // Find repo root by looking for package.json + let currentDir = __dirname; + while (currentDir !== path.parse(currentDir).root) { + if (fs.existsSync(path.join(currentDir, 'package.json'))) { + const mcpServerPath = path.join(currentDir, 'bin', 'doc-tools-mcp.js'); + if (fs.existsSync(mcpServerPath)) { + return mcpServerPath; + } + } + currentDir = path.dirname(currentDir); + } + + throw new Error('Could not find doc-tools-mcp.js. Make sure you are running this from the docs-extensions-and-macros repository.'); +} + +/** + * Main setup function using 'claude mcp add' CLI command + */ +async function setupMCP(options = {}) { + const { force = false, target = 'auto', local = false } = options; + + console.log(chalk.blue('\nπŸš€ Redpanda Doc Tools MCP Server Setup\n')); + console.log(chalk.gray(`Platform: ${detectOS()}`)); + console.log(chalk.gray(`Node: ${process.version}\n`)); + + // Determine mode: local development or npx + let mode = 'npx'; + let localPath = null; + const packageName = '@redpanda-data/docs-extensions-and-macros'; + let packageVersion = 'unknown'; + + // Only check for local installation if --local flag is set + if (local) { + try { + const mcpServerPath = getMCPServerPath(); + localPath = mcpServerPath; + const packageJsonPath = path.join(path.dirname(path.dirname(mcpServerPath)), 'package.json'); + if (fs.existsSync(packageJsonPath)) { + const pkg = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8')); + packageVersion = pkg.version; + } + mode = 'local'; + } catch (err) { + console.error(chalk.red('βœ—') + ' --local flag requires running from docs-extensions-and-macros repository'); + return { success: false, error: 'Not in local repository' }; + } + } + + if (mode === 'local') { + console.log(chalk.green('βœ“') + ` Mode: ${chalk.cyan('Local Development')}`); + console.log(chalk.gray(` Path: ${localPath}`)); + console.log(chalk.gray(` Version: ${packageVersion}\n`)); + } else { + console.log(chalk.green('βœ“') + ` Mode: ${chalk.cyan('NPX (Published Package)')}`); + console.log(chalk.gray(` Package: ${packageName}`)); + console.log(chalk.gray(` Using npx to run from node_modules or npm\n`)); + } + + // Check if MCP server is already configured + const configPath = getConfigPath(); + let alreadyConfigured = false; + let needsUpdate = false; + + if (configExists()) { + try { + const config = readConfig(configPath); + alreadyConfigured = isMCPServerConfigured(config); + + if (alreadyConfigured) { + const existingConfig = config.mcpServers['redpanda-doc-tools-assistant']; + const existingCommand = existingConfig.command; + const existingArgs = existingConfig.args || []; + + // Check if configuration matches desired mode + const isCorrectNpxConfig = mode === 'npx' && existingCommand === 'npx' && + existingArgs.includes('doc-tools-mcp'); + const isCorrectLocalConfig = mode === 'local' && existingCommand === 'node' && + existingArgs[0] === localPath; + + if (!isCorrectNpxConfig && !isCorrectLocalConfig) { + needsUpdate = true; + console.log(chalk.yellow('⚠ ') + ' MCP server configured but with different setup:'); + console.log(chalk.gray(` Current: ${existingCommand} ${existingArgs.join(' ')}`)); + const newSetup = mode === 'local' ? `node ${localPath}` : 'npx -y doc-tools-mcp'; + console.log(chalk.gray(` New: ${newSetup}\n`)); + } else { + console.log(chalk.green('βœ“') + ` MCP server already configured correctly\n`); + if (!force) { + return { + success: true, + configPath, + configType: 'Claude Code', + action: 'already-configured', + mode + }; + } + } + } + } catch (err) { + console.log(chalk.yellow('⚠ ') + ` Could not read config: ${err.message}`); + } + } + + if (alreadyConfigured && !needsUpdate && !force) { + console.log(chalk.blue('β„Ή') + ' Use --force to reconfigure\n'); + return { + success: true, + configPath, + configType: 'Claude Code', + action: 'already-configured', + mode + }; + } + + // Build the claude mcp add command + const serverName = 'redpanda-doc-tools-assistant'; + + // If server exists and we're updating, remove it first + if (alreadyConfigured && (needsUpdate || force)) { + try { + console.log(chalk.gray(`Removing existing MCP server: ${serverName}\n`)); + execSync(`claude mcp remove --scope user ${serverName}`, { stdio: 'pipe' }); + } catch (err) { + console.log(chalk.yellow('⚠ ') + ` Could not remove existing server (may not exist): ${err.message}`); + } + } + + let command; + if (mode === 'local') { + command = `claude mcp add --transport stdio --scope user ${serverName} -- node ${localPath}`; + } else { + command = `claude mcp add --transport stdio --scope user ${serverName} -- npx -y doc-tools-mcp`; + } + + // Execute the command + try { + console.log(chalk.gray(`Running: ${command}\n`)); + execSync(command, { stdio: 'inherit' }); + + console.log(chalk.green('βœ“') + ` MCP server ${alreadyConfigured ? 'updated' : 'added'}: ${chalk.cyan(serverName)}\n`); + + return { + success: true, + configPath, + configType: 'Claude Code', + action: alreadyConfigured ? 'updated' : 'added', + mode + }; + } catch (err) { + console.error(chalk.red('βœ—') + ` Failed to add MCP server: ${err.message}`); + return { success: false, error: err.message }; + } +} + +/** + * Print next steps + */ +function printNextSteps(result) { + if (!result.success) { + return; + } + + console.log(chalk.bold('πŸ“‹ Next Steps:\n')); + + if (result.configType === 'Claude Desktop') { + console.log(' 1. Restart Claude Desktop'); + console.log(' 2. The MCP server will be available automatically\n'); + } else { + console.log(' 1. Restart Claude Code (if running)'); + console.log(' 2. Navigate to any repository:'); + console.log(chalk.gray(' cd ~/repos/docs')); + console.log(' 3. Start using the MCP server:\n'); + console.log(chalk.cyan(' "Show me the Antora structure"')); + console.log(chalk.cyan(' "Generate property docs for v25.3.1"\n')); + } + + if (result.mode === 'local') { + console.log(chalk.bold('πŸ“¦ To Update (Local Mode):\n')); + console.log(' 1. Pull latest changes:'); + console.log(chalk.gray(' cd /path/to/docs-extensions-and-macros')); + console.log(chalk.gray(' git pull')); + console.log(' 2. Install dependencies:'); + console.log(chalk.gray(' npm install')); + console.log(' 3. Restart Claude Code'); + console.log(chalk.gray(' Changes will be picked up automatically!\n')); + } else { + console.log(chalk.bold('πŸ“¦ To Update (NPX Mode):\n')); + console.log(' 1. Update the package globally or in any repo:'); + console.log(chalk.gray(' npm update -g @redpanda-data/docs-extensions-and-macros')); + console.log(' 2. Restart Claude Code'); + console.log(chalk.gray(' That\'s it! npx will use the new version.\n')); + } + + console.log(chalk.bold('πŸŽ‰ Setup complete!\n')); + console.log(chalk.gray('Documentation: mcp/USER_GUIDE.adoc\n')); +} + +/** + * Show status of MCP server configuration + */ +function showStatus() { + console.log(chalk.blue('\nπŸ“Š MCP Server Configuration Status\n')); + console.log(chalk.gray(`Platform: ${detectOS()}\n`)); + + // Check Claude Code config + const configPath = getConfigPath(); + + if (configExists()) { + console.log(chalk.bold('Claude Code:')); + console.log(chalk.gray(` Config: ${configPath}`)); + + try { + const config = readConfig(configPath); + const configured = isMCPServerConfigured(config); + + if (configured) { + const server = config.mcpServers['redpanda-docs-tool-assistant']; + const mode = server.command === 'node' ? 'Local Development' : 'NPX (Published Package)'; + const details = server.command === 'node' + ? `Path: ${server.args[0]}` + : `Package: ${server.args.join(' ')}`; + + console.log(chalk.green(' βœ“ MCP server configured')); + console.log(chalk.gray(` Mode: ${mode}`)); + console.log(chalk.gray(` ${details}\n`)); + } else { + console.log(chalk.yellow(' βœ— MCP server not configured\n')); + } + } catch (err) { + console.log(chalk.red(` βœ— Error reading config: ${err.message}\n`)); + } + } else { + console.log(chalk.gray('Claude Code: Config not found\n')); + } +} + +module.exports = { + setupMCP, + showStatus, + printNextSteps, + detectOS, + getConfigPath, + configExists, + getMCPServerPath +} diff --git a/docker-compose/25.1/transactions.md b/docker-compose/25.1/transactions.md index e2309167..90e1de13 100644 --- a/docker-compose/25.1/transactions.md +++ b/docker-compose/25.1/transactions.md @@ -19,7 +19,7 @@ Each message in the `transactions` topic adheres to the following JSON schema: - **email**: The email address of the user involved in the transaction. - **index**: A numeric index associated with the transaction. This could represent the position or order of the transaction in a sequence. -- **price**: A string representing the price of the product. It includes a currency code (e.g., "XXX") followed by the amount. +- **price**: A string representing the price of the product. It includes a currency code (for example, "XXX") followed by the amount. - **product_url**: A URL that points to the product involved in the transaction. - **timestamp**: The timestamp of when the transaction occurred, formatted in ISO 8601. - **user_id**: A numeric identifier for the user. This is typically a unique ID assigned to each user in the system. diff --git a/docker-compose/transactions.md b/docker-compose/transactions.md index e2309167..90e1de13 100644 --- a/docker-compose/transactions.md +++ b/docker-compose/transactions.md @@ -19,7 +19,7 @@ Each message in the `transactions` topic adheres to the following JSON schema: - **email**: The email address of the user involved in the transaction. - **index**: A numeric index associated with the transaction. This could represent the position or order of the transaction in a sequence. -- **price**: A string representing the price of the product. It includes a currency code (e.g., "XXX") followed by the amount. +- **price**: A string representing the price of the product. It includes a currency code (for example, "XXX") followed by the amount. - **product_url**: A URL that points to the product involved in the transaction. - **timestamp**: The timestamp of when the transaction occurred, formatted in ISO 8601. - **user_id**: A numeric identifier for the user. This is typically a unique ID assigned to each user in the system. diff --git a/docs-data/connect-4.65.0.json b/docs-data/connect-4.65.0.json index f5d6bf26..748200a0 100644 --- a/docs-data/connect-4.65.0.json +++ b/docs-data/connect-4.65.0.json @@ -7294,7 +7294,7 @@ "name": "include_patterns", "type": "string", "kind": "array", - "description": "A list of file patterns to include (e.g., '**/*.md', 'configs/*.yaml'). If empty, all files will be included. Supports glob patterns: *, /**/, ?, and character ranges [a-z]. Any character with a special meaning can be escaped with a backslash.", + "description": "A list of file patterns to include (for example, '**/*.md', 'configs/*.yaml'). If empty, all files will be included. Supports glob patterns: *, /**/, ?, and character ranges [a-z]. Any character with a special meaning can be escaped with a backslash.", "is_optional": true, "default": [] }, @@ -7302,7 +7302,7 @@ "name": "exclude_patterns", "type": "string", "kind": "array", - "description": "A list of file patterns to exclude (e.g., '.git/**', '**/*.png'). These patterns take precedence over include_patterns. Supports glob patterns: *, /**/, ?, and character ranges [a-z]. Any character with a special meaning can be escaped with a backslash.", + "description": "A list of file patterns to exclude (for example, '.git/**', '**/*.png'). These patterns take precedence over include_patterns. Supports glob patterns: *, /**/, ?, and character ranges [a-z]. Any character with a special meaning can be escaped with a backslash.", "is_optional": true, "default": [] }, diff --git a/docs-data/connect-4.69.0.json b/docs-data/connect-4.69.0.json index c3d9f46a..6f663931 100644 --- a/docs-data/connect-4.69.0.json +++ b/docs-data/connect-4.69.0.json @@ -1229,7 +1229,7 @@ "name": "idempotent_write", "type": "bool", "kind": "scalar", - "description": "Enable the idempotent write producer option. When enabled, the producer initializes a producer ID and uses it to guarantee exactly-once semantics per partition (no duplicates on retries). This requires the `IDEMPOTENT_WRITE` permission on the `CLUSTER` resource. If your cluster does not grant this permission or uses ACLs restrictively, disable this option. Note: Idempotent writes are strictly a win for data integrity but may be unavailable in restricted environments (e.g., some managed Kafka services, Redpanda with strict ACLs). Disabling this option is safe and only affects retry behaviorβ€”duplicates may occur on producer retries, but the pipeline will continue to function normally.", + "description": "Enable the idempotent write producer option. When enabled, the producer initializes a producer ID and uses it to guarantee exactly-once semantics per partition (no duplicates on retries). This requires the `IDEMPOTENT_WRITE` permission on the `CLUSTER` resource. If your cluster does not grant this permission or uses ACLs restrictively, disable this option. Note: Idempotent writes are strictly a win for data integrity but may be unavailable in restricted environments (for example, some managed Kafka services, Redpanda with strict ACLs). Disabling this option is safe and only affects retry behaviorβ€”duplicates may occur on producer retries, but the pipeline will continue to function normally.", "is_advanced": true, "default": true }, @@ -7935,7 +7935,7 @@ "name": "include_patterns", "type": "string", "kind": "array", - "description": "A list of file patterns to include (e.g., '**/*.md', 'configs/*.yaml'). If empty, all files will be included. Supports glob patterns: *, /**/, ?, and character ranges [a-z]. Any character with a special meaning can be escaped with a backslash.", + "description": "A list of file patterns to include (for example, '**/*.md', 'configs/*.yaml'). If empty, all files will be included. Supports glob patterns: *, /**/, ?, and character ranges [a-z]. Any character with a special meaning can be escaped with a backslash.", "is_optional": true, "default": [] }, @@ -7943,7 +7943,7 @@ "name": "exclude_patterns", "type": "string", "kind": "array", - "description": "A list of file patterns to exclude (e.g., '.git/**', '**/*.png'). These patterns take precedence over include_patterns. Supports glob patterns: *, /**/, ?, and character ranges [a-z]. Any character with a special meaning can be escaped with a backslash.", + "description": "A list of file patterns to exclude (for example, '.git/**', '**/*.png'). These patterns take precedence over include_patterns. Supports glob patterns: *, /**/, ?, and character ranges [a-z]. Any character with a special meaning can be escaped with a backslash.", "is_optional": true, "default": [] }, @@ -30423,7 +30423,7 @@ "name": "idempotent_write", "type": "bool", "kind": "scalar", - "description": "Enable the idempotent write producer option. When enabled, the producer initializes a producer ID and uses it to guarantee exactly-once semantics per partition (no duplicates on retries). This requires the `IDEMPOTENT_WRITE` permission on the `CLUSTER` resource. If your cluster does not grant this permission or uses ACLs restrictively, disable this option. Note: Idempotent writes are strictly a win for data integrity but may be unavailable in restricted environments (e.g., some managed Kafka services, Redpanda with strict ACLs). Disabling this option is safe and only affects retry behaviorβ€”duplicates may occur on producer retries, but the pipeline will continue to function normally.", + "description": "Enable the idempotent write producer option. When enabled, the producer initializes a producer ID and uses it to guarantee exactly-once semantics per partition (no duplicates on retries). This requires the `IDEMPOTENT_WRITE` permission on the `CLUSTER` resource. If your cluster does not grant this permission or uses ACLs restrictively, disable this option. Note: Idempotent writes are strictly a win for data integrity but may be unavailable in restricted environments (for example, some managed Kafka services, Redpanda with strict ACLs). Disabling this option is safe and only affects retry behaviorβ€”duplicates may occur on producer retries, but the pipeline will continue to function normally.", "is_advanced": true, "default": true }, @@ -31280,7 +31280,7 @@ "name": "idempotent_write", "type": "bool", "kind": "scalar", - "description": "Enable the idempotent write producer option. When enabled, the producer initializes a producer ID and uses it to guarantee exactly-once semantics per partition (no duplicates on retries). This requires the `IDEMPOTENT_WRITE` permission on the `CLUSTER` resource. If your cluster does not grant this permission or uses ACLs restrictively, disable this option. Note: Idempotent writes are strictly a win for data integrity but may be unavailable in restricted environments (e.g., some managed Kafka services, Redpanda with strict ACLs). Disabling this option is safe and only affects retry behaviorβ€”duplicates may occur on producer retries, but the pipeline will continue to function normally.", + "description": "Enable the idempotent write producer option. When enabled, the producer initializes a producer ID and uses it to guarantee exactly-once semantics per partition (no duplicates on retries). This requires the `IDEMPOTENT_WRITE` permission on the `CLUSTER` resource. If your cluster does not grant this permission or uses ACLs restrictively, disable this option. Note: Idempotent writes are strictly a win for data integrity but may be unavailable in restricted environments (for example, some managed Kafka services, Redpanda with strict ACLs). Disabling this option is safe and only affects retry behaviorβ€”duplicates may occur on producer retries, but the pipeline will continue to function normally.", "is_advanced": true, "default": true }, @@ -34361,7 +34361,7 @@ "name": "idempotent_write", "type": "bool", "kind": "scalar", - "description": "Enable the idempotent write producer option. When enabled, the producer initializes a producer ID and uses it to guarantee exactly-once semantics per partition (no duplicates on retries). This requires the `IDEMPOTENT_WRITE` permission on the `CLUSTER` resource. If your cluster does not grant this permission or uses ACLs restrictively, disable this option. Note: Idempotent writes are strictly a win for data integrity but may be unavailable in restricted environments (e.g., some managed Kafka services, Redpanda with strict ACLs). Disabling this option is safe and only affects retry behaviorβ€”duplicates may occur on producer retries, but the pipeline will continue to function normally.", + "description": "Enable the idempotent write producer option. When enabled, the producer initializes a producer ID and uses it to guarantee exactly-once semantics per partition (no duplicates on retries). This requires the `IDEMPOTENT_WRITE` permission on the `CLUSTER` resource. If your cluster does not grant this permission or uses ACLs restrictively, disable this option. Note: Idempotent writes are strictly a win for data integrity but may be unavailable in restricted environments (for example, some managed Kafka services, Redpanda with strict ACLs). Disabling this option is safe and only affects retry behaviorβ€”duplicates may occur on producer retries, but the pipeline will continue to function normally.", "is_advanced": true, "default": true }, @@ -38076,7 +38076,7 @@ "name": "idempotent_write", "type": "bool", "kind": "scalar", - "description": "Enable the idempotent write producer option. When enabled, the producer initializes a producer ID and uses it to guarantee exactly-once semantics per partition (no duplicates on retries). This requires the `IDEMPOTENT_WRITE` permission on the `CLUSTER` resource. If your cluster does not grant this permission or uses ACLs restrictively, disable this option. Note: Idempotent writes are strictly a win for data integrity but may be unavailable in restricted environments (e.g., some managed Kafka services, Redpanda with strict ACLs). Disabling this option is safe and only affects retry behaviorβ€”duplicates may occur on producer retries, but the pipeline will continue to function normally.", + "description": "Enable the idempotent write producer option. When enabled, the producer initializes a producer ID and uses it to guarantee exactly-once semantics per partition (no duplicates on retries). This requires the `IDEMPOTENT_WRITE` permission on the `CLUSTER` resource. If your cluster does not grant this permission or uses ACLs restrictively, disable this option. Note: Idempotent writes are strictly a win for data integrity but may be unavailable in restricted environments (for example, some managed Kafka services, Redpanda with strict ACLs). Disabling this option is safe and only affects retry behaviorβ€”duplicates may occur on producer retries, but the pipeline will continue to function normally.", "is_advanced": true, "default": true }, @@ -38893,7 +38893,7 @@ "name": "idempotent_write", "type": "bool", "kind": "scalar", - "description": "Enable the idempotent write producer option. When enabled, the producer initializes a producer ID and uses it to guarantee exactly-once semantics per partition (no duplicates on retries). This requires the `IDEMPOTENT_WRITE` permission on the `CLUSTER` resource. If your cluster does not grant this permission or uses ACLs restrictively, disable this option. Note: Idempotent writes are strictly a win for data integrity but may be unavailable in restricted environments (e.g., some managed Kafka services, Redpanda with strict ACLs). Disabling this option is safe and only affects retry behaviorβ€”duplicates may occur on producer retries, but the pipeline will continue to function normally.", + "description": "Enable the idempotent write producer option. When enabled, the producer initializes a producer ID and uses it to guarantee exactly-once semantics per partition (no duplicates on retries). This requires the `IDEMPOTENT_WRITE` permission on the `CLUSTER` resource. If your cluster does not grant this permission or uses ACLs restrictively, disable this option. Note: Idempotent writes are strictly a win for data integrity but may be unavailable in restricted environments (for example, some managed Kafka services, Redpanda with strict ACLs). Disabling this option is safe and only affects retry behaviorβ€”duplicates may occur on producer retries, but the pipeline will continue to function normally.", "is_advanced": true, "default": true }, @@ -39329,7 +39329,7 @@ "name": "strict", "type": "bool", "kind": "scalar", - "description": "Error on unknown schema IDs. Only relevant when translate_ids is true. When false (default), unknown schema IDs are passed through unchanged, allowing migration of topics with mixed message formats. Note: messages with 0-byte prefixes (e.g., protobuf) cannot be distinguished from schema registry headers and may fail when strict is enabled.", + "description": "Error on unknown schema IDs. Only relevant when translate_ids is true. When false (default), unknown schema IDs are passed through unchanged, allowing migration of topics with mixed message formats. Note: messages with 0-byte prefixes (for example, protobuf) cannot be distinguished from schema registry headers and may fail when strict is enabled.", "default": false, "linter": "root = if this && !this.schema_registry.translate_ids { \"strict is only relevant when translate_ids is true\" }" } @@ -46931,7 +46931,7 @@ "name": "base_url", "type": "string", "kind": "scalar", - "description": "Jira instance base URL (e.g., https://your-domain.atlassian.net)" + "description": "Jira instance base URL (for example, https://your-domain.atlassian.net)" }, { "name": "username", diff --git a/extensions/DEVELOPMENT.adoc b/extensions/DEVELOPMENT.adoc new file mode 100644 index 00000000..1b91dff3 --- /dev/null +++ b/extensions/DEVELOPMENT.adoc @@ -0,0 +1,464 @@ += Antora Extensions Development Guide +:toc: +:toclevels: 3 + +Guide for developing new Antora extensions. + +Antora extensions are Node.js modules that hook into the Antora build pipeline to modify, enhance, or generate content. Extensions can run at different stages of the build process and access the content catalog, component descriptors, and more. + +== Extension architecture + +Antora extensions are JavaScript modules that export a `register` function. This function receives a context object that provides access to the Antora build lifecycle and APIs. + +=== Extension lifecycle + +Extensions register event listeners that execute at specific points in the Antora build: + +. **contentAggregated** - After content is collected but before conversion +. **uiLoaded** - After UI bundle is loaded +. **contentClassified** - After content is organized by component/module +. **documentsConverted** - After AsciiDoc conversion to HTML +. **beforePublish** - Before site is written to disk +. **sitepublished** - After site is published + +=== Extension structure + +Basic extension structure: + +[,javascript] +---- +module.exports.register = function ({ config }) { + // Extension logic here +} +---- + +With event listeners: + +[,javascript] +---- +module.exports.register = function ({ config }) { + this + .on('contentAggregated', ({ contentAggregate }) => { + // Process aggregated content + }) + .on('documentsConverted', ({ contentCatalog }) => { + // Process converted documents + }) +} +---- + +== Creating a new extension + +=== Create the file + +Create a new JavaScript file in the `extensions/` directory: + +[,bash] +---- +touch extensions/my-extension.js +---- + +=== Implement the register function + +[,javascript] +---- +/** + * My Extension - Does something useful + * + * Configuration options: + * - option1: Description of option1 + * - option2: Description of option2 + */ +module.exports.register = function ({ config }) { + const logger = this.getLogger('my-extension') + + // Access configuration + const option1 = config.option1 || 'default-value' + const option2 = config.option2 + + // Validate configuration + if (!option2) { + logger.error('option2 is required') + return + } + + // Register event listeners + this.on('contentClassified', ({ contentCatalog }) => { + logger.info('Processing content...') + + // Your extension logic here + const pages = contentCatalog.getPages() + + pages.forEach(page => { + // Do something with each page + logger.debug(`Processing ${page.src.relative}`) + }) + + logger.info('Processing complete') + }) +} +---- + +=== Add to playbook + +Test your extension by adding it to a playbook: + +[,yaml] +---- +antora: + extensions: + - require: './extensions/my-extension.js' + option1: value1 + option2: value2 +---- + +=== Test + +Run Antora and verify your extension works: + +[,bash] +---- +npx antora local-antora-playbook.yml +---- + +== Common patterns + +=== Accessing content + +Get all pages: + +[,javascript] +---- +this.on('contentClassified', ({ contentCatalog }) => { + const pages = contentCatalog.getPages() + + pages.forEach(page => { + console.log(page.src.relative) + console.log(page.asciidoc) // AsciiDoc content + console.log(page.contents) // Converted HTML + }) +}) +---- + +Filter pages by component: + +[,javascript] +---- +const pages = contentCatalog.getPages(page => + page.src.component === 'redpanda' +) +---- + +=== Creating new pages + +Add a generated page to the catalog: + +[,javascript] +---- +this.on('contentClassified', ({ contentCatalog }) => { + const newPage = contentCatalog.addFile({ + contents: Buffer.from('

Generated Page

'), + src: { + component: 'redpanda', + version: '1.0', + module: 'ROOT', + family: 'page', + relative: 'generated-page.adoc' + }, + out: { + path: 'generated-page.html' + }, + pub: { + url: '/generated-page.html' + } + }) + + // Set page attributes + newPage.asciidoc = { + attributes: { + 'page-title': 'Generated Page' + } + } +}) +---- + +=== Modifying pages + +Update page content: + +[,javascript] +---- +this.on('documentsConverted', ({ contentCatalog }) => { + const pages = contentCatalog.getPages() + + pages.forEach(page => { + if (page.asciidoc) { + // Modify AsciiDoc attributes + page.asciidoc.attributes['custom-attr'] = 'value' + + // Modify HTML content + page.contents = Buffer.from( + page.contents.toString().replace(/old/g, 'new') + ) + } + }) +}) +---- + +=== Working with components + +Access component descriptors: + +[,javascript] +---- +this.on('contentAggregated', ({ contentAggregate }) => { + contentAggregate.forEach(aggregate => { + const { name, version, title } = aggregate + console.log(`Component: ${name} v${version}`) + + // Access component files + aggregate.files.forEach(file => { + console.log(file.path) + }) + }) +}) +---- + +=== Reading external data + +Load external files or fetch data: + +[,javascript] +---- +const fs = require('fs') +const path = require('path') + +module.exports.register = function ({ config }) { + this.on('contentClassified', ({ contentCatalog, playbook }) => { + const dataPath = path.join(playbook.dir, 'data', 'my-data.json') + const data = JSON.parse(fs.readFileSync(dataPath, 'utf8')) + + // Use data to generate or modify content + }) +} +---- + +Fetch data from APIs: + +[,javascript] +---- +const https = require('https') + +function fetchData(url) { + return new Promise((resolve, reject) => { + https.get(url, (res) => { + let data = '' + res.on('data', chunk => data += chunk) + res.on('end', () => resolve(JSON.parse(data))) + res.on('error', reject) + }) + }) +} + +module.exports.register = function ({ config }) { + this.on('contentClassified', async ({ contentCatalog }) => { + const data = await fetchData('https://api.example.com/data') + // Process data + }) +} +---- + +=== Logging + +Use the logger for debugging: + +[,javascript] +---- +const logger = this.getLogger('my-extension') + +logger.error('Critical error message') +logger.warn('Warning message') +logger.info('Informational message') +logger.debug('Debug message') +---- + +Set log level in playbook: + +[,yaml] +---- +runtime: + log: + level: debug +---- + +=== Environment variables + +Access environment variables: + +[,javascript] +---- +module.exports.register = function ({ config }) { + const apiKey = process.env.API_KEY || config.api_key + + if (!apiKey) { + this.getLogger('my-extension').error('API_KEY not set') + return + } + + // Use apiKey +} +---- + +== Best practices + +=== Error handling + +Always handle errors gracefully: + +[,javascript] +---- +this.on('contentClassified', ({ contentCatalog }) => { + try { + // Extension logic + } catch (err) { + const logger = this.getLogger('my-extension') + logger.error(`Error: ${err.message}`) + logger.debug(err.stack) + // Don't throw - let build continue + } +}) +---- + +=== Performance + +* Cache expensive operations +* Use async/await for I/O operations +* Avoid processing files multiple times +* Filter content early to reduce iterations + +[,javascript] +---- +// Good: Filter early +const relevantPages = contentCatalog.getPages(page => + page.src.component === 'redpanda' && page.asciidoc +) + +relevantPages.forEach(page => { + // Process only relevant pages +}) + +// Bad: Filter during iteration +const allPages = contentCatalog.getPages() +allPages.forEach(page => { + if (page.src.component === 'redpanda' && page.asciidoc) { + // Process + } +}) +---- + +=== Testing + +Test extensions independently: + +[,javascript] +---- +// extensions/__tests__/my-extension.test.js +const myExtension = require('../my-extension') + +describe('my-extension', () => { + test('registers correctly', () => { + const context = { + on: jest.fn(), + getLogger: jest.fn(() => ({ + info: jest.fn(), + error: jest.fn() + })) + } + + myExtension.register.call(context, { config: {} }) + + expect(context.on).toHaveBeenCalled() + }) +}) +---- + +=== Documentation + +Document your extension: + +[,javascript] +---- +/** + * My Extension + * + * Description of what the extension does. + * + * Configuration: + * @param {string} option1 - Description of option1 + * @param {boolean} option2 - Description of option2 (required) + * + * Environment variables: + * - MY_VAR: Description + * + * Example: + * antora: + * extensions: + * - require: './extensions/my-extension.js' + * option1: value + * option2: true + */ +module.exports.register = function ({ config }) { + // Implementation +} +---- + +=== Code organization + +For complex extensions, split into modules: + +---- +extensions/ + my-extension/ + index.js # Main extension file + processor.js # Processing logic + generator.js # Content generation + utils.js # Utility functions +---- + +== Debugging + +=== Enable debug logging + +[,bash] +---- +ANTORA_LOG_LEVEL=debug npx antora playbook.yml +---- + +=== Use console.log sparingly + +Prefer the logger: + +[,javascript] +---- +// Good +const logger = this.getLogger('my-extension') +logger.debug('Debug info') + +// Avoid +console.log('Debug info') +---- + +=== Inspect objects + +[,javascript] +---- +const util = require('util') + +console.log(util.inspect(object, { depth: null, colors: true })) +---- + +== Related documentation + +* link:USER_GUIDE.adoc[User guide] - How to use extensions +* link:REFERENCE.adoc[Reference] - Complete extension documentation +* https://docs.antora.org/antora/latest/extend/extensions/[Antora Extensions API] diff --git a/extensions/README.adoc b/extensions/README.adoc new file mode 100644 index 00000000..3455d3c0 --- /dev/null +++ b/extensions/README.adoc @@ -0,0 +1,124 @@ += Antora Extensions +:toc: +:toclevels: 2 + +Documentation for Antora extensions that enhance the documentation build process. + +== What are Antora extensions? + +Antora extensions are Node.js modules that extend Antora's capabilities during the documentation site build. These extensions can modify content, generate new pages, integrate with external services, and customize the build pipeline. + +== Documentation + +link:USER_GUIDE.adoc[**User guide**]:: How to use and configure Antora extensions in your playbook ++ +* Installation and setup +* Configuring extensions +* Common usage patterns +* Troubleshooting + +link:REFERENCE.adoc[**Reference**]:: Complete reference for all available extensions ++ +* Extension descriptions +* Configuration options +* Environment variables +* Examples for each extension + +link:DEVELOPMENT.adoc[**Development guide**]:: How to develop new Antora extensions ++ +* Extension architecture +* Creating new extensions +* Testing and debugging +* Best practices + +== Quickstart + +=== Install the package + +[,bash] +---- +npm i @redpanda-data/docs-extensions-and-macros +---- + +=== Configure in your playbook + +[,yaml] +---- +antora: + extensions: + - '@redpanda-data/docs-extensions-and-macros/extensions/version-fetcher/set-latest-version' +---- + +IMPORTANT: Extensions must be registered under the `antora.extensions` key in your playbook, not `asciidoc.extensions`. + +== Available extensions + +=== Version management +* **set-latest-version** - Automatically fetch latest versions from GitHub releases + +=== Content generation +* **generate-index-data** - Generate searchable indexes from content +* **generate-rp-connect-categories** - Generate Redpanda Connect component categories +* **generate-rp-connect-info** - Generate Redpanda Connect component information + +=== Navigation +* **unlisted-pages** - Manage pages that appear in navigation but aren't listed +* **produce-redirects** - Generate redirect configurations +* **unpublish-pages** - Mark pages as unpublished + +=== Integrations +* **algolia-indexer** - Index content for Algolia search +* **compute-end-of-life** - Calculate and display EOL information +* **find-related-docs** - Find and link related documentation +* **find-related-labs** - Find and link related lab exercises + +=== File processing +* **archive-attachments** - Archive file attachments +* **replace-attributes-in-attachments** - Replace AsciiDoc attributes in attached files +* **collect-bloblang-samples** - Collect Bloblang code samples + +=== Content enhancement +* **add-global-attributes** - Add attributes globally to all pages +* **add-pages-to-root** - Add pages to the root navigation +* **process-context-switcher** - Handle context-dependent content +* **validate-attributes** - Validate AsciiDoc attributes + +See link:REFERENCE.adoc[Reference documentation] for complete details on each extension. + +== Common use cases + +=== Automatically update version numbers + +[,yaml] +---- +antora: + extensions: + - '@redpanda-data/docs-extensions-and-macros/extensions/version-fetcher/set-latest-version' +---- + +=== Enable Algolia search + +[,yaml] +---- +antora: + extensions: + - require: '@redpanda-data/docs-extensions-and-macros/extensions/algolia-indexer' + algolia_app_id: YOUR_APP_ID + algolia_api_key: YOUR_API_KEY + algolia_index_name: YOUR_INDEX +---- + +=== Generate redirects + +[,yaml] +---- +antora: + extensions: + - '@redpanda-data/docs-extensions-and-macros/extensions/produce-redirects.js' +---- + +== Support + +* link:USER_GUIDE.adoc#troubleshooting[Troubleshooting guide] +* https://github.com/redpanda-data/docs-extensions-and-macros/issues[Report issues] +* https://docs.antora.org/antora/latest/extend/extensions/[Antora Extensions Documentation] diff --git a/extensions/REFERENCE.adoc b/extensions/REFERENCE.adoc new file mode 100644 index 00000000..af077cb7 --- /dev/null +++ b/extensions/REFERENCE.adoc @@ -0,0 +1,768 @@ += Antora extensions reference +:toc: +:toclevels: 3 + +Reference documentation for all Antora extensions provided by this library. + +IMPORTANT: Ensure you register each extension under the `antora.extensions` key in the playbook, not the `asciidoc.extensions` key. + +== Add Bloblang samples to pages + +The `collect-bloblang-samples` extension processes Bloblang examples from YAML files in the `examples` directory of the `redpanda-connect` component. This extension ensures that these examples are accessible as structured data for use in UI components or documentation, such as sample dropdowns in a Bloblang playground. + +It validates, sorts, and attaches the processed examples as a JSON object to the Antora page attributes. The extension ensures examples have unique titles, mandatory fields (`input` and `mapping`), and are sorted in alphabetical order. + +The processed examples are added as JSON to the `page-bloblang-samples` attribute. For example: + +[,json] +---- +{ + "hello-world.yaml": { + "title": "Hello world", + "input": "{\n \"message\": \"hello world\"\n}\n", + "mapping": "root.message = this.message.uppercase()\n" + }, + "array-processing.yaml": { + "title": "Array processing", + "input": "{\n \"numbers\": [1, 2, 3, 4, 5]\n}\n", + "mapping": "root.even_numbers = this.numbers.filter(n -> n % 2 == 0)" + } +} +---- + +=== Environment variables + +This extension does not require any environment variables. + +=== Configuration options + +To enable the extension, add it to your Antora playbook under the `antora.extensions` key. No additional configuration is required. + +[,yaml] +---- +antora: + extensions: + - require: '@redpanda-data/docs-extensions-and-macros/extensions/collect-bloblang-samples' +---- + +=== Example Bloblang YAML file + +The following YAML file is an example of how to define a Bloblang sample: + +[,yaml] +---- +title: Hello world +input: | + { + "message": "hello world" + } +mapping: | + root.message = this.message.uppercase() +---- + +== Add pages to root + +The `add-pages-to-root` extension allows you to copy files from your Antora content catalog to the root of the site during the build process. This is particularly useful for files like `llms.txt` or any custom files that need to be directly accessible at the site's root level. + +This extension processes a list of file paths provided in the playbook configuration, locates those files in the Antora content catalog, and adds them to the site's root directory during the publishing phase. Each file's content and basename are preserved in the process. + +=== Environment variables + +This extension does not require any environment variables. + +=== Configuration options + +Add the `add-pages-to-root` extension to your Antora playbook under the `antora.extensions` key, and specify the list of files to process in the `files` configuration. + +[source,yaml] +---- +antora: + extensions: + - require: '@redpanda-data/docs-extensions-and-macros/extensions/add-pages-to-root' + files: + - home:ROOT:attachment$custom-file.txt +---- + +=== Registration + +[source,yaml] +---- +antora: + extensions: + - require: '@redpanda-data/docs-extensions-and-macros/extensions/add-pages-to-root' + files: + - home:ROOT:attachment$custom-file.txt +---- + +== Algolia indexer + +This extension generates an Algolia index for each version of each component. The index entries are then saved to Algolia using the `saveObjects()` method, and also saved as JSON files in the site catalog. JSON files are published to the site root using the template `algolia--.json`. + +NOTE: Only pages that include an `
` element with the `doc` class are indexed. + +=== Environment variables + +- `ALGOLIA_ADMIN_API_KEY` (required) +- `ALGOLIA_APP_ID` (required) +- `ALGOLIA_INDEX_NAME` (required) + +=== Configuration options + +The extension accepts the following configuration options: + +excludes (optional):: +Any elements, classes, or IDs that you want to exclude from the index. +index-latest-only (optional):: +Whether to index all versions or just the latest version of a component. + +=== Registration + +```yaml +antora: + extensions: + - require: '@redpanda-data/docs-extensions-and-macros/extensions/algolia-indexer/index' + excludes: ['.thumbs','script', '.page-versions','.feedback-section','.banner-container'] + index-latest-only: true +``` + +== Archive attachments + +The `archive-attachments` extension automates the packaging of specific attachment files into a compressed archive (`.tar.gz`) based on configurable patterns. This archive is then made available to the generated site, allowing users to easily download grouped resources such as Docker Compose configurations. + +This extension enables you to define which files and directories to include in the archive, ensuring that only relevant content is packaged and accessible. + +=== Environment variables + +This extension does not require any environment variables. + +=== Configuration options + +The extension accepts the following options in the Antora playbook. + +Configure the extension in your Antora playbook by defining an array of archive configurations under `data.archives`. Each archive configuration includes: + +output_archive (string, required):: The name of the generated archive file. + +component (string, required):: The name of the Antora component whose attachments should be archived. + +file_patterns (array of strings, required):: Glob patterns specifying which attachment paths to include in the archive. + +NOTE: Ensure that `file_patterns` accurately reflect the paths of the attachments you want to archive. Overly broad patterns may include unintended files, while overly restrictive patterns might exclude necessary resources. + +=== Example configuration + +Here's an example configuration to enable the extension: + +```yaml +antora: + extensions: + - require: '../docs-extensions-and-macros/extensions/archive-creation-extension.js' + data: + archives: + - output_archive: 'redpanda-quickstart.tar.gz' <1> + component: 'ROOT' <2> + file_patterns: + - '**/test-resources/**/docker-compose/**' <3> +``` + +<1> Defines the name of the generated archive placed at the site root. +<2> Defines the name of the component in which to search for attachments. +<3> Lists the glob patterns to match attachment paths for inclusion in the archive. ++ +- `**`: Matches any number of directories. +- `/test-resources/`: Specifies that the matching should occur within the `test-resources/` directory. +- `/docker-compose/`: Targets the `docker-compose/` directory and all its subdirectories. +- `**:` Ensures that all files and nested directories within `docker-compose/` are included. + +== Behavior with multiple components/versions + +*Scenario*: Multiple components and/or multiple versions of the same component contain attachments that match the defined file_patterns. + +*Outcome*: Separate archives for each component version. + +For each matching (component, version) pair, the extension creates a distinct archive named `-`. For example: +`24.3-redpanda-quickstart.tar.gz`. + +These archives are placed at the site root, ensuring they are easily accessible and do not overwrite each other. + +For the latest version of each component, the extension also adds the archive using the base `output_archive` name. As a result, the latest archives are accessible through a consistent filename, facilitating easy downloads without needing to reference version numbers. + +Because each archive has a unique filename based on the component version, there is no risk of archives overwriting each other. +The only exception is the archive for the latest version, which consistently uses the `output_archive` name. + +== Component category aggregator + +This extension maps Redpanda Connect component data into a structured format: + +- Maps original component names to common names. +- Populates `connectCategoriesData` and `flatComponentsData` attributes. +- Skips deprecated components. + +=== Environment variables + +This extension does not require any environment variables. + +=== Configuration options + +There are no configurable options for this extension. + +=== Registration + +```yaml +antora: + extensions: + - require: '@redpanda-data/docs-extensions-and-macros/extensions/generate-rp-connect-categories' +``` + +== Compute end-of-life extension + +This extension calculates and attaches metadata related to the end-of-life (EoL) status of docs pages, such as nearing EoL, past EoL, and associated EoL dates. This metadata can be used to display relevant banners or messages in docs to inform users about the lifecycle of each version. + +The extension leverages configuration settings provided in the Antora playbook to apply EoL calculations, specify the warning period, and include links to upgrade documentation and EoL policies. + +The extension computes whether a page is nearing EoL or past EoL based on the `page-release-date` attribute and configured settings. +It injects the following attributes into each page, making them available for use in UI templates: + +- `page-is-nearing-eol`: Indicates if the page is within the warning period before EoL. Calculated using `(page-release-date + supported_months) - warning_weeks`. +- `page-is-past-eol`: Indicates if the page has passed its EoL. Calculated using `today > (page-release-date + supported_months)`. +- `page-eol-date`: The calculated EoL date in a human-readable format. Calculated using `page-release-date + supported_months`. +- `page-eol-doc`: The URL to the supported versions policy or EoL documentation. +- `page-upgrade-doc`: The Antora resource ID to a document containing upgrade instructions. + +=== Environment variables + +This extension does not require any environment variables. + +=== Configuration options + +To enable and configure the extension, add it to the `antora.extensions` section of your Antora playbook. Define the EoL settings under the `data.eol_settings` key with the following options: + +`component` (required):: The component name to which the configuration applies. +`eol_doc` (required):: A link to the supported versions policy or EoL documentation. +`upgrade_doc` (required):: A link to the upgrade instructions. +`supported_months` (optional, default: 12):: The number of months after the publish date when the documentation reaches its EoL. +`warning_weeks` (optional, default: 6):: The number of weeks before EoL when the documentation is considered to be nearing EoL. Can be used to decide when to notify users of the upcoming EoL status. + +[,yaml] +---- +antora: + extensions: + - require: '@redpanda-data/docs-extensions-and-macros/extensions/compute-end-of-life' + data: + eol_settings: + - component: 'ROOT' + supported_months: 18 + warning_weeks: 8 + eol_doc: https://support.redpanda.com/hc/en-us/articles/20617574366743-Redpanda-Supported-Versions + upgrade_doc: ROOT:upgrade:index.adoc +---- + +=== Registration + +You can register the extension with a customized configuration for different components in your playbook: + +[,yaml] +---- +antora: + extensions: + - require: '@redpanda-data/docs-extensions-and-macros/extensions/compute-end-of-life' + data: + eol_settings: + - component: 'ROOT' + supported_months: 12 + warning_weeks: 6 + eol_doc: https://example.com/supported-versions + upgrade_doc: ROOT:upgrade:index.adoc + - component: 'example-docs' + supported_months: 24 + warning_weeks: 12 + eol_doc: https://example.com/example-supported-versions + upgrade_doc: example-docs:upgrade:index.adoc +---- + + +=== Example Handlebars template: + +[,handlebars] +---- +{{#if page.attributes.is-nearing-eol}} + +{{else if page.attributes.is-past-eol}} + +{{/if}} +---- + +== Generate index data + +The `generate-index-data` extension creates structured index data about doc pages based on configurable filters. The indexed data is saved to a specified attribute in all component versions, enabling the dynamic generation of categorized links and descriptions within your docs using UI templates. + +This extension allows you to define multiple indexing criteria, such as component, URL filter, and environment type. + +The generated data is an array of objects, where each object represents a component version. Each object contains the following properties: + +- `component` (string): + The name of the Antora component. + +- `version` (string): + The version of the component. + +- `pages` (array): + A list of pages that match the indexing criteria. Each page contains: +** `title` (string): The title of the doc page. +** `url` (string): The URL of the doc page relative to the site root. +** `description` (string): A brief description sourced from the `:description:` attribute in the AsciiDoc file. Defaults to an empty string if not provided. + +Example: + +```json +[ + { + "component": "ROOT", + "version": "24.3", + "pages": [ + { + "title": "Manage Debug Bundles in Redpanda Console", + "url": "/current/console/ui/generate-bundle/", + "description": "Learn how to generate, download, and delete debug bundles in Redpanda Console for comprehensive cluster diagnostics." + }, + ] + } +] +``` + +=== Environment variables + +This extension does not require any environment variables. + +=== Configuration options + +The extension accepts the following options in the Antora playbook. + +NOTE: Ensure filters are well-defined to minimize unnecessary processing. Avoid overly broad configurations in `data.sets`. + +- `data.sets` (required): An object defining one or more indexing configurations. Each configuration (or set) accepts the following options: + +** `component` (string, required): The Antora component to search for pages. + +** `attribute_name` (string, required): The attribute name to assign the generated index data. This allows pages and templates to reference the index. + +** `filter` (string, optional): A substring to match within page URLs. + +** `env_type` (string, optional): Matches pages with environment-specific attributes (for example, Docker, Kubernetes). + +** `output_file` (string, optional): Save the generated index data as a JSON file at the specified path. If not provided, no file is created. + +=== Example configuration + +Here's an example configuration to enable the generate-index-data-extension: + +```yaml +antora: + extensions: + - require: '@redpanda-data/docs-extensions-and-macros/extensions/generate-index-data-extension' + data: + sets: + console_ui: + component: ROOT # Search the ROOT component + filter: console/ui # Filter pages containing this substring in their URL + attribute_name: console-ui-index # Save the result in this attribute + output_file: redpanda-labs/console-ui-index.json # Save data to this file + docker_labs: + component: redpanda-labs + filter: docker-compose + env_type: Docker + attribute_name: docker-labs-index +``` + +=== Use the generated data + +The index data can be referenced in AsciiDoc pages by specifying the following required attributes: + +```asciidoc += CONSOLE UI +:page-index-data: console-ui-index <1> +:page-role: index-list <2> +``` + +<1> The attribute whose data you want to display on the page. This must match an attribute configured in the extension. +<2> The page role. This role specfies the UI template that renders the data in the `page-index-data` on the page. + +You can optionally display pages only if they match the component and version of the current Asciidoc page by adding the `:page-match-component-version:` attribute. + +```asciidoc += CONSOLE UI +:page-index-data: console-ui-index +:page-role: index-list +:page-match-component-version: '' +``` + +== Redpanda Connect tag modifier + +This extension updates the playbook to use the latest release tag for the Redpanda Connect documentation. It ensures that the Redpanda Connect documentation is always pulled from the latest release tag available on GitHub. + +=== Environment variables + +- `REDPANDA_GITHUB_TOKEN` (optional): A Personal access token (PAT) that has `repo` permissions for the `redpanda-data` GitHub organization. + +NOTE: If you don't set the environment variable, the latest version of Redpanda Connect may not be fetched. When the environment variable is not set, the extension sends unauthenticated requests to GitHub. Unauthenticated requests may result in hitting the API rate limit and cause GitHub to reject the request. In this case the fallback version is used. This version is defined in the playbook where the extension is registered. + +=== Configuration options + +There are no configurable options for this extension. + +=== Registration + +```yaml +antora: + extensions: + - require: '@redpanda-data/docs-extensions-and-macros/extensions/modify-connect-tag-playbook' +``` + +== Version fetcher + +This extension fetches the latest release versions from GitHub. + +The following attributes are available to all versions of all Antora components: + +`latest-console-version`: The latest release version of Redpanda Console. +`latest-connect-version`: The latest release version of Redpanda Connect. +`redpanda-beta-version`: The latest RC version of Redpanda. +`redpanda-beta-commit`: The commit hash for the latest RC version of Redpanda. + +The following attributes are available to the latest version of the `ROOT` component (Redpanda docs): + +`full-version`: The latest release version of Redpanda. +`latest-release-commit`: The commit hash for the latest release version of Redpanda. +`latest-operator-version`: The latest release version of the Redpanda Operator. +`latest-redpanda-helm-chart-version`: The latest release version of the Redpanda Helm chart. + +=== Environment variables + +- `REDPANDA_GITHUB_TOKEN` (optional): A Personal access token (PAT) that has `repo` permissions for the `redpanda-data` GitHub organization. + +NOTE: If you don't set the environment variable, the latest versions may not be fetched. When the environment variable is not set, the extension sends unauthenticated requests to GitHub. Unauthenticated requests may result in hitting the API rate limit and cause GitHub to reject the request. + +=== Registration + +```yaml +antora: + extensions: + - '@redpanda-data/docs-extensions-and-macros/extensions/version-fetcher/set-latest-version' +``` + +== Validate attributes + +This extension ensures the consistency and validity of page attributes, focusing on validating page categories against a predefined list of valid categories and subcategories. It automatically adds missing parent categories for any specified subcategories and removes any specified categories that are invalid. Additionally, it processes specific environment attributes, setting corresponding page-level attributes when environment conditions are met. + +=== Environment variables + +This extension does not require any environment variables. + +=== Configuration options + +There are no configurable options for this extension. It operates based on site attributes defined in `add-global-attributes.js` to determine valid categories and subcategories. + +=== Registration + +Register the `validate-attributes` extension in the Antora playbook under the `antora.extensions` key like so: + +[source,yaml] +---- +antora: + extensions: + - require: '@redpanda-data/docs-extensions-and-macros/extensions/validate-attributes.js' +---- + +== Related docs + +This extension enhances the connectivity between lab exercises and relevant documentation by dynamically identifying and linking related documentation pages and other lab exercises based on shared categories and deployment types. + +=== Environment variables + +This extension operates without requiring any specific environment variables. + +=== Configuration options + +This extension does not offer configurable options. It uses the inherent attributes of pages to determine relationships based on `page-categories` and deployment types (`env-kubernetes`, `env-linux`, `env-docker`, `page-cloud`). + +=== Registration + +To integrate the `related-docs-extension` into your Antora playbook, add it under the `antora.extensions` key as demonstrated below: + +[source,yaml] +---- +antora: + extensions: + - require: '@redpanda-data/docs-extensions-and-macros/extensions/related-docs-extension.js' +---- + +== Related labs + +This extension enriches documentation pages with links to related lab exercises, facilitating a deeper understanding of the content through practical application. It dynamically assigns related labs to each documentation page based on shared categories and deployment types. + +=== Environment variables + +This extension does not require any environment variables. + +=== Configuration options + +The extension operates without explicit configuration options. It automatically processes documentation pages to identify and link related labs based on shared `page-categories` attributes and deployment types (`env-kubernetes`, `env-linux`, `env-docker`, `page-cloud`). + +=== Registration + +Include the `related-labs-extension` in the Antora playbook under the `antora.extensions` key as follows: + +[source,yaml] +---- +antora: + extensions: + - require: '@redpanda-data/docs-extensions-and-macros/extensions/related-labs-extension.js' +---- + +== Global attributes + +This extension collects Asciidoc attributes from the https://github.com/redpanda-data/docs-site[`shared` component] or a local YAML file and makes them available to all component versions. Having global attributes is useful for consistent configuration of local and production builds. + +=== Environment variables + +This extension does not require any environment variables. + +=== Configuration options + +The extension accepts the following configuration options: + +attributespath (optional):: Specifies the path to a local YAML file that contains global attributes. If this is provided, the extension will load attributes from this file first. If this path is not provided or no valid attributes are found in the file, the extension will fall back to loading attributes from the `shared` component. + +=== Registration + +```yml +antora: + extensions: + - require: '@redpanda-data/docs-extensions-and-macros/extensions/add-global-attributes' + attributespath: './local-attributes.yml' +``` + +In this example, the `attributespath` option points to a local YAML file (`./local-attributes.yml`), which contains the global attributes. The extension will load attributes from this file first before falling back to the `shared` component. + +== Produce redirects (customization of core Antora) + +This extension replaces the default https://gitlab.com/antora/antora/-/tree/v3.1.x/packages/redirect-producer[`produceRedirects()` function] in Antora to handle redirect loops caused by https://docs.antora.org/antora/latest/page/page-aliases/[page aliases]. Normally, page aliases in Antora are used to resolve outdated links without causing issues. However, with https://docs.antora.org/antora/latest/playbook/urls-html-extension-style/#html-extension-style-key[`indexify`], the same URL may inadvertently be used for both the source and target of a redirect, leading to loops. This problem is https://antora.zulipchat.com/#narrow/stream/282400-users/topic/Redirect.20Loop.20Issue.20with.20Page.20Renaming.20and.20Indexify/near/433691700[recognized as a bug] in core Antora. For example, creating a page alias for `modules/manage/security/authorization.adoc` to point to `modules/manage/security/authorization/index.adoc' can lead to a redirect loop where `manage/security/authorization/` points to `manage/security/authorization/`. Furthermore, omitting the alias would lead to `xref not found` errors because Antora relies on the alias to resolve the old xrefs. This extension is necessary until such behaviors are natively supported or fixed in Antora core. + +=== Environment variables + +This extension does not require any environment variables. + +=== Configuration options + +There are no configurable options for this extension. + +=== Registration + +```yaml +antora: + extensions: + - '@redpanda-data/docs-extensions-and-macros/extensions/modify-redirects' +``` + +== Replace attributes in attachments + +This extension automates the replacement of AsciiDoc attribute placeholders with their respective values within attachment files, such as CSS, HTML, and YAML. + +[NOTE] +==== +- The `@` character is removed from attribute values to prevent potential issues with CSS or HTML syntax. +- If the same attribute placeholder is used multiple times within a file, all instances will be replaced with the attribute's value. +==== + +=== Environment variables + +This extension does not require any environment variables. + +=== Configuration options + +The extension accepts the following configuration options in the Antora playbook: + +data.replacements (required):: An array of replacement configurations. Each configuration can target multiple components and define specific file patterns and custom replacement rules. + +* `components` (array of strings, required): Lists the names of the Antora components whose attachments should undergo attribute replacement. + +* `file_patterns` (array of strings, required): Glob patterns specifying which attachment files to process. These patterns determine the files that will undergo attribute replacement based on their paths within the content catalog. + +* `custom_replacements` (array of objects, optional): Defines custom search-and-replace rules to be applied to the matched files. Each rule consists of: +** `search` (string, required): A regular expression pattern to search for within the file content. +** `replace` (string, required): The string to replace each match found by the `search` pattern. + +NOTE: Ensure that `file_patterns` accurately reflect the paths of the attachments you want to process. Overly broad patterns may include unintended files, while overly restrictive patterns might exclude necessary resources. + +=== Registration + +This is an example of how to register and configure the `replace-attributes-in-attachments` extension in your Antora playbook. This example demonstrates defining multiple replacement configurations, each targeting different components and specifying their own file patterns and custom replacements. + +```yaml +antora: + extensions: + - require: './extensions/replace-attributes-in-attachments' + data: + replacements: + - components: + - 'ROOT' + - 'redpanda-labs' + file_patterns: + - '**/docker-compose.yaml' + - '**/docker-compose.yml' + custom_replacements: + - search: ''\\$\\{CONFIG_FILE:[^}]*\\}'' + replace: 'console.yaml' + - components: + - 'API' + file_patterns: + - '**/api-docs/**/resources/**' + custom_replacements: + - search: '\\$\\{API_ENDPOINT:[^}]*\\}' + replace: 'https://api.example.com' +``` + +== Aggregate terms + +This extension aggregates all term pages from the https://github.com/redpanda-data/docs-site[`shared` component] and does the following: + +- Makes all `term-name`, `hover-text`, and `link` attributes available to the `glossterm` macro. +- Looks for glossary pages named `reference:glossary.adoc` in all versions of all components and appends the contents of each term file to the glossary in alphabetical order. +- If a glossary page is found, sets the `glossary-page` attribute of the `glossterm` macro to `reference:glossary.adoc` so that terms can be linked to the glossary page. + +=== Environment variables + +This extension does not require any environment variables. + +=== Configuration options + +The extension accepts the following configuration options: + +termspath (optional):: Specifies the path to a local directory containing term files (in `.adoc` format). If this path is provided, the extension will attempt to load terms from this directory first. If this path is not provided or no valid terms are found in the specified directory, the extension will fall back to loading terms from the `shared` component. + +Term files should follow the following structure: + +```asciidoc +:category: Documentation +:hover-text: This is a description of the term. +:link: https://example.com + +== Term Title + +This is the detailed description of the term. +``` + +=== Registration + +```yml +antora: + extensions: + - require: '@redpanda-data/docs-extensions-and-macros/extensions/aggregate-terms' + termspath: './local-terms/' +``` + +In this example, the `termspath` option points to a local directory (./local-terms/), where the term files are stored. The extension will load terms from this directory first before falling back to the `shared` component. + +== Unlisted pages + +This extension identifies and logs any pages that aren't listed in the navigation (nav) file of each version of each component. It then optionally adds these unlisted pages to the end of the navigation tree under a configurable heading. + +IMPORTANT: By default, this extension excludes components named 'api'. This behavior is hardcoded and cannot be changed in the configuration. + +=== Environment variables + +This extension does not require any environment variables. + +=== Configuration options + +This extension accepts the following configuration options: + +addToNavigation (optional):: +Whether to add unlisted pages to the navigation. The default is `false` (unlisted pages are not added). + +unlistedPagesHeading (optional):: +The heading under which to list the unlisted pages in the navigation. The default is 'Unlisted Pages'. + +=== Registration + +```yaml +antora: + extensions: + - require: '@redpanda-data/docs-extensions-and-macros/extensions/unlisted-pages' + addToNavigation: true + unlistedPagesHeading: 'Additional Resources' +``` + +== Process context switcher + +This extension processes the `page-context-switcher` attribute to enable cross-version navigation widgets in documentation pages. It automatically replaces "current" references with full resource IDs and injects the context switcher configuration to all referenced target pages, ensuring bidirectional navigation works correctly. + +The extension finds pages with the `page-context-switcher` attribute, parses the JSON configuration, and: + +1. Replaces any "current" values with the full resource ID of the current page +2. Finds all target pages referenced in the switcher configuration +3. Injects the same context switcher attribute to target pages (with appropriate resource ID mappings) +4. Builds resource IDs in the format: `version@component:module:relative-path` + +This enables UI components to render version switchers that work across different versions of the same content. + +=== Environment variables + +This extension does not require any environment variables. + +=== Configuration options + +This extension does not require any configuration options. + +=== Registration + +```yaml +antora: + extensions: + - require: '@redpanda-data/docs-extensions-and-macros/extensions/process-context-switcher' +``` + +=== Usage + +Add the `page-context-switcher` attribute to any page where you want cross-version navigation: + +```asciidoc +:page-context-switcher: [{"name": "Version 2.x", "to": "24.3@ROOT:console:config/security/authentication.adoc" },{"name": "Version 3.x", "to": "current" }] +``` + +=== Processed output + +After processing, the "current" reference is replaced with the full resource ID: + +```json +[ + {"name": "Version 2.x", "to": "24.3@ROOT:console:config/security/authentication.adoc"}, + {"name": "Version 3.x", "to": "current@ROOT:console:config/security/authentication.adoc"} +] +``` + +The target page (`24.3@ROOT:console:config/security/authentication.adoc`) will also receive the same context switcher configuration with appropriate resource ID mappings. + +=== UI integration + +The processed attribute can be used in Handlebars templates: + +```html +
+ {{#each (obj page.attributes.page-context-switcher)}} + + + + {{/each}} +
+``` diff --git a/extensions/USER_GUIDE.adoc b/extensions/USER_GUIDE.adoc new file mode 100644 index 00000000..249a8288 --- /dev/null +++ b/extensions/USER_GUIDE.adoc @@ -0,0 +1,339 @@ += Antora Extensions User Guide +:toc: +:toclevels: 3 + +Complete guide to using Antora extensions in your documentation projects. + +== Getting started + +=== Installation + +Install the package via npm: + +[,bash] +---- +npm i @redpanda-data/docs-extensions-and-macros +---- + +=== Basic configuration + +Extensions are configured in your Antora playbook under the `antora.extensions` key: + +[,yaml] +---- +antora: + extensions: + - '@redpanda-data/docs-extensions-and-macros/extensions/extension-name' +---- + +IMPORTANT: Extensions must be under `antora.extensions`, not `asciidoc.extensions`. AsciiDoc macros go under `asciidoc.extensions`. + +=== Extension with options + +Pass configuration options to extensions: + +[,yaml] +---- +antora: + extensions: + - require: '@redpanda-data/docs-extensions-and-macros/extensions/extension-name' + option1: value1 + option2: value2 +---- + +== Configuration patterns + +=== Simple registration + +For extensions without configuration options: + +[,yaml] +---- +antora: + extensions: + - '@redpanda-data/docs-extensions-and-macros/extensions/unlisted-pages.js' +---- + +=== With environment variables + +Some extensions use environment variables for sensitive data: + +[,yaml] +---- +antora: + extensions: + - require: '@redpanda-data/docs-extensions-and-macros/extensions/algolia-indexer' + algolia_app_id: $ALGOLIA_APP_ID + algolia_api_key: $ALGOLIA_API_KEY + algolia_index_name: docs +---- + +[,bash] +---- +export ALGOLIA_APP_ID=your_app_id +export ALGOLIA_API_KEY=your_api_key +npx antora local-antora-playbook.yml +---- + +=== Multiple extensions + +Register multiple extensions in order: + +[,yaml] +---- +antora: + extensions: + - '@redpanda-data/docs-extensions-and-macros/extensions/version-fetcher/set-latest-version' + - '@redpanda-data/docs-extensions-and-macros/extensions/generate-index-data.js' + - '@redpanda-data/docs-extensions-and-macros/extensions/produce-redirects.js' +---- + +Extensions run in the order they're listed. + +== Common workflows + +=== Version management + +Automatically fetch and set the latest Redpanda version: + +[,yaml] +---- +antora: + extensions: + - '@redpanda-data/docs-extensions-and-macros/extensions/version-fetcher/set-latest-version' +---- + +This sets the `latest-redpanda-version` attribute for use in your documentation. + +=== Search integration + +Enable Algolia search indexing: + +[,yaml] +---- +antora: + extensions: + - require: '@redpanda-data/docs-extensions-and-macros/extensions/algolia-indexer' + algolia_app_id: $ALGOLIA_APP_ID + algolia_api_key: $ALGOLIA_API_KEY + algolia_index_name: redpanda-docs + algolia_index_version: latest +---- + +=== Redirect management + +Generate redirects from old URLs to new ones: + +[,yaml] +---- +antora: + extensions: + - '@redpanda-data/docs-extensions-and-macros/extensions/produce-redirects.js' +---- + +Define redirects in your `antora.yml`: + +[,yaml] +---- +name: redpanda +asciidoc: + attributes: + page-aliases: old-page.adoc,another-old-page.adoc +---- + +=== Content generation + +Generate component categories for Redpanda Connect: + +[,yaml] +---- +antora: + extensions: + - '@redpanda-data/docs-extensions-and-macros/extensions/generate-rp-connect-categories.js' + - '@redpanda-data/docs-extensions-and-macros/extensions/generate-rp-connect-info.js' +---- + +=== Navigation customization + +Manage unlisted pages that appear in navigation: + +[,yaml] +---- +antora: + extensions: + - '@redpanda-data/docs-extensions-and-macros/extensions/unlisted-pages.js' +---- + +Mark pages as unlisted in their AsciiDoc header: + +[,asciidoc] +---- += Page Title +:page-unlisted: + +Content here... +---- + +== Extension order matters + +Extensions execute in registration order. Some extensions depend on others running first: + +[,yaml] +---- +antora: + extensions: + # 1. Fetch versions first + - '@redpanda-data/docs-extensions-and-macros/extensions/version-fetcher/set-latest-version' + # 2. Then generate content that uses versions + - '@redpanda-data/docs-extensions-and-macros/extensions/generate-index-data.js' + # 3. Finally index for search + - require: '@redpanda-data/docs-extensions-and-macros/extensions/algolia-indexer' + algolia_app_id: $ALGOLIA_APP_ID + algolia_api_key: $ALGOLIA_API_KEY + algolia_index_name: docs +---- + +== Troubleshooting + +=== Extension not loading + +*Problem:* Extension doesn't seem to run + +*Solutions:* + +* Check the extension is under `antora.extensions`, not `asciidoc.extensions` +* Verify the path to the extension is correct +* Ensure the package is installed (`npm install`) +* Check for typos in the extension name + +=== Environment variables not working + +*Problem:* Extensions using environment variables fail + +*Solutions:* + +* Set environment variables before running Antora: ++ +[,bash] +---- +export VAR_NAME=value +npx antora playbook.yml +---- + +* Check variable names match exactly (case-sensitive) +* Use `$VAR_NAME` syntax in playbook +* Verify variables are exported, not just set + +=== Extension execution order issues + +*Problem:* Extensions not working as expected + +*Solutions:* + +* Check extension order in playbook +* Some extensions must run before others +* Review extension dependencies in link:REFERENCE.adoc[Reference docs] + +=== Build performance + +*Problem:* Build is slow with extensions enabled + +*Solutions:* + +* Some extensions make network requests (for example, version fetcher, Algolia) +* Use caching when possible +* Disable non-essential extensions during development +* Run only necessary extensions for your workflow + +=== Extension errors + +*Problem:* Extension throws errors during build + +*Solutions:* + +* Check extension configuration options are correct +* Verify required environment variables are set +* Review extension logs for specific error messages +* Ensure your Antora version is compatible +* Check link:REFERENCE.adoc[Reference docs] for requirements + +== Best practices + +=== Development vs production + +Use different playbooks for development and production: + +*Development playbook* (`local-antora-playbook.yml`): +[,yaml] +---- +antora: + extensions: + - '@redpanda-data/docs-extensions-and-macros/extensions/version-fetcher/set-latest-version' + # Skip search indexing during development +---- + +*Production playbook* (`antora-playbook.yml`): +[,yaml] +---- +antora: + extensions: + - '@redpanda-data/docs-extensions-and-macros/extensions/version-fetcher/set-latest-version' + - '@redpanda-data/docs-extensions-and-macros/extensions/generate-index-data.js' + - require: '@redpanda-data/docs-extensions-and-macros/extensions/algolia-indexer' + algolia_app_id: $ALGOLIA_APP_ID + algolia_api_key: $ALGOLIA_API_KEY + algolia_index_name: docs +---- + +=== Organizing playbooks + +Keep extension configuration organized: + +[,yaml] +---- +antora: + extensions: + # Version management + - '@redpanda-data/docs-extensions-and-macros/extensions/version-fetcher/set-latest-version' + + # Content generation + - '@redpanda-data/docs-extensions-and-macros/extensions/generate-rp-connect-categories.js' + - '@redpanda-data/docs-extensions-and-macros/extensions/generate-rp-connect-info.js' + + # Search integration + - require: '@redpanda-data/docs-extensions-and-macros/extensions/algolia-indexer' + algolia_app_id: $ALGOLIA_APP_ID + algolia_api_key: $ALGOLIA_API_KEY + algolia_index_name: docs +---- + +=== Security + +* Never commit API keys or secrets to version control +* Use environment variables for sensitive data +* Set permissions appropriately in CI/CD +* Rotate API keys regularly + +=== Testing + +Test your playbook locally before deploying: + +[,bash] +---- +# Set test environment variables +export ALGOLIA_APP_ID=test_app_id +export ALGOLIA_API_KEY=test_api_key + +# Run Antora +npx antora local-antora-playbook.yml + +# Check the output +ls -la build/site/ +---- + +== Related documentation + +* link:REFERENCE.adoc[Extensions reference] - Complete documentation for all extensions +* link:DEVELOPMENT.adoc[Development guide] - How to create new extensions +* https://docs.antora.org/antora/latest/playbook/[Antora Playbook Reference] +* https://docs.antora.org/antora/latest/extend/extensions/[Antora Extensions] diff --git a/extensions/generate-rp-connect-info.js b/extensions/generate-rp-connect-info.js index d3b81d36..91ab1d34 100644 --- a/extensions/generate-rp-connect-info.js +++ b/extensions/generate-rp-connect-info.js @@ -13,10 +13,9 @@ module.exports.register = function ({ config }) { async function loadOctokit() { const { Octokit } = await import('@octokit/rest'); - if (!process.env.REDPANDA_GITHUB_TOKEN) return new Octokit() - return new Octokit({ - auth: process.env.REDPANDA_GITHUB_TOKEN, - }); + const { getGitHubToken } = require('../cli-utils/github-token') + const token = getGitHubToken() + return token ? new Octokit({ auth: token }) : new Octokit(); } this.once('contentClassified', async ({ contentCatalog }) => { diff --git a/extensions/version-fetcher/get-latest-console-version.js b/extensions/version-fetcher/get-latest-console-version.js index 6a1ac80f..58ffb4ce 100644 --- a/extensions/version-fetcher/get-latest-console-version.js +++ b/extensions/version-fetcher/get-latest-console-version.js @@ -1,35 +1,46 @@ +const { retryWithBackoff, isRetryableGitHubError } = require('./retry-util'); + module.exports = async (github, owner, repo) => { const semver = require('semver'); - try { - // Fetch all the releases from the repository - const releases = await github.rest.repos.listReleases({ - owner, - repo, - page: 1, - per_page: 50 - }); - // Filter valid semver tags and sort them to find the highest version - const sortedReleases = releases.data - .map(release => release.tag_name) - .filter(tag => semver.valid(tag.replace(/^v/, ''))) - .sort((a, b) => semver.rcompare(a.replace(/^v/, ''), b.replace(/^v/, ''))); + return retryWithBackoff( + async () => { + // Fetch all the releases from the repository + const releases = await github.rest.repos.listReleases({ + owner, + repo, + page: 1, + per_page: 50 + }); + + // Filter valid semver tags and sort them to find the highest version + const sortedReleases = releases.data + .map(release => release.tag_name) + .filter(tag => semver.valid(tag.replace(/^v/, ''))) + .sort((a, b) => semver.rcompare(a.replace(/^v/, ''), b.replace(/^v/, ''))); - if (sortedReleases.length > 0) { - // Find the highest versions for stable and beta releases - const latestStableReleaseVersion = sortedReleases.find(tag => !tag.includes('-beta')); - const latestBetaReleaseVersion = sortedReleases.find(tag => tag.includes('-beta')); + if (sortedReleases.length > 0) { + // Find the highest versions for stable and beta releases + const latestStableReleaseVersion = sortedReleases.find(tag => !tag.includes('-beta')); + const latestBetaReleaseVersion = sortedReleases.find(tag => tag.includes('-beta')); - return { - latestStableRelease: latestStableReleaseVersion || null, - latestBetaRelease: latestBetaReleaseVersion || null - }; - } else { - console.log("No valid semver releases found."); - return { latestStableRelease: null, latestBetaRelease: null }; + return { + latestStableRelease: latestStableReleaseVersion || null, + latestBetaRelease: latestBetaReleaseVersion || null + }; + } else { + console.log("No valid semver releases found."); + return { latestStableRelease: null, latestBetaRelease: null }; + } + }, + { + maxRetries: 3, + initialDelay: 1000, + shouldRetry: isRetryableGitHubError, + operationName: `Fetch Console version from ${owner}/${repo}` } - } catch (error) { - console.error('Failed to fetch release information:', error); + ).catch(error => { + console.error('Failed to fetch release information after retries:', error); return { latestStableRelease: null, latestBetaRelease: null }; - } + }); }; diff --git a/extensions/version-fetcher/get-latest-redpanda-version.js b/extensions/version-fetcher/get-latest-redpanda-version.js index 47af9e16..5573da86 100644 --- a/extensions/version-fetcher/get-latest-redpanda-version.js +++ b/extensions/version-fetcher/get-latest-redpanda-version.js @@ -1,64 +1,75 @@ +const { retryWithBackoff, isRetryableGitHubError } = require('./retry-util'); + module.exports = async (github, owner, repo) => { const semver = require('semver'); - try { - // Fetch all the releases from the repository - const releases = await github.rest.repos.listReleases({ - owner, - repo, - page: 1, - per_page: 50 - }); - - // Filter valid semver tags and sort them to find the highest version - const sortedReleases = releases.data - .filter(release => semver.valid(release.tag_name.replace(/^v/, ''))) - .sort((a, b) => semver.rcompare( - a.tag_name.replace(/^v/, ''), - b.tag_name.replace(/^v/, '') - )); - - // Find latest non-RC release that is NOT a draft - const latestRedpandaRelease = sortedReleases.find( - release => !release.tag_name.includes('-rc') && !release.draft - ); - - // Find latest RC release (can be draft or not, adjust if needed) - const latestRcRelease = sortedReleases.find( - release => release.tag_name.includes('-rc') - ); - let latestRedpandaReleaseCommitHash = null; - if (latestRedpandaRelease) { - const commitData = await github.rest.git.getRef({ + return retryWithBackoff( + async () => { + // Fetch all the releases from the repository + const releases = await github.rest.repos.listReleases({ owner, repo, - ref: `tags/${latestRedpandaRelease.tag_name}` + page: 1, + per_page: 50 }); - latestRedpandaReleaseCommitHash = commitData.data.object.sha; - } - let latestRcReleaseCommitHash = null; - if (latestRcRelease) { - const rcCommitData = await github.rest.git.getRef({ - owner, - repo, - ref: `tags/${latestRcRelease.tag_name}` - }); - latestRcReleaseCommitHash = rcCommitData.data.object.sha; - } + // Filter valid semver tags and sort them to find the highest version + const sortedReleases = releases.data + .filter(release => semver.valid(release.tag_name.replace(/^v/, ''))) + .sort((a, b) => semver.rcompare( + a.tag_name.replace(/^v/, ''), + b.tag_name.replace(/^v/, '') + )); + + // Find latest non-RC release that is NOT a draft + const latestRedpandaRelease = sortedReleases.find( + release => !release.tag_name.includes('-rc') && !release.draft + ); + + // Find latest RC release (can be draft or not, adjust if needed) + const latestRcRelease = sortedReleases.find( + release => release.tag_name.includes('-rc') + ); - return { - latestRedpandaRelease: latestRedpandaRelease ? { - version: latestRedpandaRelease.tag_name, - commitHash: latestRedpandaReleaseCommitHash.substring(0, 7) - } : null, - latestRcRelease: latestRcRelease ? { - version: latestRcRelease.tag_name, - commitHash: latestRcReleaseCommitHash.substring(0, 7) - } : null - }; - } catch (error) { - console.error('Failed to fetch Redpanda release information:', error); + let latestRedpandaReleaseCommitHash = null; + if (latestRedpandaRelease) { + const commitData = await github.rest.git.getRef({ + owner, + repo, + ref: `tags/${latestRedpandaRelease.tag_name}` + }); + latestRedpandaReleaseCommitHash = commitData.data.object.sha; + } + + let latestRcReleaseCommitHash = null; + if (latestRcRelease) { + const rcCommitData = await github.rest.git.getRef({ + owner, + repo, + ref: `tags/${latestRcRelease.tag_name}` + }); + latestRcReleaseCommitHash = rcCommitData.data.object.sha; + } + + return { + latestRedpandaRelease: latestRedpandaRelease ? { + version: latestRedpandaRelease.tag_name, + commitHash: latestRedpandaReleaseCommitHash.substring(0, 7) + } : null, + latestRcRelease: latestRcRelease ? { + version: latestRcRelease.tag_name, + commitHash: latestRcReleaseCommitHash.substring(0, 7) + } : null + }; + }, + { + maxRetries: 3, + initialDelay: 1000, + shouldRetry: isRetryableGitHubError, + operationName: `Fetch Redpanda version from ${owner}/${repo}` + } + ).catch(error => { + console.error('Failed to fetch Redpanda release information after retries:', error); return { latestRedpandaRelease: null, latestRcRelease: null }; - } + }); }; diff --git a/extensions/version-fetcher/retry-util.js b/extensions/version-fetcher/retry-util.js new file mode 100644 index 00000000..9b91aa62 --- /dev/null +++ b/extensions/version-fetcher/retry-util.js @@ -0,0 +1,88 @@ +/** + * Retry utility with exponential backoff + * + * @param {Function} fn - Async function to retry + * @param {Object} options - Retry options + * @param {number} options.maxRetries - Maximum number of retries (default: 3) + * @param {number} options.initialDelay - Initial delay in ms (default: 1000) + * @param {number} options.maxDelay - Maximum delay in ms (default: 10000) + * @param {Function} options.shouldRetry - Function to determine if error is retryable (default: all errors) + * @param {string} options.operationName - Name for logging + * @returns {Promise<*>} Result of the function + */ +async function retryWithBackoff(fn, options = {}) { + const { + maxRetries = 3, + initialDelay = 1000, + maxDelay = 10000, + shouldRetry = () => true, + operationName = 'operation' + } = options; + + let lastError; + + for (let attempt = 0; attempt <= maxRetries; attempt++) { + try { + return await fn(); + } catch (error) { + lastError = error; + + // Don't retry if this is the last attempt or error is not retryable + if (attempt === maxRetries || !shouldRetry(error)) { + throw error; + } + + // Calculate delay with exponential backoff and jitter + const baseDelay = Math.min(initialDelay * Math.pow(2, attempt), maxDelay); + const jitter = Math.random() * 0.3 * baseDelay; // Add up to 30% jitter + const delay = Math.floor(baseDelay + jitter); + + console.error(`⚠️ ${operationName} failed (attempt ${attempt + 1}/${maxRetries + 1}): ${error.message}`); + console.error(` Retrying in ${delay}ms...`); + + // Wait before retrying + await new Promise(resolve => setTimeout(resolve, delay)); + } + } + + throw lastError; +} + +/** + * Determine if a GitHub API error is retryable + * @param {Error} error - The error to check + * @returns {boolean} True if the error is retryable + */ +function isRetryableGitHubError(error) { + // Retry on network errors + if (error.code === 'ECONNRESET' || + error.code === 'ETIMEDOUT' || + error.code === 'ENOTFOUND' || + error.code === 'EAI_AGAIN') { + return true; + } + + // Retry on rate limit errors (though they should have a retry-after) + if (error.status === 403 && error.message?.includes('rate limit')) { + return true; + } + + // Retry on 5xx server errors + if (error.status >= 500 && error.status < 600) { + return true; + } + + // Retry on specific 4xx errors that might be transient + if (error.status === 408 || // Request Timeout + error.status === 429) { // Too Many Requests + return true; + } + + // Don't retry on other errors (4xx client errors, auth issues, etc.) + return false; +} + +module.exports = { + retryWithBackoff, + isRetryableGitHubError +}; diff --git a/extensions/version-fetcher/set-latest-version.js b/extensions/version-fetcher/set-latest-version.js index da263e55..f7528985 100644 --- a/extensions/version-fetcher/set-latest-version.js +++ b/extensions/version-fetcher/set-latest-version.js @@ -8,8 +8,11 @@ module.exports.register = function ({ config }) { const GetLatestConnectVersion = require('./get-latest-connect'); const logger = this.getLogger('set-latest-version-extension'); - if (!process.env.REDPANDA_GITHUB_TOKEN) { - logger.warn('REDPANDA_GITHUB_TOKEN environment variable not set. Attempting unauthenticated request.'); + const { getGitHubToken } = require('../../cli-utils/github-token'); + const token = getGitHubToken(); + + if (!token) { + logger.warn('GitHub token not set (REDPANDA_GITHUB_TOKEN, GITHUB_TOKEN, or GH_TOKEN). Attempting unauthenticated request.'); } this.on('contentClassified', async ({ contentCatalog }) => { @@ -22,7 +25,7 @@ module.exports.register = function ({ config }) { const githubOptions = { userAgent: 'Redpanda Docs', baseUrl: 'https://api.github.com', - auth: process.env.REDPANDA_GITHUB_TOKEN || undefined, + auth: token || undefined, }; const github = new OctokitWithRetries(githubOptions); const dockerNamespace = 'redpandadata' diff --git a/jest.config.js b/jest.config.js new file mode 100644 index 00000000..e4ec27e5 --- /dev/null +++ b/jest.config.js @@ -0,0 +1,38 @@ +module.exports = { + // Test environment + testEnvironment: 'node', + + // Test match patterns + testMatch: [ + '**/__tests__/**/*.test.js' + ], + + // Coverage configuration + collectCoverageFrom: [ + 'bin/**/*.js', + 'extensions/**/*.js', + 'macros/**/*.js', + 'tools/**/*.js', + '!**/node_modules/**', + '!**/__tests__/**' + ], + + // Reporters configuration + reporters: [ + 'default', + [ + 'jest-junit', + { + outputDirectory: 'test-results', + outputName: 'junit.xml', + classNameTemplate: '{classname}', + titleTemplate: '{title}', + ancestorSeparator: ' β€Ί ', + usePathForSuiteName: true + } + ] + ], + + // Test timeout + testTimeout: 30000 +}; diff --git a/macros/DEVELOPMENT.adoc b/macros/DEVELOPMENT.adoc new file mode 100644 index 00000000..8e1b5f2a --- /dev/null +++ b/macros/DEVELOPMENT.adoc @@ -0,0 +1,377 @@ += AsciiDoc Macros Development Guide +:toc: +:toclevels: 3 + +Guide for developing new AsciiDoc macros. + +AsciiDoc macros extend AsciiDoc syntax with custom inline and block elements. Macros are Asciidoctor.js extensions that process custom syntax and generate HTML output. + +== Macro types + +=== Inline macros + +Inline macros appear within text and generate inline HTML: + +[,asciidoc] +---- +Text with glossterm:term[] inline. +---- + +=== Block macros + +Block macros appear on their own lines and generate block-level HTML: + +[,asciidoc] +---- +component_table::[] +---- + +== Creating an inline macro + +=== Create the file + +[,bash] +---- +touch macros/my-macro.js +---- + +=== Implement the macro + +[,javascript] +---- +module.exports.register = function register (registry, context) { + // Register inline macro + registry.inlineMacro('mymacro', function () { + const self = this + + // Define macro processing + self.process(function (parent, target, attrs) { + // target: the value after the colon + // attrs: positional and named attributes + + // Generate HTML + const html = `${target}` + + // Return as passthrough + return self.createInline(parent, 'quoted', html, { type: 'unquoted' }) + }) + }) +} +---- + +=== Register in playbook + +[,yaml] +---- +asciidoc: + extensions: + - './macros/my-macro.js' +---- + +=== Use in AsciiDoc + +[,asciidoc] +---- +This is my mymacro:example[] macro. +---- + +== Creating a block macro + +[,javascript] +---- +module.exports.register = function register (registry, context) { + registry.blockMacro('myblock', function () { + const self = this + + self.process(function (parent, target, attrs) { + // Generate block HTML + const html = ` +
+

${target}

+

${attrs.content || 'No content'}

+
+ ` + + // Return as passthrough block + return self.createBlock(parent, 'pass', html) + }) + }) +} +---- + +Use in AsciiDoc: + +[,asciidoc] +---- +myblock::title[content=Some content here] +---- + +== Common patterns + +=== Accessing attributes + +[,javascript] +---- +self.process(function (parent, target, attrs) { + // Positional attributes + const first = attrs.$positional[0] + const second = attrs.$positional[1] + + // Named attributes + const name = attrs.name + const value = attrs.value || 'default' + + // Document attributes + const docAttr = parent.getDocument().getAttribute('my-attr') + + return self.createInline(parent, 'quoted', html, { type: 'unquoted' }) +}) +---- + +=== Generating links + +[,javascript] +---- +self.process(function (parent, target, attrs) { + const url = `https://example.com/${target}` + const text = attrs.text || target + + const html = `${text}` + + return self.createInline(parent, 'quoted', html, { type: 'unquoted' }) +}) +---- + +=== Accessing page context + +[,javascript] +---- +module.exports.register = function register (registry, context) { + // Access Antora context + const { contentCatalog, file } = context + + registry.inlineMacro('pagemacro', function () { + const self = this + + self.process(function (parent, target, attrs) { + // Access current page + const doc = parent.getDocument() + const component = doc.getAttribute('page-component-name') + const version = doc.getAttribute('page-component-version') + + // Access content catalog + const page = contentCatalog.getById({ + component, + version, + module: 'ROOT', + family: 'page', + relative: `${target}.adoc` + }) + + if (page) { + const html = `${page.asciidoc.doctitle}` + return self.createInline(parent, 'quoted', html, { type: 'unquoted' }) + } + + return self.createInline(parent, 'quoted', target, { type: 'unquoted' }) + }) + }) +} +---- + +=== Creating complex HTML + +[,javascript] +---- +self.process(function (parent, target, attrs) { + const items = attrs.items ? attrs.items.split(',') : [] + + const html = ` +
+
    + ${items.map(item => `
  • ${item.trim()}
    • `).join('\n')} +
+
+ ` + + return self.createBlock(parent, 'pass', html) +}) +---- + +=== Error handling + +[,javascript] +---- +self.process(function (parent, target, attrs) { + try { + if (!target) { + console.warn('mymacro: target is required') + return self.createInline(parent, 'quoted', '[Missing target]', { type: 'unquoted' }) + } + + // Process macro + const html = `${target}` + return self.createInline(parent, 'quoted', html, { type: 'unquoted' }) + + } catch (err) { + console.error(`mymacro error: ${err.message}`) + return self.createInline(parent, 'quoted', '[Macro error]', { type: 'unquoted' }) + } +}) +---- + +== Testing + +=== Unit tests + +[,javascript] +---- +// macros/__tests__/my-macro.test.js +const asciidoctor = require('@asciidoctor/core')() +const myMacro = require('../my-macro') + +describe('my-macro', () => { + let registry + + beforeEach(() => { + registry = asciidoctor.Extensions.create() + myMacro.register(registry) + }) + + test('processes inline macro', () => { + const html = asciidoctor.convert( + 'Text with mymacro:value[] inline.', + { extension_registry: registry } + ) + + expect(html).toContain('class="my-macro"') + expect(html).toContain('value') + }) +}) +---- + +=== Integration tests + +Test with Antora: + +[,bash] +---- +npx antora --fetch local-antora-playbook.yml +---- + +== Best practices + +=== Validate input + +[,javascript] +---- +if (!target) { + console.warn('Macro requires a target') + return self.createInline(parent, 'quoted', '', { type: 'unquoted' }) +} +---- + +=== Escape HTML + +[,javascript] +---- +function escapeHtml(text) { + return text + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, ''') +} + +const safeTarget = escapeHtml(target) +const html = `${safeTarget}` +---- + +=== Provide defaults + +[,javascript] +---- +const cssClass = attrs.class || 'default-class' +const text = attrs.text || target || 'Default text' +---- + +=== Use semantic HTML + +[,javascript] +---- +// Good +const html = `${target}` + +// Avoid +const html = `${target}` +---- + +=== Document your macro + +[,javascript] +---- +/** + * My Macro - Short description + * + * Syntax: mymacro:target[attr1,attr2] + * + * Parameters: + * - target: The target value (required) + * - attr1: First attribute (optional) + * - attr2: Second attribute (optional) + * + * Example: + * mymacro:example[foo,bar] + */ +module.exports.register = function register (registry, context) { + // Implementation +} +---- + +== Debugging + +=== Enable logging + +[,javascript] +---- +console.log('Macro target:', target) +console.log('Macro attrs:', JSON.stringify(attrs, null, 2)) +console.log('Document attrs:', parent.getDocument().getAttributes()) +---- + +=== Test locally + +Create a test AsciiDoc file: + +[,asciidoc] +---- += Test Page + +Testing mymacro:value[attr1,attr2] inline. +---- + +Convert directly: + +[,bash] +---- +node -e " +const asciidoctor = require('@asciidoctor/core')() +const macro = require('./macros/my-macro') +const registry = asciidoctor.Extensions.create() +macro.register(registry) +const html = asciidoctor.convertFile('test.adoc', { + extension_registry: registry, + to_file: false +}) +console.log(html) +" +---- + +== Related documentation + +* link:USER_GUIDE.adoc[User guide] - How to use macros +* link:REFERENCE.adoc[Reference] - Complete macro documentation +* https://docs.asciidoctor.org/asciidoctor.js/latest/extend/extensions/[Asciidoctor.js Extensions] +* https://docs.asciidoctor.org/asciidoctor.js/latest/extend/extensions/inline-macro-processor/[Inline Macro Processor] +* https://docs.asciidoctor.org/asciidoctor.js/latest/extend/extensions/block-macro-processor/[Block Macro Processor] diff --git a/macros/README.adoc b/macros/README.adoc new file mode 100644 index 00000000..eb6ea0f7 --- /dev/null +++ b/macros/README.adoc @@ -0,0 +1,105 @@ += AsciiDoc Macros +:toc: +:toclevels: 2 + +Documentation for AsciiDoc macros and extensions for Redpanda documentation. + +== What are AsciiDoc macros? + +AsciiDoc macros are custom inline and block elements that extend AsciiDoc syntax. These macros provide specialized formatting, references, and dynamic content generation within documentation pages. + +== Documentation + +link:USER_GUIDE.adoc[**User guide**]:: How to use AsciiDoc macros in your documentation ++ +* Installation and setup +* Using macros in AsciiDoc +* Common usage patterns +* Troubleshooting + +link:REFERENCE.adoc[**Reference**]:: Complete reference for all available macros ++ +* Macro descriptions +* Syntax and parameters +* Configuration options +* Examples for each macro + +link:DEVELOPMENT.adoc[**Development guide**]:: How to develop new AsciiDoc macros ++ +* Macro architecture +* Creating inline and block macros +* Testing and debugging +* Best practices + +== Quickstart + +=== Install the package + +[,bash] +---- +npm i @redpanda-data/docs-extensions-and-macros +---- + +=== Configure in your playbook + +[,yaml] +---- +asciidoc: + extensions: + - '@redpanda-data/docs-extensions-and-macros/macros/glossary' +---- + +IMPORTANT: Macros must be registered under the `asciidoc.extensions` key in your playbook, not `antora.extensions`. + +== Available macros + +=== Inline macros + +* **glossterm** - Reference glossary terms with tooltips +* **config_ref** - Link to configuration properties +* **helm_ref** - Link to Helm chart values +* **badge** - Display status badges + +=== Block macros + +* **components_by_category** - Display Redpanda Connect components grouped by category +* **component_table** - Generate searchable component tables +* **component_type_dropdown** - Create dropdown for component types + +See link:REFERENCE.adoc[Reference documentation] for complete details on each macro. + +== Common use cases + +=== Reference glossary terms + +[,asciidoc] +---- +The glossterm:partition[partition] contains the data. +---- + +=== Link to configuration + +[,asciidoc] +---- +Configure the config_ref:retention.ms,true,topic-properties[] property. +---- + +=== Link to Helm values + +[,asciidoc] +---- +For configuration options, see the helm_ref:storage.tieredConfig.cloud_storage_enabled[] value. +---- + +=== Display Redpanda Connect components + +[,asciidoc] +---- +components_by_category::inputs[] +---- + +== Support + +* link:USER_GUIDE.adoc#troubleshooting[Troubleshooting guide] +* https://github.com/redpanda-data/docs-extensions-and-macros/issues[Report issues] +* https://docs.asciidoctor.org/asciidoctor.js/latest/extend/extensions/[Asciidoctor.js Extensions] diff --git a/macros/REFERENCE.adoc b/macros/REFERENCE.adoc new file mode 100644 index 00000000..f32ccc6f --- /dev/null +++ b/macros/REFERENCE.adoc @@ -0,0 +1,222 @@ += AsciiDoc macros reference +:toc: +:toclevels: 3 + +Reference documentation for all AsciiDoc macros and extensions provided by this library. + +IMPORTANT: Be sure to register each extension under the `asciidoc.extensions` key in the playbook, not the `antora.extensions` key. + +== Add line numbers and highlights + +This extension adds the necessary classes to make line numbers and line highlighting work with Prism.js. + +=== Registration + +```yaml +antora: + extensions: + - '@redpanda-data/docs-extensions-and-macros/asciidoc-extensions/add-line-numbers-highlights' +``` + +== config_ref + +This inline macro is used to generate a reference to a configuration value in the Redpanda documentation. The macro's parameters allow for control over the generated reference's format and the type of output produced. + +=== Usage + +The `config_ref` macro is used in an AsciiDoc document as follows: + +[,asciidoc] +---- +config_ref:configRef,isLink,path[] +---- + +The `config_ref` macro takes three parameters: + +configRef:: +This is the configuration reference, which is also used to generate the anchor link if `isLink` is `true`. + +isLink:: +Whether the output should be a link. If `isLink` is set to `true`, the output will be a cross-reference (xref) to the relevant configuration value. + +path:: +This is the path to the document where the configuration value is defined. This parameter is used to to generate the link if `isLink` is `true`. + +IMPORTANT: The path must be the name of a document at the root of the `reference` module. + +NOTE: The `config_ref` macro is environment-aware. It checks if the document it is being used in is part of a Kubernetes environment by checking if the `env-kubernetes` attribute is set in the document's attributes. Depending on this check, it either prepends `storage.tieredConfig.` to the `configRef` or just uses the `configRef` as is. + +For example: + +[,asciidoc] +---- +config_ref:example_config,true,tunable-properties[] +---- + +=== Registration + +[,yaml] +---- +asciidoc: + extensions: + - '@redpanda-data/docs-extensions-and-macros/macros/config-ref' +---- + +== glossterm + +The `glossterm` inline macro provides a way to define and reference glossary terms in your AsciiDoc documents. + +NOTE: This macro is a customized version of https://gitlab.com/djencks/asciidoctor-glossary[`asciidoctor-glossary`]. + +=== Usage + +Use the `glossterm` inline macro to reference a term within the text of the document: + +[,asciidoc] +---- +glossterm:my term[myDefinition] +---- + +It takes two parameters: + +term:: +The term to be defined. + +definition (optional):: +The definition of the term. If the term is defined in the https://github.com/redpanda-data/docs-site[`shared` component] or the `local-terms` object of the `antora.yml` file, you can omit the definition as it will always be replaced by those definitions. + +=== Configuration options + +glossary-log-terms (optional):: +Whether to log a textual representation of a definition list item to the console. + +glossary-term-role (optional):: +Role to assign each term. By default, glossary terms are assigned the `glossary-term` role, which gives them the class `glossary-term` in generated html. + +glossary-links (optional):: +Whether to generate links to glossary entries. +By default, links to the glossary entries are generated from the glossary terms. To avoid this, set the attribute to `false` as either asciidoctor configuration or a header attribute. + +glossary-page (optional):: +Target page for glossary links. By default, links are generated to the same page as the glossary term. To specify the target page, set this attribute to the resource ID of a page where the `glossary` block macro is used. + +glossary-tooltip (optional):: +Whether to enable tooltips for the defined terms. Valid values are: +- title: This uses the browser built-in `title` attribute to display the definition. + +- true: This inserts the definition as the value of the attribute `data-glossary-tooltip`. + +- data-​: This inserts the definition as the value of the supplied attribute name, which must start with `data`. + +The last two options are intended to support js/css tooltip solutions such as tippy.js. + +=== Registration + +[,yaml] +---- +asciidoc: + extensions: + - '@redpanda-data/docs-extensions-and-macros/macros/glossary' +---- + +== helm_ref + +This is an inline macro to create links to a Helm `values.yaml` file on ArtifactHub. + +=== Usage + +In an AsciiDoc document, use the `helm_ref` macro as follows: + +[,asciidoc] +---- +helm_ref:[] +---- + +Where `` is the Helm configuration value you want to reference in the `values.yaml` file. + +For example: + +Given a Helm reference value of `myConfigValue`, you would use the macro like this: + +[,asciidoc] +---- +helm_ref:myConfigValue[] +---- + +This will generate the following output: + +[,asciidoc] +---- +For default values and documentation for configuration options, see the https://artifacthub.io/packages/helm/redpanda-data/redpanda?modal=values&path=myConfigValue[values.yaml] file. +---- + +If you do not specify a Helm reference value, the macro generates a link without specifying a path. + +=== Registration + +[,yaml] +---- +asciidoc: + extensions: + - '@redpanda-data/docs-extensions-and-macros/macros/helm-ref' +---- + +== components_by_category + +This macro generates a tabbed interface to display Redpanda Connect components by category. + +The categories are fetched from the `connectCategoriesData` that's generated in the Component category aggregator extension. + +=== Usage + +```asciidoc +components_by_category::[] +``` + +=== Registration + +```yaml +asciidoc: + extensions: + - '@redpanda-data/docs-extensions-and-macros/macros/rp-connect-components' +``` + +== component_table + +This macro generates a searchable table of all Redpanda Connect components with filters for support and type. + +The types are fetched from the `flatComponentsData` that's generated in the Component category aggregator extension. + +=== Usage + +```asciidoc +component_table::[] +``` + +=== Registration + +```yaml +asciidoc: + extensions: + - '@redpanda-data/docs-extensions-and-macros/macros/rp-connect-components' +``` + +== component_type_dropdown + +This macro generates a dropdown of other supported types for a particular component, allowing users to switch between different types. + +The types are fetched from the `flatComponentsData` that's generated in the Component category aggregator extension. + +=== Usage + +```asciidoc +component_type_dropdown::[] +``` + +=== Registration + +```yaml +asciidoc: + extensions: + - '@redpanda-data/docs-extensions-and-macros/macros/rp-connect-components' +``` diff --git a/macros/USER_GUIDE.adoc b/macros/USER_GUIDE.adoc new file mode 100644 index 00000000..202671af --- /dev/null +++ b/macros/USER_GUIDE.adoc @@ -0,0 +1,220 @@ += AsciiDoc Macros User Guide +:toc: +:toclevels: 3 + +Complete guide to using AsciiDoc macros in your documentation. + +== Getting started + +=== Installation + +Install the package: + +[,bash] +---- +npm i @redpanda-data/docs-extensions-and-macros +---- + +=== Configuration + +Register macros in your Antora playbook under `asciidoc.extensions`: + +[,yaml] +---- +asciidoc: + extensions: + - '@redpanda-data/docs-extensions-and-macros/macros/glossary' + - '@redpanda-data/docs-extensions-and-macros/macros/config-ref' + - '@redpanda-data/docs-extensions-and-macros/macros/helm-ref' +---- + +IMPORTANT: Use `asciidoc.extensions` for macros, not `antora.extensions`. + +== Using macros + +=== glossterm - Glossary terms + +Reference glossary terms with tooltips: + +[,asciidoc] +---- +The glossterm:partition[partition] stores data. +glossterm:broker[broker,A server that handles requests] +---- + +*Syntax:* `glossterm:term[definition]` + +* `term` - The term to reference (required) +* `definition` - Custom definition (optional, uses global glossary by default) + +=== config_ref - Configuration references + +Link to configuration properties: + +[,asciidoc] +---- +config_ref:log.retention.ms,true,tunable-properties[] +config_ref:cleanup.policy,false[] +---- + +*Syntax:* `config_ref:property,isLink,path[]` + +* `property` - Configuration property name (required) +* `isLink` - Whether to create a link (true/false, required) +* `path` - Document path for link (required if isLink is true) + +=== helm_ref - Helm value references + +Link to Helm chart values: + +[,asciidoc] +---- +helm_ref:storage.tieredConfig.cloud_storage_enabled[] +helm_ref:[] +---- + +*Syntax:* `helm_ref:value[]` + +* `value` - Helm value path (optional, links to values.yaml overview if omitted) + +=== components_by_category - Component display + +Display Redpanda Connect components by category: + +[,asciidoc] +---- +components_by_category::inputs[] +components_by_category::outputs[] +components_by_category::processors[] +---- + +*Syntax:* `components_by_category::[type]` + +* `type` - Component type (inputs, outputs, processors, etc.) + +=== component_table - Component table + +Generate searchable component table: + +[,asciidoc] +---- +component_table::[] +---- + +Generates a searchable table of all components with filters. + +=== component_type_dropdown - Type switcher + +Create dropdown for switching between component types: + +[,asciidoc] +---- +component_type_dropdown::[] +---- + +Generates a dropdown to switch between different implementations of the same component. + +== Configuration options + +=== Glossary configuration + +Configure glossary behavior in playbook: + +[,yaml] +---- +asciidoc: + attributes: + glossary-tooltip: data-tooltip # Enable tooltips + glossary-page: glossary.adoc # Link target + glossary-links: true # Enable links + extensions: + - '@redpanda-data/docs-extensions-and-macros/macros/glossary' +---- + +*Options:* + +* `glossary-tooltip` - Tooltip implementation (title, true, or data-attribute-name) +* `glossary-page` - Page to link glossary terms to +* `glossary-links` - Enable/disable links (default: true) +* `glossary-term-role` - CSS role for terms (default: glossary-term) + +== Troubleshooting + +=== Macro not rendering + +*Problem:* Macro appears as plain text + +*Solutions:* + +* Verify macro is registered under `asciidoc.extensions` +* Check macro syntax is correct +* Ensure package is installed +* Try rebuilding: `npx antora playbook.yml` + +=== Glossary terms not found + +*Problem:* Glossary definitions missing + +*Solutions:* + +* Define terms in shared component or local `antora.yml` +* Check term spelling matches exactly +* Provide inline definition as fallback + +=== Links not working + +*Problem:* config_ref or helm_ref links broken + +*Solutions:* + +* Verify target page exists +* Check path is relative to reference module +* Ensure property/value name is correct + +=== Component macros empty + +*Problem:* components_by_category shows no components + +*Solutions:* + +* Ensure component aggregator extension is enabled +* Check component data is generated +* Verify component type is correct + +== Best practices + +=== Use glossary terms consistently + +[,asciidoc] +---- +βœ“ Good: glossterm:partition[] +βœ— Bad: partition (without macro) +---- + +=== Provide fallback definitions + +[,asciidoc] +---- +glossterm:custom-term[A definition in case term isn't in glossary] +---- + +=== Link to configuration when relevant + +[,asciidoc] +---- +To configure retention, set config_ref:log.retention.ms,true,tunable-properties[]. +---- + +=== Keep macro usage readable + +[,asciidoc] +---- +βœ“ Good: Configure config_ref:retention.ms,true,topic-properties[]. +βœ— Bad: config_ref:retention.ms,true,topic-properties[]config_ref:cleanup.policy,true,topic-properties[]. +---- + +== Related documentation + +* link:REFERENCE.adoc[Macros reference] - Complete macro documentation +* link:DEVELOPMENT.adoc[Development guide] - How to create new macros +* https://docs.asciidoctor.org/asciidoc/latest/macros/[AsciiDoc Macro Syntax] diff --git a/macros/rp-connect-components.js b/macros/rp-connect-components.js index 00993383..79773cc4 100644 --- a/macros/rp-connect-components.js +++ b/macros/rp-connect-components.js @@ -273,10 +273,10 @@ module.exports.register = function (registry, context) { * * @returns {Object} - An object with two properties: * - `certified`: An array of SQL drivers with 'certified' support level. Each driver contains: - * - `commercialName`: The trimmed commercial name of the driver (e.g., 'PostgreSQL'). + * - `commercialName`: The trimmed commercial name of the driver (for example, 'PostgreSQL'). * - `isCloudSupported`: A boolean indicating whether the driver supports cloud. * - `community`: An array of SQL drivers with 'community' support level. Each driver contains: - * - `commercialName`: The trimmed commercial name of the driver (e.g., 'Trino'). + * - `commercialName`: The trimmed commercial name of the driver (for example, 'Trino'). * - `isCloudSupported`: A boolean indicating whether the driver supports cloud. * * Example return structure: @@ -330,7 +330,7 @@ module.exports.register = function (registry, context) { * @param {Object} connectors - An object containing the connector data, where each key is a connector name and * each value contains details about its types, licensing, and cloud support. * { - * types: Map - A map of connector types (e.g., Input, Output, Processor), with associated commercial names. + * types: Map - A map of connector types (for example, Input, Output, Processor), with associated commercial names. * isLicensed: 'Yes' or 'No' - Indicates if the connector requires an enterprise license. * isCloudConnectorSupported: true or false - Indicates if any type for this connector supports Redpanda Cloud. * } @@ -340,7 +340,7 @@ module.exports.register = function (registry, context) { * community: Array<{ commercialName: string, isCloudSupported: boolean }> * } * @param {boolean} isCloud - A flag indicating whether to filter by cloud support. If true, only cloud-supported connectors are shown. - * @param {boolean} showAllInfo - A flag indicating whether to show all information or limit the data displayed (e.g., for cloud-only views). + * @param {boolean} showAllInfo - A flag indicating whether to show all information or limit the data displayed (for example, for cloud-only views). * * @returns {string} - A string containing the generated HTML for the connectors table rows. * The output is a string of HTML rows with the following columns: @@ -831,7 +831,7 @@ module.exports.register = function (registry, context) { // Generic fallback for words ending in 's Selected' return pluralText.replace(/s Selected$/, ' Selected'); } else if (pluralText.endsWith('ies Selected')) { - // Handle words ending in 'ies' (e.g., "Categories Selected" -> "Category Selected") + // Handle words ending in 'ies' (for example, "Categories Selected" -> "Category Selected") return pluralText.replace(/ies Selected$/, 'y Selected'); } else { // If no pattern matches, return as-is @@ -984,7 +984,7 @@ module.exports.register = function (registry, context) { self.named('component_type_dropdown'); self.process((parent, target, attrs) => { const attributes = parent.getDocument().getAttributes(); - const component = attributes['page-component-title']; // Current component (e.g., 'Redpanda Cloud' or 'Redpanda Connect') + const component = attributes['page-component-title']; // Current component (for example, 'Redpanda Cloud' or 'Redpanda Connect') const name = attributes['doctitle']; const type = attributes['type']; if (!name || !type) { diff --git a/mcp/CLI_INTERFACE.adoc b/mcp/CLI_INTERFACE.adoc new file mode 100644 index 00000000..f85c0d12 --- /dev/null +++ b/mcp/CLI_INTERFACE.adoc @@ -0,0 +1,385 @@ += CLI Interface Contract +:toc: +:toclevels: 3 + +This document describes the CLI interface contract that the MCP server depends on. These contracts are verified by the CLI contract tests to ensure the MCP server continues to work as expected when the CLI changes. + +The MCP server wraps the `npx doc-tools` CLI to provide a natural language interface to Claude Code. The CLI must maintain specific commands, flags, and output formats for the MCP server to function correctly. + +== Command structure + +All commands follow the pattern: + +[,bash] +---- +npx doc-tools [subcommand] [flags] +---- + +== Top-level commands + +=== generate + +Generate various types of documentation. + +[,bash] +---- +npx doc-tools generate [flags] +---- + +Subcommands are documented in the <> section. + +=== get-redpanda-version + +Get the latest Redpanda version information. + +[,bash] +---- +npx doc-tools get-redpanda-version [--beta] +---- + +*Output format:* + +[,bash] +---- +REDPANDA_VERSION=v25.3.1 +REDPANDA_DOCKER_TAG=docker.redpanda.com/redpandadata/redpanda:v25.3.1 +REDPANDA_RELEASE_NOTES=https://github.com/redpanda-data/redpanda/releases/tag/v25.3.1 +---- + +*Flags:* + +--beta:: Get the latest beta/RC version instead of stable (optional) + +=== get-console-version + +Get the latest Redpanda Console version information. + +[,bash] +---- +npx doc-tools get-console-version +---- + +*Output format:* + +[,bash] +---- +CONSOLE_VERSION=v2.7.2 +CONSOLE_DOCKER_TAG=docker.redpanda.com/redpandadata/console:v2.7.2 +CONSOLE_RELEASE_NOTES=https://github.com/redpanda-data/console/releases/tag/v2.7.2 +---- + +=== --version + +Display the doc-tools version. + +[,bash] +---- +npx doc-tools --version +---- + +*Output format:* + +[,bash] +---- +.. +---- + +Example: `1.2.3` + +== Generate subcommands + +=== property-docs + +Generate Redpanda configuration property documentation. + +[,bash] +---- +npx doc-tools generate property-docs --tag [options] +---- + +*Required flags:* + +--tag:: Redpanda version to generate docs for (for example, `v25.3.1`, `25.3.1`, or `latest`) + +*Optional flags:* + +--generate-partials:: Generate AsciiDoc partials in addition to JSON +--cloud-support:: Include cloud support information +--overrides:: Path to property overrides JSON file + +*Output:* + +Creates one or more of: + +* `modules/reference/partials/properties.json` +* `modules/reference/partials/cluster-properties.adoc` (with `--generate-partials`) +* `modules/reference/partials/broker-properties.adoc` (with `--generate-partials`) +* `modules/reference/partials/tunable-properties.adoc` (with `--generate-partials`) +* `modules/reference/partials/topic-properties.adoc` (with `--generate-partials`) + +=== metrics-docs + +Generate Redpanda metrics documentation. + +[,bash] +---- +npx doc-tools generate metrics-docs --tag +---- + +*Required flags:* + +--tag:: Redpanda version to generate docs for + +*Output:* + +Creates `modules/reference/pages/public-metrics-reference.adoc` + +=== rpk-docs + +Generate RPK command-line documentation. + +[,bash] +---- +npx doc-tools generate rpk-docs --tag +---- + +*Required flags:* + +--tag:: Redpanda version to generate docs for + +*Output:* + +Creates AsciiDoc files in `autogenerated/v/rpk/` + +=== rpcn-connector-docs + +Generate Redpanda Connect connector documentation. + +[,bash] +---- +npx doc-tools generate rpcn-connector-docs [options] +---- + +*Optional flags:* + +--fetch-connectors:: Fetch connector definitions from Redpanda Connect repository +--draft-missing:: Create draft pages for missing connectors +--update-whats-new:: Update the what's new page +--include-bloblang:: Include Bloblang documentation +--data-dir:: Directory containing connector data +--old-data:: Path to previous connector data for comparison +--csv:: Export connector data as CSV +--overrides:: Path to overrides JSON file + +*Output:* + +Creates component documentation files in `modules/reference/pages/redpanda-connect/components/` + +=== helm-spec + +Generate Helm chart specification documentation. + +[,bash] +---- +npx doc-tools generate helm-spec [options] +---- + +*Optional flags:* + +--chart-dir:: Path to Helm chart directory +--tag:: Helm chart version +--readme:: Include README content +--output-dir:: Output directory for generated files +--output-suffix:: Suffix for output files + +=== cloud-regions + +Generate cloud regions documentation. + +[,bash] +---- +npx doc-tools generate cloud-regions [options] +---- + +*Optional flags:* + +--output:: Output file path +--format:: Output format (json, yaml, adoc) +--owner:: GitHub repository owner +--repo:: GitHub repository name +--path:: Path to regions file in repository +--ref:: Git reference (branch, tag, commit) +--template:: AsciiDoc template file +--dry-run:: Show what would be generated without writing files + +=== crd-spec + +Generate Custom Resource Definition (CRD) specification documentation. + +[,bash] +---- +npx doc-tools generate crd-spec --tag [options] +npx doc-tools generate crd-spec --branch [options] +---- + +*Required flags (one of):* + +--tag:: Redpanda Operator version tag (for released content) +--branch:: Branch name (for in-progress content) + +*Optional flags:* + +--source-path:: Path to CRD source files +--depth:: Maximum depth for nested properties +--templates-dir:: Directory containing documentation templates +--output:: Output directory + +=== bundle-openapi + +Bundle OpenAPI specifications. + +[,bash] +---- +npx doc-tools generate bundle-openapi --tag [options] +---- + +*Required flags:* + +--tag:: Redpanda version for API specifications + +*Optional flags:* + +--repo:: Repository to fetch from +--surface:: API surface to bundle (admin, connect, or both) +--out-admin:: Output path for admin API bundle +--out-connect:: Output path for connect API bundle +--admin-major:: Major version for admin API +--use-admin-major-version:: Use major version in admin API bundle +--quiet:: Suppress output messages + +== Output format contracts + +=== Version commands + +Version commands (`get-redpanda-version`, `get-console-version`) must output key-value pairs in the format: + +[,bash] +---- +KEY=value +---- + +Lines starting with `#` are treated as comments and ignored by the MCP server. + +=== Generate commands + +Generate commands should: + +* Write to stdout for informational messages +* Write to stderr for errors +* Exit with code 0 on success +* Exit with non-zero code on failure + +The MCP server parses stdout to extract: + +* File counts (for example, "Generated 342 properties") +* File paths created +* Success/failure status + +== Error handling contracts + +=== Missing required parameters + +Commands must exit with non-zero status when required parameters are missing. + +Example: + +[,bash] +---- +$ npx doc-tools generate property-docs +Error: Required flag --tag is missing +$ echo $? +1 +---- + +=== Invalid flags + +Commands must exit with non-zero status when invalid flags are provided. + +Example: + +[,bash] +---- +$ npx doc-tools generate property-docs --invalid-flag-xyz +Error: Unknown flag: --invalid-flag-xyz +$ echo $? +1 +---- + +=== Nonexistent commands + +The CLI must exit with non-zero status when nonexistent commands are called. + +Example: + +[,bash] +---- +$ npx doc-tools generate nonexistent-command +Error: Unknown command: nonexistent-command +$ echo $? +1 +---- + +=== Network errors + +Network errors (for example, GitHub API failures) are acceptable for version commands, but: + +* Must write error message to stderr +* Must exit with non-zero status +* Error message should indicate the network issue + +== Version compatibility + +The CLI must support the `--version` flag to output the current version. + +Format: `..` + +This allows the MCP server to verify compatibility and provide helpful error messages if the CLI version is incompatible. + +== Stability guarantees + +The MCP server depends on these CLI contracts remaining stable: + +*Command names and subcommands* must not change without a major version bump. + +*Required flags* must not be removed or renamed without a major version bump. + +*Output formats* for version commands must remain parseable. + +*Exit codes* must follow Unix conventions (0 = success, non-zero = failure). + +*Optional flags* may be added without breaking the MCP server. + +*New subcommands* may be added without breaking the MCP server. + +== Testing + +The `cli-contract.test.js` test suite verifies these contracts automatically. All tests must pass before releasing a new version of doc-tools that could affect the MCP server. + +Test categories: + +* Command structure - Verifies commands and subcommands exist +* Flag support - Verifies required and optional flags are present +* Output format - Verifies output can be parsed correctly +* Error handling - Verifies proper exit codes and error messages +* Version compatibility - Verifies version information is available + +== Breaking changes + +If a breaking change to the CLI is necessary: + +. Update the CLI with the breaking change +. Increment the major version of doc-tools +. Update the MCP server to handle both old and new interfaces (if possible) +. Update the CLI contract tests to verify the new interface +. Document the breaking change in release notes +. Update this document with the new contract diff --git a/mcp/DEVELOPMENT.adoc b/mcp/DEVELOPMENT.adoc new file mode 100644 index 00000000..8399e6ee --- /dev/null +++ b/mcp/DEVELOPMENT.adoc @@ -0,0 +1,726 @@ += MCP Server Development Guide +:toc: +:toclevels: 3 + +Developer guide for working on the Redpanda Doc Tools MCP server. + +The MCP (Model Context Protocol) server exposes `doc-tools` functionality to Claude Code through a writer-friendly natural language interface. This guide covers the architecture, development workflow, and how to extend the server. + +== Architecture + +=== Components + +The MCP server consists of several key components: + +*Server implementation* (`bin/doc-tools-mcp.js`):: Main MCP server that handles the protocol communication with Claude Code + +*Tool modules* (`bin/mcp-tools/`):: Individual tool implementations that wrap CLI commands ++ +* `antora.js` - Antora structure analysis +* `version.js` - Version information from GitHub +* `generate.js` - Documentation generation commands +* `cli.js` - Raw CLI command execution + +*Prompt system* (`prompts/`):: Pre-defined instructions for specific documentation workflows ++ +See link:prompts/README.adoc[Prompts documentation] + +*Setup automation* (`cli-utils/setup-mcp.js`):: Automated MCP server configuration for Claude Code/Desktop + +=== How it works + +. Claude Code starts the MCP server when you launch Claude Code in any directory +. The server detects the repository context (git root, Antora structure, doc-tools availability) +. When you ask Claude to perform a task, Claude decides which tools to use +. The server receives tool requests via the MCP protocol +. Tool modules execute CLI commands in the detected repository context +. Results are returned to Claude, which presents them to you + +=== Protocol flow + +[,mermaid] +---- +sequenceDiagram + participant User + participant Claude + participant MCP Server + participant CLI + + User->>Claude: "Generate property docs for v25.3.1" + Claude->>MCP Server: generate_property_docs(version: "v25.3.1") + MCP Server->>CLI: npx doc-tools generate property-docs --tag v25.3.1 + CLI-->>MCP Server: Output + exit code + MCP Server-->>Claude: JSON response + Claude-->>User: "βœ“ Generated 342 properties" +---- + +== Development setup + +=== Prerequisites + +* Node.js 18 or later +* Claude Code installed +* Git + +=== Initial setup + +. Clone the repository: ++ +[,bash] +---- +git clone https://github.com/redpanda-data/docs-extensions-and-macros.git +cd docs-extensions-and-macros +---- + +. Install dependencies: ++ +[,bash] +---- +npm install +---- + +. Configure MCP server for local development: ++ +[,bash] +---- +npx doc-tools setup-mcp +---- ++ +This configures Claude Code to use your local development version. + +. Restart Claude Code to load the server + +=== Project structure + +[,bash] +---- +docs-extensions-and-macros/ +β”œβ”€β”€ bin/ +β”‚ β”œβ”€β”€ doc-tools-mcp.js # Main MCP server +β”‚ └── mcp-tools/ # Tool implementations +β”‚ β”œβ”€β”€ antora.js # Antora structure analysis +β”‚ β”œβ”€β”€ cloud-regions.js # Cloud regions table generation +β”‚ β”œβ”€β”€ crd-docs.js # Kubernetes CRD docs generation +β”‚ β”œβ”€β”€ helm-docs.js # Helm chart docs generation +β”‚ β”œβ”€β”€ index.js # Main tool orchestrator +β”‚ β”œβ”€β”€ job-queue.js # Background job management +β”‚ β”œβ”€β”€ metrics-docs.js # Metrics docs generation +β”‚ β”œβ”€β”€ openapi.js # OpenAPI bundle generation +β”‚ β”œβ”€β”€ property-docs.js # Property docs generation +β”‚ β”œβ”€β”€ review.js # Documentation quality review +β”‚ β”œβ”€β”€ rpcn-docs.js # Redpanda Connect docs generation +β”‚ β”œβ”€β”€ rpk-docs.js # RPK CLI docs generation +β”‚ β”œβ”€β”€ utils.js # Shared utilities +β”‚ └── versions.js # Version information +β”œβ”€β”€ __tests__/mcp/ # MCP tests +β”‚ β”œβ”€β”€ cli-contract.test.js +β”‚ β”œβ”€β”€ doc-tools-mcp.test.js +β”‚ β”œβ”€β”€ integration.test.js +β”‚ └── README.adoc +└── cli-utils/ + └── setup-mcp.js # Setup automation +---- + +== Adding a new tool + +=== Step 1: Define the tool schema + +Edit `bin/doc-tools-mcp.js` and add your tool to the tools array: + +[,javascript] +---- +{ + name: 'my_new_tool', + description: 'Description of what this tool does for writers', + inputSchema: { + type: 'object', + properties: { + my_parameter: { + type: 'string', + description: 'Description of this parameter' + } + }, + required: ['my_parameter'] + } +} +---- + +*Design principles for tool schemas:* + +* Use writer-friendly descriptions (no technical jargon) +* Parameter names should be clear and intuitive +* Use natural language in descriptions +* Mark parameters as required only if absolutely necessary + +=== Step 2: Create the tool module + +Create a new file in `bin/mcp-tools/my-tool.js`: + +[,javascript] +---- +const { execSync } = require('child_process'); + +/** + * Execute my new tool + * @param {Object} params - Tool parameters + * @param {string} params.my_parameter - Description + * @param {Object} context - Execution context + * @param {string} context.repoRoot - Repository root path + * @returns {Object} Result object with success and data + */ +function executeMyTool(params, context) { + try { + // Validate parameters + if (!params.my_parameter) { + return { + success: false, + error: 'Parameter my_parameter is required', + suggestion: 'Provide a value like "example"' + }; + } + + // Execute CLI command + const output = execSync( + `npx doc-tools my-command --param ${params.my_parameter}`, + { + cwd: context.repoRoot, + encoding: 'utf8', + maxBuffer: 50 * 1024 * 1024, + timeout: 5 * 60 * 1000 + } + ); + + // Parse output and return structured result + return { + success: true, + output, + summary: 'Successfully executed my tool' + }; + } catch (err) { + return { + success: false, + error: `Command failed: ${err.message}`, + stdout: err.stdout || '', + stderr: err.stderr || '', + exitCode: err.status + }; + } +} + +module.exports = { executeMyTool }; +---- + +*Best practices for tool modules:* + +* Always return structured JSON objects +* Include helpful error messages and suggestions +* Use appropriate timeouts (default: 5 minutes) +* Parse CLI output to extract useful information +* Handle all error cases gracefully + +=== Step 3: Wire up the tool + +In `bin/doc-tools-mcp.js`, import your tool module and add a case to the tool handler: + +[,javascript] +---- +const { executeMyTool } = require('./mcp-tools/my-tool'); + +// In the tool handler switch statement: +case 'my_new_tool': + result = executeMyTool(params, context); + break; +---- + +=== Step 4: Add tests + +See <> section below. + +=== Step 5: Document the tool + +Update link:USER_GUIDE.adoc[USER_GUIDE.adoc] to document: + +* The tool in the "Available tools" section +* JSON response structure +* Usage examples + +Update link:CLI_INTERFACE.adoc[CLI_INTERFACE.adoc] if you added a new CLI command. + +== Adding a new prompt + +Prompts provide contextual guidance to Claude when working on specific documentation tasks. + +NOTE: This section is for *developers* adding prompts programmatically. *Technical writers* should use link:WRITER_EXTENSION_GUIDE.adoc[Writer Extension Guide] which provides step-by-step instructions with no code required. + +=== How the prompt system works + +The MCP server uses *automatic discovery* of prompts: + +. Prompts are Markdown files in `mcp/prompts/` directory +. Each prompt has YAML frontmatter defining metadata (description, version, arguments) +. At startup, the server scans the directory and loads all `.md` files +. Prompts are cached in memory for performance +. Changes are automatically reloaded in development mode (`MCP_DEV_MODE=true`) + +*No code changes needed!* Just drop a `.md` file with valid frontmatter and restart. + +=== Step 1: Create the prompt file with frontmatter + +Create a new Markdown file in `mcp/prompts/` with YAML frontmatter: + +[,bash] +---- +mcp/prompts/my-docs-guide.md +---- + +[,markdown] +---- +--- +description: Guide for working with specific documentation type +version: 1.0.0 +arguments: + - name: content + description: The documentation content to process + required: true +argumentFormat: content-append +--- + +# My Documentation Guide + +This guide explains how to work with [specific documentation type]. + +## Overview + +[Explanation of the documentation system] + +## Workflow + +1. First step +2. Second step +3. Third step + +## Important rules + +- Rule 1 +- Rule 2 +- Rule 3 + +## Common tasks + +### Task 1: Do something + +[Detailed instructions] + +### Task 2: Do something else + +[Detailed instructions] + +## Validation + +Before committing, check: + +- [ ] Checklist item 1 +- [ ] Checklist item 2 +---- + +*Required frontmatter fields:* + +description:: Clear explanation of what the prompt does (min 10 characters) +version:: Semantic version (for example, "1.0.0") +arguments:: (Optional) Array of argument definitions +* `name`: Argument name (lowercase, underscores only) +* `description`: What this argument is for +* `required`: Boolean indicating if required +argumentFormat:: (Optional) How to append arguments: +* `content-append`: Append content with separator (default) +* `structured`: Insert arguments as structured sections + +*Best practices for prompts:* + +* Write for technical writers, not developers +* Include concrete examples +* Explain the "why" behind rules +* Provide validation checklists +* Use clear, scannable structure +* Keep version updated when making significant changes + +=== Step 2: Validate the prompt + +Run validation to check frontmatter and metadata: + +[,bash] +---- +npx doc-tools validate-mcp +---- + +This checks: + +* YAML frontmatter is valid +* Required fields are present (description, version) +* Argument names use only lowercase and underscores +* Version follows semantic versioning +* No naming conflicts with existing prompts + +=== Step 3: Preview the prompt + +Test the prompt before deploying: + +[,bash] +---- +# Preview with content argument +npx doc-tools preview-prompt my-docs-guide --content "Test content" + +# See how it appears to Claude +---- + +=== Step 4: Deploy and test + +. Restart Claude Code to load the new prompt +. Test with Claude: ++ +---- +You: "List available prompts" +Claude: *shows your new prompt in the list* + +You: "Use the my-docs-guide prompt with this content: [...]" +Claude: *retrieves and applies your prompt* +---- + +=== Auto-discovery details + +The prompt discovery system (`bin/mcp-tools/prompt-discovery.js`): + +. Scans `mcp/prompts/*.md` files +. Parses YAML frontmatter using `js-yaml` +. Validates against JSON schema using `ajv` +. Extracts prompt name from filename (for example, `my-docs-guide.md` β†’ `my-docs-guide`) +. Caches content and metadata in memory +. Provides generic handler that works for all prompts + +*Development mode:* + +Enable file watching to automatically reload changes: + +[,bash] +---- +export MCP_DEV_MODE=true +# Restart Claude Code +# Now changes to prompts are automatically reloaded +---- + +=== For more information + +* link:WRITER_EXTENSION_GUIDE.adoc[Writer Extension Guide] - Complete guide for writers adding prompts (no code) +* link:TEAM_ROLLOUT_GUIDE.adoc#operational-playbook[Operational Playbook] - Managing prompts in production +* `bin/mcp-tools/prompt-discovery.js` - Auto-discovery implementation +* `bin/mcp-tools/frontmatter.js` - Frontmatter parsing and validation + +== Testing + +The MCP server has three types of tests: + +=== Integration tests + +Test MCP tools end-to-end through the tool execution layer. + +*Location:* `__tests__/mcp/integration.test.js` + +*Add tests for new tools:* + +[,javascript] +---- +test('my_new_tool returns expected result', () => { + const result = mcpTools.executeTool('my_new_tool', { + my_parameter: 'test-value' + }); + + expect(result).toBeDefined(); + expect(result.success).toBe(true); + expect(result.summary).toContain('Successfully'); +}); + +test('my_new_tool validates parameters', () => { + const result = mcpTools.executeTool('my_new_tool', {}); + + expect(result.success).toBe(false); + expect(result.error.toLowerCase()).toContain('required'); +}); +---- + +=== CLI contract tests + +Verify that the doc-tools CLI maintains the expected interface. + +*Location:* `__tests__/mcp/cli-contract.test.js` + +*Add tests for new CLI commands:* + +[,javascript] +---- +test('my-command exists', () => { + const result = executeCLI('my-command --help'); + expect(result.success).toBe(true); +}); + +test('my-command supports required flag --param', () => { + const result = executeCLI('my-command --help'); + expect(result.output).toContain('--param'); +}); +---- + +See link:CLI_INTERFACE.adoc[CLI Interface Contract] for details on the contract. + +=== Unit tests + +Test individual MCP server components in isolation. + +*Location:* `__tests__/mcp/doc-tools-mcp.test.js` + +*Add tests for tool modules:* + +[,javascript] +---- +describe('executeMyTool', () => { + test('executes successfully with valid params', () => { + const result = executeMyTool( + { my_parameter: 'test' }, + { repoRoot: '/test' } + ); + + expect(result.success).toBeDefined(); + }); + + test('returns error for missing parameters', () => { + const result = executeMyTool({}, { repoRoot: '/test' }); + + expect(result.success).toBe(false); + expect(result.error).toContain('required'); + }); +}); +---- + +=== Running tests + +Run all MCP tests: + +[,bash] +---- +npm run test:mcp +---- + +Run specific test suites: + +[,bash] +---- +npm run test:mcp:integration # Integration tests only +npm run test:mcp:contract # Contract tests only +---- + +Run with coverage: + +[,bash] +---- +npm test -- --coverage __tests__/mcp/ +---- + +Run in watch mode during development: + +[,bash] +---- +npm test -- --watch __tests__/mcp/ +---- + +See link:../__tests__/mcp/README.adoc[Test documentation] for more details. + +== Debugging + +=== Enable debug logging + +Set the `NODE_ENV` environment variable to see detailed logs: + +[,bash] +---- +NODE_ENV=development claude +---- + +This enables: + +* Stack traces in error responses +* Additional console logging +* Verbose MCP protocol messages + +=== View MCP server logs + +The server logs to stderr (not stdout) to avoid interfering with the MCP protocol. + +Claude Code captures these logs. Check: + +* macOS: `~/Library/Logs/Claude/` +* Linux: `~/.local/share/Claude/logs/` +* Windows: `%APPDATA%\Claude\logs\` + +=== Test the server directly + +You can test tool execution without Claude Code: + +[,javascript] +---- +// test-tool.js +const mcpTools = require('./bin/mcp-tools'); + +const result = mcpTools.executeTool('get_redpanda_version', {}); +console.log(JSON.stringify(result, null, 2)); +---- + +[,bash] +---- +node test-tool.js +---- + +=== Common issues + +*Tool not showing up in Claude Code* + +* Run `npx doc-tools setup-mcp --status` to verify configuration +* Check `~/.claude.json` for the `mcpServers` section +* Restart Claude Code completely +* Check logs for server startup errors + +*"doc-tools not found" errors* + +* Ensure you're in a repository that has doc-tools installed +* Check that `npx doc-tools --version` works +* Verify `repoRoot` is correctly detected + +*Timeout errors* + +* Increase timeout in tool module (default: 5 minutes) +* Check network connectivity for version tools +* Look for infinite loops in CLI commands + +== Security + +=== Command injection prevention + +All CLI commands go through validation in `cli.js`: + +* Shell metacharacters blocked: `;`, `|`, `&`, `$`, ``` ` ```, `<`, `>`, `(`, `)` +* Path traversal sequences blocked: `..`, `~` +* Commands must be non-empty strings + +*Always use the CLI validation when executing shell commands.* + +=== Sensitive data + +* Never log sensitive data (tokens, passwords, API keys) +* Don't expose internal paths in error messages +* Sanitize file paths before returning to Claude + +=== Resource limits + +All CLI commands have: + +* 5-minute timeout +* 50MB output buffer limit +* Execution in detected repository context only + +== Best practices + +=== Tool design + +* *Writer-focused*: Design tools for technical writers, not developers +* *Natural language*: Use conversational descriptions and parameter names +* *Helpful errors*: Always include suggestions for fixing errors +* *Structured output*: Return consistent JSON structures +* *Fail gracefully*: Handle all error cases with clear messages + +=== Code organization + +* One tool module per file in `bin/mcp-tools/` +* Keep modules focused and single-purpose +* Export only necessary functions +* Document function parameters and return values + +=== Error handling + +* Always catch exceptions +* Return structured error objects with `success: false` +* Include helpful suggestions in error responses +* Preserve stdout/stderr for debugging + +=== Performance + +* Use appropriate timeouts for different operations +* Parse CLI output efficiently +* Cache repository detection when possible +* Avoid unnecessary file system operations + +== Release process + +=== Before releasing + +. Run all tests: `npm run test:mcp` +. Update version in `package.json` +. Update link:USER_GUIDE.adoc[USER_GUIDE.adoc] with any changes +. Test manually with Claude Code +. Check that setup command works: `npx doc-tools setup-mcp` + +=== Publishing + +. Commit all changes +. Tag the release: `git tag -a v1.2.3 -m "Release 1.2.3"` +. Push with tags: `git push --tags` +. Publish to npm: `npm publish` + +=== After releasing + +. Update documentation if needed +. Notify writers of any breaking changes +. Monitor for issues + +== Related documentation + +*User documentation* + +* link:USER_GUIDE.adoc[User guide] - Guide for writers using the MCP server with Claude Code + +*Developer documentation* + +* link:../__tests__/mcp/README.adoc[Test documentation] - How to run and write tests +* link:CLI_INTERFACE.adoc[CLI interface contract] - CLI interface that the MCP server depends on +* link:prompts/README.adoc[Prompts documentation] - How the prompt system works + +*External documentation* + +* https://modelcontextprotocol.io/[MCP Protocol Specification] +* https://docs.anthropic.com/claude[Claude Code Documentation] +* https://nodejs.org/docs/latest/api/child_process.html[Node.js child_process] + +== Contributing + +When contributing to the MCP server: + +. Read this guide thoroughly +. Review existing tool implementations for patterns +. Write tests for all new functionality +. Update documentation +. Follow the existing code style +. Test manually with Claude Code before submitting PR + +== Getting help + +* Check Claude Code logs for errors +* Review the link:../__tests__/mcp/README.adoc[test documentation] +* Look at existing tool implementations for examples +* Open an issue in the repository + +== Roadmap + +Potential future improvements: + +* Additional documentation generation tools +* More sophisticated prompt selection +* Caching layer for expensive operations +* Better error recovery +* Streaming output for long-running operations +* Tool result validation diff --git a/mcp/README.adoc b/mcp/README.adoc new file mode 100644 index 00000000..7c0483f8 --- /dev/null +++ b/mcp/README.adoc @@ -0,0 +1,179 @@ += MCP Server Documentation +:toc: +:toclevels: 2 + +Documentation for the Redpanda Doc Tools MCP (Model Context Protocol) server. + +The MCP server exposes `doc-tools` functionality to Claude Code, allowing technical writers to automate documentation tasks through natural conversation. No CLI knowledge required. + +== Documentation + +=== For users (technical writers) + +link:USER_GUIDE.adoc[**User guide**]:: Complete guide to using the MCP server with Claude Code ++ +* Setup and installation +* Available tools, prompts, and resources +* Usage examples and workflows +* Troubleshooting + +link:WRITER_EXTENSION_GUIDE.adoc[**Writer extension guide**]:: How to extend the MCP server (no coding required!) ++ +* Adding new prompts for common tasks +* Adding new resources (style guides, templates) +* Validation and preview commands +* Best practices and examples + +link:TEAM_ROLLOUT_GUIDE.adoc[**Team rollout guide**]:: Rolling out to your documentation team ++ +* Phased rollout plan +* Training materials and best practices +* Operational playbook (rollback, emergency procedures) +* Success metrics and monitoring + +=== For developers + +link:DEVELOPMENT.adoc[**Development guide**]:: Guide for working on the MCP server ++ +* Architecture overview +* Adding new tools programmatically +* Auto-discovery system details +* Testing guidelines +* Security and best practices + +link:CLI_INTERFACE.adoc[**CLI interface contract**]:: Contract specification for the `doc-tools` CLI ++ +* Command structure and syntax +* Output format contracts +* Error handling requirements +* Stability guarantees + +link:../CLI_REFERENCE.adoc[**CLI reference**]:: Complete command reference for `doc-tools` ++ +* All commands and subcommands +* Options and usage examples +* Auto-generated from CLI help + +link:../\__tests__/mcp/README.adoc[**Test documentation**]:: How to run and write MCP tests ++ +* Test types (integration, contract, unit) +* Running tests +* Understanding failures + +=== Advanced + +link:prompts/README.adoc[**Prompts documentation**]:: Guide to the MCP prompt system ++ +* What prompts are and how they work +* Available prompts +* Adding new prompts + +link:AI_CONSISTENCY_ARCHITECTURE.adoc[**AI consistency architecture**]:: Architecture for team-wide AI consistency ++ +* Design principles +* Technical implementation +* Extension patterns + +link:AI_CONSISTENCY_SHOWCASE.adoc[**AI consistency showcase**]:: Examples and use cases ++ +* Real-world examples +* Before/after comparisons +* Best practices + +== Quickstart + +=== Install and configure + +[,bash] +---- +cd /path/to/docs-extensions-and-macros +npm install +npx doc-tools setup-mcp --local +---- + +=== Restart Claude Code + +After setup, restart Claude Code to load the MCP server. + +=== Start using it + +Navigate to any Redpanda repository and start chatting: + +---- +You: "What's the latest Redpanda version?" +Claude: *uses get_redpanda_version tool* + +You: "Generate property docs for v25.3.1" +Claude: *uses generate_property_docs tool* +---- + +== What's included + +=== Documentation generation tools + +* **Version information** - Get latest Redpanda and Console versions +* **Property docs** - Generate configuration property documentation +* **Metrics docs** - Generate metrics reference documentation +* **RPK docs** - Generate RPK CLI documentation +* **Connector docs** - Generate Redpanda Connect connector documentation +* **Antora structure** - Analyze documentation repository structure + +=== AI consistency prompts + +Pre-configured instructions that help Claude follow your team standards: + +**Review prompts:** + +* **review-for-style** - Review content for style guide compliance +* **check-terminology** - Validate terminology against approved terms +* **improve-clarity** - Refactor content for better readability + +**Content creation:** + +* **write-new-guide** - Generate new guides following team standards + +**Workflow guides:** + +* **property-docs-guide** - Workflow for updating property documentation +* **rpcn-connector-docs-guide** - Workflow for updating connector documentation + +=== Resources + +Reference materials that Claude can read: + +* **redpanda://style-guide** - Complete team style guide with formatting rules + +=== CLI validation commands + +For writers extending the MCP server: + +* **npx doc-tools validate-mcp** - Validate prompts and resources configuration +* **npx doc-tools preview-prompt** - Preview prompts with test arguments +* **npx doc-tools mcp-version** - Show versions and usage statistics + +=== Automation + +* **setup-mcp** - Automated MCP server configuration for Claude Code/Desktop +* **Auto-discovery** - Prompts automatically loaded from `mcp/prompts/` directory +* **Context detection** - Automatic repository and Antora structure detection +* **Usage telemetry** - Track prompt/resource/tool usage for adoption metrics + +== Architecture + +The MCP server consists of: + +* **Server** (`bin/doc-tools-mcp.js`) - Main MCP protocol handler +* **Tool modules** (`bin/mcp-tools/`) - Individual tool implementations +* **Prompts** (`mcp/prompts/`) - Contextual documentation guides +* **Tests** (`__tests__/mcp/`) - Integration, contract, and unit tests + +== Support + +* link:USER_GUIDE.adoc#troubleshooting[Troubleshooting guide] +* link:../\__tests__/mcp/README.adoc#understanding-test-failures[Test failures] +* https://github.com/redpanda-data/docs-extensions-and-macros/issues[Report issues] + +== External resources + +* https://modelcontextprotocol.io/[MCP Protocol Specification] +* https://docs.anthropic.com/claude[Claude Code Documentation] diff --git a/mcp/USER_GUIDE.adoc b/mcp/USER_GUIDE.adoc new file mode 100644 index 00000000..2bd6ca59 --- /dev/null +++ b/mcp/USER_GUIDE.adoc @@ -0,0 +1,1355 @@ += Redpanda Doc Tools MCP Server +:toc: +:toclevels: 3 + +An MCP (Model Context Protocol) server that provides writer-friendly documentation tools to Claude Code. This allows technical writers to manage documentation through natural conversation without needing to know CLI commands or syntax. + +== Features + +* Writer-friendly: Natural language interface, no CLI knowledge needed +* *Version information*: Get latest Redpanda and Console versions with Docker tags +* *Documentation generation*: Generate property, metrics, and RPK docs for any version +* *Background jobs*: Long-running tasks can run in background with progress updates +* *10-minute timeout*: Extended timeout for large documentation generation tasks +* *Progress streaming*: Real-time progress updates via MCP notifications +* *Context-aware*: Automatically works in any Redpanda repo based on your current directory +* *Antora intelligence*: Understands component/module structure +* *Helpful errors*: Clear error messages with suggestions +* *No API key needed*: Uses your existing Claude Code authentication + +== Quickstart + +=== Install dependencies + +From this repository: + +[,bash] +---- +npm install +---- + +=== Run setup command + +Use the built-in setup command (works on macOS, Linux, and Windows): + +[,bash] +---- +npx doc-tools setup-mcp +---- + +This automatically: + +* Detects your operating system +* Finds your Claude Code/Desktop config location +* Creates or updates the configuration +* Creates a backup of existing config +* Validates everything is correct + +*Check status:* + +[,bash] +---- +npx doc-tools setup-mcp --status +---- + +*Options:* + +[,bash] +---- +# Force update even if already configured +npx doc-tools setup-mcp --force + +# Target specific application +npx doc-tools setup-mcp --target code # Claude Code +npx doc-tools setup-mcp --target desktop # Claude Desktop +---- + +=== Restart Claude Code + +After setup completes, restart Claude Code to load the MCP server. + +=== Start using it! + +Navigate to any Redpanda repository and start chatting: + +[,bash] +---- +cd ~/repos/docs +claude +---- + +Then in Claude Code: + +---- +You: "Show me the Antora structure of this repo" +Claude: *uses get_antora_structure MCP tool* + +You: "Generate property docs for v25.3.1" +Claude: *uses run_doc_tools_command MCP tool* + *shows generated documentation* +---- + +== Available tools + +The MCP server provides *writer-friendly tools* designed for natural language interaction: + +=== Version information tools + +*get_redpanda_version*:: Get the latest Redpanda version information ++ +* Returns version number, Docker tag, and release notes URL +* Optional: Get beta/RC versions with `beta: true` +* Writers use this to find out what version to document + +*get_console_version*:: Get the latest Redpanda Console version information ++ +* Returns version number, Docker tag, and release notes URL +* Helps writers stay current with Console releases + +=== Documentation generation tools + +*generate_property_docs*:: Generate Redpanda configuration property documentation ++ +* Input: Version (for example, "25.3.1", "v25.3.1", or "latest") +* Optional: Generate AsciiDoc partials with `generate_partials: true` +* Returns: Files generated, property count, summary +* Writers use this when updating docs for a new Redpanda release + +*generate_metrics_docs*:: Generate Redpanda metrics documentation ++ +* Input: Version (for example, "25.3.1") +* Creates the public metrics reference page +* Returns: Files generated, metrics count, summary +* Writers use this when updating metrics docs for a new release + +*generate_rpk_docs*:: Generate RPK command-line documentation ++ +* Input: Version (for example, "25.3.1") +* Creates AsciiDoc files for all rpk commands +* Returns: Commands documented, files generated +* Writers use this when updating CLI docs for a new release + +*generate_rpcn_connector_docs*:: Generate Redpanda Connect connector documentation ++ +* Input: Branch (optional, defaults to "main") +* Creates component documentation for all connectors from the Redpanda Connect repository +* Returns: Branch used, connectors documented, files generated +* Writers use this when updating connector reference docs + +=== Quality review tools + +*review_generated_docs*:: Review generated documentation for quality issues ++ +* Input: Type (for example, "property-docs", "connector-docs") and version +* Analyzes generated documentation for common quality issues +* Checks for consistency, completeness, and documentation standards +* Returns: Review report with issues found, suggestions, and quality score +* Writers use this after generating documentation to ensure quality before committing + +=== Structure and discovery tools + +*get_antora_structure*:: Analyze and understand the Antora component/module structure ++ +* Shows all components, versions, and modules +* Lists available directories (pages, partials, examples, etc.) +* Detects if doc-tools is available +* Helps writers understand the documentation layout + +=== Advanced tools + +*run_doc_tools_command*:: Execute raw `npx doc-tools ` ++ +* Advanced: Only use if none of the specific tools above fit your needs +* Requires knowledge of doc-tools CLI syntax +* Only works in repos that have doc-tools installed + +=== Contextual guides (prompts) + +*property-docs-guide*:: Comprehensive guide for updating Redpanda property documentation ++ +* Explains the auto-generation system and override workflow +* Provides rules for property descriptions (no cloud conditionals, no enterprise includes) +* Lists common scenarios and validation steps +* Claude automatically retrieves this when working with property documentation +* Ensures correct workflow: update property-overrides.json, not generated .adoc files + +*rpcn-connector-docs-guide*:: Comprehensive guide for updating Redpanda Connect connector reference documentation ++ +* Explains the override system with `$ref` syntax for DRY (Don't Repeat Yourself) principles +* Describes repository structure and how autogenerated content works with overrides +* Provides workflow for improving connector documentation +* Claude automatically retrieves this when working with connector documentation +* Ensures correct workflow: update docs-data/overrides.json, use $ref for deduplication + +=== Background job management + +Long-running documentation generation tasks support background execution with progress updates: + +*get_job_status*:: Check the status and progress of a background job ++ +* Returns job status, progress percentage, and progress message +* Use the job ID returned when creating a background job +* Job statuses: `pending`, `running`, `completed`, `failed` + +*list_jobs*:: List all background jobs with optional filtering ++ +* Returns list of recent jobs with their status +* Optional filters: status (pending/running/completed/failed), tool name +* Jobs are automatically cleaned up after 1 hour + +*How to use background jobs:* + +Add `background: true` to any long-running generation tool: + +[,bash] +---- +# Synchronous (default) - waits for completion +generate_property_docs(version: "v25.3.1", generate_partials: true) + +# Background - returns job ID immediately +generate_property_docs(version: "v25.3.1", generate_partials: true, background: true) + +# Check job status +get_job_status(job_id: "abc-123-def-456") + +# List all jobs +list_jobs(status: "running") +---- + +*Progress updates:* + +Background jobs send real-time progress notifications via MCP protocol, so Claude Code can show you: + +* Current progress percentage (0-100%) +* Current operation being performed +* Estimated time remaining (when available) + +*Timeout:* + +All commands have a 10-minute timeout (increased from 5 minutes) to handle large documentation generation tasks. + +== Available prompts + +The MCP server provides *AI consistency prompts* that help Claude follow your team's documentation standards automatically. These prompts are pre-configured instructions that guide Claude's responses for common documentation tasks. + +=== Documentation review prompts + +*review-for-style*:: Review documentation content for style guide compliance ++ +* Checks: Style guide compliance, terminology consistency, voice/tone, AsciiDoc formatting +* Input: Content to review (can be file path or raw text) +* Output: Detailed feedback with critical issues, suggestions, and actionable recommendations +* Use when: Reviewing any documentation before publishing + +*check-terminology*:: Validate terminology usage against approved team terminology ++ +* Checks: Incorrect terms, deprecated terms (whitelist/blacklist, master/slave), inconsistent usage +* Input: Content to check +* Output: Critical issues with corrections, inconsistencies found, missing glossary links +* Use when: Ensuring consistent terminology across documentation + +*improve-clarity*:: Refactor existing documentation for clarity and readability ++ +* Applies: Style guide principles, simplifies complex explanations, improves structure +* Input: Content to improve +* Output: Refactored content with explanations of changes made +* Use when: Content is technically correct but difficult to understand + +=== Content creation prompts + +*write-new-guide*:: Generate a new tutorial or guide following team standards ++ +* Applies: Team templates, approved terminology, voice/tone guidelines, AsciiDoc formatting +* Input: Topic (what to write about), optional audience (beginners, experienced devs, operators) +* Output: Complete guide following team standards +* Use when: Starting a new tutorial, how-to, or getting started guide + +=== Workflow guide prompts + +*property-docs-guide*:: Comprehensive guide for updating Redpanda property documentation ++ +* Explains: Auto-generation system, override workflow, validation steps +* Rules: Never edit generated `.adoc` files, always update `property-overrides.json` +* Use when: Claude is helping with property documentation +* Claude automatically retrieves this prompt when working with properties + +*rpcn-connector-docs-guide*:: Guide for updating Redpanda Connect connector documentation ++ +* Explains: Override system with `$ref` syntax for DRY principles, repository structure +* Rules: Update `docs-data/overrides.json`, use `$ref` for deduplication +* Use when: Claude is helping with connector documentation +* Claude automatically retrieves this prompt when working with connectors + +=== How to use prompts + +Prompts work through natural conversation with Claude Code: + +*Explicit prompt usage:* + +---- +You: "Use the review-for-style prompt with this content: [paste content]" +Claude: *retrieves and applies the review-for-style prompt* + *provides structured feedback following the prompt instructions* +---- + +*Implicit prompt usage:* + +---- +You: "Review this tutorial for our style guide" +Claude: *automatically uses review-for-style prompt* + *provides feedback based on your team standards* +---- + +*Listing available prompts:* + +---- +You: "List available prompts" +Claude: *shows all prompts with descriptions* +---- + +You can also preview prompts using the CLI: + +[,bash] +---- +# Preview a review prompt +npx doc-tools preview-prompt review-for-style --content "Test content" + +# Preview a write prompt +npx doc-tools preview-prompt write-new-guide --topic "Deploy cluster" + +# List all prompts with versions +npx doc-tools mcp-version +---- + +== Available resources + +Resources are reference materials that Claude can read to understand your team's standards. + +=== Style guide + +*redpanda://style-guide*:: Redpanda Documentation Style Guide (v1.0.0) ++ +* Complete style guide based on Google Developer Documentation Style Guide +* Includes: Redpanda-specific guidelines, voice and tone, AsciiDoc formatting rules +* Contains: Terminology quick reference, formatting examples, common patterns +* Claude automatically references this when reviewing or writing documentation + +=== How to use resources + +Resources are automatically loaded by prompts, but you can also reference them explicitly: + +*Explicit reference:* + +---- +You: "Read the redpanda://style-guide resource and tell me the rules for code blocks" +Claude: *reads the style guide resource* + *explains the code block formatting rules* +---- + +*Listing available resources:* + +---- +You: "List available resources" +Claude: *shows all resources with descriptions and versions* +---- + +=== Design philosophy + +These tools are designed to be *conversational*: + +* Writers use natural language: "What's the latest Redpanda version?" +* Claude understands and calls the appropriate tool +* No need to remember CLI commands or flags +* Clear, structured responses with helpful error messages + +Claude Code's built-in tools handle everything else: + +* *File operations* (Read, Write, Edit) +* *Search* (Glob for files, Grep for content) +* *Git operations* (Bash tool) + +== JSON response structures + +=== get_antora_structure + +Returns comprehensive information about the Antora documentation structure: + +[,json] +---- +{ + "repoRoot": "/absolute/path/to/repo", + "repoInfo": { + "root": "/absolute/path/to/repo", + "detected": true, + "type": "git" // or "npm" or null + }, + "playbook": { + // Parsed playbook YAML content (if found) + "site": { /* ... */ }, + "content": { /* ... */ } + }, + "playbookPath": "/absolute/path/to/local-antora-playbook.yml", + "components": [ + { + "name": "redpanda", + "version": "25.3", + "title": "Redpanda Documentation", + "path": "/absolute/path/to/component", + "modules": [ + { + "name": "ROOT", + "path": "/absolute/path/to/component/modules/ROOT", + "pages": true, + "partials": true, + "examples": false, + "attachments": false, + "images": true + } + ] + } + ], + "hasDocTools": true // Whether doc-tools is available +} +---- + +*Field descriptions:* + +repoRoot:: Absolute path to repository root +repoInfo.detected:: Whether a repository was detected (vs using current directory) +repoInfo.type:: Repository type (`"git"`, `"npm"`, or `null`) +playbook:: Parsed YAML content of the Antora playbook (if found) +playbookPath:: Full path to the playbook file used +components[]:: Array of Antora components found +* `name`: Component name from antora.yml +* `version`: Component version +* `title`: Component display title +* `path`: Absolute path to component directory +* `modules[]`: Array of modules in this component +** Boolean flags indicate which directories exist in each module + +=== run_doc_tools_command + +==== Success response + +[,json] +---- +{ + "success": true, + "output": "Generated property docs for v25.3.1\nβœ“ Created 42 documentation files", + "command": "npx doc-tools generate property-docs --tag v25.3.1", + "repoRoot": "/absolute/path/to/repo" +} +---- + +*Field descriptions:* + +success:: Always `true` for successful executions +output:: Combined stdout from the command +command:: The full command that was executed +repoRoot:: Repository root where command was executed + +==== Error response: Command validation failed + +[,json] +---- +{ + "success": false, + "error": "Invalid command: shell metacharacters not allowed. Use simple doc-tools commands only.", + "suggestion": "Use simple doc-tools commands like: \"generate property-docs --tag v25.3.1\"" +} +---- + +==== Error response: doc-tools not found + +[,json] +---- +{ + "success": false, + "error": "doc-tools not found in this repository. This command only works in repos with doc-tools installed.", + "suggestion": "Navigate to the docs-extensions-and-macros repository or a repo that has doc-tools as a dependency.", + "repoInfo": { + "root": "/current/directory", + "detected": false, + "type": null + } +} +---- + +==== Error response: Command execution failed + +[,json] +---- +{ + "success": false, + "error": "Command failed with exit code 1", + "stdout": "Processing files...\nFound 10 properties", + "stderr": "Error: Tag v25.3.1 not found", + "exitCode": 1, + "command": "npx doc-tools generate property-docs --tag v25.3.1", + "repoRoot": "/absolute/path/to/repo" +} +---- + +*Field descriptions:* + +success:: Always `false` for errors +error:: Human-readable error message +stdout:: Standard output from the command (if any) +stderr:: Standard error from the command (if any) +exitCode:: Exit code from the command +suggestion:: Helpful suggestion for resolving the error (when applicable) + +=== Error response - Unknown tool + +[,json] +---- +{ + "success": false, + "error": "Unknown tool: invalid_tool_name", + "availableTools": ["get_antora_structure", "run_doc_tools_command"] +} +---- + +=== Error response - Internal error + +[,json] +---- +{ + "success": false, + "error": "Detailed error message", + "stack": "Error stack trace..." // Only in development mode +} +---- + +NOTE: Stack traces are only included when `NODE_ENV=development` for security reasons. + +=== get_redpanda_version + +==== Success response + +[,json] +---- +{ + "success": true, + "version": "v25.3.1", + "docker_tag": "docker.redpanda.com/redpandadata/redpanda:v25.3.1", + "is_beta": false, + "notes_url": "https://github.com/redpanda-data/redpanda/releases/tag/v25.3.1" +} +---- + +*Field descriptions:* + +success:: Always `true` for successful requests +version:: Latest Redpanda version (with 'v' prefix) +docker_tag:: Full Docker image tag for this version +is_beta:: Whether this is a beta/RC version +notes_url:: GitHub release notes URL + +==== Beta version response + +[,json] +---- +{ + "success": true, + "version": "v25.4.1-rc1", + "docker_tag": "docker.redpanda.com/redpandadata/redpanda-unstable:v25.4.1-rc1", + "is_beta": true, + "notes_url": "https://github.com/redpanda-data/redpanda/releases/tag/v25.4.1-rc1" +} +---- + +==== Error response + +[,json] +---- +{ + "success": false, + "error": "Network error: Failed to fetch version information", + "suggestion": "Make sure you have network access to fetch version information from GitHub" +} +---- + +=== get_console_version + +==== Success response + +[,json] +---- +{ + "success": true, + "version": "v2.7.2", + "docker_tag": "docker.redpanda.com/redpandadata/console:v2.7.2", + "notes_url": "https://github.com/redpanda-data/console/releases/tag/v2.7.2" +} +---- + +*Field descriptions:* + +success:: Always `true` for successful requests +version:: Latest Console version (with 'v' prefix) +docker_tag:: Full Docker image tag for Console +notes_url:: GitHub release notes URL + +==== Error response + +[,json] +---- +{ + "success": false, + "error": "Network error: Failed to fetch version information", + "suggestion": "Make sure you have network access to fetch version information from GitHub" +} +---- + +=== generate_property_docs + +==== Success response: (JSON only) + +[,json] +---- +{ + "success": true, + "version": "v25.3.1", + "files_generated": [ + "modules/reference/partials/properties.json" + ], + "property_count": 342, + "output": "Generated 342 properties\nSaved to modules/reference/partials/properties.json", + "summary": "Generated property documentation for Redpanda v25.3.1" +} +---- + +==== Success response: (with AsciiDoc partials) + +[,json] +---- +{ + "success": true, + "version": "v25.3.1", + "files_generated": [ + "modules/reference/partials/cluster-properties.adoc", + "modules/reference/partials/broker-properties.adoc", + "modules/reference/partials/tunable-properties.adoc", + "modules/reference/partials/topic-properties.adoc", + "modules/reference/partials/properties.json" + ], + "property_count": 342, + "output": "Generated 342 properties...", + "summary": "Generated property documentation for Redpanda v25.3.1" +} +---- + +*Field descriptions:* + +success:: Always `true` for successful generation +version:: Redpanda version used (normalized with 'v' prefix) +files_generated:: Array of files created +property_count:: Number of properties documented (if parseable from output) +output:: Raw output from the generation command +summary:: Human-readable summary + +==== Error response: Missing version + +[,json] +---- +{ + "success": false, + "error": "Version is required", + "suggestion": "Provide a version like \"25.3.1\", \"v25.3.1\", or \"latest\"" +} +---- + +==== Error response: doc-tools not found + +[,json] +---- +{ + "success": false, + "error": "doc-tools not found in this repository", + "suggestion": "Navigate to the docs-extensions-and-macros repository" +} +---- + +==== Error response: Generation failed + +[,json] +---- +{ + "success": false, + "error": "Command failed with exit code 1", + "stdout": "Processing files...\nFound 0 properties", + "stderr": "Error: Tag v25.3.1 not found in Redpanda repository", + "exitCode": 1, + "suggestion": "Check that the version exists in the Redpanda repository" +} +---- + +=== generate_metrics_docs + +==== Success response + +[,json] +---- +{ + "success": true, + "version": "v25.3.1", + "files_generated": [ + "modules/reference/pages/public-metrics-reference.adoc" + ], + "metrics_count": 156, + "output": "Generated 156 metrics...", + "summary": "Generated metrics documentation for Redpanda v25.3.1" +} +---- + +*Field descriptions:* + +success:: Always `true` for successful generation +version:: Redpanda version used (normalized with 'v' prefix) +files_generated:: Array of files created +metrics_count:: Number of metrics documented (if parseable) +output:: Raw output from the generation command +summary:: Human-readable summary + +==== Error responses + +Same error response structure as `generate_property_docs` (missing version, doc-tools not found, generation failed). + +=== generate_rpk_docs + +==== Success response + +[,json] +---- +{ + "success": true, + "version": "v25.3.1", + "commands_documented": 87, + "files_generated": ["autogenerated/v25.3.1/rpk/*.adoc"], + "output": "Generated 87 commands...", + "summary": "Generated RPK documentation for Redpanda v25.3.1" +} +---- + +*Field descriptions:* + +success:: Always `true` for successful generation +version:: Redpanda version used (normalized with 'v' prefix) +commands_documented:: Number of RPK commands documented +files_generated:: Array indicating where files were created +output:: Raw output from the generation command +summary:: Human-readable summary + +==== Error responses + +Same error response structure as `generate_property_docs` (missing version, doc-tools not found, generation failed). + +=== generate_rpcn_connector_docs + +==== Success response + +[,json] +---- +{ + "success": true, + "branch": "main", + "connectors_documented": 245, + "files_generated": ["modules/reference/pages/redpanda-connect/components/"], + "output": "Generated 245 connectors...", + "summary": "Generated Redpanda Connect connector documentation from branch main" +} +---- + +*Field descriptions:* + +success:: Always `true` for successful generation +branch:: Branch used to generate documentation (defaults to "main") +connectors_documented:: Number of connectors documented (if parseable, otherwise `null`) +files_generated:: Array indicating where files were created +output:: Raw output from the generation command +summary:: Human-readable summary + +==== Error responses + +Same error response structure as `generate_property_docs` (doc-tools not found, generation failed). + +=== review_generated_docs + +==== Success response + +[,json] +---- +{ + "success": true, + "doc_type": "property-docs", + "version": "v25.3.1", + "report_file": "review-properties-25.3.1-2025-11-24.adoc", + "issues_found": { + "missing_descriptions": 3, + "formatting_issues": 5, + "inconsistencies": 2 + }, + "quality_score": 92, + "summary": "Review completed for property documentation v25.3.1. Found 10 issues across 342 properties." +} +---- + +*Field descriptions:* + +success:: Always `true` for successful review +doc_type:: Type of documentation reviewed ("property-docs", "connector-docs", etc.) +version:: Version of the documentation reviewed +report_file:: Path to the generated review report file +issues_found:: Object containing counts of different issue types +quality_score:: Overall quality score (0-100) +summary:: Human-readable summary of the review + +==== Error responses + +Same error response structure as `generate_property_docs` (missing required parameters, doc-tools not found). + +== Usage examples + +=== Example 1: Get version information + +---- +You: What's the latest Redpanda version? +Claude: *uses get_redpanda_version MCP tool* + +The latest Redpanda version is v25.3.1, released on January 15, 2024. + +Docker tag: docker.redpanda.com/redpandadata/redpanda:v25.3.1 +Release notes: https://github.com/redpanda-data/redpanda/releases/tag/v25.3.1 +---- + +---- +You: What about the latest beta version? +Claude: *uses get_redpanda_version with beta: true* + +The latest beta version is v25.4.1-rc1. + +Docker tag: docker.redpanda.com/redpandadata/redpanda-unstable:v25.4.1-rc1 +---- + +=== Example 2: Generate property documentation + +---- +You: Generate property docs for version 25.3.1 +Claude: *uses generate_property_docs MCP tool* + +βœ“ Generated property documentation for Redpanda v25.3.1 + +Files created: +- modules/reference/partials/properties.json + +Documented 342 properties from Redpanda v25.3.1 +---- + +---- +You: Now generate the AsciiDoc partials too +Claude: *uses generate_property_docs with generate_partials: true* + +βœ“ Generated property documentation for Redpanda v25.3.1 + +Files created: +- modules/reference/partials/cluster-properties.adoc +- modules/reference/partials/broker-properties.adoc +- modules/reference/partials/tunable-properties.adoc +- modules/reference/partials/topic-properties.adoc +- modules/reference/partials/properties.json + +Documented 342 properties +---- + +=== Example 3: Complete documentation update workflow + +---- +You: I need to update the docs for Redpanda 25.3.1. Can you: +1. Get the latest version to confirm +2. Generate property docs +3. Generate metrics docs +4. Generate RPK docs +5. Show me what files changed + +Claude: *uses get_redpanda_version* +Confirmed: Latest version is v25.3.1 + +*uses generate_property_docs* +βœ“ Generated 342 properties + +*uses generate_metrics_docs* +βœ“ Generated 156 metrics + +*uses generate_rpk_docs* +βœ“ Generated 87 RPK commands + +*uses Bash tool to run git status* +Changes: +- modules/reference/partials/properties.json +- modules/reference/pages/public-metrics-reference.adoc +- autogenerated/v25.3.1/rpk/*.adoc + +Would you like me to review the changes or commit them? +---- + +=== Example 4: Console version information + +---- +You: What's the latest Redpanda Console version and Docker tag? +Claude: *uses get_console_version MCP tool* + +The latest Redpanda Console version is v2.7.2. + +Docker tag: docker.redpanda.com/redpandadata/console:v2.7.2 +Release notes: https://github.com/redpanda-data/console/releases/tag/v2.7.2 +---- + +=== Example 5: Understanding documentation structure + +---- +You: Show me the Antora structure of this repo +Claude: *uses get_antora_structure MCP tool* + +This repository has the following Antora structure: + +Components: +- redpanda (version: 25.3) + - Modules: + - ROOT (pages, partials, images) + - reference (pages, partials) + - deployment (pages, examples) + +The repository has doc-tools installed, so you can generate documentation. +---- + +=== Example 6: Error handling + +---- +You: Generate property docs for version 99.9.9 +Claude: *uses generate_property_docs MCP tool* + +❌ Generation failed: Tag v99.9.9 not found in Redpanda repository + +Suggestion: Check that the version exists in the Redpanda repository. You can find available versions at: +https://github.com/redpanda-data/redpanda/releases +---- + +=== Example 7: Property documentation updates (with prompt) + +---- +You: Improve the description for cleanup.policy property to explain what each value does + +Claude: *retrieves property-docs-guide prompt* + *reads property-overrides.json* + *updates the description field with enhanced explanation* + *uses generate_property_docs MCP tool with generate_partials: true* + +βœ“ Updated property-overrides.json with enhanced description for cleanup.policy +βœ“ Generated property documentation for Redpanda v25.3.1 + +Files updated: +- docs-data/property-overrides.json +- modules/reference/partials/properties/topic-properties.adoc + +The description now explains: +- `delete` - Deletes data according to retention limits +- `compact` - Keeps latest value per key +- `compact,delete` - Compaction + time-based deletion +---- + +This example shows how the prompt automatically provides context about the property documentation workflow, ensuring Claude follows the correct process (update overrides, not generated files). + +=== Example 8: Review documentation for style compliance + +---- +You: Review this tutorial for style guide compliance: + +[Content of tutorial pasted here] + +Claude: *uses review-for-style prompt* + *analyzes content against team style guide* + +Style Review for Tutorial +========================================== + +Critical Issues (3): + +1. **Deprecated Terminology** + - Location: Prerequisites section + - Found: "whitelist the following IPs" + - Should be: "allowlist the following IPs" + - Reason: "whitelist" is deprecated terminology + +2. **Inconsistent Heading Format** + - Location: Section 3 + - Found: "step 1: configure broker" + - Should be: "Step 1: Configure broker" + - Reason: Headings must use title case + +3. **Missing Glossary Link** + - Location: Introduction, first mention of "partition" + - Should use: `glossterm:partition,partitions[]` + - Reason: First mention of technical terms should link to glossary + +Suggestions (5): +- Voice could be more active in section 2 +- Code example in section 4 missing expected output +[...] + +Overall: Good structure, clear explanations. Fix critical issues before publishing. +---- + +=== Example 9: Check terminology in existing content + +---- +You: Check the terminology in this doc for any deprecated or incorrect terms + +Claude: *uses check-terminology prompt* + +Terminology Review +========================================== + +Critical Issues (2): + +**Location**: Configuration section, step 3 +**Found**: "SSL encryption" +**Should be**: "TLS encryption" +**Reason**: SSL is deprecated. TLS is the current standard. + +**Location**: Architecture diagram +**Found**: "master node" +**Should be**: "leader node" +**Reason**: master/slave terminology is deprecated. Use leader/replica. + +Summary: +- Total issues found: 2 +- Critical fixes needed: 2 +- Overall terminology health: Good (once fixed) +---- + +=== Example 10: Write a new guide with team standards + +---- +You: Write a getting started guide for deploying a 3-node Redpanda cluster + +Claude: *uses write-new-guide prompt* + *applies team templates and standards* + += Deploy a Three-Node Redpanda Cluster +:description: Learn how to deploy a production-ready Redpanda cluster with three nodes +:page-aliases: deployment:deploy-three-node.adoc + +This guide walks you through deploying a three-node glossterm:cluster[] for production use. + +== Prerequisites + +Before you begin, ensure you have: + +* Three servers or VMs with supported operating systems +* At least 2 GB RAM per server +* Network connectivity between all nodes + +[Additional sections following team structure...] + +βœ“ Generated guide following team standards: +- Uses approved terminology with glossary links +- Follows team voice/tone guidelines +- Includes proper AsciiDoc metadata +- Uses team-standard structure for tutorials +---- + +=== Example 11: Improve clarity of complex explanation + +---- +You: This explanation of compaction is too complex. Can you simplify it? + +[Paste complex explanation] + +Claude: *uses improve-clarity prompt* + +Here's a clearer version: + +**Original** (complex, nested concepts): +"Log compaction ensures the retention of the most recent update for each record key +within the log of data, while simultaneously removing older updates to the same key +in a manner that preserves the temporal ordering of messages..." + +**Improved** (clear, progressive): +Log compaction keeps only the latest value for each key in a topic: + +1. For each message key, Redpanda keeps the most recent message +2. Older messages with the same key are deleted +3. Message order is preserved + +This reduces storage while maintaining the current state of your data. + +**Changes made:** +- Simplified sentence structure +- Broke complex sentence into numbered steps +- Added concrete example +- Kept technical accuracy while improving readability +---- + +== How it works + +=== Context detection + +The MCP server automatically: + +. Detects the repository root from your current working directory +. Looks for Antora structure (components, modules) +. Checks if `doc-tools` is available +. All operations happen relative to the detected repo + +=== Multi-repo support + +You can use the *same MCP server* across all your repos: + +[,bash] +---- +# In docs-extensions-and-macros +cd ~/repos/docs-extensions-and-macros +claude # MCP server works here + +# In main docs repo +cd ~/repos/docs +claude # Same MCP server, different context + +# In cloud docs +cd ~/repos/cloud-docs +claude # Still works! +---- + +=== Tool execution + +When you ask Claude to do something: + +. Claude decides which tools to use +. Claude calls the MCP server with tool requests +. The server executes in your current repo context +. Results are returned to Claude +. Claude shows you what happened + +== Configuration + +=== Configuration locations + +The `setup-mcp` command uses the official `claude mcp add` CLI command, which automatically configures the MCP server in the correct location: + +*Claude Code:* + +* *All platforms*: `~/.claude.json` (user configuration file with `mcpServers` section) +* This works across macOS, Linux, and Windows + +*Claude Desktop:* + +* *macOS*: `~/Library/Application Support/Claude/claude_desktop_config.json` +* *Linux*: `~/.config/Claude/claude_desktop_config.json` +* *Windows*: `%APPDATA%\Claude\claude_desktop_config.json` + +=== Manual configuration (advanced) + +If you need to configure manually, use the `claude mcp add` command directly: + +*For local development:* + +[,bash] +---- +claude mcp add --transport stdio --scope user redpanda-docs-tool-assistant -- \ + node /absolute/path/to/docs-extensions-and-macros/bin/doc-tools-mcp.js +---- + +*For published package:* + +[,bash] +---- +claude mcp add --transport stdio --scope user redpanda-docs-tool-assistant -- \ + npx -y doc-tools-mcp +---- + +This adds the MCP server configuration to `~/.claude.json`. + +NOTE: We recommend using `npx doc-tools setup-mcp` instead, as it automatically detects your environment and uses the appropriate configuration! + +== Troubleshooting + +=== MCP server not showing up + +. Run `npx doc-tools setup-mcp --status` to verify configuration +. Check your config file: `~/.claude.json` (look for the `mcpServers` section) +. Verify the configuration includes `"type": "stdio"` field +. For local mode: Verify the path to `doc-tools-mcp.js` is correct and absolute +. For npx mode: Ensure the package is installed globally or locally +. Restart Claude Code completely (exit and start a new session) +. Check Claude Code logs for errors +. Try running `claude mcp list` to see if the server appears +. If still not working, try removing and re-adding with `claude mcp add` (see Manual Configuration) + +=== "doc-tools not found" error + +This means you're in a repo that doesn't have doc-tools. The `run_doc_tools_command` tool only works in repos that have doc-tools installed. + +Solution: Navigate to the docs-extensions-and-macros repo or another repo that has doc-tools. + +=== Tools not working + +. Make sure you're in a git repository +. Check that you have the required permissions +. For doc-tools commands, ensure dependencies are installed (`npm install`) + +=== Understanding repository structure + +* Use `get_antora_structure` MCP tool to understand the Antora layout +* Then use Claude's built-in Glob/Grep tools to find specific files +* Claude's Read/Write/Edit tools handle all file operations + +== Security + +=== Command injection protection + +The MCP server includes protection against command injection attacks: + +* *Input validation*: All commands are validated before execution +* *Shell metacharacter blocking*: Characters like `;`, `|`, `&`, `$`, ``` ` ```, `<`, `>`, `()` are blocked +* *Path traversal prevention*: Sequences like `..` and `~` are rejected +* *Type checking*: Commands must be non-empty strings + +Safe commands: + +[,bash] +---- +generate property-docs --tag v25.3.1 βœ“ +generate metrics-docs --branch main βœ“ +help βœ“ +---- + +Blocked commands: + +[,bash] +---- +generate property-docs; rm -rf / βœ— (semicolon) +generate property-docs | malicious βœ— (pipe) +generate ../../../etc/passwd βœ— (path traversal) +---- + +=== General security + +* The MCP server is read-only for analysis (get_antora_structure) +* Only runs commands you explicitly request (run_doc_tools_command) +* Commands are restricted to `npx doc-tools` only +* All operations are local to your machine +* File modifications are handled by Claude's built-in tools (which you control) +* 10-minute timeout on all commands +* 50MB output buffer limit + +== Advanced usage + +=== Using with multiple Claude Code instances + +You can run Claude Code in different directories simultaneously, and each will use the same MCP server but operate in its own repo context. + +=== Combining with other MCP servers + +You can have multiple MCP servers configured. Claude will use the appropriate tools from each server as needed. + +=== Scripting workflows + +Since this integrates with Claude Code, you can create complex multi-step workflows through conversation: + +---- +You: I need to: +1. Generate property docs for v25.3.1 +2. Review the changes +3. Create a branch called 'props-v25.3.1' +4. Commit with an appropriate message +5. Give me a summary for the PR description + +Can you do all of that? + +Claude: *executes each step sequentially* + *asks for confirmation at key points* + *provides final summary* +---- + +== Status messages + +The MCP server logs to stderr (not stdout) so it doesn't interfere with the protocol: + +* Server startup message +* Working directory +* Repository root detected + +These appear in Claude Code's logs but won't clutter your chat. + +== Best practices + +. *Start with structure*: Use `get_antora_structure` MCP tool to understand the repo layout +. *Let Claude work naturally*: It will use its built-in tools (Read, Write, Edit, Bash, Glob, Grep) automatically +. *Run automation explicitly*: Use `run_doc_tools_command` for generating docs +. *Trust the tools*: Claude knows when to use MCP tools vs built-in tools +. *Be specific about goals*: Tell Claude what you want to achieve, not how to do it + +== Updates + +To update the MCP server: + +[,bash] +---- +cd /path/to/docs-extensions-and-macros +git pull +npm install +---- + +Then restart Claude Code. + +== Getting help + +If you encounter issues: + +. Check Claude Code logs +. Verify your MCP server configuration +. Test with simple commands first (for example, "Show me the Antora structure") +. Check that you're in a git repository +. Open an issue in the repository + +== Learning more + +* https://modelcontextprotocol.io/[MCP Documentation] +* https://docs.anthropic.com/claude[Claude Code Documentation] +* https://docs.antora.org/[Antora Documentation] + +== What's next + +With the MCP server set up, you can use natural language to: + +=== Version information + +* "What's the latest Redpanda version?" +* "Get me the Redpanda Docker tag" +* "What's the latest Console version?" +* "Show me the beta version" + +=== Documentation generation + +* "Generate property docs for 25.3.1" +* "Generate property docs with AsciiDoc partials" +* "Generate metrics docs for the latest version" +* "Generate RPK command docs for 25.3.1" +* "Update all docs for the latest version" + +=== Understanding your repo + +* "Show me the Antora structure" +* "What modules do we have?" +* "Is doc-tools available here?" + +=== Complete workflows + +* "Update docs for 25.3.1 and show me what changed" +* "Generate all docs for latest version and create a PR" +* "Get the latest version info and update the Docker compose file" + +*No CLI commands to remember. No flags to look up. Just talk naturally!* + +The MCP server provides writer-friendly tools, Claude provides the intelligence. You focus on the content! diff --git a/mcp/WRITER_EXTENSION_GUIDE.adoc b/mcp/WRITER_EXTENSION_GUIDE.adoc new file mode 100644 index 00000000..80a8ddd3 --- /dev/null +++ b/mcp/WRITER_EXTENSION_GUIDE.adoc @@ -0,0 +1,814 @@ += How to Extend the MCP Server (For Technical Writers) +:toc: +:toc-title: Contents + +== Overview + +This guide shows technical writers how to add new prompts, resources, and tools to the doc-tools MCP server. Most extensions require only creating or editing simple files - no coding required! + +== What you can add + +[cols="1,2,1"] +|=== +|Type |What It Does |Difficulty + +|Prompts +|Pre-configured instructions for common tasks (review, write, check, etc.) +|Easy (just Markdown) + +|Resources +|Reference materials (style guides, templates, terminology) that AI can read +|Easy (Markdown/JSON) + +|Tools +|Custom commands that execute doc-tools operations +|Medium (requires JavaScript) +|=== + +== Adding new prompts + +=== When to add a new prompt + +Create a new prompt when you have a repeated task that would benefit from standardized instructions: + +* Review content for a specific doc type (API reference, tutorial, troubleshooting) +* Generate content following a specific template +* Check for specific quality issues +* Refactor content in a consistent way + +=== How auto-discovery works + +The MCP server uses automatic prompt discovery, no code editing required. + +* Save a `.md` file with YAML frontmatter in `mcp/prompts/`. +* The server automatically loads it at startup. +* No need to edit `bin/doc-tools-mcp.js` or any JavaScript files. +* Just validate, restart Claude Code, and use it! + +=== Add a new prompt + +==== Create the prompt file + +Create a new `.md` file in `mcp/prompts/`: + +[source,bash] +---- +cd mcp/prompts +touch review-api-docs.md +---- + +==== Add YAML frontmatter + +At the very top of your file, add metadata using YAML frontmatter: + +[source,markdown] +---- +--- +description: "Review API documentation for completeness, clarity, and consistency with team standards" +version: "1.0.0" +arguments: + content: + description: "The API documentation content to review" + required: true +--- + +# Review API Documentation + +You are reviewing API documentation for the Redpanda documentation team. + +## Your Task + +Review API documentation and check for: +1. [Specific thing to check] +2. [Another thing to check] +3. [etc.] + +## Resources Available + +Read these resources first: +- `redpanda://style-guide` - Complete style guide +- `redpanda://terminology` - Approved terminology + +## API Documentation Standards + +[Explain what makes good API docs for your team] + +### Required Sections + +All API documentation must include: +- Overview (what the API does) +- Authentication requirements +- Request format (with example) +- Response format (with example) +- Error responses +- Code examples + +### Format Requirements + +[Your specific formatting rules] + +## Output Format + +Provide feedback in this structure: + +### Critical Issues +[Format for issues] + +### Suggestions +[Format for suggestions] + +--- + +Please provide the API documentation you'd like me to review. +---- + +==== Validate your prompt + +Run the validation command to check for errors: + +[source,bash] +---- +npx doc-tools validate-mcp +---- + +If validation passes, you're ready to use it! + +==== Test it + +Restart Claude Code and test your prompt: + +[source,text] +---- +Use the review-api-docs prompt with this content: +[paste some API docs] +---- + +=== Prompt template library + +Copy and customize these templates for common prompt types: + +==== Review prompt template + +[source,markdown] +---- +--- +description: "Review [content type] for [specific aspects]" +version: "1.0.0" +arguments: + content: + description: "The content to review" + required: true +--- + +# [Type] Review Prompt + +You are reviewing [content type] for the Redpanda documentation team. + +## Your Task + +Review the provided content and check for: +1. [Check 1] +2. [Check 2] +3. [Check 3] + +## Resources + +- `redpanda://style-guide` +- `redpanda://terminology` + +## Standards for [Content Type] + +[Specific standards] + +## Output Format + +### Critical Issues +[Format] + +### Suggestions +[Format] + +--- + +Please provide the content to review. +---- + +==== Writing prompt template + +[source,markdown] +---- +--- +description: "Generate [content type] following team standards" +version: "1.0.0" +arguments: + topic: + description: "The topic to write about" + required: true + context: + description: "Additional context or requirements" + required: false +--- + +# Write [Content Type] Prompt + +You are writing [content type] for the Redpanda documentation team. + +## Your Task + +Create [content type] that follows our standards. + +## Resources + +- `redpanda://style-guide` +- `redpanda://terminology` +- `redpanda://[content-type]-template` (if you have one) + +## Standards + +[Specific standards for this content type] + +## Structure + +[Required sections and format] + +## Quality Checklist + +- [Requirement 1] +- [Requirement 2] +- [Requirement 3] + +--- + +Please provide: +1. **Topic**: [What to write about] +2. **[Other context]**: [Additional info needed] +---- + +== Adding new resources + +=== When to add a new resource + +Add a resource when you have reference material that AI should access: + +* Templates for specific doc types +* Checklists for quality reviews +* Reference tables or lists +* Code style guides +* Workflow documentation + +=== How auto-discovery works for resources + +Resources are automatically discovered too! + +* Add `.md` or `.json` files to `mcp/team-standards/` +* The server automatically makes them available +* No code editing required +* The filename becomes the resource URI (e.g., `api-template.md` β†’ `redpanda://api-template`) + +=== Step-by-step: add a new resource + +Step 1: Create the resource file** + +Add your file to `mcp/team-standards/`: + +[source,bash] +---- +cd mcp/team-standards +touch api-template.md +---- + +Step 2: Write the content** + +Use Markdown or JSON format: + +[source,markdown] +---- +# API Documentation Template + +Use this template when documenting API endpoints. + +## Structure + += API Endpoint Name +:description: Brief description + +Overview paragraph explaining what this endpoint does. + +== Authentication + +[Authentication requirements] + +== Request + +[Request format with example] + +== Response + +[Response format with example] + +== Examples + +[Code examples] + +== Error responses + +[Common errors] +---- + +Step 3: Validate the resource** + +Run validation to ensure the file is accessible: + +[source,bash] +---- +npx doc-tools validate-mcp +---- + +Step 4: Reference it in prompts** + +Update relevant prompts to reference your new resource. The URI is `redpanda://[filename-without-extension]`: + +[source,markdown] +---- +## Resources Available + +- `redpanda://style-guide` +- `redpanda://terminology` +- `redpanda://api-template` - Use this template for structure +---- + +Step 5: Test it** + +Restart Claude Code and verify: + +[source,text] +---- +List available resources +---- + +Your new resource should appear in the list. + +=== Resource types and formats + +==== Markdown resources (.md) + +Best for: +- Templates +- Guidelines +- Instructions +- Examples + +[source,markdown] +---- +# Resource Title + +## Section + +Content here with examples. + +## Another Section + +More content. +---- + +==== JSON resources (.json) + +Best for: +- Structured data +- Terminology lists +- Configuration examples +- Reference tables + +[source,json] +---- +{ + "category": { + "item1": { + "description": "Description here", + "example": "Example here" + }, + "item2": { + "description": "Description here", + "example": "Example here" + } + } +} +---- + +== Updating existing resources + +=== Update style guide + +Edit `mcp/team-standards/style-guide.md`: + +[source,bash] +---- +# Add new section or update existing content +vi mcp/team-standards/style-guide.md +---- + +Changes take effect immediately - just restart Claude Code. + +=== Update terminology + +Terminology is maintained in the official glossary sources: + +- GitHub: https://github.com/redpanda-data/docs/tree/shared/modules/terms/partials (each term is a separate file) +- Published glossary: https://docs.redpanda.com/current/reference/glossary/ + +The style guide (`mcp/team-standards/style-guide.md`) includes a quick reference to common terms that points to the official glossary. + +To add or update terms, update the glossary source files in the docs repository. + +=== Update templates + +Edit template files in `mcp/team-standards/`: + +[source,bash] +---- +# Example: Update a template resource +vi mcp/team-standards/api-template.md +---- + +Changes are immediately available to AI after restarting Claude Code. + +== Adding new tools + +Tools require JavaScript knowledge, but the pattern is straightforward. + +=== When to add a new tool + +Add a tool when you need to: +- Execute a doc-tools command that isn't covered +- Automate a multi-step process +- Integrate with external systems +- Perform custom validation + +=== Tool template + +Create a new file in `bin/mcp-tools/my-new-tool.js`: + +[source,javascript] +---- +const { executeCommand, findRepoRoot } = require('./utils'); + +/** + * Execute my new tool + * @param {Object} args - Tool arguments + * @returns {Object} Result with success flag and data + */ +function myNewTool(args) { + try { + // Validate arguments + if (!args.required_param) { + return { + success: false, + error: 'Missing required parameter: required_param' + }; + } + + // Build command + const command = ['my-command', args.required_param]; + + if (args.optional_param) { + command.push('--flag', args.optional_param); + } + + // Execute + const result = executeCommand(command); + + if (result.code === 0) { + return { + success: true, + message: 'Operation completed successfully', + output: result.stdout + }; + } else { + return { + success: false, + error: result.stderr, + suggestion: 'Check the parameters and try again' + }; + } + } catch (error) { + return { + success: false, + error: error.message + }; + } +} + +module.exports = { myNewTool }; +---- + +Then register it in `bin/mcp-tools/index.js` and `bin/doc-tools-mcp.js`. + +For tools, consider asking a developer for help or use this guide as a starting point. + +== Testing your changes + +=== Validation command + +Before committing changes, **always run validation**: + +[source,bash] +---- +npx doc-tools validate-mcp +---- + +This command checks: + +* All prompts load correctly and have valid frontmatter +* All resources are accessible +* Metadata is complete (descriptions, versions) +* No syntax errors in YAML or JSON +* No naming conflicts + +Expected output: + +[source,text] +---- +Loading prompts... +Found 6 prompts + +Validating configuration... + +MCP Configuration Validation +============================================================ + +Prompts found: 6 +Resources found: 1 + +βœ“ Validation passed +---- + +If validation fails, fix the errors before proceeding. + +=== Preview command + +Test a prompt without Claude Code: + +[source,bash] +---- +# Preview a review prompt +npx doc-tools preview-prompt review-for-style --content "Test content" + +# Preview a write prompt +npx doc-tools preview-prompt write-new-guide --topic "Deploy cluster" +---- + +This shows exactly how your prompt will appear with arguments. + +=== Test checklist + +After adding or updating anything: + +. Run validation: ++ +[source,bash] +---- +npx doc-tools validate-mcp +---- + +. Preview your prompt (if you added/changed one): ++ +[source,bash] +---- +npx doc-tools preview-prompt [prompt-name] --content "Sample text" +---- + +. Restart Claude Code to load changes + +. List available items in Claude Code: ++ +[source,text] +---- +List available prompts +List available resources +List available tools +---- + +. Test the new item: ++ +[source,text] +---- +Use the [prompt-name] prompt +Read resource redpanda://[resource-name] +Use the [tool-name] tool +---- + +. Verify output is what you expected +. Check for errors in Claude Code console + +=== Common issues + +Validation fails with "Invalid frontmatter": +- Check YAML syntax in the frontmatter block +- Ensure required fields are present (description) +- Verify argument names use only lowercase and underscores +- Check that version follows semantic versioning (for example, "1.0.0") + +Prompt not appearing: +- Run `npx doc-tools validate-mcp` to check for errors +- Verify file is in `mcp/prompts/` directory and ends with `.md` +- Check that frontmatter is valid YAML +- Restart Claude Code + +Resource not loading: +- Run `npx doc-tools validate-mcp` to check for errors +- Verify file is in `mcp/team-standards/` directory +- Check for JSON syntax errors (if JSON file) +- Restart Claude Code + +Prompt content not updating: +- Files are cached - restart Claude Code +- Check for syntax errors in Markdown or frontmatter +- Run validation to catch issues + +== Best practices + +=== For prompts + +. Be specific - Tell AI exactly what to check or create +. Provide examples - Show good vs bad +. Reference resources - Point to style guide and terminology +. Define output format - Specify structure for consistency +. Include context - Explain why rules matter + +=== For resources + +. Keep them current - Update when standards change +. Make them scannable - Use clear headings and examples +. Show, don't just tell - Include examples +. Link to sources - Reference official docs when relevant +. Version control - Commit changes with clear descriptions + +=== For the team + +. Document changes - Update this guide when you learn something +. Share examples - Add good prompts to examples gallery +. Test thoroughly - Don't skip the testing checklist +. Ask for help - Reach out when you need developer support +. Iterate - Improve prompts based on usage and feedback + +== Quick reference + +=== File locations + +[source,text] +---- +mcp/ +β”œβ”€β”€ prompts/ # Prompt files (.md) - Auto-discovered! +β”‚ β”œβ”€β”€ review-for-style.md +β”‚ β”œβ”€β”€ write-new-guide.md +β”‚ └── [your-prompt].md +β”œβ”€β”€ team-standards/ # Resource files (.md) - Auto-discovered! +β”‚ β”œβ”€β”€ style-guide.md +β”‚ └── [your-resource].md +└── WRITER_EXTENSION_GUIDE.adoc # This file! + +bin/ +β”œβ”€β”€ doc-tools-mcp.js # MCP server (no editing needed!) +└── mcp-tools/ # Tool implementations (advanced) +---- + +=== Common edits + +Add a prompt: +. Create `mcp/prompts/my-prompt.md` with YAML frontmatter +. Run `npx doc-tools validate-mcp` to check for errors +. Restart Claude Code +. Test it! + +Add a resource: +. Create `mcp/team-standards/my-resource.md` +. Run `npx doc-tools validate-mcp` to verify it's accessible +. Restart Claude Code +. Reference it in prompts using `redpanda://my-resource` + +Update content: +. Edit the file in `mcp/prompts/` or `mcp/team-standards/` +. Run `npx doc-tools validate-mcp` to catch any errors +. Restart Claude Code +. Test the changes + +=== Getting help + +For writers: +- Review examples in existing prompts +- Check the showcase document for patterns +- Ask team lead for guidance + +For developers: +- See `mcp/DEVELOPMENT.adoc` for technical details +- Check MCP SDK documentation +- Review tool implementation examples + +== Examples gallery + +=== Example 1: Simple review prompt + +File: `mcp/prompts/review-troubleshooting.md` + +[source,markdown] +---- +--- +description: "Review troubleshooting guides for required elements and clarity" +version: "1.0.0" +arguments: + content: + description: "Troubleshooting content to review" + required: true +--- + +# Review Troubleshooting Guide + +Review troubleshooting documentation for clarity and completeness. + +## Required Elements + +Every troubleshooting guide must have: +1. Clear problem statement +2. Symptoms users would see +3. Root cause explanation +4. Step-by-step solution +5. Verification steps + +## Check For + +- Problem statement is specific (not vague) +- Symptoms are observable by users +- Solution steps are sequential and complete +- Commands include expected output +- Uses glossterm macro for technical terms + +## Output Format + +List missing elements and unclear sections. + +--- + +Provide the troubleshooting content to review. +---- + +That's it! Save the file, run `npx doc-tools validate-mcp`, restart Claude Code, and it's ready to use. + +=== Example 2: Reference resource + +File: `mcp/team-standards/code-examples-guide.md` + +[source,markdown] +---- +# Code Examples Guide + +## General Rules + +- Always include context before code +- Show expected output after commands +- Use realistic examples (not foo/bar) +- Test all code before publishing + +## AsciiDoc Format + +Use appropriate roles: + +[source,bash] +---- +rpk topic create orders --partitions 3 +---- + +[source,bash,role=no-copy] +---- +TOPIC STATUS +orders OK +---- + +## Language-Specific Rules + +### Bash +- Include comments for non-obvious commands +- Show full command with all flags +- Use long-form flags when clearer + +### Python +- Follow PEP 8 style +- Include import statements +- Show complete, runnable examples + +### YAML +- Use 2-space indentation +- Include comments for complex sections +- Validate syntax before publishing +---- + +That's it! Save the file, run `npx doc-tools validate-mcp`, restart Claude Code, and it's available as `redpanda://code-examples-guide`. + +== What's next? + +. Try adding a simple prompt using the templates above +. Update existing resources to match your current standards +. Share your additions with the team +. Document new patterns you discover +. Iterate and improve based on team feedback + +Remember: The goal is to make AI work consistently for your entire docs team. Every prompt and resource you add makes that easier! diff --git a/mcp/prompts/README.adoc b/mcp/prompts/README.adoc new file mode 100644 index 00000000..40f0cc79 --- /dev/null +++ b/mcp/prompts/README.adoc @@ -0,0 +1,183 @@ += MCP Server Prompts + +This directory contains prompt files for the Redpanda Docs MCP Server. Prompts provide contextual guides and instructions to LLMs when working with specific documentation tasks. + +== What are prompts? + +MCP Prompts are pre-defined instructions that LLMs can access when needed. They provide: + +* Detailed context about complex workflows +* Rules and best practices +* Step-by-step instructions +* Quality checks and validation + +== How prompts work + +. **Writer makes a request**: "Improve the description for cleanup.policy" +. **Claude detects the topic**: Recognizes this is about property documentation +. **Retrieves the prompt**: Automatically loads `property-docs-guide.md` +. **Follows instructions**: Updates overrides correctly, uses proper workflow +. **Ensures quality**: Applies all rules and validation checks + +== Available prompts + +=== property-docs-guide.md + +**Purpose**: Comprehensive guide for updating Redpanda property documentation + +**When used**: Automatically retrieved when working with configuration properties + +**Key content**: + +* Override system explanation +* Mandatory rules (no cloud conditionals, no enterprise includes) +* Workflow instructions (update property-overrides.json, use generate_property_docs tool) +* Common scenarios and examples +* Validation steps + +=== rpcn-connector-docs-guide.md + +**Purpose**: Comprehensive guide for updating Redpanda Connect connector reference documentation + +**When used**: Automatically retrieved when working with connector documentation + +**Key content**: + +* Override system with `$ref` syntax explanation +* DRY (Don't Repeat Yourself) principles +* Repository structure and folder organization +* Workflow instructions (update overrides.json, use generate_rpcn_connector_docs tool) +* Quality checks and validation +* Examples of `$ref` usage + +== Adding new prompts + +The MCP server uses automatic prompt discovery. No code editing required! + +=== 1. Create the markdown file + +Create a new file in this directory: + +[,bash] +---- +prompts/your-prompt-name.md +---- + +=== 2. Add YAML frontmatter + +At the top of your file, add metadata: + +[,markdown] +---- +--- +description: "Brief description for LLM to understand when to use this" +version: "1.0.0" +arguments: + content: + description: "The content to process (if needed)" + required: true +--- + +# Your Prompt Title + +This guide explains how to... + +## Your workflow: +1. Do this +2. Then do that +3. Use the `tool_name` MCP tool to... + +## Rules you must follow: +- Never do X +- Always do Y +---- + +Required frontmatter fields: +- description: What the prompt does +- version: Semantic version (1.0.0) +- arguments: (optional) Input parameters + +=== 3. Validate the prompt + +Check for errors: + +[,bash] +---- +npx doc-tools validate-mcp +---- + +=== 4. Test it + +Restart Claude Code and test: + +[,text] +---- +You: "List available prompts" +Claude: *shows your new prompt* + +You: "Use the your-prompt-name prompt" +Claude: *loads and follows your instructions* +---- + +== Writing good prompts + +**Do:** + +* **Write for LLMs**: Instructions should be direct ("You should..." not "Tell the user to...") +* **Be specific**: Clear, actionable steps +* **Include examples**: Show the correct way to do things +* **Define rules**: List what must/must not be done +* **Provide context**: Explain why rules exist +* **Use workflow sections**: Step-by-step instructions + +**Don't:** + +* **Don't write for users**: Prompts are consumed by LLMs +* **Don't be vague**: "Handle it appropriately" is not helpful +* **Don't skip validation**: Always include quality checks +* **Don't forget edge cases**: Cover error scenarios +* **Don't embed in code**: Keep content in .md files + +== Prompt structure template + +[,markdown] +---- +# Title - What This Prompt Does + +Brief explanation of the purpose. + +## Overview + +Context about the system/workflow. + +## Critical Rules (MANDATORY) + +- Never do X +- Always do Y +- Must include Z + +## Your workflow: + +1. First step +2. Second step +3. Use the `tool_name` MCP tool to... +4. Validate the result + +## Common scenarios + +### Scenario 1: Description +Steps to handle this case. + +### Scenario 2: Description +Steps to handle this case. + +## Quality checks you must perform: + +- Check for A +- Verify B +- Ensure C + +## Validation + +How to verify everything worked correctly. +---- diff --git a/mcp/prompts/check-terminology.md b/mcp/prompts/check-terminology.md new file mode 100644 index 00000000..b211f395 --- /dev/null +++ b/mcp/prompts/check-terminology.md @@ -0,0 +1,134 @@ +--- +description: Validate terminology usage against the team's approved terminology from official glossary sources. Identifies incorrect terms, deprecated terms, and inconsistent usage. +version: 1.0.0 +arguments: + - name: content + description: The documentation content to check + required: true +argumentFormat: content-append +--- + +# Check Terminology Prompt + +You are validating terminology for the Redpanda documentation team. + +## Your task + +Check documentation content against our approved terminology and identify: + +1. **Incorrect terms** (wrong capitalization, spelling, or usage) +2. **Deprecated terms** (outdated terms that should be replaced) +3. **Inconsistent usage** (same concept referred to differently) +4. **Missing terms** (concepts that should use approved terminology) + +## Terminology resources + +Our approved terms are maintained in these official sources: +- **GitHub**: https://github.com/redpanda-data/docs/tree/shared/modules/terms/partials (each term is a separate file) +- **Published glossary**: https://docs.redpanda.com/current/reference/glossary/ + +**Always check the published glossary or GitHub terms directory for official definitions and approved usage.** + +You can also reference the style guide (`redpanda://style-guide`) which includes a quick reference of common terms. + +## What to check + +### Product names +- Redpanda (not RedPanda, red panda, or redpanda in prose) +- Redpanda Cloud (not Redpanda cloud or redpanda Cloud) +- Redpanda Console (not Redpanda console) +- Redpanda Connect (not Benthos, which is deprecated) + +### Kafka concepts +- topic, partition, broker, cluster (lowercase) +- consumer, producer (lowercase) +- leader, replica (not master, slave) +- ISR, ACL (uppercase acronyms) + +### Security terms +- TLS (not SSL, which is deprecated) +- SASL, mTLS (uppercase) +- allowlist, denylist (not whitelist, blacklist - deprecated) + +### Deprecated terms to flag + +These should NEVER appear: +- master/slave β†’ Use leader/replica or primary/follower +- whitelist/blacklist β†’ Use allowlist/denylist or blocklist +- SSL β†’ Use TLS +- Benthos β†’ Use Redpanda Connect +- sanity check β†’ Use validation or verification + +### Commonly confused terms + +Check that these are used correctly (refer to glossary for definitions): +- Topic vs Partition (topic is logical category, partition is storage unit) +- Broker vs Node (broker is Redpanda process, node is server/machine) +- Replica vs Replication (replica is copy, replication is process) + +## Output format + +Provide a structured report: + +### Critical issues (incorrect or deprecated terms) + +For each issue: +- **Location**: [Section/paragraph reference] +- **Found**: [The incorrect term used] +- **Should be**: [The correct term] +- **Reason**: [Why it's wrong - incorrect, deprecated, etc.] + +### Inconsistencies (same concept, different terms) + +- **Concept**: [What's being referred to] +- **Variations found**: [List of different terms used] +- **Recommended term**: [Which one to use consistently (link to glossary entry if available)] +- **Locations**: [Where each variation appears] + +### Missing glossary links + +If terms are defined in the glossary but not linked with the `glossterm` macro: + +- **Term**: [The term that should be linked] +- **Location**: [Where it appears] +- **Suggested macro**: `glossterm:[]` or `glossterm:,[]` + +### Suggestions (minor improvements) + +- **Location**: [Where it appears] +- **Current**: [Current phrasing] +- **Suggested**: [Better phrasing using approved terminology] + +### Summary + +- Total issues found: [number] +- Critical fixes needed: [number] +- Inconsistencies: [number] +- Missing glossary links: [number] +- Overall terminology health: [Good/Fair/Needs Work] + +## Example output + +### Critical issues + +**Location**: Prerequisites section, paragraph 2 +**Found**: "whitelist the broker IP addresses" +**Should be**: "allowlist the broker IP addresses" +**Reason**: "whitelist" is deprecated terminology. Use "allowlist" instead. + +**Location**: Configuration section, step 3 +**Found**: "Enable SSL encryption" +**Should be**: "Enable TLS encryption" +**Reason**: SSL is deprecated. TLS is the current standard. + +### Missing glossary links + +**Term**: "partition" +**Location**: Introduction, paragraph 1 +**Suggested macro**: `glossterm:partition[]` (first mention should link to glossary using glossterm macro) + +**Example**: Change "A topic is divided into partitions" to "A glossterm:topic[] is divided into glossterm:partition,partitions[]" + +--- + +Please provide the content you'd like me to check for terminology issues. diff --git a/mcp/prompts/improve-clarity.md b/mcp/prompts/improve-clarity.md new file mode 100644 index 00000000..deebbe5e --- /dev/null +++ b/mcp/prompts/improve-clarity.md @@ -0,0 +1,109 @@ +--- +description: Refactor existing documentation for clarity and readability while maintaining technical accuracy. Applies style guide principles and simplifies complex explanations. +version: 1.0.0 +arguments: + - name: content + description: The documentation content to improve + required: true +argumentFormat: content-append +--- + +# Improve Clarity Prompt + +You are refactoring documentation for the Redpanda documentation team. + +## Your task + +Take existing documentation content and improve it for clarity and readability while maintaining technical accuracy. + +## Resources to reference + +- `redpanda://style-guide` - Style guide with clarity principles +- Official glossary for terminology: + - GitHub: https://github.com/redpanda-data/docs/tree/shared/modules/terms/partials + - Published: https://docs.redpanda.com/current/reference/glossary/ + +## Clarity principles + +### Simplify without losing accuracy +- Break long sentences into shorter ones +- Replace complex words with simpler alternatives +- Remove unnecessary jargon +- Add explanations for technical concepts + +### Improve structure +- Use clear headings that describe content +- Break content into scannable sections +- Use lists for related items +- Add transitions between sections + +### Make instructions clear +- Use imperative form (no gerunds) +- One action per step +- Include context for why steps matter +- Show expected results + +## Common improvements + +### Replace passive with active voice + +- Before: "Data is replicated by the broker across partitions" +- After: "The broker replicates data across partitions" + +### Simplify complex sentences + +- Before: "In order to configure TLS encryption for secure communication between clients and brokers, you'll need to generate certificates and configure the broker settings accordingly" +- After: "To enable TLS encryption, generate certificates and update your broker configuration" + +### Remove gerunds from instructions + +- Before: "Creating a topic involves specifying the partition count" +- After: "To create a topic, specify the partition count" + +### Break up long paragraphs + +- Before: One dense 8-sentence paragraph +- After: Three focused 2-3 sentence paragraphs with clear topics + +### Add context for commands + +- Before: +``` +rpk topic create orders --partitions 3 +``` + +- After: +``` +Create a topic named "orders" with 3 partitions for parallel processing: + +[source,bash] +---- +rpk topic create orders --partitions 3 +---- +``` + +## What not to change + +- Technical accuracy +- Correct terminology +- Command syntax +- Configuration values +- Code examples (unless they're wrong) + +## Output format + +Provide: + +1. **Improved content** in AsciiDoc format +2. **Summary of changes** explaining what you improved and why +3. **Key improvements** (3-5 bullet points highlighting major clarity gains) + +Keep: +- All xrefs (ensure they include module names) +- Custom macro usage +- Code block roles (`.no-copy`, `.no-wrap`, etc.) +- Correct terminology + +--- + +Please provide the content you'd like me to improve for clarity. diff --git a/mcp/prompts/property-docs-guide.md b/mcp/prompts/property-docs-guide.md new file mode 100644 index 00000000..ca59711b --- /dev/null +++ b/mcp/prompts/property-docs-guide.md @@ -0,0 +1,283 @@ +--- +description: Comprehensive guide for updating Redpanda property documentation using the override system. Explains how to work with auto-generated property files and the property-overrides.json system. +version: 1.0.0 +--- + +# Property documentation update guide for LLMs + +This guide explains how to update Redpanda property documentation when all property reference pages are auto-generated. + +Critical rule: Never directly edit files in `/modules/reference/partials/properties/` - they are auto-generated and will be overwritten. + +## The auto-generation system + +All property documentation files are automatically generated from source code metadata. + +Generated files (do not edit): +- `/modules/reference/partials/properties/broker-properties.adoc` +- `/modules/reference/partials/properties/cluster-properties.adoc` +- `/modules/reference/partials/properties/object-storage-properties.adoc` +- `/modules/reference/partials/properties/topic-properties.adoc` + +Override file (edit this): +- `/docs-data/property-overrides.json` + +## Why this matters + +When a user asks you to: +- "Improve the description of cleanup.policy" +- "Add an example for kafka_qdc_enable" +- "Fix the documentation for compression.type" +- "Add related topics to retention.ms" + +You must: +1. Update `/docs-data/property-overrides.json` +2. Run the doc-tools CLI to regenerate +3. Do not edit the generated `.adoc` files directly + +## The override system + +The override file provides human-curated content that supplements or replaces auto-generated content. + +## File structure + +Location: `docs-data/property-overrides.json` + +Basic structure: +```json +{ + "properties": { + "property_name": { + "description": "Enhanced description text", + "config_scope": "broker|cluster|topic", + "category": "category-name", + "example": [ + "Line 1 of example", + "Line 2 of example" + ], + "related_topics": [ + "xref:path/to/doc.adoc[Link Text]", + "xref:another/doc.adoc#anchor[Another Link]" + ], + "exclude_from_docs": false + } + } +} +``` + +## What can be overridden + +Fields you can override: +- `description` - Enhance or replace property description +- `config_scope` - Specify broker/cluster/topic scope +- `category` - Categorize property +- `example` - Add YAML configuration examples (array of strings) +- `related_topics` - Add cross-references (array of AsciiDoc xref links) +- `exclude_from_docs` - Hide internal/deprecated properties +- `type` - Override detected type +- `default` - Override default value +- `accepted_values` - Override accepted values + +## How to update overrides + +### Step 1: Read the current override file +Always read the file first to preserve existing overrides. + +### Step 2: Add or update property overrides +Modify the properties object. + +### Step 3: Write back to file +Save the updated JSON. + +### Step 4: Verify JSON is valid +Run: `python -c "import json; json.load(open('docs-data/property-overrides.json'))"` + +## Regenerating documentation + +### Prerequisites +Before running doc-tools, you must have: +1. A valid GitHub token with repo access to cloudv2 in the redpandadata organization +2. The token set as the GITHUB_TOKEN environment variable + +### The doc-tools CLI +After updating overrides, regenerate documentation: + +```bash +npx doc-tools generate property-docs \ + --tag "" \ + --generate-partials \ + --cloud-support \ + --overrides docs-data/property-overrides.json +``` + +Important notes: +- Always use `npx doc-tools` (not just `doc-tools`) +- The `--tag` flag specifies which Redpanda version to generate docs for +- The `--generate-partials` flag generates files in the partials directory +- The `--cloud-support` flag must ALWAYS be included - never exclude it +- The `--overrides` flag points to the property overrides JSON file + +## Property description rules (mandatory) + +### Never add enterprise license includes +Do not include enterprise license markers in descriptions. + +Wrong: +``` +Enable shadow linking for disaster recovery. + +include::reference:partial$enterprise-licensed-property.adoc[] +``` + +Right: +``` +Enable shadow linking for disaster recovery. +``` + +Reason: Enterprise licensing information is displayed in the metadata table. + +### Never add descriptions for deprecated properties +Do not add or update descriptions for properties marked as deprecated. + +Process: +1. Check if the property is deprecated +2. If deprecated, remove any existing override +3. Never add new overrides for deprecated properties + +### Keep descriptions focused +Descriptions should explain: +- What the property does +- When to use it +- How it relates to other properties +- Important behavioral details + +Descriptions should NOT include: +- Version availability (metadata) +- Cloud availability (metadata) +- Enterprise license requirements (metadata) +- Requires restart (metadata) +- Default values (metadata) +- Type information (metadata) + +### Use consistent formatting +Use AsciiDoc formatting in descriptions: +- `` `property_name` `` for property names +- `xref:module:path/to/doc.adoc[Link Text]` for cross-references (always use full resource IDs with module prefix) +- `<>` for internal document references +- `\n\n` for paragraph breaks + +Important: Always use full Antora resource IDs with module prefixes in xref links, never relative paths. + +Wrong: +```json +{ + "description": "When `segment.bytes` is set, it overrides xref:./cluster-properties.adoc#log_segment_size[`log_segment_size`]." +} +``` + +Right: +```json +{ + "description": "When `segment.bytes` is set, it overrides xref:reference:properties/cluster-properties.adoc#log_segment_size[`log_segment_size`]." +} +``` + +Common module prefixes: +- `reference:` for reference documentation +- `manage:` for management documentation +- `deploy:` for deployment documentation +- `get-started:` for getting started guides + +### Prefix self-managed-only links +Some documentation pages only exist in self-managed deployments. Prefix these with `self-managed-only:`. + +Example: +```json +{ + "kafka_connections_max": { + "related_topics": [ + "self-managed-only:xref:manage:cluster-maintenance/configure-client-connections.adoc[Limit client connections]" + ] + } +} +``` + +### Remove duplicate links +Always remove duplicates from related_topics lists. + +### Normalize xref links to full resource IDs +After updating overrides, normalize all xref links to use full Antora resource IDs. + +## Common scenarios + +### Improve a property description +1. Read the override file +2. Update the description field +3. Write back to override file +4. Use the `generate_property_docs` MCP tool to regenerate the documentation +5. If that tool is not available, tell the user to run: + `npx doc-tools generate property-docs --tag "" --generate-partials --cloud-support --overrides docs-data/property-overrides.json` + +### Add an example +1. Add an `example` array with YAML lines to the property override +2. Use the `generate_property_docs` MCP tool to regenerate + +### Add related topics +1. Add `related_topics` array with AsciiDoc xref links +2. Use the `generate_property_docs` MCP tool to regenerate + +### Fix incorrect metadata +1. Override specific fields like `default` or `type` +2. Use the `generate_property_docs` MCP tool to regenerate + +### Hide internal properties +1. Set `exclude_from_docs: true` +2. Use the `generate_property_docs` MCP tool to regenerate + +## Validation + +After updating overrides: +1. Validate JSON syntax +2. Check for common mistakes (example/related_topics are arrays, xref format) +3. Verify after regeneration + +## Summary for LLMs + +When asked to update property documentation: + +1. Update `/docs-data/property-overrides.json` +2. Run the doc-tools CLI with the correct command (including all required flags) +3. Never edit `/modules/reference/partials/properties/*.adoc` directly + +Critical requirements: +- Must have GITHUB_TOKEN environment variable set +- Must ALWAYS include `--cloud-support` flag +- Must use `npx doc-tools` +- Must include all flags: `--tag`, `--generate-partials`, `--cloud-support`, `--overrides` + +Property description rules (mandatory): +- Never add enterprise license includes +- Never add descriptions for deprecated properties +- Keep descriptions focused on behavior, not metadata +- Use AsciiDoc formatting +- Always use full Antora resource IDs with module prefixes in xref links +- Prefix self-managed-only links with `self-managed-only:` +- Remove duplicate links + +Your workflow: +1. Always read the override file first to preserve existing overrides +2. Make your changes to the property overrides +3. Validate JSON syntax after changes +4. Use the `generate_property_docs` MCP tool to regenerate the documentation + - Set version parameter to the Redpanda version + - Set generate_partials to true +5. If the tool is not available, provide the user with the command: + `npx doc-tools generate property-docs --tag "" --generate-partials --cloud-support --overrides docs-data/property-overrides.json` +6. Explain what was changed and what files will be regenerated +7. If generation fails, remind the user they need GITHUB_TOKEN set with cloudv2 repo access + +Quality checks you must perform: +- Clean up any inappropriate content from descriptions (no enterprise includes, no cloud conditionals) +- Remove any overrides for deprecated properties +- Normalize all xref links to full Antora resource IDs with module prefixes +- Remove duplicate links from related_topics arrays diff --git a/mcp/prompts/review-for-style.md b/mcp/prompts/review-for-style.md new file mode 100644 index 00000000..feb2206e --- /dev/null +++ b/mcp/prompts/review-for-style.md @@ -0,0 +1,123 @@ +--- +description: Review documentation content for style guide compliance, terminology consistency, and voice/tone. Provides detailed, actionable feedback based on team standards. +version: 1.0.0 +arguments: + - name: content + description: The documentation content to review (can be a file path or raw content) + required: true +argumentFormat: content-append +--- + +# Style Review Prompt + +You are reviewing documentation for the Redpanda documentation team. + +## Your task + +Review the provided content against our style guide and provide detailed, actionable feedback on: + +1. **Style violations** with specific line/section references +2. **Terminology issues** (incorrect terms, inconsistent usage, deprecated terms) +3. **Voice and tone** feedback (too formal, too casual, inconsistent) +4. **Structural issues** (missing sections, poor organization, heading hierarchy) +5. **Formatting issues** (overuse of bold, em dashes, code formatting) +6. **Accessibility issues** (missing alt text, poor heading structure) +7. **Actionable fixes** for each issue found + +## Style guide reference + +You have access to: +- `redpanda://style-guide` - Complete style guide with all standards + +**Terminology sources:** +- GitHub: https://github.com/redpanda-data/docs/tree/shared/modules/terms/partials +- Published glossary: https://docs.redpanda.com/current/reference/glossary/ + +**Important**: Read the style guide before starting your review. Reference the official glossary for term definitions and usage. + +## Key style principles to check + +Based on Google Developer Documentation Style Guide: +- Present tense for describing how things work +- Active voice (not passive) +- Second person ("you" not "the user") +- **Sentence case for ALL headings except the title (H1)** + - Check that H2, H3, H4 only capitalize first word and proper nouns + - Example: "Configure TLS encryption" not "Configure TLS Encryption" +- Clear, conversational tone + +Redpanda-specific preferences: +- Avoid overuse of bold text (only for UI elements and important warnings) +- Avoid em dashes (use parentheses or commas instead) +- Use realistic examples (not foo/bar placeholders) +- Proper product name capitalization (Redpanda, not RedPanda) + +## Terminology to verify + +Check for: +- Correct product names (Redpanda, Redpanda Cloud, Redpanda Console, Redpanda Connect) +- Lowercase concepts (topic, partition, broker, cluster, consumer, producer) +- Deprecated terms (master/slave, whitelist/blacklist, SSL) +- Consistent terminology throughout + +## Output format + +Provide your review in this structure: + +### Critical issues (must fix before publishing) + +For each critical issue: +- **Location**: [Section/heading or line reference] +- **Issue**: [What's wrong] +- **Fix**: [Specific correction to make] +- **Reason**: [Why it matters] + +### Style suggestions (should consider) + +For each suggestion: +- **Location**: [Section/heading or line reference] +- **Current**: [What it says now] +- **Suggested**: [Better way to phrase it] +- **Reason**: [Why the suggestion improves the content] + +### Terminology issues + +List all terminology problems: +- **Incorrect term**: [What was used] +- **Correct term**: [What should be used] +- **Location**: [Where it appears] + +### Positive elements + +What works well in this content: +- [List 2-3 things the author did well] +- [Acknowledge good examples, clear explanations, proper structure, etc.] + +## Review guidelines + +- Be constructive and specific +- Focus on high-impact improvements first +- Acknowledge what's working well +- Provide clear examples of fixes +- Reference specific style guide sections when relevant +- Don't just point out problems; explain why they matter +- Consider the reader's experience + +## Example of good feedback + +- Poor feedback: "This section has style issues." + +- Good feedback: +**Location**: Introduction, paragraph 2 +**Issue**: Uses passive voice - "Data is encrypted by Redpanda" +**Fix**: "Redpanda encrypts data at rest" +**Reason**: Active voice is clearer and more direct. Per our style guide, always prefer active voice for describing what software does. + +**Location**: Section heading +**Issue**: Title case used for H2 heading - "Configure TLS Encryption" +**Fix**: "Configure TLS encryption" +**Reason**: All headings except the page title (H1) must use sentence case. Only capitalize the first word and proper nouns. + +--- + +Please provide the content you'd like me to review. diff --git a/mcp/prompts/rpcn-connector-docs-guide.md b/mcp/prompts/rpcn-connector-docs-guide.md new file mode 100644 index 00000000..1fdcba4b --- /dev/null +++ b/mcp/prompts/rpcn-connector-docs-guide.md @@ -0,0 +1,126 @@ +--- +description: Comprehensive guide for updating Redpanda Connect connector reference documentation using the overrides system with $ref syntax. Explains the DRY approach for connector documentation. +version: 1.0.0 +--- + +# Redpanda Connect Reference Documentation: LLM Prompt Guide + +Redpanda Connect reference documentation is generated using a combination of: + +- **Autogenerated content**: Produced from source code and schemas, providing a baseline of all available fields, types, and options. +- **Overrides**: A JSON file (`overrides.json`) that allows technical writers to override, deduplicate, clarify, and cross-reference field descriptions using a DRY (Don't Repeat Yourself) approach. + +## Repository folder structure + +- `/docs`: Built documentation site (HTML, assets, and static files for all Redpanda Connect docs versions) +- `/docs-data`: Source data for reference docs, including `overrides.json` and autogenerated JSON files +- `/modules`: Modular AsciiDoc content for guides, cookbooks, and reference topics +- `/package.json`, `README.adoc`, `antora.yml`: Project configuration and metadata +- `/.github`: GitHub workflows and Copilot custom instructions + +## `$ref` syntax + +- The `$ref` syntax is used in the overrides file to avoid repeating identical or similar descriptions for fields that appear in multiple places. +- Instead of duplicating a description, a field can reference a central definition using: + + ```json + "$ref": "#/definitions/field_name" + ``` + +- The referenced definition is found in the `definitions` section at the top of `overrides.json`. +- This approach ensures consistency, simplifies updates, and enables cross-referencing between related fields. + +## How reference docs are structured + +- **Autogenerated files** (such as `.adoc` or `.json`): List all fields, types, options, and sometimes examples, but may lack clarity, deduplication, or cross-references. +- **Overrides file** (`overrides.json`): + - Contains a `definitions` section for reusable field descriptions. + - Uses `$ref` to point fields in the autogenerated structure to these definitions. + - Allows for cross-references between related fields (such as "see also", "must be used with", etc.). + - Supports enhancements for clarity, user-friendliness, and technical accuracy. + +## Example + +```json +"definitions": { + "client_certs": { + "description": "A list of client certificates for mutual TLS (mTLS) authentication. ... (full description here)" + } +}, +"inputs": [ + { + "name": "amqp_0_9", + "config": { + "children": [ + { + "name": "tls", + "$ref": "#/definitions/tls", + "children": [ + { + "name": "client_certs", + "$ref": "#/definitions/client_certs" + } + ] + } + ] + } + } +] +``` + +## Your workflow + +When asked to improve or work with Redpanda Connect reference documentation: + +1. **Read the overrides file**: Always start by reading `docs-data/overrides.json` to understand the current structure and definitions. + +2. **Understand the `$ref` system**: Do not duplicate descriptions; suggest improvements to central definitions and update references as needed. + +3. **Make improvements**: + - Suggest improvements for clarity, consistency, and user experience, especially in the `definitions` section + - Identify opportunities for new or improved cross-references between related fields + - Create new definition objects for missing references or enhance existing ones + - Recommend deduplication: If you find repeated inline descriptions, propose moving them to a definition and referencing them with `$ref` + +4. **Update the overrides file**: Use the Edit or Write tool to update `docs-data/overrides.json` with your changes. + +5. **Regenerate documentation**: Use the `generate_rpcn_connector_docs` MCP tool to regenerate the documentation with your changes. + - Default branch is "main" + - Can specify a different branch if needed + +6. **Explain your changes**: Provide a clear explanation of what you changed and why. + +## Quality checks you must perform + +- **Preserve technical accuracy**: Ensure that all field descriptions are correct, actionable, and user-friendly +- **Maintain DRY principles**: Avoid unnecessary repetition in the documentation +- **Validate JSON syntax**: Ensure the overrides file remains valid JSON after your changes +- **Check $ref references**: Ensure all `$ref` references point to valid definitions +- **Maintain consistency**: Use consistent terminology and formatting across all descriptions + +## Output expectations + +When asked for improvements, your output should include: + +- Suggested changes to the `definitions` section +- Proposed new or updated `$ref` references +- Rationale for changes (clarity, deduplication, cross-linking, etc.) +- Any additional user experience or technical writing enhancements +- Summary of what will be regenerated + +## Summary for LLMs + +When working with Redpanda Connect reference documentation: + +1. Read `docs-data/overrides.json` first to understand the current structure +2. Make your improvements to the definitions and $ref references +3. Update the overrides file using Edit or Write tool +4. Use the `generate_rpcn_connector_docs` MCP tool to regenerate documentation +5. Explain what was changed and why + +Critical requirements: +- Always respect the `$ref` system - don't duplicate descriptions +- Maintain DRY principles throughout +- Validate JSON syntax after changes +- Preserve technical accuracy +- Ensure all `$ref` references are valid diff --git a/mcp/prompts/write-new-guide.md b/mcp/prompts/write-new-guide.md new file mode 100644 index 00000000..e066fa7e --- /dev/null +++ b/mcp/prompts/write-new-guide.md @@ -0,0 +1,214 @@ +--- +description: Generate a new tutorial or guide following team templates and style standards. Automatically applies approved terminology and voice/tone guidelines. +version: 1.0.0 +arguments: + - name: topic + description: What the guide should teach (for example, "Configure TLS encryption", "Deploy a three-node cluster") + required: true + - name: audience + description: Target audience (for example, "beginners", "experienced developers", "operators") + required: false +argumentFormat: structured +--- + +# Write New Documentation Prompt + +You are writing documentation for the Redpanda documentation team. + +## Resources to read first + +- `redpanda://style-guide` - Complete style guide with Google Developer Documentation Style Guide principles +- Official glossary sources for terminology: + - GitHub: https://github.com/redpanda-data/docs/tree/shared/modules/terms/partials + - Published: https://docs.redpanda.com/current/reference/glossary/ +- `/lib` directory - Custom AsciiDoc macros and extensions documentation +- `/extensions` directory - Custom Antora extensions + +Use `get_antora_structure` tool to understand the documentation organization. + +## Documentation format + +We use **AsciiDoc** with **Antora**. Key formatting rules: + +### Cross-references (CRITICAL) + +Always include the module name in xrefs, even within the same component: + +Correct: +```asciidoc +xref:manage:kubernetes/configure-cluster.adoc[Configure your cluster] +xref:reference:properties/cluster-properties.adoc[Cluster properties] +``` + +Never use: +```asciidoc +xref:./configure-cluster.adoc[...] // No relative paths +xref:configure-cluster.adoc[...] // Missing module +``` + +### Glossary terms (CRITICAL) + +Use the `glossterm` macro for terms defined in the glossary (first mention in a document): + +Correct: +```asciidoc +A glossterm:topic[] is divided into glossterm:partition,partitions[] +The glossterm:broker[] handles data replication +Configure glossterm:TLS[] encryption for secure communication +``` + +Don't link terms manually: +```asciidoc +A xref:reference:glossary.adoc#topic[topic] is divided... // Wrong syntax +A topic is divided into partitions // Not linked at all +``` + +**Glossary terms location**: https://github.com/redpanda-data/docs/tree/shared/modules/terms/partials + +### Code blocks + +Use custom roles for different code block types: + +**For commands:** +```asciidoc +[source,bash] +---- +rpk topic create orders --partitions 3 +---- +``` + +**For output (no copy button):** +```asciidoc +[source,bash,role=no-copy] +---- +TOPIC STATUS +orders OK +---- +``` + +**For long code (no wrapping, scrollbar instead):** +```asciidoc +[source,yaml,role=no-wrap] +---- +very long configuration here... +---- +``` + +**For code with `` syntax (prevent frontend editing):** +```asciidoc +[source,bash,role=no-placeholders] +---- +rpk topic create +---- +``` + +**Combine roles with comma:** +```asciidoc +[source,bash,role="no-copy,no-wrap"] +---- +long output here... +---- +``` + +### Custom macros + +Check `/lib` and `/extensions` directories for available custom macros before writing. Use them appropriately. + +## Writing standards + +### Style (from Google Developer Documentation Style Guide) +- Present tense +- Active voice +- Second person ("you") +- **Sentence case for ALL headings except the title (H1)** + - H2, H3, H4, etc.: Only capitalize first word and proper nouns + - Example: "Configure TLS encryption" not "Configure TLS Encryption" +- Clear, conversational tone +- **Never use gerunds for instructions** (use imperative: "Create a topic" not "Creating a topic") + +### Redpanda-specific +- Avoid overuse of bold text (only for UI elements, critical warnings) +- Avoid em dashes (use parentheses or break into sentences) +- Use realistic examples (no foo/bar placeholders) + +### Terminology +Check the official glossary for approved terms: +- GitHub: https://github.com/redpanda-data/docs/tree/shared/modules/terms/partials +- Published: https://docs.redpanda.com/current/reference/glossary/ + +Quick reference: +- Product names: Redpanda, Redpanda Cloud, Redpanda Console, Redpanda Connect +- Lowercase concepts: topic, partition, broker, cluster +- Deprecated terms to avoid: master/slave, whitelist/blacklist, SSL (use TLS) + +## Page structure + +```asciidoc += Page Title +:description: Brief description + +Opening paragraph explaining what this page covers. + +== First Section + +Content here. + +=== Subsection + +More content. +``` + +## Code example pattern + +Provide context, show the command, show the output: + +```asciidoc +Create a topic for order events: + +[source,bash] +---- +rpk topic create orders --partitions 3 --replicas 3 +---- + +Expected output: + +[source,bash,role=no-copy] +---- +TOPIC STATUS +orders OK +---- +``` + +## Instructions format + +- Correct (imperative): +- "Create a topic" +- "Configure TLS" +- "Deploy the cluster" + +- Incorrect (gerund): +- "Creating a topic" +- "Configuring TLS" +- "Deploying the cluster" + +## Quality checklist + +- All xrefs include module name +- Glossary terms use `glossterm` macro on first mention +- Code blocks use appropriate roles (`.no-copy` for outputs) +- Custom macros used correctly +- Terminology follows approved dictionary +- Voice is clear and conversational +- No overuse of bold or em dashes +- Present tense, active voice, second person +- Imperative form for instructions (no gerunds) +- Code examples are realistic and tested + +--- + +Please provide: +1. **Topic**: What to document +2. **Type**: Tutorial, concept, reference, or guide +3. **Target module** (optional): Where in Antora structure this belongs + +I'll create documentation following all team standards. diff --git a/mcp/team-standards/style-guide.md b/mcp/team-standards/style-guide.md new file mode 100644 index 00000000..6689f205 --- /dev/null +++ b/mcp/team-standards/style-guide.md @@ -0,0 +1,321 @@ +# Redpanda Documentation Style Guide + +## Base Style Guide + +This guide is based on the [Google Developer Documentation Style Guide](https://developers.google.com/style). All Google style guidelines apply unless explicitly overridden below. + +Key Google style principles we emphasize: + +- Use present tense +- Use active voice +- Use second person ("you" instead of "the user") +- Write in a conversational tone +- Use sentence case for all headings except the page title + +## Redpanda team preferences + +### Formatting conventions + +Avoid overuse of bold text: +- Use bold sparingly, primarily for UI elements and important warnings +- Don't bold every important word or phrase +- Prefer clear writing over heavy formatting +- Good: "To enable TLS, set the `enable_tls` flag to true" +- Avoid: "To **enable TLS**, set the **`enable_tls`** flag to **true**" + +Avoid em dashes: +- Use parentheses or commas for asides instead of em dashes +- Use colons to introduce explanations or lists +- Break long sentences into shorter ones rather than using em dashes +- Good: "TLS encryption (enabled by default) protects data in transit" +- Good: "TLS encryption protects data in transit. It's enabled by default" +- Avoid: "TLS encryptionβ€”enabled by defaultβ€”protects data in transit" + +### When bold is appropriate + +Use bold only for: + +- UI element names: Click the Start button +- Important warnings or cautions (sparingly) +- Glossary terms on first use (optional, if using a glossary system) + +### When parentheses are better than em dashes + +Use parentheses for: + +- Clarifications or asides +- Abbreviations: "Transport Layer Security (TLS)" +- Version numbers or optional information + +## Redpanda-specific guidelines + +### Voice and tone + +General principles: + +- Clear and direct: Use simple, straightforward language +- Technically accurate: Precision matters, but don't sacrifice clarity +- Helpful: Anticipate questions and provide context +- Conversational but professional: Friendly without being casual +- Action-oriented: Focus on what users can do + +### Voice examples + +Good: "Configure your cluster to use TLS encryption for secure communication." +Too Casual: "Let's slap some TLS on your cluster!" +Too Formal: "It is advisable that one should configure the cluster such that TLS encryption protocols are employed." + +## Writing standards + +### Headings + +**Title (H1):** Can use title case or sentence case based on content type. + +**All other headings (H2, H3, H4, etc.):** Use sentence case only. + +- Capitalize only the first word and proper nouns +- Make headings descriptive and scannable +- Use verb phrases for task-based headings + +Examples: + +```asciidoc += Page Title Can Use Title Case (H1) + +== Configure TLS encryption (H2 - sentence case) + +=== Set up certificates (H3 - sentence case) + +== Deploy a three-node cluster (H2 - sentence case) +``` + +Good: "Configure TLS encryption" +Good: "Deploy a three-node cluster" +Bad: "Configure TLS Encryption" (don't capitalize "Encryption") +Bad: "Deploy A Three-Node Cluster" (don't capitalize every word) +Bad: "TLS" (not descriptive enough) +Bad: "Configuration" (too vague) + +### Lists + +- Use parallel structure (all items start with same part of speech) +- Use numbered lists for sequential steps +- Use bulleted lists for non-sequential items +- Capitalize the first word of each list item +- Use periods for complete sentences, omit for fragments + +### Code examples +- Always provide context before code examples +- Include comments explaining non-obvious behavior +- Show both the command and expected output when relevant +- Use realistic examples, not foo/bar placeholders when possible + +### Links and cross-references + +We use AsciiDoc with Antora. Important rules: + +**Internal links (xref):** +- Always include the module name, even within the same component +- Never use relative paths (like `./page.adoc`) +- Use descriptive link text (never "click here") + +```asciidoc +Good: xref:security:tls-config.adoc[TLS configuration guide] +Good: xref:manage:kubernetes/configure.adoc[Configure your cluster] +Bad: xref:./tls-config.adoc[guide] // No relative paths! +Bad: xref:tls-config.adoc[guide] // Missing module! +Bad: xref:security:tls-config.adoc[Click here] // Poor link text +``` + +**Glossary terms (glossterm macro):** +- Use the `glossterm` macro to reference terms defined in the glossary +- Link terms on first mention in a document +- Glossary terms are in: https://github.com/redpanda-data/docs/tree/shared/modules/terms/partials + +```asciidoc +βœ… Good: A glossterm:topic[] is divided into glossterm:partition,partitions[] +βœ… Good: The glossterm:broker[] handles data storage +❌ Bad: A topic is divided into partitions // Terms not linked +``` + +**External links:** +- Use descriptive link text +- Include brief context for where the link goes + +```asciidoc +βœ… Good: See the https://kafka.apache.org/documentation/[Apache Kafka documentation] +❌ Bad: See https://kafka.apache.org/documentation/[here] +``` + +## Structure standards + +### Prerequisites section +Always include when users need: +- Specific software installed +- Prior configuration completed +- Specific permissions or access +- Understanding of certain concepts + +### Procedure format +1. Start with what the user will accomplish +2. List prerequisites +3. Provide numbered steps +4. Include verification steps +5. Suggest next steps or related topics + +### Examples section +Include examples that: +- Cover common use cases +- Show real-world scenarios +- Include expected output +- Explain what's happening + +## Terminology + +### Official glossary sources + +Our approved terminology is maintained in these locations: + +- **GitHub source**: https://github.com/redpanda-data/docs/tree/shared/modules/terms/partials (each term is a separate file) +- **Published glossary**: https://docs.redpanda.com/current/reference/glossary/ + +**Always reference the official glossary when you need term definitions or approved usage.** + +### Quick reference: Common terms + +Use these as a quick reference, but check the official glossary for complete definitions: + +**Product names:** +- Redpanda (never RedPanda or red panda) +- Redpanda Cloud +- Redpanda Console +- Redpanda Connect (formerly Benthos) +- Kafka (when referring to Apache Kafka) + +**Kafka concepts (lowercase):** +- topic +- partition +- broker +- cluster +- consumer +- producer +- leader (not master) +- replica (not slave) + +**Security terms:** +- TLS (not SSL - deprecated) +- SASL +- mTLS +- ACL +- allowlist (not whitelist - deprecated) +- denylist (not blacklist - deprecated) + +**Command names (lowercase):** +- `rpk` +- `docker` +- `kubectl` + +### Using the glossterm macro + +When writing documentation, link to glossary terms using the `glossterm` macro: + +```asciidoc +A glossterm:topic[] is divided into glossterm:partition,partitions[] +The glossterm:broker[] handles data replication +``` + +See the official glossary for the complete list of terms and their definitions. + +### Command documentation +- Show the full command with all required flags +- Explain what each flag does +- Provide example output +- Mention common errors and solutions + +### API documentation +- Start with a one-sentence description +- Document all parameters (name, type, required/optional, description) +- Include request and response examples +- Document error responses + +## Accessibility + +### Images and diagrams +- Always include alt text +- Use diagrams to supplement text, not replace it +- Describe the diagram in surrounding text + +### Structure +- Use proper heading hierarchy (don't skip levels) +- One H1 per page +- Logical heading progression (H2 β†’ H3, never H2 β†’ H4) + +### Code blocks +- Always specify the language for syntax highlighting +- Include descriptive titles when helpful +- Ensure code is keyboard-navigable + +## Grammar and mechanics + +### Active voice +Prefer active voice over passive voice for clarity. + +Good: "Redpanda encrypts data at rest" +Avoid: "Data is encrypted at rest by Redpanda" + +### Present tense +Use present tense for describing how things work. + +Good: "The broker replicates data across partitions" +Avoid: "The broker will replicate data across partitions" + +### Second person +Address the reader directly as "you". + +Good: "You can configure TLS by..." +Avoid: "Users can configure TLS by..." or "We can configure TLS by..." + +### Contractions +Use contractions sparingly in technical content. Avoid in: +- Command documentation +- API references +- Configuration guides + +Acceptable in: +- Tutorials +- Conceptual overviews +- Blog-style content + +## Error messages and troubleshooting + +### Format +1. State what went wrong +2. Explain why it happened +3. Provide specific steps to fix it + +### Example + +```markdown +**Problem**: `rpk` returns "connection refused" + +**Cause**: The Redpanda broker isn't running or isn't accessible on the specified port. + +**Solution**: +1. Verify the broker is running: `systemctl status redpanda` +2. Check the broker address: `rpk cluster info` +3. Ensure no firewall is blocking port 9092 +``` + +## Review checklist + +Before publishing, verify: +- [ ] All code examples tested and working +- [ ] Links are valid and point to correct destinations +- [ ] Images have alt text +- [ ] Headings follow hierarchy +- [ ] Terminology is consistent and correct +- [ ] Voice and tone are appropriate +- [ ] Prerequisites are listed +- [ ] Examples are realistic and helpful +- [ ] Grammar and spelling are correct +- [ ] Accessibility standards met diff --git a/mcp/templates/README.adoc b/mcp/templates/README.adoc new file mode 100644 index 00000000..b4c4f988 --- /dev/null +++ b/mcp/templates/README.adoc @@ -0,0 +1,212 @@ += MCP Extension Templates +:toc: + +== Overview + +This directory contains ready-to-use templates for extending the MCP server. Copy these templates and customize them for your needs. + +== Available templates + +[cols="1,3"] +|=== +|Template |Use For + +|`prompt-review-template.md` +|Creating review prompts for specific content types + +|`prompt-write-template.md` +|Creating writing prompts that generate new content + +|`resource-template.md` +|Creating reference resources (guides, checklists, templates) +|=== + +== How to use these templates + +=== Quick start + +. Copy the template you need +. Replace `[PLACEHOLDERS]` with your content +. Add YAML frontmatter (for prompts) +. Validate with `npx doc-tools validate-mcp` +. Restart Claude Code and test + +=== Detailed steps + +**Step 1: Choose a template** + +Pick the template that matches what you're creating: +- Review prompts β†’ `prompt-review-template.md` +- Writing prompts β†’ `prompt-write-template.md` +- Resources β†’ `resource-template.md` + +**Step 2: Copy and rename** + +[source,bash] +---- +# For a prompt +cp mcp/templates/prompt-review-template.md mcp/prompts/review-api-docs.md + +# For a resource +cp mcp/templates/resource-template.md mcp/team-standards/api-guide.md +---- + +**Step 3: Add frontmatter (prompts only)** + +Prompts need YAML frontmatter at the top. The templates include placeholder frontmatter - customize it: + +[source,yaml] +---- +--- +description: "Your prompt description" +version: "1.0.0" +arguments: + content: + description: "The content to process" + required: true +--- +---- + +**Step 4: Customize the content** + +Replace all `[PLACEHOLDERS]` with your specific content: + +- `[CONTENT-TYPE]` β†’ "API documentation", "troubleshooting guide", etc. +- `[First thing to check]` β†’ Specific checks for your use case +- `[Section names]` β†’ Appropriate section names + +**Step 5: Validate** + +Check your prompt or resource for errors: + +[source,bash] +---- +npx doc-tools validate-mcp +---- + +**Step 6: Test** + +Restart Claude Code and test your new prompt or resource. + +No code editing required - the MCP server auto-discovers files in `mcp/prompts/` and `mcp/team-standards/`! + +== Customization tips + +=== For review prompts + +Focus on: +- **Specific checks** relevant to the content type +- **Common mistakes** you see in reviews +- **Required elements** that must be present +- **Output format** that's easy to act on + +Good checklist items: +- Concrete and testable +- Focused on one aspect +- Clear about what success looks like + +=== For writing prompts + +Focus on: +- **Clear structure** that matches your templates +- **Specific requirements** for the content type +- **Quality standards** that must be met +- **Examples** showing what good looks like + +Good requirements: +- Specific and actionable +- Based on real patterns +- Include rationale + +=== For resources + +Focus on: +- **Practical examples** showing how to apply rules +- **Good vs bad** comparisons +- **Checklists** for quick verification +- **Clear organization** that's easy to scan + +Good resources: +- Scannable with clear headings +- More examples than rules +- Actionable guidance + +== Examples + +=== Example: Review prompt for API docs + +[source,bash] +---- +# Copy template +cp mcp/templates/prompt-review-template.md mcp/prompts/review-api-docs.md + +# Edit and customize +vi mcp/prompts/review-api-docs.md +---- + +Key customizations: +- Replace `[CONTENT-TYPE]` with "API documentation" +- Add checks for: authentication section, request/response examples, error codes +- Add common issues: missing status codes, unclear parameter descriptions +- Define output format for API-specific feedback + +=== Example: Resource for code examples + +[source,bash] +---- +# Copy template +cp mcp/templates/resource-template.md mcp/team-standards/code-examples-guide.md + +# Edit and customize +vi mcp/team-standards/code-examples-guide.md +---- + +Key customizations: +- Add language-specific guidelines (bash, Python, YAML) +- Include examples of good vs bad code formatting +- Add checklist for code review +- Show proper use of code block roles + +== Template placeholders reference + +Common placeholders you'll find in templates: + +[cols="1,2"] +|=== +|Placeholder |Replace With + +|`[CONTENT-TYPE]` +|Specific type: "API docs", "tutorials", "troubleshooting guides" + +|`[First thing to check]` +|Specific checks relevant to your use case + +|`[Required section X]` +|Sections that must be present + +|`[Resource Name]` +|Name of your resource + +|`[Main Section X]` +|Appropriate section names for your content + +|`[Check item X]` +|Specific checklist items +|=== + +== Getting help + +- See link:../WRITER_EXTENSION_GUIDE.adoc[Writer Extension Guide] for complete instructions +- Check existing prompts in `mcp/prompts/` for examples +- Review existing resources in `mcp/team-standards/` for patterns +- Ask your team lead for guidance + +== Contributing + +When you create a great prompt or resource: + +. Add it to the examples in link:../WRITER_EXTENSION_GUIDE.adoc[Writer Extension Guide] +. Share it with the team +. Consider if it should become a new template + +The more prompts and resources we build, the more consistent and efficient our docs team becomes! diff --git a/mcp/templates/prompt-review-template.md b/mcp/templates/prompt-review-template.md new file mode 100644 index 00000000..a191855e --- /dev/null +++ b/mcp/templates/prompt-review-template.md @@ -0,0 +1,80 @@ +--- +description: "Review [CONTENT-TYPE] for [specific aspects you're checking]" +version: "1.0.0" +arguments: + content: + description: "The [CONTENT-TYPE] content to review" + required: true +--- + +# Review [CONTENT-TYPE] + +You are reviewing [CONTENT-TYPE] for the Redpanda documentation team. + +## Your Task + +Review the provided content and check for: + +1. [First thing to check] +2. [Second thing to check] +3. [Third thing to check] +4. [Additional checks...] + +## Resources to Reference + +Read these resources before reviewing: +- `redpanda://style-guide` - Complete style guide +- Official glossary for terminology: + - GitHub: https://github.com/redpanda-data/docs/tree/shared/modules/terms/partials + - Published: https://docs.redpanda.com/current/reference/glossary/ +- [Add other relevant resources] + +## Standards for [CONTENT-TYPE] + +[Describe what makes good content of this type] + +### Required Sections + +All [CONTENT-TYPE] documentation must include: +- [Required section 1] +- [Required section 2] +- [Required section 3] + +### Format Requirements + +[Specific formatting rules for this content type] + +### Common Issues to Check + +- [Common mistake 1] +- [Common mistake 2] +- [Common mistake 3] + +## Output Format + +Provide a structured review: + +### Critical Issues (must fix) + +For each critical issue: +- **Location**: [Section/paragraph reference] +- **Issue**: [What's wrong] +- **Fix**: [Specific correction] +- **Reason**: [Why it matters] + +### Suggestions (should consider) + +For each suggestion: +- **Location**: [Where it appears] +- **Current**: [What it says now] +- **Suggested**: [Better alternative] +- **Reason**: [Why the suggestion helps] + +### Positive Elements + +What works well: +- [Good thing 1] +- [Good thing 2] +- [Good thing 3] + +--- \ No newline at end of file diff --git a/mcp/templates/prompt-write-template.md b/mcp/templates/prompt-write-template.md new file mode 100644 index 00000000..4e74746a --- /dev/null +++ b/mcp/templates/prompt-write-template.md @@ -0,0 +1,110 @@ +--- +description: "Generate [CONTENT-TYPE] following team standards" +version: "1.0.0" +arguments: + topic: + description: "The topic to write about" + required: true + target_module: + description: "Where in Antora structure this belongs (optional)" + required: false + context: + description: "Additional context or requirements" + required: false +--- + +# Write [CONTENT-TYPE] + +You are writing [CONTENT-TYPE] for the Redpanda documentation team. + +## Your Task + +Create [CONTENT-TYPE] that follows our team standards. + +## Resources to Read First + +- `redpanda://style-guide` - Complete style guide +- Official glossary for terminology: + - GitHub: https://github.com/redpanda-data/docs/tree/shared/modules/terms/partials + - Published: https://docs.redpanda.com/current/reference/glossary/ +- `redpanda://[content-type]-template` - Template for this content type (if available) +- [Add other relevant resources] + +Use `get_antora_structure` tool to understand the documentation organization. + +## Standards for [CONTENT-TYPE] + +### Structure + +[CONTENT-TYPE] should follow this structure: + +1. [Section 1] + - [What goes here] +2. [Section 2] + - [What goes here] +3. [Section 3] + - [What goes here] + +### Content Requirements + +[What must be included:] +- [Requirement 1] +- [Requirement 2] +- [Requirement 3] + +### Writing Style + +Follow these style guidelines: +- Present tense +- Active voice +- Second person ("you") +- Sentence case for all headings except title +- Imperative form for instructions (no gerunds) +- Avoid overuse of bold or em dashes + +### AsciiDoc Format + +- All xrefs must include module name +- Use `glossterm` macro for glossary terms on first mention +- Code blocks use appropriate roles (`.no-copy` for outputs) +- Custom macros used correctly + +## Quality Checklist + +Before finalizing, verify: +- [ ] Structure follows template +- [ ] All required sections included +- [ ] Code examples are realistic and tested +- [ ] Terminology uses approved terms +- [ ] Glossary terms use `glossterm` macro +- [ ] All xrefs include module names +- [ ] Voice is clear and conversational +- [ ] Headings use sentence case + +## Output Format + +Provide complete [CONTENT-TYPE] in AsciiDoc format: + +```asciidoc += Page Title +:description: Brief description + +Opening content explaining what this covers. + +== First section + +Content here. + +== Second section + +More content. +``` + +--- + +Please provide: +1. **Topic**: What this [CONTENT-TYPE] should cover +2. **Target module** (optional): Where in Antora structure this belongs +3. **[Other context]**: [Additional information needed] + +I'll create complete [CONTENT-TYPE] following all team standards. diff --git a/mcp/templates/resource-template.md b/mcp/templates/resource-template.md new file mode 100644 index 00000000..972ed00c --- /dev/null +++ b/mcp/templates/resource-template.md @@ -0,0 +1,76 @@ +# [Resource Name] + +[Brief description of what this resource provides] + +## Overview + +[1-2 paragraphs explaining when and how to use this resource] + +## [Main Section 1] + +[Content for this section] + +### Subsection + +[Details with examples] + +Example: + +```asciidoc +[Example code or content here] +``` + +## [Main Section 2] + +[Content for this section] + +### Key Points + +- [Important point 1] +- [Important point 2] +- [Important point 3] + +### Examples + +Good: +``` +[Good example] +``` + +Bad: +``` +[Bad example - what to avoid] +``` + +## [Main Section 3] + +[Additional content] + +## Checklist + +Use this checklist to verify compliance: + +- [ ] [Check item 1] +- [ ] [Check item 2] +- [ ] [Check item 3] +- [ ] [Check item 4] + +## Common Mistakes + +[List common mistakes and how to avoid them] + +### Mistake 1 + +Problem: [What people do wrong] +Solution: [How to do it right] + +### Mistake 2 + +Problem: [What people do wrong] +Solution: [How to do it right] + +## References + +- [Link to related resource 1] +- [Link to related resource 2] +- [Link to external documentation] diff --git a/package-lock.json b/package-lock.json index c6abac1f..52a47c76 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,16 +1,17 @@ { "name": "@redpanda-data/docs-extensions-and-macros", - "version": "4.12.5", + "version": "4.13.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@redpanda-data/docs-extensions-and-macros", - "version": "4.12.5", + "version": "4.13.0", "license": "ISC", "dependencies": { "@asciidoctor/tabs": "^1.0.0-beta.6", "@bufbuild/buf": "^1.28.1", + "@modelcontextprotocol/sdk": "^1.0.4", "@octokit/core": "^6.1.2", "@octokit/plugin-retry": "^7.1.1", "@octokit/rest": "^21.0.1", @@ -19,6 +20,7 @@ "chalk": "4.1.2", "cheerio": "^1.1.2", "commander": "^14.0.0", + "glob": "^11.0.0", "gulp": "^4.0.2", "gulp-connect": "^5.7.0", "handlebars": "^4.7.8", @@ -40,13 +42,15 @@ "yargs": "^17.7.2" }, "bin": { - "doc-tools": "bin/doc-tools.js" + "doc-tools": "bin/doc-tools.js", + "doc-tools-mcp": "bin/doc-tools-mcp.js" }, "devDependencies": { "@antora/cli": "3.1.4", "@antora/site-generator": "3.1.4", "@web/dev-server": "^0.2.5", - "jest": "^29.7.0" + "jest": "^29.7.0", + "jest-junit": "^16.0.0" } }, "node_modules/@algolia/cache-browser-local-storage": { @@ -1712,6 +1716,28 @@ } } }, + "node_modules/@jest/reporters/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "dev": true, + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/@jest/schemas": { "version": "29.6.3", "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz", @@ -1906,6 +1932,120 @@ "integrity": "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==", "license": "BSD-2-Clause" }, + "node_modules/@modelcontextprotocol/sdk": { + "version": "1.22.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.22.0.tgz", + "integrity": "sha512-VUpl106XVTCpDmTBil2ehgJZjhyLY2QZikzF8NvTXtLRF1CvO5iEE2UNZdVIUer35vFOwMKYeUGbjJtvPWan3g==", + "license": "MIT", + "dependencies": { + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "content-type": "^1.0.5", + "cors": "^2.8.5", + "cross-spawn": "^7.0.5", + "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.0", + "express": "^5.0.1", + "express-rate-limit": "^7.5.0", + "pkce-challenge": "^5.0.0", + "raw-body": "^3.0.0", + "zod": "^3.23.8", + "zod-to-json-schema": "^3.24.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@cfworker/json-schema": "^4.1.1" + }, + "peerDependenciesMeta": { + "@cfworker/json-schema": { + "optional": true + } + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/http-errors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "license": "MIT", + "dependencies": { + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/iconv-lite": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.0.tgz", + "integrity": "sha512-cf6L2Ds3h57VVmkZe+Pn+5APsT7FpqJtEhhieDCvrE2MK5Qk9MyffgQyuxQTm6BChfeZNtcOLHp9IcWRVcIcBQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/raw-body": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.1.tgz", + "integrity": "sha512-9G8cA+tuMS75+6G/TzW8OtLzmBDMo8p1JRxN5AZ+LAp8uxGA8V8GZm4GQ4/N5QNQEnLmg6SS7wyuSmbKepiKqA==", + "license": "MIT", + "dependencies": { + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.7.0", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/@noble/hashes": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.8.0.tgz", @@ -2449,84 +2589,6 @@ "node": ">= 6" } }, - "node_modules/@redocly/cli/node_modules/glob": { - "version": "11.0.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-11.0.3.tgz", - "integrity": "sha512-2Nim7dha1KVkaiF4q6Dj+ngPPMdfvLJEOpZk/jKiUAkqKebpGAWQXAq9z1xu9HKu5lWfqw/FASuccEjyznjPaA==", - "license": "ISC", - "dependencies": { - "foreground-child": "^3.3.1", - "jackspeak": "^4.1.1", - "minimatch": "^10.0.3", - "minipass": "^7.1.2", - "package-json-from-dist": "^1.0.0", - "path-scurry": "^2.0.0" - }, - "bin": { - "glob": "dist/esm/bin.mjs" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/@redocly/cli/node_modules/jackspeak": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.1.1.tgz", - "integrity": "sha512-zptv57P3GpL+O0I7VdMJNBZCu+BPHVQUk55Ft8/QCJjTVxrnJHuVuX/0Bl2A6/+2oyR/ZMEuFKwmzqqZ/U5nPQ==", - "license": "BlueOak-1.0.0", - "dependencies": { - "@isaacs/cliui": "^8.0.2" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/@redocly/cli/node_modules/lru-cache": { - "version": "11.2.2", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.2.tgz", - "integrity": "sha512-F9ODfyqML2coTIsQpSkRHnLSZMtkU8Q+mSfcaIyKwy58u+8k5nvAYeiNhsyMARvzNcXJ9QfWVrcPsC9e9rAxtg==", - "license": "ISC", - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/@redocly/cli/node_modules/minimatch": { - "version": "10.0.3", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.0.3.tgz", - "integrity": "sha512-IPZ167aShDZZUMdRk66cyQAW3qr0WzbHkPdMYa8bzZhlHhO3jALbKdxcaak7W9FfT2rZNpQuUu4Od7ILEpXSaw==", - "license": "ISC", - "dependencies": { - "@isaacs/brace-expansion": "^5.0.0" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/@redocly/cli/node_modules/path-scurry": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.0.tgz", - "integrity": "sha512-ypGJsmGtdXUOeM5u93TyeIEfEhM6s+ljAhrk5vAvSx8uyY/02OvrZnA0YNGUrPXfpJMgI1ODd3nwz8Npx4O4cg==", - "license": "BlueOak-1.0.0", - "dependencies": { - "lru-cache": "^11.0.0", - "minipass": "^7.1.2" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/@redocly/cli/node_modules/yargs": { "version": "17.0.1", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.0.1.tgz", @@ -3568,6 +3630,25 @@ "node": ">=8.11" } }, + "node_modules/asciidoctor-opal-runtime/node_modules/glob": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", + "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "dev": true, + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + } + }, "node_modules/asciidoctor-opal-runtime/node_modules/unxhr": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/unxhr/-/unxhr-1.0.1.tgz", @@ -3917,25 +3998,222 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/bindings": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", - "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, + "node_modules/body": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/body/-/body-5.1.0.tgz", + "integrity": "sha512-chUsBxGRtuElD6fmw1gHLpvnKdVLK302peeFa9ZqAEk8TyzZ3fygLyUEDDPTJvL9+Bor0dIwn6ePOsRM2y0zQQ==", + "dependencies": { + "continuable-cache": "^0.3.1", + "error": "^7.0.0", + "raw-body": "~1.1.0", + "safe-json-parse": "~1.0.1" + } + }, + "node_modules/body-parser": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.0.tgz", + "integrity": "sha512-02qvAaxv8tp7fBa/mw1ga98OGm+eCbqzJOKoRt70sLmfEEi+jyBYVTDGfCL/k06/4EMk/z01gCe7HoCH/f2LTg==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^1.0.5", + "debug": "^4.4.0", + "http-errors": "^2.0.0", + "iconv-lite": "^0.6.3", + "on-finished": "^2.4.1", + "qs": "^6.14.0", + "raw-body": "^3.0.0", + "type-is": "^2.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/body-parser/node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/body-parser/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/body-parser/node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/body-parser/node_modules/media-typer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/body-parser/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/body-parser/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/body-parser/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/body-parser/node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/body-parser/node_modules/raw-body": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.1.tgz", + "integrity": "sha512-9G8cA+tuMS75+6G/TzW8OtLzmBDMo8p1JRxN5AZ+LAp8uxGA8V8GZm4GQ4/N5QNQEnLmg6SS7wyuSmbKepiKqA==", + "license": "MIT", + "dependencies": { + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.7.0", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/body-parser/node_modules/raw-body/node_modules/http-errors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "license": "MIT", + "dependencies": { + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/body-parser/node_modules/raw-body/node_modules/iconv-lite": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.0.tgz", + "integrity": "sha512-cf6L2Ds3h57VVmkZe+Pn+5APsT7FpqJtEhhieDCvrE2MK5Qk9MyffgQyuxQTm6BChfeZNtcOLHp9IcWRVcIcBQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/body-parser/node_modules/raw-body/node_modules/statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/body-parser/node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", "license": "MIT", - "optional": true, - "dependencies": { - "file-uri-to-path": "1.0.0" + "engines": { + "node": ">= 0.8" } }, - "node_modules/body": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/body/-/body-5.1.0.tgz", - "integrity": "sha512-chUsBxGRtuElD6fmw1gHLpvnKdVLK302peeFa9ZqAEk8TyzZ3fygLyUEDDPTJvL9+Bor0dIwn6ePOsRM2y0zQQ==", + "node_modules/body-parser/node_modules/type-is": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.0.1.tgz", + "integrity": "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==", + "license": "MIT", "dependencies": { - "continuable-cache": "^0.3.1", - "error": "^7.0.0", - "raw-body": "~1.1.0", - "safe-json-parse": "~1.0.1" + "content-type": "^1.0.5", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 0.6" } }, "node_modules/boolbase": { @@ -4165,6 +4443,22 @@ "node": ">= 0.4" } }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/call-me-maybe": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-me-maybe/-/call-me-maybe-1.0.2.tgz", @@ -4931,7 +5225,6 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.6" @@ -4971,6 +5264,15 @@ "node": ">= 0.6" } }, + "node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, "node_modules/cookies": { "version": "0.9.1", "resolved": "https://registry.npmjs.org/cookies/-/cookies-0.9.1.tgz", @@ -5022,6 +5324,19 @@ "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", "license": "MIT" }, + "node_modules/cors": { + "version": "2.8.5", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", + "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==", + "license": "MIT", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + } + }, "node_modules/crc-32": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", @@ -5842,6 +6157,27 @@ "node": ">=0.8.x" } }, + "node_modules/eventsource": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", + "integrity": "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==", + "license": "MIT", + "dependencies": { + "eventsource-parser": "^3.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/eventsource-parser": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.0.6.tgz", + "integrity": "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/execa": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", @@ -5975,6 +6311,282 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, + "node_modules/express": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/express/-/express-5.1.0.tgz", + "integrity": "sha512-DT9ck5YIRU+8GYzzU5kT3eHGA5iL+1Zd0EutOmTE9Dtk+Tvuzd23VBU+ec7HPNSTxXYO55gPV/hq4pSBJDjFpA==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.0", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express-rate-limit": { + "version": "7.5.1", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-7.5.1.tgz", + "integrity": "sha512-7iN8iPMDzOMHPUYllBEsQdWVB6fPDMPqwjBaFrgr4Jgr/+okjvzAy+UHlYYL/Vs0OsOrMkwS6PJDkFlJwoxUnw==", + "license": "MIT", + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, + "node_modules/express/node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express/node_modules/content-disposition": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.1.tgz", + "integrity": "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/express/node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/express/node_modules/finalhandler": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.0.tgz", + "integrity": "sha512-/t88Ty3d5JWQbWYgaOGCCYfXRwV1+be02WqYYlL6h0lEiUAMPM8o8qKGO01YIkOHzka2up08wvgYD0mDiI+q3Q==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/express/node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/express/node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express/node_modules/media-typer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/express/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/express/node_modules/negotiator": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express/node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/express/node_modules/send": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.0.tgz", + "integrity": "sha512-uaW0WwXKpL9blXE2o0bRhoL2EGXIrZxQ2ZQ4mgcfoBxdFmQold+qWsD2jLrfZ0trjKL6vOw0j//eAwcALFjKSw==", + "license": "MIT", + "dependencies": { + "debug": "^4.3.5", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "mime-types": "^3.0.1", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/express/node_modules/serve-static": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.0.tgz", + "integrity": "sha512-61g9pCh0Vnh7IutZjtLGGpTA355+OPn2TyDv/6ivP2h/AdAVX9azsoxmg2/M6nZeQZNYBEwIcsne1mJd9oQItQ==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/express/node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/express/node_modules/type-is": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.0.1.tgz", + "integrity": "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==", + "license": "MIT", + "dependencies": { + "content-type": "^1.0.5", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/ext": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/ext/-/ext-1.7.0.tgz", @@ -6542,6 +7154,15 @@ "node": ">=12.20.0" } }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/fragment-cache": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz", @@ -6703,21 +7324,26 @@ } }, "node_modules/glob": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", - "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", - "deprecated": "Glob versions prior to v9 are no longer supported", - "license": "ISC", + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-11.1.0.tgz", + "integrity": "sha512-vuNwKSaKiqm7g0THUBu2x7ckSs3XJLXE+2ssL7/MfTGPLLcrJQ/4Uq1CjPTtO5cCIiRxqvN6Twy1qOwhL0Xjcw==", + "license": "BlueOak-1.0.0", "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" + "foreground-child": "^3.3.1", + "jackspeak": "^4.1.1", + "minimatch": "^10.1.1", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^2.0.0" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "engines": { + "node": "20 || >=22" }, - "engines": { - "node": "*" + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, "node_modules/glob-parent": { @@ -7098,6 +7724,61 @@ "node": ">=0.10.0" } }, + "node_modules/glob/node_modules/jackspeak": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.1.1.tgz", + "integrity": "sha512-zptv57P3GpL+O0I7VdMJNBZCu+BPHVQUk55Ft8/QCJjTVxrnJHuVuX/0Bl2A6/+2oyR/ZMEuFKwmzqqZ/U5nPQ==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob/node_modules/lru-cache": { + "version": "11.2.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.2.tgz", + "integrity": "sha512-F9ODfyqML2coTIsQpSkRHnLSZMtkU8Q+mSfcaIyKwy58u+8k5nvAYeiNhsyMARvzNcXJ9QfWVrcPsC9e9rAxtg==", + "license": "ISC", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/glob/node_modules/minimatch": { + "version": "10.1.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.1.tgz", + "integrity": "sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/brace-expansion": "^5.0.0" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob/node_modules/path-scurry": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.1.tgz", + "integrity": "sha512-oWyT4gICAu+kaA7QWk/jvCHWarMKNs6pXOGWKDTr7cw4IGcUbW+PeTfbaQiLGheFRpjo6O9J0PmyMfQPjH71oA==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/global-modules": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/global-modules/-/global-modules-1.0.0.tgz", @@ -7973,6 +8654,15 @@ "dev": true, "license": "MIT" }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, "node_modules/is-absolute": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-absolute/-/is-absolute-1.0.0.tgz", @@ -8223,6 +8913,12 @@ "node": ">=0.10.0" } }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, "node_modules/is-relative": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-relative/-/is-relative-1.0.0.tgz", @@ -8622,6 +9318,28 @@ } } }, + "node_modules/jest-config/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "dev": true, + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/jest-config/node_modules/parse-json": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", @@ -8739,6 +9457,35 @@ "fsevents": "^2.3.2" } }, + "node_modules/jest-junit": { + "version": "16.0.0", + "resolved": "https://registry.npmjs.org/jest-junit/-/jest-junit-16.0.0.tgz", + "integrity": "sha512-A94mmw6NfJab4Fg/BlvVOUXzXgF0XIH6EmTgJ5NDPp4xoKq0Kr7sErb+4Xs9nZvu58pJojz5RFGpqnZYJTrRfQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "mkdirp": "^1.0.4", + "strip-ansi": "^6.0.1", + "uuid": "^8.3.2", + "xml": "^1.0.1" + }, + "engines": { + "node": ">=10.12.0" + } + }, + "node_modules/jest-junit/node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true, + "license": "MIT", + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/jest-leak-detector": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-29.7.0.tgz", @@ -8934,6 +9681,28 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, + "node_modules/jest-runtime/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "dev": true, + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/jest-runtime/node_modules/strip-bom": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", @@ -9897,6 +10666,18 @@ "node": ">= 0.6" } }, + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/merge-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", @@ -10561,9 +11342,9 @@ } }, "node_modules/object-inspect": { - "version": "1.13.2", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.2.tgz", - "integrity": "sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==", + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", "license": "MIT", "engines": { "node": ">= 0.4" @@ -11130,6 +11911,16 @@ "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", "license": "ISC" }, + "node_modules/path-to-regexp": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.3.0.tgz", + "integrity": "sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/path-type": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-1.1.0.tgz", @@ -11326,6 +12117,15 @@ "node": ">= 6" } }, + "node_modules/pkce-challenge": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.0.tgz", + "integrity": "sha512-ueGLflrrnvwB3xuo/uGob5pd5FN7l0MsLf0Z87o/UQmRtwjvfylfc9MurIxRAWywCYTgrvpXBcqjV4OfCYGCIQ==", + "license": "MIT", + "engines": { + "node": ">=16.20.0" + } + }, "node_modules/pkg-dir": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", @@ -11600,6 +12400,19 @@ "node": ">=12.0.0" } }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, "node_modules/pump": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", @@ -11651,12 +12464,12 @@ "license": "MIT" }, "node_modules/qs": { - "version": "6.13.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", - "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", + "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", "license": "BSD-3-Clause", "dependencies": { - "side-channel": "^1.0.6" + "side-channel": "^1.1.0" }, "engines": { "node": ">=0.6" @@ -12288,6 +13101,45 @@ "fsevents": "~2.3.2" } }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/router/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/router/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -12784,15 +13636,69 @@ "license": "MIT" }, "node_modules/side-channel": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", - "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", "license": "MIT", "dependencies": { - "call-bind": "^1.0.7", "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.4", - "object-inspect": "^1.13.1" + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", + "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" }, "engines": { "node": ">= 0.4" @@ -14278,6 +15184,16 @@ "node": ">= 0.4.0" } }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "dev": true, + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/v8-to-istanbul": { "version": "9.3.0", "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz", @@ -14335,7 +15251,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.8" @@ -14398,6 +15313,27 @@ "stream-shift": "^1.0.0" } }, + "node_modules/vinyl-fs/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/vinyl-fs/node_modules/glob-parent": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-3.1.0.tgz", @@ -14736,6 +15672,13 @@ "node": ">=4" } }, + "node_modules/xml": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/xml/-/xml-1.0.1.tgz", + "integrity": "sha512-huCv9IH9Tcf95zuYCsQraZtWnJvBtLVE0QHMOs8bWyZAFZNDcYjsPq1nEx8jKA9y+Beo9v+7OBPRisQTjinQMw==", + "dev": true, + "license": "MIT" + }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", @@ -14860,6 +15803,24 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.0", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.0.tgz", + "integrity": "sha512-HvWtU2UG41LALjajJrML6uQejQhNJx+JBO9IflpSja4R03iNWfKXrj6W2h7ljuLyc1nKS+9yDyL/9tD1U/yBnQ==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25 || ^4" + } } } } diff --git a/package.json b/package.json index fef3f6ad..8ac6fe80 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@redpanda-data/docs-extensions-and-macros", - "version": "4.12.5", + "version": "4.13.0", "description": "Antora extensions and macros developed for Redpanda documentation.", "keywords": [ "antora", @@ -13,7 +13,8 @@ "name": "Redpanda Docs Team" }, "bin": { - "doc-tools": "./bin/doc-tools.js" + "doc-tools": "./bin/doc-tools.js", + "doc-tools-mcp": "./bin/doc-tools-mcp.js" }, "scripts": { "install-test-dependencies": "doc-tools install-test-dependencies", @@ -22,8 +23,12 @@ "build": "antora --to-dir docs --fetch local-antora-playbook.yml", "serve": "wds --node-resolve --open preview/test/ --watch --root-dir docs", "test": "jest", + "test:mcp": "jest __tests__/mcp/", + "test:mcp:integration": "jest __tests__/mcp/integration.test.js", + "test:mcp:contract": "jest __tests__/mcp/cli-contract.test.js", "test:python": "./__tests__/tools/property-extractor/setup-and-test.sh", "test:all": "npm run test && npm run test:python", + "generate:cli-docs": "node tools/generate-cli-docs.js", "bundle:admin": "doc-tools generate bundle-openapi --surface admin", "bundle:connect": "doc-tools generate bundle-openapi --surface connect", "bundle:both": "doc-tools generate bundle-openapi --surface both" @@ -85,6 +90,7 @@ "dependencies": { "@asciidoctor/tabs": "^1.0.0-beta.6", "@bufbuild/buf": "^1.28.1", + "@modelcontextprotocol/sdk": "^1.0.4", "@octokit/core": "^6.1.2", "@octokit/plugin-retry": "^7.1.1", "@octokit/rest": "^21.0.1", @@ -93,6 +99,7 @@ "chalk": "4.1.2", "cheerio": "^1.1.2", "commander": "^14.0.0", + "glob": "^11.0.0", "gulp": "^4.0.2", "gulp-connect": "^5.7.0", "handlebars": "^4.7.8", @@ -117,6 +124,7 @@ "@antora/cli": "3.1.4", "@antora/site-generator": "3.1.4", "@web/dev-server": "^0.2.5", - "jest": "^29.7.0" + "jest": "^29.7.0", + "jest-junit": "^16.0.0" } } diff --git a/tools/bundle-openapi.js b/tools/bundle-openapi.js index dfd39c16..46661350 100644 --- a/tools/bundle-openapi.js +++ b/tools/bundle-openapi.js @@ -12,8 +12,8 @@ const yaml = require('yaml'); * and validates that the result matches MAJOR.MINOR.PATCH with optional pre-release/build metadata. * Throws if the input is not a non-empty string or does not conform to the expected version format. * - * @param {string} tag - Git tag (e.g., 'v25.1.1', '25.1.1', or 'dev'). - * @returns {string} Normalized version (e.g., '25.1.1' or 'dev'). + * @param {string} tag - Git tag (for example, 'v25.1.1', '25.1.1', or 'dev'). + * @returns {string} Normalized version (for example, '25.1.1' or 'dev'). * @throws {Error} If `tag` is not a non-empty string or does not match the semantic version pattern. */ function normalizeTag(tag) { @@ -44,8 +44,8 @@ function normalizeTag(tag) { * * Accepts a semantic version like `25.1.1` and yields `25.1`. The special value * `'dev'` is returned unchanged. - * @param {string} version - Semantic version (e.g., `'25.1.1'`) or `'dev'`. - * @returns {string} The `major.minor` string (e.g., `'25.1'`) or `'dev'`. + * @param {string} version - Semantic version (for example, `'25.1.1'`) or `'dev'`. + * @returns {string} The `major.minor` string (for example, `'25.1'`) or `'dev'`. * @throws {Error} If `version` is not a non-empty string, lacks major/minor parts, or if major/minor are not numeric. */ function getMajorMinor(version) { @@ -554,13 +554,13 @@ function postProcessBundle(filePath, options, quiet = false) { * Bundle OpenAPI fragments for the specified API surface(s) from a repository tag and write the resulting bundled YAML files to disk. * * @param {Object} options - Configuration options. - * @param {string} options.tag - Git tag to checkout (e.g., 'v25.1.1'). + * @param {string} options.tag - Git tag to checkout (for example, 'v25.1.1'). * @param {'admin'|'connect'|'both'} options.surface - API surface to process. * @param {string} [options.output] - Standalone output file path; when provided, used for the single output file. * @param {string} [options.outAdmin] - Output path for the admin API when integrating with doc-tools mode. * @param {string} [options.outConnect] - Output path for the connect API when integrating with doc-tools mode. * @param {string} [options.repo] - Repository URL to clone (defaults to https://github.com/redpanda-data/redpanda.git). - * @param {string} [options.adminMajor] - Admin API major version string used for metadata (e.g., 'v2.0.0'). + * @param {string} [options.adminMajor] - Admin API major version string used for metadata (for example, 'v2.0.0'). * @param {boolean} [options.useAdminMajorVersion] - When true and processing the admin surface, use `adminMajor` for the bundle info.version. * @param {boolean} [options.quiet=false] - Suppress logging to stdout/stderr when true. * @returns {Object|Object[]} An object (for a single surface) or an array of objects (for both surfaces) with fields: @@ -625,9 +625,19 @@ async function bundleOpenAPI(options) { if (!quiet) { console.log('πŸ“₯ Cloning redpanda repository...'); } - - const repositoryUrl = repo || 'https://github.com/redpanda-data/redpanda.git'; - + + const { getAuthenticatedGitHubUrl, hasGitHubToken } = require('../cli-utils/github-token'); + + let repositoryUrl = repo || 'https://github.com/redpanda-data/redpanda.git'; + + // Use token if available for better rate limits and reliability + if (hasGitHubToken() && repositoryUrl.includes('github.com')) { + repositoryUrl = getAuthenticatedGitHubUrl(repositoryUrl); + if (!quiet) { + console.log('πŸ”‘ Using authenticated clone (token provided)'); + } + } + try { execSync(`git clone --depth 1 --branch ${tag} ${repositoryUrl} redpanda`, { cwd: tempDir, @@ -778,7 +788,7 @@ if (require.main === module) { program .name('bundle-openapi') .description('Bundle OpenAPI fragments from Redpanda repository') - .requiredOption('-t, --tag ', 'Git tag to checkout (e.g., v25.1.1)') + .requiredOption('-t, --tag ', 'Git tag to checkout (for example, v25.1.1)') .requiredOption('-s, --surface ', 'API surface', (value) => { if (!['admin', 'connect', 'both'].includes(value)) { throw new Error('Invalid API surface. Must be "admin", "connect", or "both"'); diff --git a/tools/cloud-regions/generate-cloud-regions.js b/tools/cloud-regions/generate-cloud-regions.js index 9b822f38..c57468ed 100644 --- a/tools/cloud-regions/generate-cloud-regions.js +++ b/tools/cloud-regions/generate-cloud-regions.js @@ -203,7 +203,7 @@ function processCloudRegions(yamlText) { * @param {string} options.repo - GitHub repository name. * @param {string} options.path - Path to the YAML file within the repository. * @param {string} [options.ref='main'] - Git reference (branch, tag, or commit SHA). - * @param {string} [options.format='md'] - The output format (e.g., 'md' for Markdown). + * @param {string} [options.format='md'] - The output format (for example, 'md' for Markdown). * @param {string} [options.token] - Optional GitHub token for authentication. * @param {string} [options.template] - Optional path to custom Handlebars template. * @returns {string} The rendered cloud regions output. diff --git a/tools/fetch-from-github.js b/tools/fetch-from-github.js index b812cd49..708edfd2 100644 --- a/tools/fetch-from-github.js +++ b/tools/fetch-from-github.js @@ -27,20 +27,34 @@ async function saveFile(content, saveDir, filename) { console.log(`Saved: ${target}`); } -async function fetchFromGithub(owner, repo, remotePath, saveDir, customFilename) { +/** + * Fetch file or directory from GitHub repository + * @param {string} owner - Repository owner (for example, "redpanda-data") + * @param {string} repo - Repository name (for example, "redpanda-operator") + * @param {string} remotePath - Path to file or directory in repository + * @param {string} saveDir - Local directory to save files to + * @param {string} [customFilename] - Optional custom filename (for single files) + * @param {string} [ref=null] - Git ref (branch, tag, or commit SHA). Defaults to repository's default branch if null + * @returns {Promise} + */ +async function fetchFromGithub(owner, repo, remotePath, saveDir, customFilename, ref = null) { const octokit = await loadOctokit(); try { - const resp = await octokit.repos.getContent({ owner, repo, path: remotePath }); + const params = { owner, repo, path: remotePath }; + if (ref) { + params.ref = ref; + } + const resp = await octokit.repos.getContent(params); if (Array.isArray(resp.data)) { // directory for (const item of resp.data) { if (item.type === 'file') { - await fetchFromGithub(owner, repo, item.path, saveDir, customFilename); + await fetchFromGithub(owner, repo, item.path, saveDir, customFilename, ref); } else if (item.type === 'dir') { // For directories, maintain the directory structure const nestedDir = path.join(saveDir, path.basename(item.path)); - await fetchFromGithub(owner, repo, item.path, nestedDir); + await fetchFromGithub(owner, repo, item.path, nestedDir, null, ref); } } } else { diff --git a/tools/gen-rpk-ascii.py b/tools/gen-rpk-ascii.py index ea2b6eb0..e1f42195 100644 --- a/tools/gen-rpk-ascii.py +++ b/tools/gen-rpk-ascii.py @@ -18,7 +18,9 @@ cmd_dict = {} -basic_commands_docker = ["docker","exec","-it"] +# Use 'docker exec' without -it flags for non-interactive/background execution +# -it flags require a TTY and fail in CI/background jobs +basic_commands_docker = ["docker", "exec"] basic_commands_docker.append("redpanda-1") rpk_basic_command = ["rpk"] diff --git a/tools/generate-cli-docs.js b/tools/generate-cli-docs.js new file mode 100755 index 00000000..f15a0271 --- /dev/null +++ b/tools/generate-cli-docs.js @@ -0,0 +1,325 @@ +#!/usr/bin/env node + +/** + * Generate CLI Reference Documentation + * + * This script automatically generates AsciiDoc documentation for the doc-tools CLI + * by executing commands with --help and parsing the output, then enhancing it with + * JSDoc comments from the source code. + */ + +const { execSync } = require('child_process'); +const fs = require('fs'); +const path = require('path'); + +/** + * Execute a CLI command and capture help output + */ +function getHelpOutput(command) { + try { + return execSync(`npx doc-tools ${command} --help`, { + encoding: 'utf8', + stdio: ['pipe', 'pipe', 'pipe'] + }); + } catch (error) { + // Commander exits with code 0 for help, but some shells treat it as error + return error.stdout || ''; + } +} + +/** + * Sanitize path values in option descriptions to remove user-specific absolute paths + */ +function sanitizePathInDescription(description) { + // Get the repository root path for relativization + const repoRoot = path.resolve(__dirname, '..'); + + let sanitized = description; + + // First, handle repository root paths specifically + const repoRootEscaped = repoRoot.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const repoRootPattern = new RegExp(repoRootEscaped, 'g'); + + sanitized = sanitized.replace(repoRootPattern, ''); + + // Then handle any remaining long absolute paths that contain our repo structure + // This regex matches paths that look like they're part of our repository + const longPathPattern = /\/[^\/\s"')]*docs-extensions-and-macros[^\/\s"')]*(?:\/[^\/\s"')\]]+)*/g; + + sanitized = sanitized.replace(longPathPattern, (match) => { + // If this path wasn't already replaced and looks like a subpath, make it relative + if (!match.includes('')) { + const relativePath = path.relative(repoRoot, match); + if (relativePath && !relativePath.startsWith('..') && relativePath !== match) { + return `./${relativePath}`; + } + return ''; + } + return match; + }); + + // Finally, handle generic home directory patterns for any remaining absolute paths + const homePattern = /\/(?:Users|home)\/[^\/\s"')]+/g; + sanitized = sanitized.replace(homePattern, '~'); + + return sanitized; +} + +/** + * Parse command help output into structured data + */ +function parseHelp(helpText) { + const lines = helpText.split('\n'); + const result = { + usage: '', + description: '', + options: [], + commands: [] + }; + + let currentSection = null; + let currentItem = null; + + for (const line of lines) { + // Usage line + if (line.startsWith('Usage:')) { + result.usage = line.replace('Usage:', '').trim(); + currentSection = null; + } + // Options section + else if (line === 'Options:') { + currentSection = 'options'; + } + // Commands section + else if (line === 'Commands:') { + currentSection = 'commands'; + } + // Option or command line (starts with spaces and has content) + else if (line.match(/^ \S/) && currentSection) { + // Match option/command name (everything up to 2+ spaces) and description + const match = line.match(/^ (.+?)\s{2,}(.*)/); + if (match) { + currentItem = { + name: match[1].trim(), + description: match[2].trim() + }; + result[currentSection].push(currentItem); + } + } + // Continuation of description (more indentation than option lines) + else if (line.match(/^\s{10,}/) && currentItem) { + currentItem.description += ' ' + line.trim(); + } + // Description (first non-empty line that's not a section) + else if (line.trim() && !currentSection && !result.description && !line.startsWith('Usage:')) { + result.description = line.trim(); + } + } + + return result; +} + +/** + * Parse JSDoc comments from source file + */ +function parseJSDocComments(sourceFile) { + const content = fs.readFileSync(sourceFile, 'utf8'); + const comments = {}; + + // Regex to match JSDoc comments followed by command definitions + // Matches both top-level commands and automation.command() + const pattern = /\/\*\*\s*([\s\S]*?)\s*\*\/\s*(?:programCli|automation)\s*\.command\(['"]([^'"]+)['"]\)/g; + + let match; + while ((match = pattern.exec(content)) !== null) { + const [, commentText, commandName] = match; + + // Parse the comment into sections + const parsed = { + description: '', + why: '', + example: '', + requirements: '' + }; + + // Extract sections + const descMatch = commentText.match(/@description\s*([\s\S]*?)(?=@\w+|$)/); + if (descMatch) { + parsed.description = descMatch[1] + .split('\n') + .map(line => line.replace(/^\s*\*\s*/, '').trim()) + .filter(line => line) + .join(' '); + } + + const whyMatch = commentText.match(/@why\s*([\s\S]*?)(?=@\w+|$)/); + if (whyMatch) { + parsed.why = whyMatch[1] + .split('\n') + .map(line => line.replace(/^\s*\*\s*/, '').trim()) + .filter(line => line) + .join(' '); + } + + const exampleMatch = commentText.match(/@example\s*([\s\S]*?)(?=@\w+|$)/); + if (exampleMatch) { + parsed.example = exampleMatch[1] + .split('\n') + .map(line => line.replace(/^\s*\*\s?/, '')) + .join('\n') + .trim(); + } + + const reqMatch = commentText.match(/@requirements\s*([\s\S]*?)(?=@\w+|$)/); + if (reqMatch) { + parsed.requirements = reqMatch[1] + .split('\n') + .map(line => line.replace(/^\s*\*\s?/, '')) + .join('\n') + .trim(); + } + + comments[commandName] = parsed; + } + + return comments; +} + +/** + * Generate AsciiDoc for a command + */ +function generateCommandDoc(commandName, helpData, jsdoc, level = 2) { + const heading = '='.repeat(level); + let doc = `${heading} ${commandName || 'doc-tools'}\n\n`; + + // Add extended description from JSDoc if available + if (jsdoc && jsdoc.description) { + doc += `${jsdoc.description}\n\n`; + } else if (helpData.description) { + doc += `${helpData.description}\n\n`; + } + + // Add "Why use it" section if available + if (jsdoc && jsdoc.why) { + doc += `*Why use it:*\n\n${jsdoc.why}\n\n`; + } + + if (helpData.usage) { + doc += `*Usage:*\n\n`; + doc += `[,bash]\n----\n${helpData.usage}\n----\n\n`; + } + + if (helpData.options.length > 0) { + doc += `*Options:*\n\n`; + helpData.options.forEach(opt => { + const name = opt.name.replace(/\s+/g, ' '); + const sanitizedDescription = sanitizePathInDescription(opt.description); + doc += `\`${name}\`::\n${sanitizedDescription}\n\n`; + }); + } + + if (helpData.commands.length > 0) { + doc += `*Commands:*\n\n`; + helpData.commands.forEach(cmd => { + const cmdName = cmd.name.split(' ')[0]; + doc += `* \`${cmdName}\` - ${cmd.description}\n`; + }); + doc += `\n`; + } + + // Add examples from JSDoc if available + if (jsdoc && jsdoc.example) { + doc += `*Examples:*\n\n[,bash]\n----\n${jsdoc.example}\n----\n\n`; + } + + // Add requirements from JSDoc if available + if (jsdoc && jsdoc.requirements) { + doc += `*Requirements:*\n\n${jsdoc.requirements}\n\n`; + } + + return doc; +} + +/** + * Main generation function + */ +function generateDocs() { + console.log('Generating CLI documentation...'); + + // Parse JSDoc comments from source + const sourceFile = path.join(__dirname, '..', 'bin', 'doc-tools.js'); + console.log(' Parsing JSDoc comments from source...'); + const jsdocs = parseJSDocComments(sourceFile); + console.log(` Found ${Object.keys(jsdocs).length} documented commands`); + + let doc = `= Doc Tools CLI Reference +:toc: +:toclevels: 3 + +Auto-generated reference documentation for the \`doc-tools\` command-line interface. + +IMPORTANT: This documentation is auto-generated. Do not edit manually. Run \`npm run generate:cli-docs\` to regenerate. + +`; + + // Get main help + const mainHelp = getHelpOutput(''); + const mainData = parseHelp(mainHelp); + doc += generateCommandDoc('', mainData, null, 2); + + // Top-level commands (excluding 'generate' which has subcommands) + const topLevelCommands = [ + 'install-test-dependencies', + 'get-redpanda-version', + 'get-console-version', + 'link-readme', + 'fetch', + 'setup-mcp' + ]; + + topLevelCommands.forEach(cmd => { + console.log(` Generating docs for: ${cmd}`); + const help = getHelpOutput(cmd); + const data = parseHelp(help); + const jsdoc = jsdocs[cmd]; + doc += generateCommandDoc(cmd, data, jsdoc, 2); + }); + + // Generate command and its subcommands + console.log(' Generating docs for: generate'); + const generateHelp = getHelpOutput('generate'); + const generateData = parseHelp(generateHelp); + doc += generateCommandDoc('generate', generateData, null, 2); + + // Generate subcommands + const generateSubcommands = [ + 'property-docs', + 'metrics-docs', + 'rpk-docs', + 'rpcn-connector-docs', + 'helm-spec', + 'cloud-regions', + 'crd-spec', + 'bundle-openapi' + ]; + + generateSubcommands.forEach(subcmd => { + console.log(` Generating docs for: generate ${subcmd}`); + const help = getHelpOutput(`generate ${subcmd}`); + const data = parseHelp(help); + const jsdoc = jsdocs[subcmd]; + doc += generateCommandDoc(`generate ${subcmd}`, data, jsdoc, 3); + }); + + // Write to file + const outputPath = path.join(__dirname, '..', 'CLI_REFERENCE.adoc'); + fs.writeFileSync(outputPath, doc); + console.log(`βœ“ Generated: ${outputPath}`); +} + +// Run if executed directly +if (require.main === module) { + generateDocs(); +} + +module.exports = { generateDocs }; diff --git a/tools/get-console-version.js b/tools/get-console-version.js index 31dccdfe..c437637b 100644 --- a/tools/get-console-version.js +++ b/tools/get-console-version.js @@ -24,9 +24,11 @@ module.exports = async function getConsoleVersion({ beta = false, fromAntora = f } // Initialize GitHub client + const { getGitHubToken } = require('../cli-utils/github-token'); const { Octokit } = await import('@octokit/rest'); - const octokit = process.env.REDPANDA_GITHUB_TOKEN - ? new Octokit({ auth: process.env.REDPANDA_GITHUB_TOKEN }) + const token = getGitHubToken(); + const octokit = token + ? new Octokit({ auth: token }) : new Octokit(); // Fetch latest release info diff --git a/tools/get-redpanda-version.js b/tools/get-redpanda-version.js index 218e5b81..8074e0c6 100644 --- a/tools/get-redpanda-version.js +++ b/tools/get-redpanda-version.js @@ -20,9 +20,11 @@ module.exports = async function getRedpandaVersion({ beta = false, fromAntora = } // Load Octokit + const { getGitHubToken } = require('../cli-utils/github-token'); const { Octokit } = await import('@octokit/rest'); - const octokit = process.env.REDPANDA_GITHUB_TOKEN - ? new Octokit({ auth: process.env.REDPANDA_GITHUB_TOKEN }) + const token = getGitHubToken(); + const octokit = token + ? new Octokit({ auth: token }) : new Octokit(); // Fetch version data diff --git a/tools/metrics/metrics.py b/tools/metrics/metrics.py index c6460a93..5822e5ad 100644 --- a/tools/metrics/metrics.py +++ b/tools/metrics/metrics.py @@ -158,9 +158,6 @@ def ensure_directory_exists(directory): # Resolve the base autogenerated folder at the repo root repo_root = os.getcwd() gen_path = os.path.join(repo_root, "autogenerated") - if not os.path.isdir(gen_path): - logging.error(f"autogenerated folder not found at: {gen_path}") - sys.exit(1) # Build the output directory using the already provided tag_modified. output_dir = os.path.join(gen_path, tag_modified, "metrics") diff --git a/tools/property-extractor/Makefile b/tools/property-extractor/Makefile index d5d0a001..392d1d07 100644 --- a/tools/property-extractor/Makefile +++ b/tools/property-extractor/Makefile @@ -78,7 +78,13 @@ redpanda-git: @if [ -d "$(REDPANDA_SRC)" ]; then \ git -C "$(REDPANDA_SRC)" fetch --all --tags -q; \ else \ - git clone -q https://github.com/redpanda-data/redpanda.git "$(REDPANDA_SRC)"; \ + GH_TOKEN=$${REDPANDA_GITHUB_TOKEN:-$${GITHUB_TOKEN:-$${GH_TOKEN}}}; \ + if [ -n "$$GH_TOKEN" ]; then \ + echo "πŸ”‘ Using authenticated clone (token provided)"; \ + git clone -q https://$$GH_TOKEN@github.com/redpanda-data/redpanda.git "$(REDPANDA_SRC)"; \ + else \ + git clone -q https://github.com/redpanda-data/redpanda.git "$(REDPANDA_SRC)"; \ + fi; \ fi; \ if git -C "$(REDPANDA_SRC)" rev-parse --verify -q "$(TAG)" >/dev/null; then \ echo "πŸ”– Checking out '$(TAG)'"; \ diff --git a/tools/property-extractor/cloud_config.py b/tools/property-extractor/cloud_config.py index 9693a2f4..e57fbad2 100644 --- a/tools/property-extractor/cloud_config.py +++ b/tools/property-extractor/cloud_config.py @@ -117,7 +117,7 @@ def fetch_cloud_config(github_token: Optional[str] = None) -> CloudConfig: GitHubAuthError: Authentication or access problems with the GitHub API (including 401/403 responses). NetworkError: Network connectivity or timeout failures when contacting the GitHub API. CloudConfigParsingError: Failure to parse or validate the repository YAML files or their expected structure. - CloudConfigError: Generic configuration error (e.g., missing token) or unexpected internal failures. + CloudConfigError: Generic configuration error (for example, missing token) or unexpected internal failures. """ if not github_token: github_token = os.environ.get('GITHUB_TOKEN') or os.environ.get('REDPANDA_GITHUB_TOKEN') @@ -246,10 +246,10 @@ def fetch_cloud_config(github_token: Optional[str] = None) -> CloudConfig: logger.warning(f"Skipping file with missing name/url: {file}") continue - # Look for version YAML files (e.g., "25.1.yml", "25.2.yml") + # Look for version YAML files (for example, "25.1.yml", "25.2.yml") if file_name.endswith('.yml'): version_part = file_name.replace('.yml', '') - # Check if it looks like a version number (e.g., "25.1", "25.2.1") + # Check if it looks like a version number (for example, "25.1", "25.2.1") if version_part.replace('.', '').isdigit(): version_files.append((version_part, download_url)) logger.debug(f"Found version file: {file_name} -> {version_part}") @@ -282,7 +282,7 @@ def fetch_cloud_config(github_token: Optional[str] = None) -> CloudConfig: "No valid version files found in cloudv2/install-pack directory.\n" f"Found {len(version_files)} files but none had valid version formats.\n" f"Available files: {[v[0] for v in version_files]}\n" - "Expected version format: 'X.Y' or 'X.Y.Z' (e.g., '25.1', '25.2.1')\n" + "Expected version format: 'X.Y' or 'X.Y.Z' (for example, '25.1', '25.2.1')\n" "Contact cloud team to verify configuration file naming convention." ) logger.error(error_msg) diff --git a/tools/property-extractor/constant_resolver.py b/tools/property-extractor/constant_resolver.py index a233b45a..bb48c117 100644 --- a/tools/property-extractor/constant_resolver.py +++ b/tools/property-extractor/constant_resolver.py @@ -2,7 +2,7 @@ """ Resolves C++ constant references to their actual values. -For properties that use constants as default values (e.g., `ss::sstring{net::tls_v1_2_cipher_suites}`), +For properties that use constants as default values (for example, `ss::sstring{net::tls_v1_2_cipher_suites}`), this module looks up the constant definition and extracts the actual string value. """ @@ -30,7 +30,7 @@ def resolve_constant(self, constant_name: str) -> Optional[str]: Resolve a C++ constant name to its actual string value. Args: - constant_name: The constant name (e.g., "net::tls_v1_2_cipher_suites" or "tls_v1_2_cipher_suites") + constant_name: The constant name (for example, "net::tls_v1_2_cipher_suites" or "tls_v1_2_cipher_suites") Returns: The actual string value, or None if not found @@ -72,8 +72,8 @@ def _search_in_files(self, patterns: list[str], identifier: str) -> Optional[str Search for a constant definition in files matching the given patterns. Args: - patterns: List of file patterns to search (e.g., ['net/tls.cc']) - identifier: The constant identifier (e.g., 'tls_v1_2_cipher_suites') + patterns: List of file patterns to search (for example, ['net/tls.cc']) + identifier: The constant identifier (for example, 'tls_v1_2_cipher_suites') Returns: The constant's string value, or None if not found @@ -111,7 +111,7 @@ def resolve_array_constant(self, array_name: str) -> Optional[list]: - constexpr std::array array_name = {val1, val2, val3}; Args: - array_name: The array constant name (e.g., "supported_sasl_mechanisms") + array_name: The array constant name (for example, "supported_sasl_mechanisms") Returns: List of string values from the array, or None if not found @@ -230,7 +230,7 @@ def _resolve_authenticator_name(self, class_ref: str) -> Optional[dict]: Finds the class definition and extracts the `static constexpr const char* name` value. Args: - class_ref: Qualified class name (e.g., "security::scram_sha256_authenticator") + class_ref: Qualified class name (for example, "security::scram_sha256_authenticator") Returns: Dict with 'value' and 'is_enterprise' keys, or None if not found @@ -420,12 +420,12 @@ def resolve_validator_enum_constraint(validator_name: str, resolver: ConstantRes For validators like validate_sasl_mechanisms, this function: 1. Finds the validator function in validators.cc - 2. Parses it to find what constant array it validates against (e.g., supported_sasl_mechanisms) + 2. Parses it to find what constant array it validates against (for example, supported_sasl_mechanisms) 3. Resolves that array to get the actual enum values - 4. Checks for enterprise values (e.g., enterprise_sasl_mechanisms) + 4. Checks for enterprise values (for example, enterprise_sasl_mechanisms) Args: - validator_name: Name of the validator function (e.g., "validate_sasl_mechanisms") + validator_name: Name of the validator function (for example, "validate_sasl_mechanisms") resolver: ConstantResolver instance Returns: @@ -521,8 +521,8 @@ def resolve_runtime_validation_enum_constraint(property_name: str, defined_in: s } Args: - property_name: Name of the property (e.g., "sasl_mechanism") - defined_in: Path where property is defined (e.g., "src/v/kafka/client/configuration.cc") + property_name: Name of the property (for example, "sasl_mechanism") + defined_in: Path where property is defined (for example, "src/v/kafka/client/configuration.cc") resolver: ConstantResolver instance Returns: diff --git a/tools/property-extractor/property_extractor.py b/tools/property-extractor/property_extractor.py index a6893ed0..1d4d0322 100644 --- a/tools/property-extractor/property_extractor.py +++ b/tools/property-extractor/property_extractor.py @@ -233,7 +233,7 @@ def _extract_namespace_for_function(self, content, position): position (int): Position in the file Returns: - str: Namespace (e.g., "model" or "config::tls") + str: Namespace (for example, "model" or "config::tls") """ # Look backwards from position to find namespace declaration preceding = content[:position] @@ -403,7 +403,7 @@ def process_enterprise_value(enterprise_str): 2. C++ scoped enum-like tokens (foo::bar::BAZ) β†’ "BAZ". 3. Lambda expressions (strings starting with "[](" and ending with "}") β†’ a short human-readable hint such as "Enterprise feature enabled" or context-specific text. - 4. Simple literal values (e.g., "true", "false", "OIDC", or quoted strings) β†’ returned as-is. + 4. Simple literal values (for example, "true", "false", "OIDC", or quoted strings) β†’ returned as-is. Parameters: enterprise_str (str): Raw C++ expression text to be converted. @@ -496,7 +496,7 @@ def resolve_cpp_function_call(function_name): functions from source using general patterns. No hardcoded patterns needed. Parameters: - function_name (str): Fully-qualified C++ function name to resolve (e.g., "model::kafka_audit_logging_topic") + function_name (str): Fully-qualified C++ function name to resolve (for example, "model::kafka_audit_logging_topic") Returns: str or None: The literal string returned by the C++ function, or None if not found in cache @@ -509,7 +509,7 @@ def resolve_cpp_function_call(function_name): logger.debug(f"Resolved function '{function_name}' -> '{cached_result}' from cache") return cached_result - # Also try without namespace qualifier (e.g., "kafka_audit_logging_topic") + # Also try without namespace qualifier (for example, "kafka_audit_logging_topic") if '::' in function_name: simple_name = function_name.split('::')[-1] cached_result = _constexpr_cache.lookup_function(simple_name) @@ -529,7 +529,7 @@ def resolve_constexpr_identifier(identifier): Searches common Redpanda source locations for constexpr string or string_view definitions matching the given identifier and returns the literal if found. Parameters: - identifier (str): The identifier name to resolve (e.g., "scram" or "net::tls_v1_2_cipher_suites"). + identifier (str): The identifier name to resolve (for example, "scram" or "net::tls_v1_2_cipher_suites"). Returns: str or None: The resolved literal string value if found, otherwise `None`. @@ -546,7 +546,7 @@ def resolve_constexpr_identifier(identifier): logger.debug(f"Could not find Redpanda source directory to resolve identifier: {identifier}") return None - # Strip namespace qualifier if present (e.g., "net::tls_v1_2_cipher_suites" -> "tls_v1_2_cipher_suites") + # Strip namespace qualifier if present (for example, "net::tls_v1_2_cipher_suites" -> "tls_v1_2_cipher_suites") search_identifier = identifier.split('::')[-1] if '::' in identifier else identifier # Pattern to match constexpr string_view definitions @@ -1567,7 +1567,7 @@ def resolve_type_and_default(properties, definitions): └───────────────────────────────────────────────────────────────────────── HOW TO ADD NEW TYPE DEFINITIONS: - 1. Identify the C++ type that needs a definition (e.g., new_endpoint_type) + 1. Identify the C++ type that needs a definition (for example, new_endpoint_type) 2. Analyze the C++ struct/class to determine JSON schema structure 3. Add entry to definitions.json with appropriate JSON Schema: { @@ -1662,10 +1662,10 @@ def process_cpp_patterns(arg_str): This function recognises common C++ patterns produced by the extractor and maps them to values suitable for JSON schema defaults and examples. Handled cases include: - std::nullopt -> null - - zero-argument functions (e.g., model::kafka_audit_logging_topic()) resolved from source when possible - - enum tokens (e.g., fips_mode_flag::disabled -> "disabled") + - zero-argument functions (for example, model::kafka_audit_logging_topic()) resolved from source when possible + - enum tokens (for example, fips_mode_flag::disabled -> "disabled") - constexpr identifiers and simple string constructors resolved to their literal strings when available - - known default constructors and truncated type names mapped to sensible defaults (e.g., duration -> 0, path -> "") + - known default constructors and truncated type names mapped to sensible defaults (for example, duration -> 0, path -> "") - simple heuristics for unknown constructors and concatenated expressions Returns: @@ -1722,7 +1722,7 @@ def process_cpp_patterns(arg_str): base = base + "s" return f'"{num} {base}"' - # Evaluate arithmetic in duration constructors (e.g., "60 * 5" -> "300 seconds") + # Evaluate arithmetic in duration constructors (for example, "60 * 5" -> "300 seconds") if "*" in value: try: result = safe_arithmetic_eval(value) @@ -1854,12 +1854,12 @@ def expand_default(type_name, default_str): - Integer and boolean literals β†’ Python int and bool. - Object constructors (Type(arg1, arg2) or Type{...}) β†’ dict mapping constructor arguments to the object's properties when a corresponding type definition exists. - Nested constructors β†’ nested dicts with their fields expanded. - - Array initializer lists (e.g., {Type(...), Type(...)}) β†’ Python list with each element expanded. + - Array initializer lists (for example, {Type(...), Type(...)}) β†’ Python list with each element expanded. - Special-case mappings for known type patterns (for example, an address-type constructor expanded into {"address", "port"} when the target type expects that shape). If a default cannot be resolved or the type is an enum, the original input is returned unchanged; the string "null" is converted to None. If default_str is not a string, it is returned as-is. Parameters: - type_name (str): The resolved type name for the default value (e.g., "model::broker_endpoint" or a primitive type like "string"). + type_name (str): The resolved type name for the default value (for example, "model::broker_endpoint" or a primitive type like "string"). default_str (str | any): The C++ default expression to expand, or a non-string value already decoded. Returns: @@ -1881,7 +1881,7 @@ def expand_default(type_name, default_str): else: return processed - # Handle string type with constructor syntax (e.g., ss::sstring{scram}) + # Handle string type with constructor syntax (for example, ss::sstring{scram}) if type_name == "string" and ("{" in default_str or "(" in default_str): tname, args = parse_constructor(default_str) if tname and args: @@ -1897,7 +1897,7 @@ def expand_default(type_name, default_str): type_def = resolve_definition_type(resolve_type_with_namespace(type_name, definitions)) if "enum" in type_def: # Strip C++ namespace qualifiers from enum values - # e.g., model::partition_autobalancing_mode::continuous β†’ continuous + # for example, model::partition_autobalancing_mode::continuous β†’ continuous if isinstance(default_str, str) and '::' in default_str: return default_str.split('::')[-1] return default_str @@ -2775,7 +2775,7 @@ def generate_options(): # Load overrides file (contains both property and definition overrides) overrides = None - if options.overrides: + if options.overrides and os.path.exists(options.overrides): try: with open(options.overrides) as f: overrides = json.load(f) @@ -2790,6 +2790,8 @@ def generate_options(): except Exception as e: logging.error(f"Failed to load overrides file: {e}") sys.exit(1) + elif options.overrides: + logger.info(f"Overrides file not found: {options.overrides} (skipping)") # DEPRECATED: Support legacy --definitions flag for backward compatibility # Users should migrate to putting definitions in overrides.json under "definitions" key diff --git a/tools/property-extractor/topic_property_extractor.py b/tools/property-extractor/topic_property_extractor.py index d46964a4..6a623575 100644 --- a/tools/property-extractor/topic_property_extractor.py +++ b/tools/property-extractor/topic_property_extractor.py @@ -444,8 +444,8 @@ def _is_valid_topic_property(self, prop_name: str) -> bool: if not re.match(r'^[a-zA-Z][a-zA-Z0-9._-]*$', prop_name): return False - # Reject Java-style package names (e.g., "redpanda.core.admin.Service") - # Topic properties use lowercase with dots (e.g., "cleanup.policy", "segment.ms") + # Reject Java-style package names (for example, "redpanda.core.admin.Service") + # Topic properties use lowercase with dots (for example, "cleanup.policy", "segment.ms") # Split by dots and check each segment - reject if any segment after first has uppercase segments = prop_name.split('.') for i, segment in enumerate(segments): diff --git a/tools/property-extractor/transformers.py b/tools/property-extractor/transformers.py index 465038ac..0e355e8e 100644 --- a/tools/property-extractor/transformers.py +++ b/tools/property-extractor/transformers.py @@ -23,7 +23,7 @@ - info["declaration"]: Full C++ type declaration - info["params"]: List of parsed constructor parameters - info["name_in_file"]: C++ variable name -- info["type"]: Property template type (e.g., "property", "enterprise_property") +- info["type"]: Property template type (for example, "property", "enterprise_property") OUTPUT FORMAT (PropertyBag): - Complete JSON schema-compatible property definition @@ -511,7 +511,7 @@ def get_meta_value(info, key, default=None): Args: info (dict): Property info dictionary to search for metadata - key (str): Metadata key to extract (e.g., "needs_restart", "visibility") + key (str): Metadata key to extract (for example, "needs_restart", "visibility") default (any): Default value if key not found or metadata missing Returns: @@ -1254,7 +1254,7 @@ def extract_template_content(text, template_name): Parameters: text (str): The string to search for the template. - template_name (str): The template name (e.g., "std::vector" or "property"). + template_name (str): The template name (for example, "std::vector" or "property"). Returns: str or None: The substring inside the outermost angle brackets for the matched template (excluding the brackets), @@ -2068,7 +2068,7 @@ def _clean_value(self, val): if len(val) >= 2 and val[0] == '"' and val[-1] == '"': val = val[1:-1] # Strip C++ namespace qualifiers from enum values - # e.g., model::partition_autobalancing_mode::continuous β†’ continuous + # for example, model::partition_autobalancing_mode::continuous β†’ continuous if '::' in val: val = val.split('::')[-1] return val @@ -2286,7 +2286,7 @@ class ValidatorEnumExtractor: Analyzes validator functions to extract enum constraints for array-typed properties. For example, if sasl_mechanisms uses validate_sasl_mechanisms, this transformer: 1. Finds the validator function in validators.cc - 2. Identifies the constraint array (e.g., supported_sasl_mechanisms) + 2. Identifies the constraint array (for example, supported_sasl_mechanisms) 3. Resolves that array to get the actual enum values 4. Adds them to property['items']['enum'] @@ -2294,7 +2294,7 @@ class ValidatorEnumExtractor: 1. Detects array properties (type="array") with validator parameters 2. Extracts validator function name from params 3. Parses validator to find constraint array - 4. Resolves array to get enum values (e.g., ["SCRAM", "GSSAPI", "OAUTHBEARER", "PLAIN"]) + 4. Resolves array to get enum values (for example, ["SCRAM", "GSSAPI", "OAUTHBEARER", "PLAIN"]) 5. Sets property['items']['enum'] with the discovered values DOWNSTREAM USAGE: @@ -2408,7 +2408,7 @@ class RuntimeValidationEnumExtractor: PROCESSING: 1. Detects string properties (not arrays) without validator parameters 2. Searches the source file for validation functions that reference the property - 3. Parses comparison patterns (e.g., property != constant1 && property != constant2) + 3. Parses comparison patterns (for example, property != constant1 && property != constant2) 4. Resolves constants to actual string values 5. Sets property['enum'] with discovered values diff --git a/tools/property-extractor/type_definition_extractor.py b/tools/property-extractor/type_definition_extractor.py index 64885a85..5b965064 100644 --- a/tools/property-extractor/type_definition_extractor.py +++ b/tools/property-extractor/type_definition_extractor.py @@ -281,7 +281,7 @@ def search_content(search_content): if cc_path.exists(): files_to_search.append(cc_path) - # Also look for parent directory's main .cc file (e.g., model/model.cc) + # Also look for parent directory's main .cc file (for example, model/model.cc) parent_dir = file_path.parent parent_cc = parent_dir / f"{parent_dir.name}.cc" if parent_cc.exists() and parent_cc != cc_path: @@ -364,7 +364,7 @@ def _resolve_alias_type(self, alias_type): Resolve a C++ type alias to a JSON schema type. Args: - alias_type (str): The C++ type expression (e.g., "named_type") + alias_type (str): The C++ type expression (for example, "named_type") Returns: str: JSON schema type (integer, string, etc.) or None if unknown @@ -525,7 +525,7 @@ def _extract_fields(self, struct_body): # Convert C++ type to JSON schema type json_type = self._cpp_type_to_json_type(return_type) - # Use method name as field name (e.g., host() becomes "host") + # Use method name as field name (for example, host() becomes "host") properties[method_name] = {"type": json_type} continue @@ -555,7 +555,7 @@ def _extract_namespace(self, content, position): position (int): Position in the file Returns: - str: Namespace (e.g., "model" or "config::tls") + str: Namespace (for example, "model" or "config::tls") """ # Look backwards from position to find namespace declaration preceding = content[:position] diff --git a/tools/redpanda-connect/README.adoc b/tools/redpanda-connect/README.adoc index 70fa8092..3f51a28f 100644 --- a/tools/redpanda-connect/README.adoc +++ b/tools/redpanda-connect/README.adoc @@ -525,7 +525,7 @@ Overrides support: * *Manual override maintenance*: Custom descriptions must be maintained in overrides.json * *Example quality*: Auto-generated examples may not represent best practices -* *Duplicate YAML keys*: Some examples contain multiple configurations in one block, creating invalid YAML with duplicate top-level keys (e.g., multiple `processors:` keys) +* *Duplicate YAML keys*: Some examples contain multiple configurations in one block, creating invalid YAML with duplicate top-level keys (for example, multiple `processors:` keys) * *CSV catalog dependency*: Relies on manually maintained CSV in Connect repository * *Version detection timing*: Must be called during Antora build to access component attributes diff --git a/tools/redpanda-connect/generate-rpcn-connector-docs.js b/tools/redpanda-connect/generate-rpcn-connector-docs.js index 12deb112..4932f26b 100644 --- a/tools/redpanda-connect/generate-rpcn-connector-docs.js +++ b/tools/redpanda-connect/generate-rpcn-connector-docs.js @@ -195,7 +195,7 @@ function resolveReferences(obj, root) { * * @param {Object} options - Configuration options for documentation generation. * @param {string} options.data - Path to the connector data file (JSON or YAML). - * @param {string} [options.overrides] - Optional path to a JSON file with override data. Supports $ref references in JSON Pointer format (e.g., "#/definitions/client_certs"). + * @param {string} [options.overrides] - Optional path to a JSON file with override data. Supports $ref references in JSON Pointer format (for example, "#/definitions/client_certs"). * @param {string} options.template - Path to the main Handlebars template. * @param {string} [options.templateIntro] - Path to the intro partial template (used in full draft mode). * @param {string} [options.templateFields] - Path to the fields partial template. @@ -253,7 +253,7 @@ async function generateRpcnConnectorDocs(options) { markBeta(dataObj); // Apply overrides if provided - if (overrides) { + if (overrides && fs.existsSync(overrides)) { const ovRaw = fs.readFileSync(overrides, 'utf8'); const ovObj = JSON.parse(ovRaw); // Resolve any $ref references in the overrides @@ -281,9 +281,11 @@ async function generateRpcnConnectorDocs(options) { } } } + } else if (overrides) { + console.log(`Overrides file not found: ${overrides} (skipping)`); } - // Compile the β€œmain” template (used when writeFullDrafts = true) + // Compile the "main" template (used when writeFullDrafts = true) const compiledTemplate = handlebars.compile(fs.readFileSync(template, 'utf8')); // Determine which templates to use for β€œfields” and β€œexamples”