diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 912fad9a7d770..9b7fa0951b6f1 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,13 +2,14 @@ .github/workflows/regression.yml @vectordotdev/vector @vectordotdev/single-machine-performance regression/config.yaml @vectordotdev/vector @vectordotdev/single-machine-performance -docs/ @vectordotdev/ux-team @vectordotdev/documentation -website/ @vectordotdev/ux-team -website/content @vectordotdev/documentation -website/cue/reference @vectordotdev/documentation -website/js @vectordotdev/vector-website -website/layouts @vectordotdev/vector-website -website/scripts @vectordotdev/vector-website -website/data @vectordotdev/vector-website -website/* @vectordotdev/vector-website +docs/ @vectordotdev/vector @vectordotdev/ux-team @vectordotdev/documentation +website/ @vectordotdev/vector @vectordotdev/ux-team +website/content @vectordotdev/vector @vectordotdev/documentation +website/cue/reference @vectordotdev/vector @vectordotdev/documentation + +website/js @vectordotdev/vector @vectordotdev/vector-website +website/layouts @vectordotdev/vector @vectordotdev/vector-website +website/scripts @vectordotdev/vector @vectordotdev/vector-website +website/data @vectordotdev/vector @vectordotdev/vector-website +website/* @vectordotdev/vector @vectordotdev/vector-website diff --git a/.github/DISCUSSION_TEMPLATE/q-a.yml b/.github/DISCUSSION_TEMPLATE/q-a.yml new file mode 100644 index 0000000000000..23cb1f8467745 --- /dev/null +++ b/.github/DISCUSSION_TEMPLATE/q-a.yml @@ -0,0 +1,33 @@ +title: "Q&A" +labels: [ q-a ] +body: + - type: markdown + attributes: + value: | + Please fill out the following fields to help us assist you effectively. + + - type: textarea + id: question + attributes: + label: Question + description: What are you trying to do? What issue are you encountering? + + - type: textarea + id: config + attributes: + label: Vector Config + description: Your Vector configuration (please redact sensitive data) + placeholder: | + ```yaml + # your config + ``` + + - type: textarea + id: logs + attributes: + label: Vector Logs + description: Paste any relevant Vector logs or error messages. + placeholder: | + ```sh + Jul 10 14:32:02 vector[1234]: ERROR ... + ``` diff --git a/.github/ISSUE_TEMPLATE/minor-release.md b/.github/ISSUE_TEMPLATE/minor-release.md index 50dabfcc72139..5d34d0c63143f 100644 --- a/.github/ISSUE_TEMPLATE/minor-release.md +++ b/.github/ISSUE_TEMPLATE/minor-release.md @@ -5,25 +5,46 @@ title: "Vector [version] release" labels: "domain: releasing" --- -The week before the release: + +# Setup and Automation + +Note the preparation steps are now automated. First, alter/create release.env + +```shell +export NEW_VECTOR_VERSION= # replace this with the actual new version +export MINOR_VERSION=$(echo "NEW_VECTOR_VERSION" | cut -d. -f2) +export PREP_BRANCH=prepare-v-0-"${MINOR_VERSION}"-"${NEW_VECTOR_VERSION}"-website +export RELEASE_BRANCH=v0."${MINOR_VERSION}" +export NEW_VRL_VERSION= # replace this with the actual new VRL version +``` + +and then source it by running `source ./release.env` + +# The week before the release + +## 1. Manual Steps - [ ] Cut a new release of [VRL](https://github.com/vectordotdev/vrl) if needed -- [ ] Check for any outstanding deprecation actions in [DEPRECATIONS.md](https://github.com/vectordotdev/vector/blob/master/docs/DEPRECATIONS.md) and - take them (or have someone help you take them) + - VRL release steps: https://github.com/vectordotdev/vrl/blob/main/release/README.md + +## 2. Automated Steps + +Run the following: + +```shell +cargo vdev release prepare --version "${NEW_VECTOR_VERSION}" --vrl-version "${NEW_VRL_VERSION}" +``` + +Automated steps include: - [ ] Create a new release branch from master to freeze commits - - `git fetch && git checkout origin/master && git checkout -b v0. && git push -u` + - `git fetch && git checkout origin/master && git checkout -b "{RELEASE_BRANCH}" && git push -u` - [ ] Create a new release preparation branch from `master` - - `git checkout -b website-prepare-v0. && git push -u` + - `git checkout -b "${PREP_BRANCH}" && git push -u` - [ ] Pin VRL to latest released version rather than `main` - [ ] Check if there is a newer version of [Alpine](https://alpinelinux.org/releases/) or [Debian](https://www.debian.org/releases/) available to update the release images in `distribution/docker/`. Update if so. - [ ] Run `cargo vdev build release-cue` to generate a new cue file for the release - - [ ] Add description key to the generated cue file with a description of the release (see - previous releases for examples). - - [ ] Ensure any breaking changes are highlighted in the release upgrade guide - - [ ] Ensure any deprecations are highlighted in the release upgrade guide - - [ ] Review generated changelog entries to ensure they are understandable to end-users - [ ] Copy VRL changelogs from the VRL version in the last Vector release as a new changelog entry ([example](https://github.com/vectordotdev/vector/blob/9c67bba358195f5018febca2f228dfcb2be794b5/website/cue/reference/releases/0.41.0.cue#L33-L64)) - [ ] Update version number in `website/cue/reference/administration/interfaces/kubectl.cue` @@ -32,35 +53,48 @@ The week before the release: - [ ] Create new release md file by copying an existing one in `./website/content/en/releases/` and updating version number - [ ] Commit these changes -- [ ] Open PR against the release branch (`v0.`) for review -- [ ] PR approval +- [ ] Open PR against the release branch (`"${RELEASE_BRANCH}"`) for review + +## 3. Manual Steps + +- [ ] Edit `website/cue/reference/releases/"${NEW_VECTOR_VERSION}".cue` + - [ ] Add description key to the generated cue file with a description of the release (see + previous releases for examples). + - [ ] Ensure any breaking changes are highlighted in the release upgrade guide + - [ ] Ensure any deprecations are highlighted in the release upgrade guide + - [ ] Review generated changelog entries to ensure they are understandable to end-users +- [ ] Check for any outstanding deprecation actions in [DEPRECATIONS.md](https://github.com/vectordotdev/vector/blob/master/docs/DEPRECATIONS.md) and + take them (or have someone help you take them) +- [ ] PR review & approval -On the day of release: +# On the day of release - [ ] Rebase the release preparation branch on the release branch - [ ] Squash the release preparation commits (but not the cherry-picked commits!) to a single - commit. This makes it easier to cherry-pick to master after the release.  + commit. This makes it easier to cherry-pick to master after the release. - [ ] Ensure release date in cue matches current date. - [ ] Merge release preparation branch into the release branch - - `git co v0. && git merge --ff-only prepare-v0.` + - `git switch "${RELEASE_BRANCH}" && git merge --ff-only "${PREP_BRANCH}"` - [ ] Tag new release - - [ ] `git tag v0..0 -a -m v0..0` - - [ ] `git push origin v0..0` + - [ ] `git tag v"${NEW_VECTOR_VERSION}" -a -m v"${NEW_VECTOR_VERSION}"` + - [ ] `git push origin v"${NEW_VECTOR_VERSION}"` - [ ] Wait for release workflow to complete - - Discoverable via [https://github.com/timberio/vector/actions/workflows/release.yml](https://github.com/timberio/vector/actions/workflows/release.yml) + - Discoverable via [release.yml](https://github.com/vectordotdev/vector/actions/workflows/release.yml) - [ ] Reset the `website` branch to the `HEAD` of the release branch to update https://vector.dev - - [ ] `git checkout website && git reset --hard origin/v0. && git push` + - [ ] `git switch website && git reset --hard origin/"${RELEASE_BRANCH}" && git push` - [ ] Confirm that the release changelog was published to https://vector.dev/releases/ - The deployment is done by Amplify. You can see the [deployment logs here](https://dd-corpsite.datadoghq.com/logs?query=service%3Awebsites-vector%20branch%3Awebsite&agg_m=count&agg_m_source=base&agg_t=count&cols=host%2Cservice&fromUser=true&messageDisplay=inline&refresh_mode=sliding&storage=hot&stream_sort=time%2Casc&viz=stream). - [ ] Release Linux packages. See [`vector-release` usage](https://github.com/DataDog/vector-release#usage). - - Note: the pipeline inputs are the version number `v0.` and a personal GitHub token. + - Note: the pipeline inputs are the version number `v"${NEW_VECTOR_VERSION}"` and a personal GitHub token. - [ ] Manually trigger the `trigger-package-release-pipeline-prod-stable` job. - [ ] Release updated Helm chart. See [releasing Helm chart](https://github.com/vectordotdev/helm-charts#releasing). - [ ] Once Helm chart is released, updated Vector manifests - Run `cargo vdev build manifests` and open a PR with changes -- [ ] Add docker images to [https://github.com/DataDog/images](https://github.com/DataDog/images/tree/master/vector) to have them available internally. -- [ ] Cherry-pick any release commits from the release branch that are not on `master`, to `master` -- [ ] Bump the release number in the `Cargo.toml` on master to the next major release. - - Also, update `Cargo.lock` with: `cargo update -p vector` +- [ ] Add docker images to [https://github.com/DataDog/images](https://github.com/DataDog/images/tree/master/vector) to have them available internally. ([Example PR](https://github.com/DataDog/images/pull/7104)) +- [ ] Create a new PR with title starting as `chore(releasing):` + - [ ] Cherry-pick any release commits from the release branch that are not on `master`, to `master` + - [ ] Bump the release number in the `Cargo.toml` on master to the next minor release. + - [ ] Also, update `Cargo.lock` with: `cargo update -p vector` + - [ ] If there is a VRL version update, revert it and make it track the git `main` branch and then run `cargo update -p vrl`. - [ ] Kick-off post-mortems for any regressions resolved by the release diff --git a/.github/ISSUE_TEMPLATE/patch-release.md b/.github/ISSUE_TEMPLATE/patch-release.md index 185d5beff7917..afc5d8d826423 100644 --- a/.github/ISSUE_TEMPLATE/patch-release.md +++ b/.github/ISSUE_TEMPLATE/patch-release.md @@ -5,10 +5,22 @@ title: "Vector [version] release" labels: "domain: releasing" --- -Before the release: +# Setup environment + +```shell +export CURRENT_MINOR_VERSION = # e.g. 47 +export CURRENT_PATCH_VERSION = # e.g. 0 +export CURRENT_VERSION="${RELEASE_BRANCH}"."${CURRENT_PATCH_VERSION}" +export NEW_PATCH_VERSION = # e.g. 1 +export NEW_VERSION="${RELEASE_BRANCH}"."${NEW_PATCH_VERSION}" +export RELEASE_BRANCH=v0."${CURRENT_MINOR_VERSION}" +export PREP_BRANCH=prepare-v-0-"${CURRENT_MINOR_VERSION}"-"${NEW_PATCH_VERSION}"-website +``` + +# Before the release - [ ] Create a new release preparation branch from the current release branch - - `git fetch --all && git checkout v0. && git checkout -b website-prepare-v0-- + - `git fetch --all && git checkout "${RELEASE_BRANCH}" && git checkout -b "${PREP_BRANCH}""` - [ ] Cherry-pick in all commits to be released from the associated release milestone - If any merge conflicts occur, attempt to solve them and if needed enlist the aid of those familiar with the conflicting commits. - [ ] Bump the release number in the `Cargo.toml` to the current version number @@ -18,31 +30,31 @@ Before the release: - [ ] Update version number in `distribution/install.sh` - [ ] Add new version to `website/cue/reference/versions.cue` - [ ] Create new release md file by copying an existing one in `./website/content/en/releases/`. - - Update the version number to `v0..` and increase the `weight` by 1. + - Update the version number to `"${NEW_VERSION}"` and increase the `weight` by 1. - [ ] Run `cargo check` to regenerate `Cargo.lock` file - [ ] Commit these changes -- [ ] Open PR against the release branch (`v0.`) for review +- [ ] Open PR against the release branch (`"${RELEASE_BRANCH}"`) for review - [ ] PR approval -On the day of release: +# On the day of release - [ ] Ensure release date in cue matches current date. - [ ] Rebase the release preparation branch on the release branch - Squash the release preparation commits (but not the cherry-picked commits!) to a single commit. This makes it easier to cherry-pick to master after the release. - - `git fetch --all && git checkout website-prepare-v0-- && git rebase -i v0.` + - `git fetch --all && git checkout website-prepare-v0-"${CURRENT_MINOR_VERSION}"-"${NEW_PATCH_VERSION}" && git rebase -i "${RELEASE_BRANCH}"` - [ ] Merge release preparation branch into the release branch - - `git checkout v0. && git merge --ff-only website-prepare-v0--` + - `git checkout "${RELEASE_BRANCH}" && git merge --ff-only website-prepare-v0-"${CURRENT_MINOR_VERSION}"-"${NEW_PATCH_VERSION}"` - [ ] Tag new release - - [ ] `git tag v0.. -a -m v0..` - - [ ] `git push origin v0..` + - [ ] `git tag "${NEW_VERSION}" -a -m "${NEW_VERSION}"` + - [ ] `git push origin "${NEW_VERSION}"` - [ ] Wait for release workflow to complete - Discoverable via [https://github.com/timberio/vector/actions/workflows/release.yml](https://github.com/timberio/vector/actions/workflows/release.yml) - [ ] Release Linux packages. See [`vector-release` usage](https://github.com/DataDog/vector-release#usage). - - Note: the pipeline inputs are the version number `v0.` and a personal GitHub token. + - Note: the pipeline inputs are the version number `"${CURRENT_VERSION}"` and a personal GitHub token. - [ ] Manually trigger the `trigger-package-release-pipeline-prod-stable` job. - [ ] Push the release branch to update the remote (This should close the preparation branch PR). - - `git checkout v0. && git push` + - `git checkout "${RELEASE_BRANCH}" && git push` - [ ] Release updated Helm chart. See [releasing Helm chart](https://github.com/vectordotdev/helm-charts#releasing). - [ ] Once Helm chart is released, updated Vector manifests - Run `cargo vdev build manifests` and open a PR with changes @@ -50,5 +62,5 @@ On the day of release: - Follow the [instructions at the top of the mirror.yaml file](https://github.com/DataDog/images/blob/fbf12868e90d52e513ebca0389610dea8a3c7e1a/mirror.yaml#L33-L49). - [ ] Cherry-pick any release commits from the release branch that are not on `master`, to `master` - [ ] Reset the `website` branch to the `HEAD` of the release branch to update https://vector.dev - - [ ] `git checkout website && git reset --hard origin/v0.. && git push` + - [ ] `git checkout website && git reset --hard origin/"${RELEASE_BRANCH}" && git push` - [ ] Kick-off post-mortems for any regressions resolved by the release diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index c5c0550653c74..76781a822e55d 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,26 +1,13 @@ - - ## Summary +## Vector configuration + + +## How did you test this PR? + + ## Change Type - [ ] Bug fix - [ ] New feature @@ -31,38 +18,50 @@ This should help the reviewers give feedback faster and with higher quality. --> - [ ] Yes - [ ] No -## How did you test this PR? - - ## Does this PR include user facing changes? - [ ] Yes. Please add a changelog fragment based on our [guidelines](https://github.com/vectordotdev/vector/blob/master/changelog.d/README.md). -- [ ] No. A maintainer will apply the "no-changelog" label to this PR. +- [ ] No. A maintainer will apply the `no-changelog` label to this PR. + +## References + + ## Notes - Please read our [Vector contributor resources](https://github.com/vectordotdev/vector/tree/master/docs#getting-started). - Do not hesitate to use `@vectordotdev/vector` to reach out to us regarding this PR. -- The CI checks run only after we manually approve them. +- Some CI checks run only after we manually approve them. - We recommend adding a `pre-push` hook, please see [this template](https://github.com/vectordotdev/vector/blob/master/CONTRIBUTING.md#Pre-push). - Alternatively, we recommend running the following locally before pushing to the remote branch: - `cargo fmt --all` - `cargo clippy --workspace --all-targets -- -D warnings` - `cargo nextest run --workspace` (alternatively, you can run `cargo test --all`) - - `./scripts/check_changelog_fragments.sh` - After a review is requested, please avoid force pushes to help us review incrementally. - Feel free to push as many commits as you want. They will be squashed into one before merging. - For example, you can run `git merge origin master` and `git push`. - If this PR introduces changes Vector dependencies (modifies `Cargo.lock`), please run `cargo vdev build licenses` to regenerate the [license inventory](https://github.com/vectordotdev/vrl/blob/main/LICENSE-3rdparty.csv) and commit the changes (if any). More details [here](https://crates.io/crates/dd-rust-license-tool). -## References - - + Your PR title must conform to the conventional commit spec: + https://www.conventionalcommits.org/en/v1.0.0/ + + ()!: - + * `type` = chore, enhancement, feat, fix, docs, revert + * `!` = OPTIONAL: signals a breaking change + * `scope` = Optional when `type` is "chore" or "docs", available scopes https://github.com/vectordotdev/vector/blob/master/.github/workflows/semantic.yml#L31 + * `description` = short description of the change + +Examples: + + * enhancement(file source): Add `sort` option to sort discovered files + * feat(new source): Initial `statsd` source + * fix(file source): Fix a bug discovering new files + * chore(external docs): Clarify `batch_size` option +--> diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index 36e53e7b8f9af..301c272018cb1 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -1,26 +1,43 @@ -ANSIX -APAC -APPSIGNAL +acmecorp Acro +addonmanager Ainol +aiohttp Airis Airpad +AIXM Alcatel -Alexey alertmanager +Alexey +algoliasearch Alibaba Allfine Allview Allwinner +alpinejs +altostrat Amarok Amaway +amazonlinux Amoi +ansible +ANSIX Aoc Aoson +APAC +apachectl +apachepulsar Apanda +apikey +apimachinery +apiserver +APPSIGNAL Appsignal +appsignal +archlinux Archos Arival +armhf Arnova Asus Atlassian @@ -28,221 +45,17 @@ atleastonce atmostonce Attab Audiosonic +AUTOSAR +aviv avsc Axioo Azend -Bedove -Benss -Blaupunkt -Blusens -Casio -Cantarell -Celkon -Ceph -Chromecast -Citrix -Cloudflare -Cloudfone -Cmx -Coby -Collectd -Comcast -Consolas -Coolpad -BADDCAFE -DEBHELPER -Danew -dkr -Dockerfiles -DOOV -Douban -emqx -eventloop -Enot -Evercoss -exactlyonce -Explay -FAQs -FDO -FQDNs -Fabro -Figma -Flipboard -Foto -Freescale -Galapad -Garmin -Geeksphone -Gfive -Ghemawat -Gionee -HMACs -HTTPDATE -Haier -Haipad -Hannspree -Hena -Hisense -Huawei -Hyundai -Ideapad -Infinix -Instacart -Intenso -Itamar -Ivio -JXD -Jacq -Jameel -Jaytech -Jia -Jiayu -Joda -KBytes -KDL -KTtech -Karbonn -Kingcom -Kolkata -Kruno -Ktouch -Kurio -Kyros -LGE -LYF -Lenco -Lexibook -Lifetab -Lifetouch -Lumia -Malata -manden -Maxthon -Mediacom -Medion -Meizu -Mertz -metakey -Metakey -Micromax -Mito -Mobistel -Modecom -Moto -Mpman -Multilaser -Mumbai -musleabi -Mytab -NLB -Nabi -Netflix -Neue -Nextbook -Nextcloud -OVH -Odys -Openpeak -Oppo -Ovi -Owncloud -PHILIPSTV -POCs -Panasonic -Papyre -pbs -Phicomm -Phototechnik -Pingdom -Pinterest -Pinterestbot -Pipo -Ployer -Podcast -Podkicker -Positivo -Prestigio -pront -Proscan -Qmobilevn -RPZ -RRSIGs -Rackspace -Rathi -Regza -Rijs -Roboto -Rockchip -Roku -Roundcube -Rowling -rumqttc -SBT -SKtelesys -Salesforce -Samsung -Sega -Segoe -Shopify -SIGINTs -Simvalley -Skype -Skytex -Smartbitt -Snapchat -Softbank -Sogou -Soref -Tagi -Tecmobile -Telstra -Tencent -Texet -Thl -Tizen -Tmobile -Tomtec -Tooky -Touchmate -Traefik -Trekstor -Treq -Umeox -Verizon -Videocon -Viewsonic -Wellcom -Wii -Wiko -Woxter -Xeon -Xianghe -Xolo -Xoro -Xperia -Yarvik -Yifang -ZTE -Zopo -Zync -acmecorp -addonmanager -aiohttp -algoliasearch -alpinejs -altostrat -amazonlinux -ansible -apachectl -apachepulsar -apikey -apimachinery -apiserver -appsignal -archlinux -armhf backpressure backticks +BADDCAFE +BADVERS +Bedove +Benss bigendian bindir binfmt @@ -252,16 +65,36 @@ bitflags bitnami bitwidth blackbox +Blaupunkt +Blusens buildname buildroot bytestream callsites +Cantarell +Casio +cbor +CDMA cdnskey +Celkon +Ceph +Chromecast +Citrix +cksum +Cloudflare +Cloudfone +Cmx cncf +Coby codepath codepaths +Collectd +Comcast commandline compiletime +Consolas +contactless +Coolpad coredns corejs coreutils @@ -269,46 +102,83 @@ csync curta daemonset dalek +dammam +Danew databend datacenter datadog datadoghq datanode ddog +DEBHELPER debian +DECT demuxing dfs discriminants distro distroless +dkr +DNP dnslookup dnssec dnstap dnsutils dockercmd +Dockerfiles +doha +DOOV +Douban downsides downwardapi +DVB ede emoji +emqx +enableable +Enot +EPC esbuild etld +eventloop +Evercoss +exactlyonce +Explay +Fabro fakeintake +FAQs fargate +FDO fibonacci +Figma fileapi filebeat finalizers findstring firefox +FLEXRAY +Flipboard fluentbit fluentd +Foto +FQDNs +Freescale +FUJITSU fuzzers +Galapad +Garmin gce gcloud gcp gcr gcs gdpr +Geeksphone +GENIBUS +Gfive +Ghemawat +gifs +Gionee github gnueabi gnueabihf @@ -327,43 +197,79 @@ graphiql greptime greptimecloud greptimedb +GSM gvisor gws hadoop +Haier +Haipad +Hannspree hdfs healthcheck healthchecks +Hena heroicon heroicons heroku herokuapp +Hisense +hitag hitech +HMACs hogwarts htc htmltest +HTTPDATE https +Huawei humungus +Hyundai icecream +Ideapad idn ifeq ifneq imobile +Infinix influxd +Instacart +Intenso +INTERLAKEN ionik ipallowlist ipod ircd +Itamar +Ivio +Jacq +JAMCRC +Jameel +Jaytech jemalloc jemallocator +jetbrains +JetBrains +Jia +Jiayu jndi +Joda journalctl jsonnet jsontag jvm -kenton +JXD +Karbonn +KBytes +KDL keepappkey keephq +kenton +Kingcom +Kolkata konqueror +Kruno +Ktouch +KTtech kube kubeadm kubeconfig @@ -371,160 +277,268 @@ kubectl kubelet kubernetes kubeval +Kurio kustomization kustomize kyocera +Kyros +Lenco lenovo levenstein +Lexibook +LGE +Lifetab +Lifetouch linkerd localdomain localstack lucene +Lumia +LYF macbook +Malata +manden maxmind maxminddb +Maxthon +MCRF +Mediacom +Medion +MEF +Meizu +meme +Mertz messagebird +metakey +Metakey +microcontroller +Micromax +MIFARE minikube minimalistic minio minishift +miniz +Mito mkfile +Mobistel +modbus +Modecom mongod +Moto motorola mountpoint mozilla +Mpman msiexec +Multilaser +Mumbai +mumbai +musleabi +Mytab +Nabi namenode netcat netdata +Netflix netlify +Neue +Nextbook +Nextcloud nintendo nio nixos nixpkg nixpkgs +NLB nokia +NRSC nslookup nsupdate ntapi ntfs +Odys +onig opendal +Openpeak +OPENPGP +OPENSAFETY opensearch opentelemetry +Oppo oss +otel +otelcol +OVH +Ovi +Owncloud pacman +Panasonic pantech papertrail papertrailapp +Papyre +paulo +pbs petabytes +Phicomm philips +PHILIPSTV +Phototechnik +Pingdom +Pinterest +Pinterestbot +Pipo +Ployer +POCs +Podcast podinfo +Podkicker podman +Positivo postgresql +Prestigio +PROFIBUS +pront +Proscan proxyuser pseudocode psl pushgateway +QA'd +Qmobilevn qwerty rabbitmq +Rackspace +Rathi rclone +Regza +requestline rfcs +RFID +riello +Rijs roadmap +Roboto +Rockchip +ROHC +Roku rootfs +Roundcube roundrobin +Rowling rpmbuild rpms +RPZ +RRSIGs rstest rsyslog rsyslogd +rumqttc +Salesforce +Samsung +SBT scriptblock +Sega +Segoe servlet -Sinjo -sublocation shannon -sundar -svcb +Shopify +SIGINTs +simdutf +Simvalley +Sinjo +siv +SKtelesys +Skype +Skytex +Smartbitt +SMBUS +Snapchat snyk socketaddr +Softbank +Sogou solarwinds +Soref +sortedset splunk ssh staticuser statsd +sublocation +sundar +svcb symbian +Tagi tanushri +Tecmobile +teledisk +Telstra +Tencent +Texet +Thl timeframe timeseries timespan timestamped +Tizen +Tmobile +Tomtec +Tooky +Touchmate +Traefik +Trekstor +Treq +turin typesense tzdata ubuntu +Umeox +UMTS unchunked upstreaminfo +urlencoding useragents usergroups userguide -urlencoding +Verizon +vhosts +Videocon +Viewsonic +WCDMA webhdfs +Wellcom +Wii +Wiko winapi workarounds +Woxter XCHACHA +Xeon +Xianghe +XMODEM +Xolo +Xoro +Xperia XSALSA yandex +Yarvik +Yifang +zadd zeek zookeeper +Zopo zst zstandard -otel -otelcol -siv -onig -aviv -dammam -doha -mumbai -paulo -turin -AIXM -AUTOSAR -CDMA -cksum -contactless -DECT -DNP -DVB -EPC -FLEXRAY -FUJITSU -GENIBUS -GSM -hitag -INTERLAKEN -JAMCRC -MCRF -MEF -microcontroller -MIFARE -modbus -NRSC -OPENPGP -OPENSAFETY -PROFIBUS -RFID -riello -ROHC -SMBUS -teledisk -UMTS -WCDMA -XMODEM -cbor -requestline -jetbrains -JetBrains -gifs -meme -simdutf -vhosts -miniz +ZTE +Zync +envsubst +jimmystewpot +esensar +jchap-pnnl +jhbigler-pnnl +jlambatl +jorgehermo9 +notchairmk +yjagdale diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index ee745e5c30d5e..af010d0a4d5d9 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -174,7 +174,6 @@ eeyun efgh Elhage emerg -Enableable endianess endler eni diff --git a/.github/audit.yml b/.github/audit.yml index 77ae1a5548fd5..2544a1bad0530 100644 --- a/.github/audit.yml +++ b/.github/audit.yml @@ -11,7 +11,8 @@ jobs: security_audit: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v1 - - uses: actions-rs/audit-check@v1 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + # TODO: replace this action - abandoned since 2020 + - uses: actions-rs/audit-check@35b7b53b1e25b55642157ac01b4adceb5b9ebef3 # v1.2.0 with: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/changelog.yaml b/.github/workflows/changelog.yaml index 57d7013f3b2d6..9c6fe296d9733 100644 --- a/.github/workflows/changelog.yaml +++ b/.github/workflows/changelog.yaml @@ -6,63 +6,66 @@ # - new commits pushed # - label is added or removed # Runs on merge queues, but just passes, because it is a required check. - name: Changelog on: - pull_request: + pull_request_target: types: [opened, synchronize, reopened, labeled, unlabeled] - # Due to merge queue requiring same status checks as PRs, must pass by default + + # Required by GitHub merge queue due to branch protection rules. Should always be successful merge_group: types: [checks_requested] jobs: validate-changelog: + permissions: + contents: read + pull-requests: none + runs-on: ubuntu-24.04 - if: github.event_name == 'pull_request' + env: - PR_HAS_LABEL: ${{ contains( github.event.pull_request.labels.*.name, 'no-changelog') }} - steps: - # Checkout full depth because in the check_changelog_fragments script, we need to specify a - # merge base. If we only shallow clone the repo, git may not have enough history to determine - # the base. - - uses: actions/checkout@v4 - with: - fetch-depth: 0 + NO_CHANGELOG: ${{ contains(github.event.pull_request.labels.*.name, 'no-changelog') }} + SHOULD_RUN: ${{ !contains(github.event.pull_request.labels.*.name, 'no-changelog') && github.event_name != 'merge_group' }} - - name: Generate authentication token - # Don't run this step if the PR is from a fork or dependabot since the secrets won't exist - if: ${{ github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' }} - id: generate_token - uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a - with: - app_id: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_ID }} - private_key: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_PRIVATE_KEY }} + steps: + - name: Bypass when no‑changelog label is present + if: env.NO_CHANGELOG == 'true' + run: | + echo "'no-changelog' label detected – skipping changelog validation." + exit 0 - - run: | - if [[ $PR_HAS_LABEL == 'true' ]] ; then - echo "'no-changelog' label detected." - exit 0 - fi + - name: Merge queue + if: ${{ github.event_name == 'merge_group' }} + run: | + echo "merge_group event – passing without running changelog validation." + exit 0 - # Helper script needs to reference the master branch to compare against - git fetch origin master:refs/remotes/origin/master + # Checkout changelog script and changelog.d/ from master + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + if: env.SHOULD_RUN == 'true' + with: + ref: master + sparse-checkout: | + scripts/check_changelog_fragments.sh + changelog.d/ + sparse-checkout-cone-mode: false - ./scripts/check_changelog_fragments.sh + # Checkout PR's changelog.d/ into tmp/ + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + if: env.SHOULD_RUN == 'true' + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + path: tmp + sparse-checkout: changelog.d/ - check-changelog: - name: Changelog - runs-on: ubuntu-24.04 - needs: validate-changelog - if: always() - env: - FAILED: ${{ contains(needs.*.result, 'failure') }} - steps: - - name: exit + - name: Run changelog fragment checker + if: env.SHOULD_RUN == 'true' run: | - echo "failed=${{ env.FAILED }}" - if [[ "$FAILED" == "true" ]] ; then - exit 1 - else - exit 0 - fi + # Overwrite changelog.d/*.md + rm -rf changelog.d/*.md && mv tmp/changelog.d/*.md changelog.d/ + + # Add files and then compare with HEAD instead of origin/master + git add changelog.d/ + MERGE_BASE=HEAD ./scripts/check_changelog_fragments.sh diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index 682dded4a7b21..111ab22e97c9b 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -10,10 +10,10 @@ on: # merge_group context. inputs: base_ref: - required: true + required: false type: string head_ref: - required: true + required: false type: string int_tests: required: false @@ -40,12 +40,12 @@ on: value: ${{ jobs.source.outputs.component_docs }} markdown: value: ${{ jobs.source.outputs.markdown }} + website_only: + value: ${{ jobs.source.outputs.website_only }} install: value: ${{ jobs.source.outputs.install }} k8s: value: ${{ jobs.source.outputs.k8s }} - all-int: - value: ${{ jobs.int_tests.outputs.all-tests }} amqp: value: ${{ jobs.int_tests.outputs.amqp }} appsignal: @@ -119,12 +119,15 @@ on: all-changes-json: value: ${{ jobs.int_tests.outputs.all-changes-json }} # e2e tests - all-e2e: - value: ${{ jobs.e2e_tests.outputs.all-tests }} e2e-datadog-logs: value: ${{ jobs.e2e_tests.outputs.datadog-logs }} e2e-datadog-metrics: value: ${{ jobs.e2e_tests.outputs.datadog-metrics }} + e2e-opentelemetry-logs: + value: ${{ jobs.e2e_tests.outputs.opentelemetry-logs }} +env: + BASE_SHA: ${{ inputs.base_ref || (github.event_name == 'merge_group' && github.event.merge_group.base_sha) || github.event.pull_request.base.sha }} + HEAD_SHA: ${{ inputs.head_ref || (github.event_name == 'merge_group' && github.event.merge_group.head_sha) || github.event.pull_request.head.sha }} jobs: # Detects changes that are not specific to integration tests @@ -141,14 +144,15 @@ jobs: markdown: ${{ steps.filter.outputs.markdown }} install: ${{ steps.filter.outputs.install }} k8s: ${{ steps.filter.outputs.k8s }} + website_only: ${{ steps.filter.outputs.website == 'true' && steps.filter.outputs.not_website == 'false' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: dorny/paths-filter@v3 + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 id: filter with: - base: ${{ inputs.base_ref }} - ref: ${{ inputs.head_ref }} + base: ${{ env.BASE_SHA }} + ref: ${{ env.HEAD_SHA }} filters: | source: - ".github/workflows/test.yml" @@ -170,13 +174,12 @@ jobs: - 'Cargo.toml' - 'Cargo.lock' - 'rust-toolchain.toml' - - '.github/workflows/pr.yml' - 'Makefile' - 'scripts/cross/**' - "vdev/**" cue: - 'website/cue/**' - - "vdev" + - "vdev/**" component_docs: - 'scripts/generate-component-docs.rb' - "vdev/**" @@ -195,6 +198,10 @@ jobs: - "distribution/install.sh" k8s: - "src/sources/kubernetes_logs/**" + website: + - "website/**" + not_website: + - "!website/**" # Detects changes that are specific to integration tests int_tests: @@ -202,7 +209,6 @@ jobs: timeout-minutes: 5 if: ${{ inputs.int_tests }} outputs: - all-tests: ${{ steps.filter.outputs.all-tests}} amqp: ${{ steps.filter.outputs.amqp }} appsignal: ${{ steps.filter.outputs.appsignal}} aws: ${{ steps.filter.outputs.aws }} @@ -240,7 +246,7 @@ jobs: webhdfs: ${{ steps.filter.outputs.webhdfs }} all-changes-json: ${{ steps.aggregate.outputs.all-changes-json }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 # creates a yaml file that contains the filters for each integration, # extracted from the output of the `vdev int ci-paths` command, which @@ -248,11 +254,11 @@ jobs: - name: Create filter rules for integrations run: cargo vdev int ci-paths > int_test_filters.yaml - - uses: dorny/paths-filter@v3 + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 id: filter with: - base: ${{ inputs.base_ref }} - ref: ${{ inputs.head_ref }} + base: ${{ env.BASE_SHA }} + ref: ${{ env.HEAD_SHA }} filters: int_test_filters.yaml # This JSON hack was introduced because GitHub Actions does not support dynamic expressions in the @@ -304,7 +310,7 @@ jobs: echo "$json" > int_tests_changes.json - name: Upload JSON artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: int_tests_changes path: int_tests_changes.json @@ -315,11 +321,10 @@ jobs: timeout-minutes: 5 if: ${{ inputs.e2e_tests }} outputs: - all-tests: ${{ steps.filter.outputs.all-tests}} datadog-logs: ${{ steps.filter.outputs.datadog-logs }} datadog-metrics: ${{ steps.filter.outputs.datadog-metrics }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 # creates a yaml file that contains the filters for each test, # extracted from the output of the `vdev int ci-paths` command, which @@ -327,9 +332,9 @@ jobs: - name: Create filter rules for e2e tests run: cargo vdev e2e ci-paths > int_test_filters.yaml - - uses: dorny/paths-filter@v3 + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 id: filter with: - base: ${{ inputs.base_ref }} - ref: ${{ inputs.head_ref }} + base: ${{ env.BASE_SHA }} + ref: ${{ env.HEAD_SHA }} filters: int_test_filters.yaml diff --git a/.github/workflows/ci-integration-review.yml b/.github/workflows/ci-integration-review.yml index 1371bac1ed337..5c47a53afe430 100644 --- a/.github/workflows/ci-integration-review.yml +++ b/.github/workflows/ci-integration-review.yml @@ -36,8 +36,6 @@ permissions: statuses: write env: - AWS_ACCESS_KEY_ID: "dummy" - AWS_SECRET_ACCESS_KEY: "dummy" AXIOM_TOKEN: ${{ secrets.AXIOM_TOKEN }} TEST_APPSIGNAL_PUSH_API_KEY: ${{ secrets.TEST_APPSIGNAL_PUSH_API_KEY }} TEST_DATADOG_API_KEY: ${{ secrets.CI_TEST_DATADOG_API_KEY }} @@ -65,7 +63,7 @@ jobs: private_key: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_PRIVATE_KEY }} - name: Get PR review author id: comment - uses: tspascoal/get-user-teams-membership@v3 + uses: tspascoal/get-user-teams-membership@57e9f42acd78f4d0f496b3be4368fc5f62696662 # v3.0.0 with: username: ${{ github.actor }} team: 'Vector' @@ -76,7 +74,7 @@ jobs: run: exit 1 - name: (PR review) Set latest commit status as pending - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} @@ -96,12 +94,12 @@ jobs: "redis", "splunk", "webhdfs" ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: submodules: "recursive" ref: ${{ github.event.review.commit_id }} - - run: sudo npm -g install @datadog/datadog-ci + - run: bash scripts/environment/prepare.sh --modules=datadog-ci - run: docker image prune -af ; docker container prune -f @@ -109,44 +107,50 @@ jobs: if: ${{ startsWith(github.event.review.body, '/ci-run-integration-all') || startsWith(github.event.review.body, '/ci-run-all') || startsWith(github.event.review.body, format('/ci-run-integration-{0}', matrix.service)) }} - uses: nick-fields/retry@v3 + uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int ${{ matrix.service }} + command: bash scripts/int-e2e-test.sh int ${{ matrix.service }} e2e-tests: needs: prep-pr runs-on: ubuntu-24.04-8core timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: submodules: "recursive" ref: ${{ github.event.review.commit_id }} - - run: sudo npm -g install @datadog/datadog-ci + - run: bash scripts/environment/prepare.sh --modules=datadog-ci - run: docker image prune -af ; docker container prune -f - name: e2e-datadog-logs if: ${{ startsWith(github.event.review.body, '/ci-run-e2e-datadog-logs') - || startsWith(github.event.review.body, '/ci-run-integration-all') + || startsWith(github.event.review.body, '/ci-run-e2e-all') || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 + uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 with: timeout_minutes: 35 max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh e2e datadog-logs + command: bash scripts/int-e2e-test.sh e2e datadog-logs - name: datadog-e2e-metrics if: ${{ startsWith(github.event.review.body, '/ci-run-e2e-datadog-metrics') || startsWith(github.event.review.body, '/ci-run-e2e-all') || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 + uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 with: timeout_minutes: 35 max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh e2e datadog-metrics + command: bash scripts/int-e2e-test.sh e2e datadog-metrics + + - name: e2e-opentelemetry-logs + if: ${{ startsWith(github.event.review.body, '/ci-run-e2e-opentelemetry-logs') + || startsWith(github.event.review.body, '/ci-run-e2e-all') + || startsWith(github.event.review.body, '/ci-run-all') }} + run: bash scripts/int-e2e-test.sh e2e opentelemetry-logs update-pr-status: name: Signal result to PR @@ -157,7 +161,7 @@ jobs: - e2e-tests if: always() && (startsWith(github.event.review.body, '/ci-run-integration') || contains(github.event.review.body, '/ci-run-all')) env: - FAILED: ${{ contains(needs.*.result, 'failure') }} + FAILED: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} steps: - name: Generate authentication token id: generate_token @@ -168,7 +172,7 @@ jobs: - name: Validate issue comment if: github.event_name == 'pull_request_review' - uses: tspascoal/get-user-teams-membership@v3 + uses: tspascoal/get-user-teams-membership@57e9f42acd78f4d0f496b3be4368fc5f62696662 # v3.0.0 with: username: ${{ github.actor }} team: 'Vector' @@ -176,16 +180,17 @@ jobs: - name: (PR review) Submit PR result as success if: github.event_name == 'pull_request_review' && env.FAILED != 'true' - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} status: 'success' - - run: | - echo "failed=${{ env.FAILED }}" - if [[ "$FAILED" == "true" ]] ; then + - name: Check all jobs status + run: | + if [[ "${{ env.FAILED }}" == "true" ]]; then + echo "One or more jobs failed or were cancelled" exit 1 else - exit 0 + echo "All jobs completed successfully" fi diff --git a/.github/workflows/ci-review-trigger.yml b/.github/workflows/ci-review-trigger.yml index 06f638da5fbab..6d2ef24c4c9a7 100644 --- a/.github/workflows/ci-review-trigger.yml +++ b/.github/workflows/ci-review-trigger.yml @@ -73,7 +73,7 @@ jobs: private_key: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_PRIVATE_KEY }} - name: Get PR review author id: comment - uses: tspascoal/get-user-teams-membership@v3 + uses: tspascoal/get-user-teams-membership@57e9f42acd78f4d0f496b3be4368fc5f62696662 # v3.0.0 with: username: ${{ github.actor }} team: 'Vector' diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml index e2833e17a027c..d6ce3ab9bb255 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/cli.yml @@ -15,7 +15,7 @@ jobs: steps: - name: (PR review) Set latest commit status as pending if: ${{ github.event_name == 'pull_request_review' }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} @@ -24,16 +24,16 @@ jobs: - name: (PR review) Checkout review SHA if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.event.review.commit_id }} - name: Checkout branch if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Cache Cargo registry + index - uses: actions/cache@v4 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: | ~/.cargo/bin/ @@ -45,7 +45,7 @@ jobs: ${{ runner.os }}-cargo- - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - - run: bash scripts/environment/prepare.sh + - run: bash scripts/environment/prepare.sh --modules=cargo-nextest,datadog-ci - run: echo "::add-matcher::.github/matchers/rust.json" - run: make test-cli - name: Upload test results @@ -53,7 +53,7 @@ jobs: if: always() - name: (PR review) Set latest commit status as ${{ job.status }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 if: always() && github.event_name == 'pull_request_review' with: sha: ${{ github.event.review.commit_id }} diff --git a/.github/workflows/compilation-timings.yml b/.github/workflows/compilation-timings.yml index 4b0bfe5745ca0..9546f44471c0c 100644 --- a/.github/workflows/compilation-timings.yml +++ b/.github/workflows/compilation-timings.yml @@ -15,10 +15,10 @@ jobs: name: "Release Build (optimized)" runs-on: ubuntu-24.04-8core steps: - - uses: colpal/actions-clean@v1 - - uses: actions/checkout@v4 + - uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - - run: bash scripts/environment/prepare.sh + - run: bash scripts/environment/prepare.sh --modules=rustup - run: cargo clean - run: cargo build --release @@ -31,10 +31,10 @@ jobs: # with full LTO / single codegen unit. PROFILE: debug steps: - - uses: colpal/actions-clean@v1 - - uses: actions/checkout@v4 + - uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - - run: bash scripts/environment/prepare.sh + - run: bash scripts/environment/prepare.sh --modules=rustup - run: cargo clean - run: cargo build --release @@ -42,10 +42,10 @@ jobs: name: "Debug Build" runs-on: ubuntu-24.04-8core steps: - - uses: colpal/actions-clean@v1 - - uses: actions/checkout@v4 + - uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - - run: bash scripts/environment/prepare.sh + - run: bash scripts/environment/prepare.sh --modules=rustup - run: cargo clean - run: cargo build @@ -53,10 +53,10 @@ jobs: name: "Debug Rebuild" runs-on: ubuntu-24.04-8core steps: - - uses: colpal/actions-clean@v1 - - uses: actions/checkout@v4 + - uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - - run: bash scripts/environment/prepare.sh + - run: bash scripts/environment/prepare.sh --modules=rustup - run: cargo clean - run: cargo build - run: touch src/app.rs @@ -66,9 +66,9 @@ jobs: name: "Cargo Check" runs-on: ubuntu-24.04-8core steps: - - uses: colpal/actions-clean@v1 - - uses: actions/checkout@v4 + - uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - - run: bash scripts/environment/prepare.sh + - run: bash scripts/environment/prepare.sh --modules=rustup - run: cargo clean - run: cargo check diff --git a/.github/workflows/component_features.yml b/.github/workflows/component_features.yml index 6af5c66dd2f68..b6537c076621b 100644 --- a/.github/workflows/component_features.yml +++ b/.github/workflows/component_features.yml @@ -27,7 +27,7 @@ jobs: steps: - name: (PR review) Set latest commit status as pending if: github.event_name == 'pull_request_review' - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} @@ -36,22 +36,22 @@ jobs: - name: (PR review) Checkout PR branch if: github.event_name == 'pull_request_review' - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.event.review.commit_id }} - name: Checkout branch if: github.event_name != 'pull_request_review' - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - - run: bash scripts/environment/prepare.sh + - run: bash scripts/environment/prepare.sh --modules=rustup - run: echo "::add-matcher::.github/matchers/rust.json" - run: make check-component-features - name: (PR review) Set latest commit status as ${{ job.status }} if: always() && github.event_name == 'pull_request_review' - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/create_preview_sites.yml b/.github/workflows/create_preview_sites.yml index cd943c5f4f8be..47ce7bced9f80 100644 --- a/.github/workflows/create_preview_sites.yml +++ b/.github/workflows/create_preview_sites.yml @@ -35,7 +35,7 @@ jobs: steps: # Get the artifacts with the PR number and branch name - name: Download artifact - uses: actions/github-script@v7.0.1 + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 with: script: | const fs = require('fs'); @@ -55,7 +55,7 @@ jobs: # Extract the info from the artifact and set variables - name: Extract PR info from artifact - uses: actions/github-script@v7.0.1 + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 with: script: | const fs = require('fs'); @@ -82,7 +82,7 @@ jobs: # Validate the integrity of the artifact - name: Validate Artifact Integrity - uses: actions/github-script@v7.0.1 + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 with: script: | const crypto = require('crypto'); @@ -101,7 +101,7 @@ jobs: # Kick off the job in amplify - name: Deploy Site - uses: actions/github-script@v7.0.1 + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 env: APP_ID: ${{ inputs.APP_ID }} APP_NAME: ${{ inputs.APP_NAME }} @@ -168,7 +168,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} APP_ID: ${{ inputs.APP_ID }} APP_NAME: ${{ inputs.APP_NAME }} - uses: actions/github-script@v7.0.1 + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 with: script: | const fs = require('fs'); diff --git a/.github/workflows/cross.yml b/.github/workflows/cross.yml index fed5655fa1b61..77d88f5bea906 100644 --- a/.github/workflows/cross.yml +++ b/.github/workflows/cross.yml @@ -27,7 +27,7 @@ jobs: steps: - name: (PR review) Set latest commit status as pending if: ${{ github.event_name == 'pull_request_review' }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} @@ -36,15 +36,15 @@ jobs: - name: (PR review) Checkout PR branch if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.event.review.commit_id }} - name: Checkout branch if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/cache@v4 + - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 name: Cache Cargo registry + index with: path: | @@ -64,13 +64,13 @@ jobs: # aarch64 and musl in particular are notoriously hard to link. # While it may be tempting to slot a `check` in here for quickness, please don't. - run: make cross-build-${{ matrix.target }} - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: "vector-debug-${{ matrix.target }}" path: "./target/${{ matrix.target }}/debug/vector" - name: (PR review) Set latest commit status as failed - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 if: failure() && github.event_name == 'pull_request_review' with: sha: ${{ steps.comment-branch.outputs.head_sha }} @@ -86,7 +86,7 @@ jobs: if: needs.cross-linux.result == 'success' && github.event_name == 'pull_request_review' steps: - name: (PR review) Submit PR result as success - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/deny.yml b/.github/workflows/deny.yml index 287a1c05e8441..e1c11f2068448 100644 --- a/.github/workflows/deny.yml +++ b/.github/workflows/deny.yml @@ -28,7 +28,7 @@ jobs: steps: - name: (PR review) Set latest commit status as pending if: ${{ github.event_name == 'pull_request_review' }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} @@ -37,15 +37,15 @@ jobs: - name: (PR review) Checkout PR branch if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.event.review.commit_id }} - name: Checkout branch if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/cache@v4 + - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 name: Cache Cargo registry + index with: path: | @@ -58,13 +58,13 @@ jobs: ${{ runner.os }}-cargo- - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - - run: bash scripts/environment/prepare.sh + - run: bash scripts/environment/prepare.sh --modules=cargo-deny - run: echo "::add-matcher::.github/matchers/rust.json" - name: Check cargo deny advisories/licenses run: make check-deny - name: (PR review) Set latest commit status as ${{ job.status }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 if: always() && github.event_name == 'pull_request_review' with: sha: ${{ github.event.review.commit_id }} diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index cdba1faec680f..edd6353e04c9c 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -4,10 +4,12 @@ # - PRs if there are code changes to the source files that are noted in `vdev e2e ci_paths` # - MQ (always pass) # - Scheduled: at midnight UTC Tues-Sat +# - Manual trigger via Actions UI. name: E2E Test Suite on: + workflow_dispatch: pull_request: # Needs to pass by default in MQ merge_group: @@ -37,15 +39,12 @@ env: # observing issues fetching boringssl via HTTPS in the OSX build, seeing if this helps # can be removed when we switch back to the upstream openssl-sys crate CARGO_NET_GIT_FETCH_WITH_CLI: true - jobs: changes: if: github.event_name == 'pull_request' uses: ./.github/workflows/changes.yml with: - base_ref: ${{ github.event.pull_request.base.ref }} - head_ref: ${{ github.event.pull_request.head.ref }} source: false e2e_tests: true secrets: inherit @@ -56,18 +55,18 @@ jobs: timeout-minutes: 45 needs: changes if: always() && ( - github.event_name == 'schedule' || ( - needs.changes.outputs.all-e2e == 'true' - || needs.changes.outputs.e2e-datadog-logs == 'true' - || needs.changes.outputs.e2e-datadog-metrics == 'true' - ) + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + needs.changes.outputs.e2e-datadog-logs == 'true' || + needs.changes.outputs.e2e-datadog-metrics == 'true' || + needs.changes.outputs.e2e-opentelemetry-logs == 'true' ) steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: submodules: "recursive" - - run: sudo npm -g install @datadog/datadog-ci + - run: bash scripts/environment/prepare.sh --modules=datadog-ci - run: docker image prune -af ; docker container prune -f @@ -82,23 +81,34 @@ jobs: echo "PR_HAS_ACCESS_TO_SECRETS=false" >> "$GITHUB_ENV" fi - - if: (github.event_name == 'schedule' || needs.changes.outputs.all-e2e == 'true' || needs.changes.outputs.e2e-datadog-logs == 'true') && + - if: (github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + needs.changes.outputs.e2e-datadog-logs == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') name: e2e-datadog-logs - uses: nick-fields/retry@v3 + uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 with: timeout_minutes: 35 max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh e2e datadog-logs + command: bash scripts/int-e2e-test.sh e2e datadog-logs - - if: (github.event_name == 'schedule' || needs.changes.outputs.all-e2e == 'true' || needs.changes.outputs.e2e-datadog-metrics == 'true') && + - if: (github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + needs.changes.outputs.e2e-datadog-metrics == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') name: e2e-datadog-metrics - uses: nick-fields/retry@v3 + uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 with: timeout_minutes: 35 max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh e2e datadog-metrics + command: bash scripts/int-e2e-test.sh e2e datadog-metrics + + - if: (github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + needs.changes.outputs.e2e-opentelemetry-logs == 'true') && + (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') + name: e2e-opentelemetry-logs + run: bash scripts/int-e2e-test.sh e2e opentelemetry-logs e2e-test-suite: @@ -107,13 +117,12 @@ jobs: timeout-minutes: 5 if: always() needs: e2e-tests - env: - FAILED: ${{ contains(needs.*.result, 'failure') }} steps: - - run: | - echo "failed=${{ env.FAILED }}" - if [[ "$FAILED" == "true" ]] ; then + - name: Check all jobs status + run: | + if [[ "${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}" == "true" ]]; then + echo "One or more jobs failed or were cancelled" exit 1 else - exit 0 + echo "All jobs completed successfully" fi diff --git a/.github/workflows/environment.yml b/.github/workflows/environment.yml index 500e7db8c8425..4540484612559 100644 --- a/.github/workflows/environment.yml +++ b/.github/workflows/environment.yml @@ -21,7 +21,7 @@ jobs: steps: - name: (PR review) Set latest commit status as pending if: ${{ github.event_name == 'pull_request_review' }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} @@ -30,27 +30,27 @@ jobs: - name: (PR review) Checkout PR branch if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.event.review.commit_id }} - name: Checkout branch if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Set up QEMU - uses: docker/setup-qemu-action@v3.6.0 + uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.10.0 + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - name: Login to DockerHub - uses: docker/login-action@v3 + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 if: github.ref == 'refs/heads/master' with: username: ${{ secrets.CI_DOCKER_USERNAME }} password: ${{ secrets.CI_DOCKER_PASSWORD }} - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 + uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 with: images: timberio/vector-dev flavor: | @@ -62,7 +62,7 @@ jobs: org.opencontainers.image.title=Vector development environment org.opencontainers.image.url=https://github.com/vectordotdev/vector - name: Build and push - uses: docker/build-push-action@v6.16.0 + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: context: . file: ./scripts/environment/Dockerfile @@ -71,7 +71,7 @@ jobs: labels: ${{ steps.meta.outputs.labels }} - name: (PR review) Set latest commit status as ${{ job.status }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 if: always() && github.event_name == 'pull_request_review' with: sha: ${{ github.event.review.commit_id }} diff --git a/.github/workflows/gardener_issue_comment.yml b/.github/workflows/gardener_issue_comment.yml index 1e38590b7ca7d..b9936a1edecc4 100644 --- a/.github/workflows/gardener_issue_comment.yml +++ b/.github/workflows/gardener_issue_comment.yml @@ -26,7 +26,7 @@ jobs: - name: Get PR comment author id: comment - uses: tspascoal/get-user-teams-membership@v3 + uses: tspascoal/get-user-teams-membership@57e9f42acd78f4d0f496b3be4368fc5f62696662 # v3.0.0 with: username: ${{ github.actor }} team: 'Vector' diff --git a/.github/workflows/gardener_open_issue.yml b/.github/workflows/gardener_open_issue.yml index 303e8d5147fce..c8ba082d3ed63 100644 --- a/.github/workflows/gardener_open_issue.yml +++ b/.github/workflows/gardener_open_issue.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 5 steps: - - uses: actions/add-to-project@v1.0.2 + - uses: actions/add-to-project@244f685bbc3b7adfa8466e08b698b5577571133e # v1.0.2 with: project-url: https://github.com/orgs/vectordotdev/projects/49 github-token: ${{ secrets.GH_PROJECT_PAT }} diff --git a/.github/workflows/gardener_open_pr.yml b/.github/workflows/gardener_open_pr.yml index c299c9318a42a..873c9da7e26ba 100644 --- a/.github/workflows/gardener_open_pr.yml +++ b/.github/workflows/gardener_open_pr.yml @@ -20,13 +20,13 @@ jobs: with: app_id: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_ID }} private_key: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_PRIVATE_KEY }} - - uses: tspascoal/get-user-teams-membership@v3 + - uses: tspascoal/get-user-teams-membership@57e9f42acd78f4d0f496b3be4368fc5f62696662 # v3.0.0 id: checkVectorMember with: username: ${{ github.actor }} team: vector GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }} - - uses: actions/add-to-project@v1.0.2 + - uses: actions/add-to-project@244f685bbc3b7adfa8466e08b698b5577571133e # v1.0.2 if: ${{ steps.checkVectorMember.outputs.isTeamMember == 'false' }} with: project-url: https://github.com/orgs/vectordotdev/projects/49 @@ -37,7 +37,7 @@ jobs: timeout-minutes: 5 if: ${{ github.actor == 'dependabot[bot]' }} steps: - - uses: actions/add-to-project@v1.0.2 + - uses: actions/add-to-project@244f685bbc3b7adfa8466e08b698b5577571133e # v1.0.2 with: project-url: https://github.com/orgs/vectordotdev/projects/49 github-token: ${{ secrets.GH_PROJECT_PAT }} diff --git a/.github/workflows/gardener_remove_waiting_author.yml b/.github/workflows/gardener_remove_waiting_author.yml index a9e7f52cb3660..9a739fb2bf6b9 100644 --- a/.github/workflows/gardener_remove_waiting_author.yml +++ b/.github/workflows/gardener_remove_waiting_author.yml @@ -1,15 +1,23 @@ name: Remove Awaiting Author Label +permissions: + pull-requests: write + on: + pull_request_target: + types: [synchronize, review_requested] + pull_request: types: [synchronize, review_requested] jobs: remove_label: + if: "contains(github.event.pull_request.labels.*.name, 'meta: awaiting author')" runs-on: ubuntu-24.04 timeout-minutes: 5 steps: - - uses: actions/checkout@v4 - - uses: actions-ecosystem/action-remove-labels@v1 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 # v1.3.0 with: labels: "meta: awaiting author" + fail_on_error: true diff --git a/.github/workflows/install-sh.yml b/.github/workflows/install-sh.yml index 431ae1243fb1e..bc06f366026c9 100644 --- a/.github/workflows/install-sh.yml +++ b/.github/workflows/install-sh.yml @@ -14,12 +14,12 @@ jobs: steps: - name: (PR comment) Get PR branch if: ${{ github.event_name == 'issue_comment' }} - uses: xt0rted/pull-request-comment-branch@v3 + uses: xt0rted/pull-request-comment-branch@e8b8daa837e8ea7331c0003c9c316a64c6d8b0b1 # v3.0.0 id: comment-branch - name: (PR comment) Set latest commit status as pending if: ${{ github.event_name == 'issue_comment' }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ steps.comment-branch.outputs.head_sha }} token: ${{ secrets.GITHUB_TOKEN }} @@ -28,13 +28,13 @@ jobs: - name: (PR comment) Checkout PR branch if: ${{ github.event_name == 'issue_comment' }} - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ steps.comment-branch.outputs.head_ref }} - name: Checkout branch if: ${{ github.event_name != 'issue_comment' }} - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - run: sudo apt-get install --yes bc - run: bash distribution/install.sh -- -y @@ -42,7 +42,7 @@ jobs: - name: (PR comment) Set latest commit status as ${{ job.status }} if: github.event_name == 'issue_comment' - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ steps.comment-branch.outputs.head_sha }} token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index b27ff5617bdd0..2fa4abe3966c2 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -15,8 +15,6 @@ on: type: string env: - AWS_ACCESS_KEY_ID: "dummy" - AWS_SECRET_ACCESS_KEY: "dummy" AXIOM_TOKEN: ${{ secrets.AXIOM_TOKEN }} TEST_APPSIGNAL_PUSH_API_KEY: ${{ secrets.TEST_APPSIGNAL_PUSH_API_KEY }} CONTAINER_TOOL: "docker" @@ -36,20 +34,20 @@ jobs: steps: - name: (PR comment) Get PR branch if: ${{ github.event_name == 'issue_comment' }} - uses: xt0rted/pull-request-comment-branch@v3 + uses: xt0rted/pull-request-comment-branch@e8b8daa837e8ea7331c0003c9c316a64c6d8b0b1 # v3.0.0 id: comment-branch - name: (PR comment) Checkout PR branch if: ${{ github.event_name == 'issue_comment' }} - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ steps.comment-branch.outputs.head_ref }} - name: Checkout branch if: ${{ github.event_name != 'issue_comment' }} - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - run: sudo npm -g install @datadog/datadog-ci + - run: bash scripts/environment/prepare.sh --modules=rustup,datadog-ci - run: make test-integration-${{ inputs.test_name }} env: diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index f8920e80f5d08..81f7d08287568 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -18,8 +18,6 @@ concurrency: cancel-in-progress: true env: - AWS_ACCESS_KEY_ID: "dummy" - AWS_SECRET_ACCESS_KEY: "dummy" CONTAINER_TOOL: "docker" DD_ENV: "ci" DD_API_KEY: ${{ secrets.DD_API_KEY }} @@ -40,64 +38,23 @@ jobs: if: github.event_name == 'pull_request' || github.event_name == 'merge_group' uses: ./.github/workflows/changes.yml with: - base_ref: ${{ github.event.pull_request.base.ref || github.event.merge_group.base_ref }} - head_ref: ${{ github.event.pull_request.head.ref || github.event.merge_group.head_ref }} - source: false + source: true int_tests: true secrets: inherit - check-all: + check-secrets: runs-on: ubuntu-latest - needs: changes outputs: - should_run: ${{ steps.check.outputs.should_run }} + can_access_secrets: ${{ steps.secret_check.outputs.can_access_secrets }} steps: - - name: Download JSON artifact from changes.yml - uses: actions/download-artifact@v4 - with: - name: int_tests_changes - - - name: Check if all values are false - id: check - run: | - # Always run the suite when the PR is in the merge queue - if [[ "${{ github.event_name }}" == "merge_group" ]] ; then - echo "should_run=true" >> $GITHUB_OUTPUT - exit 0 - fi - - # Check if all values are 'false' - json=$(cat int_tests_changes.json) - all_false=$(echo "$json" | jq -r 'to_entries | all(.value == false)') - - if [[ "$all_false" == "true" ]]; then - echo "No changes detected. Skipping integration tests." - echo "should_run=false" >> $GITHUB_OUTPUT - else - echo "Detected changes. Proceeding with integration tests." - echo "should_run=true" >> $GITHUB_OUTPUT - fi - - setup: - runs-on: ubuntu-latest - needs: check-all - if: ${{ needs.check-all.outputs.should_run == 'true' }} - outputs: - can_access_secrets: ${{ steps.secret_access_check.outputs.can_access_secrets }} - steps: - - uses: actions/checkout@v4 - - - run: sudo npm -g install @datadog/datadog-ci - - run: sudo -E bash scripts/ci-free-disk-space.sh - - run: docker image prune -af ; docker container prune -f + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Determine if secrets are defined (PR author is team member) - id: secret_access_check - if: github.event_name == 'pull_request' + id: secret_check env: GH_APP_DATADOG_VECTOR_CI_APP_ID: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_ID }} run: | - if [[ "$GH_APP_DATADOG_VECTOR_CI_APP_ID" != "" ]] ; then + if [[ "$GH_APP_DATADOG_VECTOR_CI_APP_ID" != "" ]]; then echo "can_access_secrets=true" >> $GITHUB_OUTPUT else echo "can_access_secrets=false" >> $GITHUB_OUTPUT @@ -107,15 +64,15 @@ jobs: runs-on: ubuntu-24.04 needs: - changes - - setup - if: ${{ needs.setup.outputs.can_access_secrets == 'true' }} + - check-secrets + + if: ${{ !failure() && !cancelled() && (needs.check-secrets.outputs.can_access_secrets == 'true' || github.event_name == 'merge_group') }} strategy: matrix: - # TODO: Add "azure" back one https://github.com/vectordotdev/vector/issues/22777 is fixed. - # TODO: Add "splunk" back once https://github.com/vectordotdev/vector/issues/22379 is fixed. + # TODO: Add "splunk" back once https://github.com/vectordotdev/vector/issues/23474 is fixed. # If you modify this list, please also update the `int_tests` job in changes.yml. service: [ - "amqp", "appsignal", "axiom", "aws", "clickhouse", "databend", "datadog-agent", + "amqp", "appsignal", "axiom", "aws", "azure", "clickhouse", "databend", "datadog-agent", "datadog-logs", "datadog-metrics", "datadog-traces", "dnstap", "docker-logs", "elasticsearch", "eventstoredb", "fluent", "gcp", "greptimedb", "http-client", "influxdb", "kafka", "logstash", "loki", "mongodb", "nats", "nginx", "opentelemetry", "postgres", "prometheus", "pulsar", @@ -123,32 +80,44 @@ jobs: ] timeout-minutes: 90 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: submodules: "recursive" - - run: docker image prune -af ; docker container prune -f - - name: Download JSON artifact from changes.yml - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + if: github.event_name == 'pull_request' || github.event_name == 'merge_group' with: name: int_tests_changes - name: Run Integration Tests for ${{ matrix.service }} - uses: nick-fields/retry@v3 + uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 with: timeout_minutes: 30 max_attempts: 3 command: | - # Parse the JSON and check if the specific integration test should run. - should_run=$(jq '."${{ matrix.service }}"' int_tests_changes.json) + if [[ -f int_tests_changes.json ]]; then + # Parse the JSON and check if the specific integration test should run. + should_run=$(jq -r '."${{ matrix.service }}" // false' int_tests_changes.json) + else + # The `changes` job did not run (manual run) or the file is missing, default to false. + should_run=false + fi + + if [[ "${{ needs.changes.outputs.website_only }}" == "true" ]]; then + echo "Skipping ${{ matrix.service }} test since only website changes were detected" + exit 0 + fi # Check if any of the three conditions is true if [[ "${{ github.event_name }}" == "merge_group" || \ - "${{ needs.changes.outputs.all-int }}" == "true" || \ + "${{ github.event_name }}" == "workflow_dispatch" || \ + "${{ needs.changes.outputs.dependencies }}" == "true" || \ "$should_run" == "true" ]]; then + # Only install dep if test runs + bash scripts/environment/prepare.sh --modules=datadog-ci echo "Running test for ${{ matrix.service }}" - bash scripts/ci-int-e2e-test.sh int ${{ matrix.service }} + bash scripts/int-e2e-test.sh int ${{ matrix.service }} else echo "Skipping ${{ matrix.service }} test as the value is false or conditions not met." fi @@ -157,20 +126,14 @@ jobs: integration-test-suite: name: Integration Test Suite runs-on: ubuntu-24.04 - timeout-minutes: 5 if: always() needs: - - changes - - check-all - - setup - integration-tests - env: - FAILED: ${{ contains(needs.*.result, 'failure') }} steps: - run: | - echo "failed=${{ env.FAILED }}" - if [[ "$FAILED" == "true" ]] ; then + if [[ "${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}" == "true" ]]; then + echo "One or more jobs failed or were cancelled" exit 1 else - exit 0 + echo "All jobs completed successfully" fi diff --git a/.github/workflows/k8s_e2e.yml b/.github/workflows/k8s_e2e.yml index 244e727f446c6..602308378ca72 100644 --- a/.github/workflows/k8s_e2e.yml +++ b/.github/workflows/k8s_e2e.yml @@ -40,8 +40,6 @@ concurrency: cancel-in-progress: true env: - AWS_ACCESS_KEY_ID: "dummy" - AWS_SECRET_ACCESS_KEY: "dummy" CONTAINER_TOOL: "docker" RUST_BACKTRACE: full TEST_LOG: vector=debug @@ -53,11 +51,8 @@ env: jobs: changes: # Only evaluate files changed on pull request trigger - if: github.event_name == 'pull_request' + if: ${{ github.event_name == 'pull_request' || github.event_name == 'merge_group' }} uses: ./.github/workflows/changes.yml - with: - base_ref: ${{ github.event.pull_request.base.ref }} - head_ref: ${{ github.event.pull_request.head.ref }} secrets: inherit build-x86_64-unknown-linux-gnu: @@ -65,8 +60,8 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 45 needs: changes - # Run this job even if `changes` job is skipped (non- pull request trigger) - if: ${{ !failure() && !cancelled() && (github.event_name != 'pull_request' || needs.changes.outputs.k8s == 'true') }} + # Run this job even if `changes` job is skipped + if: ${{ !failure() && !cancelled() && needs.changes.outputs.website_only != 'true' && needs.changes.outputs.k8s != 'false' }} # cargo-deb requires a release build, but we don't need optimizations for tests env: CARGO_PROFILE_RELEASE_OPT_LEVEL: 0 @@ -75,7 +70,7 @@ jobs: steps: - name: (PR review) Set latest commit status as pending if: ${{ github.event_name == 'pull_request_review' }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} @@ -83,15 +78,15 @@ jobs: - name: (PR review) Checkout PR branch if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.event.review.commit_id }} - name: Checkout branch if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/cache@v4 + - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: | ~/.cargo/registry @@ -100,18 +95,18 @@ jobs: - run: sudo -E bash scripts/ci-free-disk-space.sh - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - - run: bash scripts/environment/prepare.sh - - run: ~/.cargo/bin/rustup target add x86_64-unknown-linux-gnu + - run: bash scripts/environment/prepare.sh --modules=rustup,cross + - run: rustup target add x86_64-unknown-linux-gnu - run: echo "::add-matcher::.github/matchers/rust.json" - run: VECTOR_VERSION="$(cargo vdev version)" make package-deb-x86_64-unknown-linux-gnu - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: e2e-test-deb-package path: target/artifacts/* - name: (PR review) Set latest commit status as 'failure' - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 if: failure() && github.event_name == 'pull_request_review' with: sha: ${{ github.event.review.commit_id }} @@ -130,11 +125,11 @@ jobs: timeout-minutes: 5 needs: changes # Run this job even if `changes` job is skipped - if: ${{ !failure() && !cancelled() && (github.event_name != 'pull_request' || needs.changes.outputs.k8s == 'true') }} + if: ${{ !failure() && !cancelled() && needs.changes.outputs.website_only != 'true' && needs.changes.outputs.k8s != 'false' }} outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - - uses: actions/github-script@v7.0.1 + - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 id: set-matrix with: script: | @@ -194,20 +189,20 @@ jobs: steps: - name: (PR review) Get PR branch if: ${{ github.event_name == 'pull_request_review' }} - uses: xt0rted/pull-request-comment-branch@v3 + uses: xt0rted/pull-request-comment-branch@e8b8daa837e8ea7331c0003c9c316a64c6d8b0b1 # v3.0.0 id: comment-branch - name: (PR review) Checkout PR branch if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ steps.comment-branch.outputs.head_ref }} - name: Checkout branch if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: e2e-test-deb-package path: target/artifacts @@ -221,7 +216,7 @@ jobs: # TODO: This job has been quite flakey. Need to investigate further and then remove the retries. - name: Run tests - uses: nick-fields/retry@v3 + uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 env: USE_MINIKUBE_CACHE: "true" SKIP_PACKAGE_DEB: "true" @@ -232,7 +227,7 @@ jobs: command: make test-e2e-kubernetes - name: (PR review) Set latest commit status as failure - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 if: failure() && github.event_name == 'pull_request_review' with: sha: ${{ github.event.review.commit_id }} @@ -250,25 +245,26 @@ jobs: - test-e2e-kubernetes if: always() env: - FAILED: ${{ contains(needs.*.result, 'failure') }} + FAILED: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} steps: - name: (PR review) Get PR branch if: github.event_name == 'pull_request_review' && env.FAILED != 'true' - uses: xt0rted/pull-request-comment-branch@v3 + uses: xt0rted/pull-request-comment-branch@e8b8daa837e8ea7331c0003c9c316a64c6d8b0b1 # v3.0.0 id: comment-branch - name: (PR review) Submit PR result as success if: github.event_name == 'pull_request_review' && env.FAILED != 'true' - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} status: 'success' - - run: | - echo "failed=${{ env.FAILED }}" - if [[ "$FAILED" == "true" ]] ; then + - name: Check all jobs status + run: | + if [[ "${{ env.FAILED }}" == "true" ]]; then + echo "One or more jobs failed or were cancelled" exit 1 else - exit 0 + echo "All jobs completed successfully" fi diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index a230b8569a94b..44171cc3746d7 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -10,7 +10,7 @@ jobs: contents: read pull-requests: write steps: - - uses: actions/labeler@v5 + - uses: actions/labeler@8558fd74291d67161a8a78ce36a881fa63b766a9 # v5.0.0 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" sync-labels: true diff --git a/.github/workflows/master_merge_queue.yml b/.github/workflows/master_merge_queue.yml index 034eb5914c624..b6cf9486e1c38 100644 --- a/.github/workflows/master_merge_queue.yml +++ b/.github/workflows/master_merge_queue.yml @@ -28,8 +28,6 @@ concurrency: cancel-in-progress: true env: - AWS_ACCESS_KEY_ID: "dummy" - AWS_SECRET_ACCESS_KEY: "dummy" CONTAINER_TOOL: "docker" DD_ENV: "ci" DD_API_KEY: ${{ secrets.DD_API_KEY }} @@ -49,9 +47,6 @@ jobs: changes: if: ${{ github.event_name == 'merge_group' }} uses: ./.github/workflows/changes.yml - with: - base_ref: ${{ github.event.merge_group.base_ref }} - head_ref: ${{ github.event.merge_group.head_ref }} secrets: inherit test-cli: @@ -119,14 +114,12 @@ jobs: - unit-mac - unit-windows - install-sh - env: - FAILED: ${{ contains(needs.*.result, 'failure') }} steps: - - name: exit + - name: Check all jobs status run: | - echo "failed=${{ env.FAILED }}" - if [[ "$FAILED" == "true" ]] ; then + if [[ "${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}" == "true" ]]; then + echo "One or more jobs failed or were cancelled" exit 1 else - exit 0 + echo "All jobs completed successfully" fi diff --git a/.github/workflows/misc.yml b/.github/workflows/misc.yml index 6af0fca39b788..fead79447cd80 100644 --- a/.github/workflows/misc.yml +++ b/.github/workflows/misc.yml @@ -15,7 +15,7 @@ jobs: steps: - name: (PR review) Set latest commit status as pending if: ${{ github.event_name == 'pull_request_review' }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} @@ -24,15 +24,15 @@ jobs: - name: (PR review) Checkout review SHA if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.event.review.commit_id }} - name: Checkout branch if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/cache@v4 + - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 name: Cache Cargo registry + index with: path: | @@ -46,14 +46,14 @@ jobs: - run: sudo -E bash scripts/ci-free-disk-space.sh - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - - run: bash scripts/environment/prepare.sh + - run: bash scripts/environment/prepare.sh --modules=rustup - run: echo "::add-matcher::.github/matchers/rust.json" - run: make test-behavior - run: make check-examples - run: make test-docs - name: (PR review) Set latest commit status as ${{ job.status }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 if: always() && github.event_name == 'pull_request_review' with: sha: ${{ github.event.review.commit_id }} diff --git a/.github/workflows/msrv.yml b/.github/workflows/msrv.yml index 18a70126da711..451f14b705099 100644 --- a/.github/workflows/msrv.yml +++ b/.github/workflows/msrv.yml @@ -1,7 +1,13 @@ -name: Check minimum supported Rust version +name: Check minimum supported Rust version (MSRV) on: workflow_call: + workflow_dispatch: + inputs: + checkout_ref: + description: "Git ref (branch, tag or SHA) to check out" + required: false + type: string env: RUST_BACKTRACE: full @@ -16,10 +22,9 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 20 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ inputs.checkout_ref }} - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - - run: | - # We usually run `scripts/environment/prepare.sh` but in this case we only need the toolchain. - rustup show active-toolchain || rustup toolchain install - - run: cargo install cargo-msrv --version 0.15.1 + - run: bash scripts/environment/prepare.sh --modules=cargo-msrv - run: cargo msrv verify diff --git a/.github/workflows/preview_site_trigger.yml b/.github/workflows/preview_site_trigger.yml index 9bae486e39849..811f5925ae86c 100644 --- a/.github/workflows/preview_site_trigger.yml +++ b/.github/workflows/preview_site_trigger.yml @@ -7,63 +7,45 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 5 # Only run for PRs with 'website' in the branch name - if: ${{ contains(github.head_ref, 'website') && contains(github.head_ref, '-') }} + if: ${{ contains(github.head_ref, 'website') }} steps: - - name: Echo approval + # Validate branch name + - name: Validate branch name and set output + id: validate run: | - echo "Workflow has been allowed to run for PR ${{ github.event.number }}. Setting artifacts and then continuing workflow runs" - - # Use GitHub Action to safely validate and store PR information + BRANCH="${{ github.head_ref }}" + if [[ ! "$BRANCH" =~ ^[a-zA-Z0-9_-]+$ ]]; then + echo "valid=false" >> $GITHUB_OUTPUT + else + echo "valid=true" >> $GITHUB_OUTPUT + fi + + # Save PR information (only if branch is valid) - name: Validate and save PR information - uses: actions/github-script@v7.0.1 + if: steps.validate.outputs.valid == 'true' + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 with: script: | const fs = require('fs').promises; const path = require('path'); const crypto = require('crypto'); + const prNumber = context.payload.number; + const branchName = context.payload.pull_request.head.ref; - async function createAndValidateArtifact() { - try { - // Create directory for artifact - await fs.mkdir('./pr', { recursive: true }); - - // Get PR number and validate - const prNumber = context.payload.number; - if (typeof prNumber !== 'number' || !Number.isInteger(prNumber) || prNumber <= 0) { - core.setFailed(`Invalid PR number: ${prNumber}`); - return; - } - - // Get branch name and validate - const branchName = context.payload.pull_request.head.ref; - // Validate branch name (only allow alphanumeric, dash, and underscore) - const branchNameRegex = /^[a-zA-Z0-9_\-]+$/; - if (!branchNameRegex.test(branchName)) { - core.setFailed(`Invalid branch name detected: ${branchName}`); - return; - } - - // Write validated information to files - await fs.writeFile('./pr/number', prNumber.toString()); - await fs.writeFile('./pr/branch', branchName); - - // Log success - core.info(`Successfully validated and saved PR #${prNumber} with branch ${branchName}`); + await fs.mkdir('./pr', { recursive: true }); + await fs.writeFile('./pr/number', prNumber.toString()); + await fs.writeFile('./pr/branch', branchName); - // Create hash signature of the data - const numberHash = crypto.createHash('sha256').update(prNumber.toString()).digest('hex'); - const branchHash = crypto.createHash('sha256').update(branchName).digest('hex'); - await fs.writeFile('./pr/integrity', `${numberHash}:${branchHash}`); - } catch (error) { - core.setFailed(`An error occurred: ${error.message}`); - } - } + const numberHash = crypto.createHash('sha256').update(prNumber.toString()).digest('hex'); + const branchHash = crypto.createHash('sha256').update(branchName).digest('hex'); + await fs.writeFile('./pr/integrity', `${numberHash}:${branchHash}`); - createAndValidateArtifact(); + core.info(`Saved PR #${prNumber} and branch ${branchName}`); - # Upload the artifact using latest version + # Upload the artifact using latest version (only if branch is valid) - name: Upload PR information artifact - uses: actions/upload-artifact@v4 + if: steps.validate.outputs.valid == 'true' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: pr path: pr/ diff --git a/.github/workflows/protobuf.yml b/.github/workflows/protobuf.yml index 20fa9f864f351..836c0f19b29dc 100644 --- a/.github/workflows/protobuf.yml +++ b/.github/workflows/protobuf.yml @@ -1,5 +1,8 @@ name: Protobuf Compatibility +permissions: + contents: read + on: pull_request: paths: @@ -19,10 +22,12 @@ jobs: timeout-minutes: 5 steps: # Run `git checkout` - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 # Install the `buf` CLI - - uses: bufbuild/buf-setup-action@v1.50.0 + - uses: bufbuild/buf-setup-action@a47c93e0b1648d5651a065437926377d060baa99 # v1.50.0 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} # Perform breaking change detection against the `master` branch - - uses: bufbuild/buf-breaking-action@v1.1.4 + - uses: bufbuild/buf-breaking-action@c57b3d842a5c3f3b454756ef65305a50a587c5ba # v1.1.4 with: against: "https://github.com/vectordotdev/vector.git#branch=master" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 099fb27b09146..321542e678cd5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -40,7 +40,7 @@ jobs: vector_release_channel: ${{ steps.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.git_ref }} - name: Generate publish metadata @@ -58,17 +58,17 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - name: Bootstrap runner environment (generic) - run: bash scripts/environment/prepare.sh + run: bash scripts/environment/prepare.sh --modules=rustup,cross - name: Build Vector run: make package-x86_64-unknown-linux-musl-all - name: Stage package artifacts for publish - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-musl path: target/artifacts/vector* @@ -76,25 +76,25 @@ jobs: build-x86_64-unknown-linux-gnu-packages: name: Build Vector for x86_64-unknown-linux-gnu (.tar.gz, DEB, RPM) runs-on: release-builder-linux - needs: generate-publish-metadata timeout-minutes: 60 + needs: generate-publish-metadata env: VECTOR_VERSION: ${{ needs.generate-publish-metadata.outputs.vector_version }} VECTOR_BUILD_DESC: ${{ needs.generate-publish-metadata.outputs.vector_build_desc }} CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - name: Bootstrap runner environment (generic) - run: bash scripts/environment/prepare.sh + run: bash scripts/environment/prepare.sh --modules=rustup,cross - name: Build Vector run: make package-x86_64-unknown-linux-gnu-all - name: Stage package artifacts for publish - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-gnu path: target/artifacts/vector* @@ -110,19 +110,19 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - name: Bootstrap runner environment (generic) - run: bash scripts/environment/prepare.sh + run: bash scripts/environment/prepare.sh --modules=rustup,cross - name: Build Vector env: DOCKER_PRIVILEGED: "true" run: make package-aarch64-unknown-linux-musl-all - name: Stage package artifacts for publish - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-musl path: target/artifacts/vector* @@ -138,19 +138,19 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - name: Bootstrap runner environment (generic) - run: bash scripts/environment/prepare.sh + run: bash scripts/environment/prepare.sh --modules=rustup,cross - name: Build Vector env: DOCKER_PRIVILEGED: "true" run: make package-aarch64-unknown-linux-gnu-all - name: Stage package artifacts for publish - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-gnu path: target/artifacts/vector* @@ -166,19 +166,19 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - name: Bootstrap runner environment (generic) - run: bash scripts/environment/prepare.sh + run: bash scripts/environment/prepare.sh --modules=rustup,cross - name: Build Vector env: DOCKER_PRIVILEGED: "true" run: make package-armv7-unknown-linux-gnueabihf-all - name: Stage package artifacts for publish - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-gnueabihf path: target/artifacts/vector* @@ -194,19 +194,19 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - name: Bootstrap runner environment (generic) - run: bash scripts/environment/prepare.sh + run: bash scripts/environment/prepare.sh --modules=rustup,cross - name: Build Vector env: DOCKER_PRIVILEGED: "true" run: make package-armv7-unknown-linux-musleabihf - name: Stage package artifacts for publish - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-musleabihf path: target/artifacts/vector* @@ -222,19 +222,19 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - name: Bootstrap runner environment (generic) - run: bash scripts/environment/prepare.sh + run: bash scripts/environment/prepare.sh --modules=rustup,cross - name: Build Vector env: DOCKER_PRIVILEGED: "true" run: make package-arm-unknown-linux-gnueabi-all - name: Stage package artifacts for publish - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-gnueabi path: target/artifacts/vector* @@ -242,6 +242,7 @@ jobs: build-arm-unknown-linux-musleabi-packages: name: Build Vector for arm-unknown-linux-musleabi (.tar.gz) runs-on: release-builder-linux + timeout-minutes: 60 needs: generate-publish-metadata env: VECTOR_VERSION: ${{ needs.generate-publish-metadata.outputs.vector_version }} @@ -249,19 +250,19 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - name: Bootstrap runner environment (generic) - run: bash scripts/environment/prepare.sh + run: bash scripts/environment/prepare.sh --modules=rustup,cross - name: Build Vector env: DOCKER_PRIVILEGED: "true" run: make package-arm-unknown-linux-musleabi - name: Stage package artifacts for publish - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-musleabi path: target/artifacts/vector* @@ -279,10 +280,11 @@ jobs: matrix: include: # Refer to https://docs.github.com/en/actions/using-github-hosted-runners/using-larger-runners/about-larger-runners#about-macos-larger-runners. + # and to https://github.com/actions/runner-images - architecture: x86_64 - runner: macos-latest-large + runner: macos-14-large - architecture: arm64 - runner: macos-latest-xlarge + runner: macos-14-xlarge steps: - name: Verify Runner Architecture run: | @@ -293,15 +295,13 @@ jobs: exit 1 fi - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (macOS-specific) run: | bash scripts/environment/bootstrap-macos.sh - # We usually run `scripts/environment/prepare.sh` but in this case we only need the toolchain. - rustup show active-toolchain || rustup toolchain install - rustup show + bash scripts/environment/prepare.sh --modules=rustup - name: Build Vector env: TARGET: "${{ matrix.architecture }}-apple-darwin" @@ -310,7 +310,7 @@ jobs: export PATH="$HOME/.cargo/bin:$PATH" make package - name: Stage package artifacts for publish - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: vector-${{ env.VECTOR_VERSION }}-${{ matrix.architecture }}-apple-darwin path: target/artifacts/vector* @@ -329,7 +329,7 @@ jobs: RELEASE_BUILDER: "true" steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Windows-specific) @@ -358,7 +358,7 @@ jobs: export PATH="/c/wix:$PATH" ./scripts/package-msi.sh - name: Stage package artifacts for publish - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-pc-windows-msvc path: target/artifacts/vector* @@ -379,7 +379,6 @@ jobs: - ubuntu:20.04 - ubuntu:22.04 - ubuntu:24.04 - - debian:10 - debian:11 - debian:12 container: @@ -399,11 +398,11 @@ jobs: - name: Fix Git safe directories issue when in containers (actions/checkout#760) run: git config --global --add safe.directory /__w/vector/vector - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.git_ref }} - name: Download staged package artifacts (x86_64-unknown-linux-gnu) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-gnu path: target/artifacts @@ -450,11 +449,11 @@ jobs: - name: Fix Git safe directories issue when in containers (actions/checkout#760) run: git config --global --add safe.directory /__w/vector/vector - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.git_ref }} - name: Download staged package artifacts (x86_64-unknown-linux-gnu) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-gnu path: target/artifacts @@ -474,17 +473,18 @@ jobs: strategy: matrix: include: + # Refer to https://github.com/actions/runner-images - target: x86_64-apple-darwin - runner: macos-latest-large + runner: macos-14-large - target: arm64-apple-darwin - runner: macos-latest-xlarge + runner: macos-14 steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.git_ref }} - name: Download staged package artifacts (${{ matrix.target }}) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-${{ matrix.target }} path: target/artifacts @@ -514,67 +514,67 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.git_ref }} - name: Login to DockerHub - uses: docker/login-action@v3 + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 with: username: ${{ secrets.CI_DOCKER_USERNAME }} password: ${{ secrets.CI_DOCKER_PASSWORD }} - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Set up QEMU - uses: docker/setup-qemu-action@v3.6.0 + uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0 with: platforms: all - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v3.10.0 + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 with: version: latest install: true - name: Download staged package artifacts (aarch64-unknown-linux-gnu) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-gnu path: target/artifacts - name: Download staged package artifacts (aarch64-unknown-linux-musl) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-musl path: target/artifacts - name: Download staged package artifacts (x86_64-unknown-linux-gnu) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-gnu path: target/artifacts - name: Download staged package artifacts (x86_64-unknown-linux-musl) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-musl path: target/artifacts - name: Download staged package artifacts (armv7-unknown-linux-gnueabihf) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-gnueabihf path: target/artifacts - name: Download staged package artifacts (armv7-unknown-linux-musleabihf) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-musleabihf path: target/artifacts - name: Download staged package artifacts (arm-unknown-linux-gnueabi) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-gnueabi path: target/artifacts - name: Download staged package artifacts (arm-unknown-linux-musleabi) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-musleabi path: target/artifacts @@ -582,7 +582,7 @@ jobs: env: PLATFORM: "linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6" REPOS: "timberio/vector,ghcr.io/vectordotdev/vector" - uses: nick-fields/retry@v3 + uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 with: timeout_minutes: 15 max_attempts: 3 @@ -613,61 +613,61 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.git_ref }} - name: Download staged package artifacts (aarch64-unknown-linux-gnu) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-gnu path: target/artifacts - name: Download staged package artifacts (aarch64-unknown-linux-musl) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-musl path: target/artifacts - name: Download staged package artifacts (x86_64-unknown-linux-gnu) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-gnu path: target/artifacts - name: Download staged package artifacts (x86_64-unknown-linux-musl) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-musl path: target/artifacts - name: Download staged package artifacts (x86_64-apple-darwin) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-apple-darwin path: target/artifacts - name: Download staged package artifacts (arm64-apple-darwin) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm64-apple-darwin path: target/artifacts - name: Download staged package artifacts (x86_64-pc-windows-msvc) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-pc-windows-msvc path: target/artifacts - name: Download staged package artifacts (armv7-unknown-linux-gnueabihf) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-gnueabihf path: target/artifacts - name: Download staged package artifacts (armv7-unknown-linux-musleabihf) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-musleabihf path: target/artifacts - name: Download staged package artifacts (arm-unknown-linux-gnueabi) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-gnueabi path: target/artifacts - name: Download staged package artifacts (arm-unknown-linux-musleabi) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-musleabi path: target/artifacts @@ -678,7 +678,7 @@ jobs: run: make release-s3 publish-github: - name: Publish to GitHub + name: Publish release to GitHub # We only publish to GitHub for versioned releases, not nightlies. if: inputs.channel == 'release' runs-on: ubuntu-24.04 @@ -703,66 +703,66 @@ jobs: VECTOR_VERSION: ${{ needs.generate-publish-metadata.outputs.vector_version }} steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.git_ref }} - name: Download staged package artifacts (aarch64-unknown-linux-gnu) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-gnu path: target/artifacts - name: Download staged package artifacts (aarch64-unknown-linux-musl) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-musl path: target/artifacts - name: Download staged package artifacts (x86_64-unknown-linux-gnu) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-gnu path: target/artifacts - name: Download staged package artifacts (x86_64-unknown-linux-musl) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-musl path: target/artifacts - name: Download staged package artifacts (x86_64-apple-darwin) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-apple-darwin path: target/artifacts - name: Download staged package artifacts (arm64-apple-darwin) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm64-apple-darwin path: target/artifacts - name: Download staged package artifacts (x86_64-pc-windows-msvc) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-pc-windows-msvc path: target/artifacts - name: Download staged package artifacts (armv7-unknown-linux-gnueabihf) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-gnueabihf path: target/artifacts - name: Download staged package artifacts (armv7-unknown-linux-musleabihf) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-musleabihf path: target/artifacts - name: Download artifact checksums - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-SHA256SUMS path: target/artifacts - name: Download staged package artifacts (arm-unknown-linux-gnueabi) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-gnueabi path: target/artifacts - name: Download staged package artifacts (arm-unknown-linux-musleabi) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-musleabi path: target/artifacts @@ -784,7 +784,7 @@ jobs: VECTOR_VERSION: ${{ needs.generate-publish-metadata.outputs.vector_version }} steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.git_ref }} - name: Publish update to Homebrew tap @@ -812,68 +812,68 @@ jobs: VECTOR_VERSION: ${{ needs.generate-publish-metadata.outputs.vector_version }} steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.git_ref }} - name: Download staged package artifacts (aarch64-unknown-linux-gnu) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-gnu path: target/artifacts - name: Download staged package artifacts (aarch64-unknown-linux-musl) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-musl path: target/artifacts - name: Download staged package artifacts (x86_64-unknown-linux-gnu) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-gnu path: target/artifacts - name: Download staged package artifacts (x86_64-unknown-linux-musl) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-musl path: target/artifacts - name: Download staged package artifacts (x86_64-apple-darwin) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-apple-darwin path: target/artifacts - name: Download staged package artifacts (arm64-apple-darwin) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm64-apple-darwin path: target/artifacts - name: Download staged package artifacts (x86_64-pc-windows-msvc) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-pc-windows-msvc path: target/artifacts - name: Download staged package artifacts (armv7-unknown-linux-gnueabihf) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-gnueabihf path: target/artifacts - name: Download staged package artifacts (armv7-unknown-linux-musleabihf) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-musleabihf path: target/artifacts - name: Download staged package artifacts (arm-unknown-linux-gnueabi) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-gnueabi path: target/artifacts - name: Download staged package artifacts (arm-unknown-linux-musleabi) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-musleabi path: target/artifacts - name: Generate SHA256 checksums for artifacts run: make sha256sum - name: Stage checksum for publish - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: vector-${{ env.VECTOR_VERSION }}-SHA256SUMS path: target/artifacts/vector-${{ env.VECTOR_VERSION }}-SHA256SUMS diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index af2426ce30429..680ddddafa318 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -51,7 +51,7 @@ jobs: smp-version: ${{ steps.experimental-meta.outputs.SMP_CRATE_VERSION }} steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 0 # need to pull repository history to find merge bases @@ -105,7 +105,7 @@ jobs: - name: Set SMP version id: experimental-meta run: | - export SMP_CRATE_VERSION="0.21.0" + export SMP_CRATE_VERSION="0.22.0" echo "smp crate version: ${SMP_CRATE_VERSION}" echo "SMP_CRATE_VERSION=${SMP_CRATE_VERSION}" >> $GITHUB_OUTPUT @@ -117,11 +117,11 @@ jobs: outputs: source_changed: ${{ steps.filter.outputs.SOURCE_CHANGED }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Collect file changes id: changes - uses: dorny/paths-filter@v3 + uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 with: base: ${{ needs.resolve-inputs.outputs.baseline-sha }} ref: ${{ needs.resolve-inputs.outputs.comparison-sha }} @@ -190,21 +190,21 @@ jobs: - should-run-gate - resolve-inputs steps: - - uses: colpal/actions-clean@v1 + - uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1 - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ needs.resolve-inputs.outputs.baseline-sha }} path: baseline-vector - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v3.10.0 + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - name: Build 'vector' target image - uses: docker/build-push-action@v6.16.0 + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: context: baseline-vector/ cache-from: type=gha @@ -216,7 +216,7 @@ jobs: vector:${{ needs.resolve-inputs.outputs.baseline-tag }} - name: Upload image as artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: baseline-image path: "${{ runner.temp }}/baseline-image.tar" @@ -229,21 +229,21 @@ jobs: - should-run-gate - resolve-inputs steps: - - uses: colpal/actions-clean@v1 + - uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1 - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ needs.resolve-inputs.outputs.comparison-sha }} path: comparison-vector - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v3.10.0 + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - name: Build 'vector' target image - uses: docker/build-push-action@v6.16.0 + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: context: comparison-vector/ cache-from: type=gha @@ -255,7 +255,7 @@ jobs: vector:${{ needs.resolve-inputs.outputs.comparison-tag }} - name: Upload image as artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: comparison-image path: "${{ runner.temp }}/comparison-image.tar" @@ -269,7 +269,7 @@ jobs: - resolve-inputs steps: - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4.1.0 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -294,7 +294,7 @@ jobs: - build-baseline steps: - name: 'Download baseline image' - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: baseline-image @@ -303,7 +303,7 @@ jobs: docker load --input baseline-image.tar - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4.1.0 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -311,10 +311,10 @@ jobs: - name: Login to Amazon ECR id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 + uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 - name: Docker Login to ECR - uses: docker/login-action@v3 + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 with: registry: ${{ steps.login-ecr.outputs.registry }} @@ -334,7 +334,7 @@ jobs: - build-comparison steps: - name: 'Download comparison image' - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: comparison-image @@ -343,7 +343,7 @@ jobs: docker load --input comparison-image.tar - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4.1.0 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -351,10 +351,10 @@ jobs: - name: Login to Amazon ECR id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 + uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 - name: Docker Login to ECR - uses: docker/login-action@v3 + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 with: registry: ${{ steps.login-ecr.outputs.registry }} @@ -373,12 +373,12 @@ jobs: - upload-baseline-image-to-ecr - upload-comparison-image-to-ecr steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ needs.resolve-inputs.outputs.comparison-sha }} - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4.1.0 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -386,7 +386,7 @@ jobs: - name: Login to Amazon ECR id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 + uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 - name: Download SMP binary run: | @@ -408,7 +408,7 @@ jobs: --submission-metadata ${{ runner.temp }}/submission-metadata \ --replicas ${{ env.SMP_REPLICAS }} - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: vector-submission-metadata path: ${{ runner.temp }}/submission-metadata @@ -448,10 +448,10 @@ jobs: - should-run-gate - resolve-inputs steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4.1.0 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -462,7 +462,7 @@ jobs: aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp - name: Download submission metadata - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-submission-metadata path: ${{ runner.temp }}/ @@ -485,12 +485,12 @@ jobs: - submit-job - resolve-inputs steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ needs.resolve-inputs.outputs.comparison-sha }} - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4.1.0 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -501,7 +501,7 @@ jobs: aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp - name: Download submission metadata - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-submission-metadata path: ${{ runner.temp }}/ @@ -518,12 +518,12 @@ jobs: - name: Read regression report id: read-analysis - uses: juliangruber/read-file-action@v1 + uses: juliangruber/read-file-action@b549046febe0fe86f8cb4f93c24e284433f9ab58 # v1.1.7 with: path: ${{ runner.temp }}/outputs/report.md - name: Upload regression report to artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: capture-artifacts path: ${{ runner.temp }}/outputs/* @@ -544,12 +544,10 @@ jobs: - submit-job - detect-regression - analyze-experiment - env: - FAILED: ${{ contains(needs.*.result, 'failure') }} steps: - name: Download capture-artifacts continue-on-error: true - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: capture-artifacts @@ -563,11 +561,11 @@ jobs: echo "Did not find ${REPORT_MD} file." fi - - name: exit + - name: Check all jobs status run: | - echo "failed=${{ env.FAILED }}" - if [[ "$FAILED" == "true" ]] ; then + if [[ "${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}" == "true" ]]; then + echo "One or more jobs failed or were cancelled" exit 1 else - exit 0 + echo "All jobs completed successfully" fi diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index ba19d832ca3fb..70ba6da880d76 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -32,12 +32,12 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@v2.4.1 + uses: ossf/scorecard-action@05b42c624433fc40578a4040d5cf5e36ddca8cde # v2.4.2 with: results_file: results.sarif results_format: sarif @@ -59,7 +59,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: SARIF file path: results.sarif @@ -68,6 +68,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard (optional). # Commenting out will disable upload of results to your repo's Code Scanning dashboard - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@v3 + uses: github/codeql-action/upload-sarif@51f77329afa6477de8c49fc9c7046c15b9a4e79d # v3.29.5 with: sarif_file: results.sarif diff --git a/.github/workflows/semantic.yml b/.github/workflows/semantic.yml index 47a53618e4eef..99c6cf0d66b83 100644 --- a/.github/workflows/semantic.yml +++ b/.github/workflows/semantic.yml @@ -4,18 +4,17 @@ name: "PR Title Semantic Check" on: + pull_request_target: + types: [opened, edited, synchronize] pull_request: - types: - - opened - - edited - - synchronize + types: [opened, edited, synchronize] jobs: main: name: Check Semantic PR runs-on: ubuntu-24.04 steps: - - uses: amannn/action-semantic-pull-request@v5 + - uses: amannn/action-semantic-pull-request@0723387faaf9b38adef4775cd42cfd5155ed6017 # v5.5.3 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: @@ -29,28 +28,29 @@ jobs: revert scopes: | - new source - new transform - new sink - ARC administration api + ARC architecture auth buffers ci cli codecs + codecs protobuf compression config + config provider core data model delivery deployment + deprecations deps dev durability enriching + enrichment tables enterprise exceptions external @@ -62,6 +62,9 @@ jobs: logs metrics networking + new sink + new source + new transform observability parsing performance @@ -86,7 +89,9 @@ jobs: traces transforms unit tests + vdev vrl + amazon-linux platform apt platform arm platform @@ -111,7 +116,6 @@ jobs: x86_64 platform yum platform - service providers aws service azure service confluent service @@ -127,8 +131,10 @@ jobs: new relic service papertrail service sematext service + service providers splunk service yandex service + apache_metrics source aws_ecs_metrics source aws_kinesis_firehose source @@ -145,7 +151,7 @@ jobs: gcp_pubsub source heroku_logs source host_metrics source - http source + http_client source http_scrape source internal_logs source internal_metrics source @@ -168,8 +174,11 @@ jobs: stdin source syslog source vector source + websocket source + aws_ec2_metadata transform dedupe transform + exclusive_route transform filter transform geoip transform log_to_metric transform @@ -180,10 +189,10 @@ jobs: reduce transform remap transform route transform - exclusive_route transform sample transform tag_cardinality_limit transform throttle transform + amqp sink apex sink aws_cloudwatch_logs sink @@ -192,12 +201,14 @@ jobs: aws_kinesis_streams sink aws_s3 sink aws_sqs sink + axiom sink azure_blob sink azure_monitor_logs sink blackhole sink clickhouse sink console sink datadog_archives sink + datadog_common sink datadog_events sink datadog_logs sink datadog_metrics sink @@ -208,8 +219,8 @@ jobs: gcp_pubsub sink gcp_stackdriver_logs sink gcp_stackdriver_metrics sink - greptimedb_metrics sink greptimedb_logs sink + greptimedb_metrics sink honeycomb sink http sink humio_logs sink @@ -238,6 +249,7 @@ jobs: vector sink websocket sink websocket_server sink + blog website css website guides website @@ -248,4 +260,4 @@ jobs: website website deps - config provider + opentelemetry lib diff --git a/.github/workflows/spelling.yml b/.github/workflows/spelling.yml index 5976f1a9fedb4..810fb03867dd3 100644 --- a/.github/workflows/spelling.yml +++ b/.github/workflows/spelling.yml @@ -87,7 +87,7 @@ jobs: steps: - name: check-spelling id: spelling - uses: check-spelling/check-spelling@v0.0.24 + uses: check-spelling/check-spelling@c635c2f3f714eec2fcf27b643a1919b9a811ef2e # v0.0.25 with: suppress_push_for_open_pull_request: ${{ github.actor != 'dependabot[bot]' && 1 }} checkout: true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c30215f33bf49..fb96784f22c40 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,8 +11,6 @@ concurrency: cancel-in-progress: true env: - AWS_ACCESS_KEY_ID: "dummy" - AWS_SECRET_ACCESS_KEY: "dummy" CONTAINER_TOOL: "docker" DD_ENV: "ci" DD_API_KEY: ${{ secrets.DD_API_KEY }} @@ -29,53 +27,43 @@ jobs: changes: uses: ./.github/workflows/changes.yml secrets: inherit - with: - base_ref: ${{ github.event.merge_group.base_ref || github.event.pull_request.base.ref }} - head_ref: ${{ github.event.merge_group.head_ref || github.event.pull_request.head.ref }} - checks: - name: Checks - runs-on: ubuntu-24.04-8core - timeout-minutes: 60 + check-fmt: + name: Check code format + runs-on: ubuntu-24.04 needs: changes - env: - CARGO_INCREMENTAL: 0 steps: - - uses: actions/checkout@v4 - with: - # check-version needs tags - fetch-depth: 0 # fetch everything - - - uses: actions/cache@v4 - name: Cache Cargo registry + index - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo- - - - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - - - uses: ruby/setup-ruby@v1 - - - run: bash scripts/environment/prepare.sh - + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Enable Rust matcher run: echo "::add-matcher::.github/matchers/rust.json" + - run: make check-fmt - - name: Check code format - run: make check-fmt + check-clippy: + name: Check clippy + runs-on: ubuntu-24.04-8core + if: needs.changes.outputs.source == 'true' + needs: changes + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Enable Rust matcher + run: echo "::add-matcher::.github/matchers/rust.json" + - run: sudo bash ./scripts/environment/install-protoc.sh + - run: sudo apt-get update && sudo apt-get install -y libsasl2-dev + - run: make check-clippy - - name: Check clippy - if: needs.changes.outputs.source == 'true' - run: make check-clippy + test: + name: Unit and Component Validation tests - x86_64-unknown-linux-gnu + runs-on: ubuntu-24.04-8core + if: needs.changes.outputs.source == 'true' + needs: changes + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Enable Rust matcher + run: echo "::add-matcher::.github/matchers/rust.json" + - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh + - run: bash ./scripts/environment/prepare.sh --modules=cargo-nextest,datadog-ci - - name: Unit - x86_64-unknown-linux-gnu - if: needs.changes.outputs.source == 'true' + - name: Unit Test run: make test env: CARGO_BUILD_JOBS: 5 @@ -88,61 +76,141 @@ jobs: run: scripts/upload-test-results.sh if: always() - - name: Check version - run: make check-version + check-version: + name: Check version + runs-on: ubuntu-24.04 + needs: changes + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - run: make check-version - - name: Check scripts - run: make check-scripts + check-scripts: + name: Check scripts + runs-on: ubuntu-24.04 + needs: changes + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - run: make check-scripts - - name: Check events - if: needs.changes.outputs.source == 'true' - run: make check-events + check-events: + name: Check events + runs-on: ubuntu-24.04-8core + if: needs.changes.outputs.source == 'true' + needs: changes + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Enable Rust matcher + run: echo "::add-matcher::.github/matchers/rust.json" + - run: make check-events - - name: Check that the 3rd-party license file is up to date - if: needs.changes.outputs.dependencies == 'true' - run: make check-licenses + check-licenses: + name: Check that the 3rd-party license file is up to date + runs-on: ubuntu-24.04 + if: needs.changes.outputs.dependencies == 'true' + needs: changes + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - run: bash ./scripts/environment/prepare.sh --modules=dd-rust-license-tool + - run: make check-licenses - - name: Check Cue docs - if: needs.changes.outputs.cue == 'true' - run: make check-docs + check-docs: + name: Check Cue docs + runs-on: ubuntu-24.04 + if: needs.changes.outputs.cue == 'true' + needs: changes + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh + - run: make check-docs - - name: Check Markdown - if: needs.changes.outputs.markdown == 'true' - run: make check-markdown + check-markdown: + name: Check Markdown + runs-on: ubuntu-24.04 + if: needs.changes.outputs.markdown == 'true' + needs: changes + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - run: bash ./scripts/environment/prepare.sh --modules=markdownlint + - run: make check-markdown - - name: Check Component Docs - if: needs.changes.outputs.source == 'true' || needs.changes.outputs.component_docs == 'true' - run: make check-component-docs + check-component-docs: + name: Check Component Docs + runs-on: ubuntu-24.04-8core + if: needs.changes.outputs.source == 'true' || needs.changes.outputs.component_docs == 'true' + needs: changes + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - run: sudo bash ./scripts/environment/install-protoc.sh + - run: sudo apt-get update && sudo apt-get install -y libsasl2-dev + - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh + - run: make check-component-docs - - name: Check Rust Docs - if: needs.changes.outputs.source == 'true' - run: cd rust-doc && make docs + check-rust-docs: + name: Check Rust Docs + runs-on: ubuntu-24.04-8core + if: needs.changes.outputs.source == 'true' + needs: changes + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Enable Rust matcher + run: echo "::add-matcher::.github/matchers/rust.json" + - run: cd rust-doc && make docs - - name: VRL - Linux - if: needs.changes.outputs.source == 'true' || needs.changes.outputs.cue == 'true' - run: cargo vdev test-vrl + test-vrl: + name: VRL - Linux + runs-on: ubuntu-24.04-8core + if: needs.changes.outputs.source == 'true' || needs.changes.outputs.cue == 'true' + needs: changes + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Enable Rust matcher + run: echo "::add-matcher::.github/matchers/rust.json" + - run: sudo bash ./scripts/environment/install-protoc.sh + - run: bash ./scripts/environment/prepare.sh --modules=wasm-pack + - run: make test-vrl - - name: Build VRL Playground - if: needs.changes.outputs.source == 'true' || needs.changes.outputs.dependencies == 'true' - run: | + build-vrl-playground: + name: Build VRL Playground + runs-on: ubuntu-24.04-8core + if: needs.changes.outputs.source == 'true' || needs.changes.outputs.dependencies == 'true' + needs: changes + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Enable Rust matcher + run: echo "::add-matcher::.github/matchers/rust.json" + - run: bash ./scripts/environment/prepare.sh --modules=wasm-pack + - run: | cd lib/vector-vrl/web-playground/ - ~/.cargo/bin/rustup target add wasm32-unknown-unknown + rustup target add wasm32-unknown-unknown wasm-pack build --target web --out-dir public/pkg - # This is a required status check, so it always needs to run if prior jobs failed, in order to mark the status correctly. all-checks: name: Test Suite runs-on: ubuntu-24.04 - timeout-minutes: 5 if: always() - needs: [changes, checks] - env: - FAILED: ${{ contains(needs.*.result, 'failure') }} + needs: + - changes + - check-fmt + - check-clippy + - test + - check-version + - check-scripts + - check-events + - check-licenses + - check-docs + - check-markdown + - check-component-docs + - check-rust-docs + - test-vrl + - build-vrl-playground steps: - - run: | - echo "failed=${{ env.FAILED }}" - if [[ "$FAILED" == "true" ]] ; then + - name: Check all jobs status + run: | + if [[ "${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}" == "true" ]]; then + echo "One or more jobs failed or were cancelled" exit 1 else - exit 0 + echo "All jobs completed successfully" fi diff --git a/.github/workflows/unit_mac.yml b/.github/workflows/unit_mac.yml index 5426f6bc51b40..67c21d25ba58d 100644 --- a/.github/workflows/unit_mac.yml +++ b/.github/workflows/unit_mac.yml @@ -8,14 +8,14 @@ permissions: jobs: unit-mac: - runs-on: macos-13 + runs-on: macos-14-xlarge timeout-minutes: 90 env: CARGO_INCREMENTAL: 0 steps: - name: (PR review) Set latest commit status as pending if: ${{ github.event_name == 'pull_request_review' }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} @@ -24,15 +24,15 @@ jobs: - name: (PR review) Checkout PR branch if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.event.review.commit_id }} - name: Checkout branch if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/cache@v4 + - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 name: Cache Cargo registry + index with: path: | @@ -45,12 +45,12 @@ jobs: ${{ runner.os }}-cargo- - run: bash scripts/environment/bootstrap-macos.sh - - run: bash scripts/environment/prepare.sh + - run: bash scripts/environment/prepare.sh --modules=cargo-nextest - run: echo "::add-matcher::.github/matchers/rust.json" # Some tests e.g. `reader_exits_cleanly_when_writer_done_and_in_flight_acks` are flaky. - name: Run tests - uses: nick-fields/retry@v3 + uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 with: timeout_minutes: 45 max_attempts: 3 @@ -59,7 +59,7 @@ jobs: - run: make test-behavior - name: (PR review) Set latest commit status as ${{ job.status }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 if: always() && github.event_name == 'pull_request_review' with: sha: ${{ github.event.review.commit_id }} diff --git a/.github/workflows/unit_windows.yml b/.github/workflows/unit_windows.yml index 357ce89557a74..127a727d27875 100644 --- a/.github/workflows/unit_windows.yml +++ b/.github/workflows/unit_windows.yml @@ -14,7 +14,7 @@ jobs: steps: - name: (PR review) Set latest commit status as pending if: ${{ github.event_name == 'pull_request_review' }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} @@ -23,19 +23,19 @@ jobs: - name: (PR review) Checkout PR branch if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.event.review.commit_id }} - name: Checkout branch if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - run: .\scripts\environment\bootstrap-windows-2022.ps1 - run: make test - name: (PR review) Set latest commit status as ${{ job.status }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 if: always() && github.event_name == 'pull_request_review' with: sha: ${{ github.event.review.commit_id }} diff --git a/.gitignore b/.gitignore index 24b0593d51d6c..1963fcca25fcc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,24 +1,66 @@ +# Rust cargo-timing*.html +**/*.rs.bk +target +tests/data/wasm/*/target +bench_output.txt + +# Python *.pyc + +# Ruby +miniodat + +# Node.js / JavaScript / TypeScript +node_modules +scripts/package-lock.json + +# Yarn +yarn.lock + +# JetBrains IDEs +.idea/ *.iml -*.tmp -*~ -**/*.rs.bk + +# macOS .DS_Store + +# Emacs .dir-locals.el -checkpoints/* -miniodat -bench_output.txt + +# Temporary files +tmp/ +*.tmp +*~ sample.log -scripts/package-lock.json -target -node_modules -tests/data/wasm/*/target + +# Profiling tools heaptrack.* massif.* -# tilt +# Checkpoints +checkpoints/* + +# Tilt tilt_modules/ -# Jetbrains -.idea/ +# Docker +**/Dockerfile~ +**/Dockerfile.*~ +.dockerignore + +# Docker Compose +docker-compose.override.yml +docker-compose.override.*.yml +**/compose-temp-*.yaml + +# Compose project volume mounts +volumes/ +*.volume/ + +# Environment variable files (can contain secrets) +*.env +.env.* + +# LLM tools +copilot-instructions.md diff --git a/Cargo.lock b/Cargo.lock index e207d60a00ce8..b6cd3eb1ad6a6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -102,6 +102,7 @@ dependencies = [ "cfg-if", "getrandom 0.2.15", "once_cell", + "serde", "version_check", "zerocopy 0.7.31", ] @@ -132,9 +133,9 @@ dependencies = [ [[package]] name = "allocator-api2" -version = "0.2.16" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "amq-protocol" @@ -146,7 +147,7 @@ dependencies = [ "amq-protocol-types", "amq-protocol-uri", "cookie-factory", - "nom", + "nom 7.1.3", "serde", ] @@ -168,7 +169,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7412353b58923fa012feb9a64ccc0c811747babee2e5a2fd63eb102dc8054c3" dependencies = [ "cookie-factory", - "nom", + "nom 7.1.3", "serde", "serde_json", ] @@ -421,9 +422,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.23" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b37fc50485c4f3f736a4fb14199f6d5f5ba008d7f28fe710306c92780f004c07" +checksum = "ddb939d66e4ae03cee6091612804ba446b12878410cfa17f785f4dd67d4014e8" dependencies = [ "brotli", "flate2", @@ -489,9 +490,9 @@ dependencies = [ [[package]] name = "async-graphql" -version = "7.0.16" +version = "7.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3ee559e72d983e7e04001ba3bf32e6b71c1d670595780723727fd8a29d36e87" +checksum = "036618f842229ba0b89652ffe425f96c7c16a49f7e3cb23b56fca7f61fd74980" dependencies = [ "async-graphql-derive", "async-graphql-parser", @@ -505,7 +506,7 @@ dependencies = [ "futures-timer", "futures-util", "http 1.1.0", - "indexmap 2.9.0", + "indexmap 2.10.0", "mime", "multer", "num-traits", @@ -520,9 +521,9 @@ dependencies = [ [[package]] name = "async-graphql-derive" -version = "7.0.16" +version = "7.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29db05b624fb6352fc11bfe30c54ab1b16a1fe937d7c05a783f4e88ef1292b3b" +checksum = "fd45deb3dbe5da5cdb8d6a670a7736d735ba65b455328440f236dfb113727a3d" dependencies = [ "Inflector", "async-graphql-parser", @@ -531,15 +532,15 @@ dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", "strum 0.26.3", - "syn 2.0.101", + "syn 2.0.104", "thiserror 1.0.68", ] [[package]] name = "async-graphql-parser" -version = "7.0.16" +version = "7.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4904895044116aab098ca82c6cec831ec43ed99efd04db9b70a390419bc88c5b" +checksum = "60b7607e59424a35dadbc085b0d513aa54ec28160ee640cf79ec3b634eba66d3" dependencies = [ "async-graphql-value", "pest", @@ -549,21 +550,21 @@ dependencies = [ [[package]] name = "async-graphql-value" -version = "7.0.16" +version = "7.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0cde74de18e3a00c5dd5cfa002ab6f532e1a06c2a79ee6671e2fc353b400b92" +checksum = "34ecdaff7c9cffa3614a9f9999bf9ee4c3078fe3ce4d6a6e161736b56febf2de" dependencies = [ "bytes 1.10.1", - "indexmap 2.9.0", + "indexmap 2.10.0", "serde", "serde_json", ] [[package]] name = "async-graphql-warp" -version = "7.0.16" +version = "7.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e662bc1abaf791b2b21b54f679bb2cd34ba456c6c27b908c3f2b18ec5ce30a8b" +checksum = "ff6008a33c32d5a048aa72437821eb864dd56a80c0d80c8df48f11f12154db6c" dependencies = [ "async-graphql", "futures-util", @@ -632,25 +633,25 @@ dependencies = [ [[package]] name = "async-nats" -version = "0.33.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc1f1a75fd07f0f517322d103211f12d757658e91676def9a2e688774656c60" +checksum = "08f6da6d49a956424ca4e28fe93656f790d748b469eaccbc7488fec545315180" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "bytes 1.10.1", "futures 0.3.31", - "http 0.2.9", "memchr", - "nkeys 0.3.2", + "nkeys", "nuid", "once_cell", + "pin-project", + "portable-atomic", "rand 0.8.5", "regex", "ring", - "rustls 0.21.11", - "rustls-native-certs 0.6.3", - "rustls-pemfile 1.0.4", - "rustls-webpki 0.101.7", + "rustls-native-certs 0.7.0", + "rustls-pemfile 2.1.0", + "rustls-webpki 0.102.8", "serde", "serde_json", "serde_nanos", @@ -658,9 +659,11 @@ dependencies = [ "thiserror 1.0.68", "time", "tokio", - "tokio-retry", - "tokio-rustls 0.24.1", + "tokio-rustls 0.26.2", + "tokio-util", + "tokio-websockets", "tracing 0.1.41", + "tryhard", "url", ] @@ -692,25 +695,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "async-process" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63255f1dc2381611000436537bbedfe83183faa303a5a0edaf191edef06526bb" -dependencies = [ - "async-channel 2.3.1", - "async-io 2.4.0", - "async-lock 3.4.0", - "async-signal", - "async-task", - "blocking", - "cfg-if", - "event-listener 5.3.1", - "futures-lite 2.6.0", - "rustix 0.38.40", - "tracing 0.1.41", -] - [[package]] name = "async-reactor-trait" version = "1.1.0" @@ -731,7 +715,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -771,7 +755,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -788,7 +772,7 @@ checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -844,9 +828,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.3" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "687bc16bc431a8533fe0097c7f0182874767f920989d7260950172ae8e3c4465" +checksum = "b68c2194a190e1efc999612792e25b1ab3abfefe4306494efaaabc25933c0cbe" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -856,9 +840,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.5.6" +version = "1.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aff45ffe35196e593ea3b9dd65b320e51e2dda95aff4390bc459e461d09c6ad" +checksum = "b2090e664216c78e766b6bac10fe74d2f451c02441d43484cd76ac9a295075f7" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -873,7 +857,6 @@ dependencies = [ "fastrand 2.3.0", "http 0.2.9", "http-body 0.4.5", - "once_cell", "percent-encoding", "pin-project-lite", "tracing 0.1.41", @@ -1003,9 +986,9 @@ dependencies = [ [[package]] name = "aws-sdk-kms" -version = "1.65.0" +version = "1.75.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5325c5e2badf4148e850017cc56cc205888c6e0b52c9e29d3501ec577005230" +checksum = "bb89d6ae47f03ca664f604571d0f29165112543ba1a39878347815b8028c235b" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1019,7 +1002,6 @@ dependencies = [ "bytes 1.10.1", "fastrand 2.3.0", "http 0.2.9", - "once_cell", "regex-lite", "tracing 0.1.41", ] @@ -1061,9 +1043,9 @@ dependencies = [ [[package]] name = "aws-sdk-secretsmanager" -version = "1.68.0" +version = "1.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c18fb03c6aad6a5de2a36e44eebc83640eea9a025bf407579f503b5dc4cd0b0" +checksum = "eb99bf4d3be2b4598ad26eed5da8d0c930b8d47d76b279a03e47d160151eb0fb" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1077,16 +1059,15 @@ dependencies = [ "bytes 1.10.1", "fastrand 2.3.0", "http 0.2.9", - "once_cell", "regex-lite", "tracing 0.1.41", ] [[package]] name = "aws-sdk-sns" -version = "1.65.0" +version = "1.73.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61cedc4d189aa87ced9b8362cc453e3c3bdcdb73c1020578a0506ed75a280117" +checksum = "9e936a9af3eccbd24452a57bb8206d2f8e1e483d38c52b1a2901fcb892d98866" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1101,7 +1082,6 @@ dependencies = [ "aws-types", "fastrand 2.3.0", "http 0.2.9", - "once_cell", "regex-lite", "tracing 0.1.41", ] @@ -1177,9 +1157,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.65.0" +version = "1.73.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96a78a8f50a1630db757b60f679c8226a8a70ee2ab5f5e6e51dc67f6c61c7cfd" +checksum = "f1e9c3c24e36183e2f698235ed38dcfbbdff1d09b9232dc866c4be3011e0b47e" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1194,16 +1174,15 @@ dependencies = [ "aws-types", "fastrand 2.3.0", "http 0.2.9", - "once_cell", "regex-lite", "tracing 0.1.41", ] [[package]] name = "aws-sigv4" -version = "1.3.1" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3503af839bd8751d0bdc5a46b9cac93a003a353e635b0c12cf2376b5b53e41ea" +checksum = "ddfb9021f581b71870a17eac25b52335b82211cdc092e02b6876b2bcefa61666" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -1274,9 +1253,9 @@ dependencies = [ [[package]] name = "aws-smithy-eventstream" -version = "0.60.8" +version = "0.60.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c45d3dddac16c5c59d553ece225a88870cf81b7b813c9cc17b78cf4685eac7a" +checksum = "604c7aec361252b8f1c871a7641d5e0ba3a7f5a586e51b66bc9510a5519594d9" dependencies = [ "aws-smithy-types", "bytes 1.10.1", @@ -1285,9 +1264,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.62.1" +version = "0.62.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99335bec6cdc50a346fda1437f9fefe33abf8c99060739a546a16457f2862ca9" +checksum = "43c82ba4cab184ea61f6edaafc1072aad3c2a17dcf4c0fce19ac5694b90d8b5f" dependencies = [ "aws-smithy-eventstream", "aws-smithy-runtime-api", @@ -1306,20 +1285,21 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.0.1" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aff1159006441d02e57204bf57a1b890ba68bedb6904ffd2873c1c4c11c546b" +checksum = "f108f1ca850f3feef3009bdcc977be201bca9a91058864d9de0684e64514bee0" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", - "h2 0.4.9", + "h2 0.3.26", + "h2 0.4.11", "http 0.2.9", "http-body 0.4.5", "hyper 0.14.28", "hyper-rustls 0.24.2", "pin-project-lite", - "rustls 0.21.11", + "rustls 0.21.12", "tokio", "tracing 0.1.41", ] @@ -1354,9 +1334,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.8.3" +version = "1.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14302f06d1d5b7d333fd819943075b13d27c7700b414f574c3c35859bfb55d5e" +checksum = "c3aaec682eb189e43c8a19c3dab2fe54590ad5f2cc2d26ab27608a20f2acf81c" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -1378,9 +1358,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.8.0" +version = "1.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e5d9e3a80a18afa109391fb5ad09c3daf887b516c6fd805a157c6ea7994a57" +checksum = "937a49ecf061895fca4a6dd8e864208ed9be7546c0527d04bc07d502ec5fba1c" dependencies = [ "aws-smithy-async", "aws-smithy-types", @@ -1395,9 +1375,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.3.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40076bd09fadbc12d5e026ae080d0930defa606856186e31d83ccc6a255eeaf3" +checksum = "d498595448e43de7f4296b7b7a18a8a02c61ec9349128c80a368f7c3b4ab11a8" dependencies = [ "base64-simd", "bytes 1.10.1", @@ -1430,9 +1410,9 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.3.7" +version = "1.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a322fec39e4df22777ed3ad8ea868ac2f94cd15e1a55f6ee8d8d6305057689a" +checksum = "b069d19bf01e46298eaedd7c6f283fe565a59263e53eebec945f3e6398f42390" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -1466,7 +1446,7 @@ dependencies = [ "serde", "sync_wrapper 0.1.2", "tokio", - "tower", + "tower 0.4.13", "tower-layer", "tower-service", ] @@ -1493,7 +1473,7 @@ dependencies = [ "rustversion", "serde", "sync_wrapper 1.0.1", - "tower", + "tower 0.4.13", "tower-layer", "tower-service", ] @@ -1537,110 +1517,94 @@ dependencies = [ [[package]] name = "azure_core" -version = "0.21.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b552ad43a45a746461ec3d3a51dfb6466b4759209414b439c165eb6a6b7729e" +checksum = "4ccd63c07d1fbfb3d4543d7ea800941bf5a30db1911b9b9e4db3b2c4210a434f" dependencies = [ "async-trait", - "base64 0.22.1", + "base64 0.21.7", "bytes 1.10.1", "dyn-clone", "futures 0.3.31", "getrandom 0.2.15", - "hmac", "http-types", - "once_cell", + "log", "paste", "pin-project", "quick-xml 0.31.0", "rand 0.8.5", - "reqwest 0.12.9", + "reqwest 0.11.26", "rustc_version 0.4.1", "serde", "serde_json", - "sha2", "time", - "tracing 0.1.41", "url", "uuid", ] [[package]] name = "azure_identity" -version = "0.21.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ddd80344317c40c04b603807b63a5cefa532f1b43522e72f480a988141f744" +checksum = "8bd7ea32ca7eb66ff4757f83baac702ff11d469e5de365b6bc6f79f9c25d3436" dependencies = [ "async-lock 3.4.0", - "async-process 2.3.0", "async-trait", "azure_core", "futures 0.3.31", + "log", "oauth2", "pin-project", "serde", + "serde_json", "time", - "tracing 0.1.41", + "tz-rs", "url", "uuid", ] [[package]] name = "azure_storage" -version = "0.21.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59f838159f4d29cb400a14d9d757578ba495ae64feb07a7516bf9e4415127126" +checksum = "83ca0a07f89fd72a006da4713e93af3d6c44a693e61a1c3c2e7985de39c182e8" dependencies = [ "RustyXML", - "async-lock 3.4.0", "async-trait", "azure_core", "bytes 1.10.1", + "futures 0.3.31", + "hmac", + "log", "serde", "serde_derive", + "serde_json", + "sha2", "time", - "tracing 0.1.41", "url", "uuid", ] [[package]] name = "azure_storage_blobs" -version = "0.21.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97e83c3636ae86d9a6a7962b2112e3b19eb3903915c50ce06ff54ff0a2e6a7e4" +checksum = "8096c04d370118323c42b2752aa1883e4880a56ef65239f317b359f263b6e194" dependencies = [ "RustyXML", "azure_core", "azure_storage", - "azure_svc_blobstorage", "bytes 1.10.1", "futures 0.3.31", + "log", "serde", "serde_derive", "serde_json", "time", - "tracing 0.1.41", "url", "uuid", ] -[[package]] -name = "azure_svc_blobstorage" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e6c6f20c5611b885ba94c7bae5e02849a267381aecb8aee577e8c35ff4064c6" -dependencies = [ - "azure_core", - "bytes 1.10.1", - "futures 0.3.31", - "log", - "once_cell", - "serde", - "serde_json", - "time", -] - [[package]] name = "backoff" version = "0.4.0" @@ -1654,9 +1618,9 @@ dependencies = [ [[package]] name = "backon" -version = "1.5.0" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd0b50b1b78dbadd44ab18b3c794e496f3a139abb9fbc27d9c94c4eebbb96496" +checksum = "592277618714fbcecda9a02ba7a8781f319d26532a88553bbacc77ba5d2b3a8d" dependencies = [ "fastrand 2.3.0", "gloo-timers", @@ -1739,15 +1703,9 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" dependencies = [ - "bit-vec 0.8.0", + "bit-vec", ] -[[package]] -name = "bit-vec" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" - [[package]] name = "bit-vec" version = "0.8.0" @@ -1776,7 +1734,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6cbbb8f56245b5a479b30a62cdc86d26e2f35c2b9f594bc4671654b03851380" dependencies = [ "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -1831,14 +1789,14 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "489d2af57852b78a86478273ac6a1ef912061b6af3a439694c49f309f6ea3bdd" dependencies = [ - "siphasher", + "siphasher 0.3.11", ] [[package]] name = "bollard" -version = "0.16.1" +version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aed08d3adb6ebe0eff737115056652670ae290f177759aac19c30456135f94c" +checksum = "899ca34eb6924d6ec2a77c6f7f5c7339e60fd68235eaf91edd5a15f12958bb06" dependencies = [ "base64 0.22.1", "bollard-stubs", @@ -1852,13 +1810,13 @@ dependencies = [ "http-body-util", "hyper 1.4.1", "hyper-named-pipe", - "hyper-rustls 0.26.0", + "hyper-rustls 0.27.5", "hyper-util", - "hyperlocal-next", + "hyperlocal", "log", "pin-project-lite", - "rustls 0.22.4", - "rustls-native-certs 0.7.0", + "rustls 0.23.23", + "rustls-native-certs 0.8.1", "rustls-pemfile 2.1.0", "rustls-pki-types", "serde", @@ -1866,7 +1824,7 @@ dependencies = [ "serde_json", "serde_repr", "serde_urlencoded", - "thiserror 1.0.68", + "thiserror 2.0.3", "tokio", "tokio-util", "tower-service", @@ -1876,16 +1834,23 @@ dependencies = [ [[package]] name = "bollard-stubs" -version = "1.44.0-rc.2" +version = "1.48.3-rc.28.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "709d9aa1c37abb89d40f19f5d0ad6f0d88cb1581264e571c9350fc5bb89cf1c5" +checksum = "64ea257e555d16a2c01e5593f40b73865cdf12efbceda33c6d14a2d8d1490368" dependencies = [ "chrono", "serde", + "serde_json", "serde_repr", - "serde_with 3.12.0", + "serde_with 3.14.0", ] +[[package]] +name = "borrow-or-share" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eeab4423108c5d7c744f4d234de88d18d636100093ae04caf4825134b9c3a32" + [[package]] name = "brotli" version = "8.0.0" @@ -1978,6 +1943,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + [[package]] name = "bytemuck" version = "1.21.0" @@ -2033,7 +2004,7 @@ checksum = "c06acb4f71407ba205a07cb453211e0e6a67b21904e47f6ba1f9589e38f2e454" dependencies = [ "semver 1.0.26", "serde", - "toml", + "toml 0.8.23", "url", ] @@ -2095,9 +2066,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" [[package]] name = "cfg_aliases" @@ -2156,26 +2127,15 @@ dependencies = [ [[package]] name = "chrono-tz" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efdce149c370f133a071ca8ef6ea340b7b88748ab0810097a9e2976eaa34b4f3" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" dependencies = [ "chrono", - "chrono-tz-build", - "phf", + "phf 0.12.1", "serde", ] -[[package]] -name = "chrono-tz-build" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94fea34d77a245229e7746bd2beb786cd2a896f306ff491fb8cecb3074b10a7" -dependencies = [ - "parse-zoneinfo", - "phf_codegen", -] - [[package]] name = "ciborium" version = "0.2.2" @@ -2222,9 +2182,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.37" +version = "4.5.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eccb054f56cbd38340b380d4a8e69ef1f02f1af43db2f0cc817a4774d80ae071" +checksum = "ed87a9d530bb41a67537289bafcac159cb3ee28460e0a4571123d2a778a6a882" dependencies = [ "clap_builder", "clap_derive", @@ -2232,9 +2192,9 @@ dependencies = [ [[package]] name = "clap-verbosity-flag" -version = "3.0.2" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2678fade3b77aa3a8ff3aae87e9c008d3fb00473a41c71fbf74e91c8c7b37e84" +checksum = "eeab6a5cdfc795a05538422012f20a5496f050223c91be4e5420bfd13c641fb1" dependencies = [ "clap", "log", @@ -2242,9 +2202,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.37" +version = "4.5.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd9466fac8543255d3b1fcad4762c5e116ffe808c8a3043d4263cd4fd4862a2" +checksum = "64f4f3f3c77c94aff3c7e9aac9a2ca1974a5adf392a8bb751e827d6d127ab966" dependencies = [ "anstream", "anstyle", @@ -2255,23 +2215,23 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.5.48" +version = "4.5.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be8c97f3a6f02b9e24cadc12aaba75201d18754b53ea0a9d99642f806ccdb4c9" +checksum = "a5abde44486daf70c5be8b8f8f1b66c49f86236edf6fa2abadb4d961c4c6229a" dependencies = [ "clap", ] [[package]] name = "clap_derive" -version = "4.5.32" +version = "4.5.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09176aae279615badda0765c0c0b3f6ed53f4709118af73cf4655d85d1530cd7" +checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491" dependencies = [ "heck 0.5.0", "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -2327,15 +2287,15 @@ dependencies = [ "ordered-float 4.6.0", "prost 0.12.6", "prost-reflect", - "rand 0.9.1", + "rand 0.9.2", "regex", "rstest", "serde", "serde_json", - "serde_with 3.12.0", + "serde_with 3.14.0", "similar-asserts", "smallvec", - "snafu 0.7.5", + "snafu 0.8.6", "syslog_loose", "tokio", "tokio-util", @@ -2442,14 +2402,14 @@ dependencies = [ [[package]] name = "confy" -version = "0.6.1" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45b1f4c00870f07dc34adcac82bb6a72cc5aabca8536ba1797e01df51d2ce9a0" +checksum = "f29222b549d4e3ded127989d523da9e928918d0d0d7f7c1690b439d0d538bae9" dependencies = [ - "directories 5.0.1", + "directories", "serde", - "thiserror 1.0.68", - "toml", + "thiserror 2.0.3", + "toml 0.8.23", ] [[package]] @@ -2461,10 +2421,22 @@ dependencies = [ "encode_unicode 0.3.6", "lazy_static", "libc", - "unicode-width 0.1.13", "windows-sys 0.45.0", ] +[[package]] +name = "console" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e09ced7ebbccb63b4c65413d821f2e00ce54c5ca4514ddc6b3c892fdbcbc69d" +dependencies = [ + "encode_unicode 1.0.0", + "libc", + "once_cell", + "unicode-width 0.2.0", + "windows-sys 0.60.2", +] + [[package]] name = "console-api" version = "0.8.1" @@ -2472,8 +2444,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8030735ecb0d128428b64cd379809817e620a40e5001c54465b99ec5feec2857" dependencies = [ "futures-core", - "prost 0.13.3", - "prost-types 0.13.3", + "prost 0.13.5", + "prost-types 0.13.5", "tonic 0.12.3", "tracing-core 0.1.33", ] @@ -2491,8 +2463,8 @@ dependencies = [ "hdrhistogram", "humantime", "hyper-util", - "prost 0.13.3", - "prost-types 0.13.3", + "prost 0.13.5", + "prost-types 0.13.5", "serde", "serde_json", "thread_local", @@ -2510,6 +2482,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" +[[package]] +name = "const_fn" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f8a2ca5ac02d09563609681103aada9e1777d54fc57a5acd7a41404f9c93b6e" + [[package]] name = "convert_case" version = "0.4.0" @@ -2525,12 +2503,50 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "convert_case" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baaaa0ecca5b51987b9423ccdc971514dd8b0bb7b4060b983d3664dad3f1f89f" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "cookie" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + [[package]] name = "cookie-factory" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "396de984970346b0d9e93d1415082923c679e5ae5c3ee3dcbd104f5610af126b" +[[package]] +name = "cookie_store" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eac901828f88a5241ee0600950ab981148a18f2f756900ffba1b125ca6a3ef9" +dependencies = [ + "cookie", + "document-features", + "idna 1.0.3", + "log", + "publicsuffix", + "serde", + "serde_derive", + "serde_json", + "time", + "url", +] + [[package]] name = "core-foundation" version = "0.9.3" @@ -2541,11 +2557,21 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" -version = "0.8.4" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "core2" @@ -2567,9 +2593,9 @@ dependencies = [ [[package]] name = "crc" -version = "3.2.1" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" dependencies = [ "crc-catalog", ] @@ -2591,9 +2617,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ "cfg-if", ] @@ -2609,26 +2635,22 @@ dependencies = [ [[package]] name = "criterion" -version = "0.5.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +checksum = "e1c047a62b0cc3e145fa84415a3191f628e980b194c2755aa12300a4e6cbd928" dependencies = [ "anes", "cast", "ciborium", "clap", "criterion-plot", - "futures 0.3.31", - "is-terminal", - "itertools 0.10.5", + "itertools 0.13.0", "num-traits", - "once_cell", "oorandom", "plotters", "rayon", "regex", "serde", - "serde_derive", "serde_json", "tinytemplate", "tokio", @@ -2637,12 +2659,12 @@ dependencies = [ [[package]] name = "criterion-plot" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +checksum = "9b1bcc0dc7dfae599d84ad0b1a55f80cde8af3725da8313b528da95ef783e338" dependencies = [ "cast", - "itertools 0.10.5", + "itertools 0.13.0", ] [[package]] @@ -2856,7 +2878,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -2904,7 +2926,7 @@ dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", "strsim 0.11.1", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -2926,7 +2948,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -2935,19 +2957,6 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7762d17f1241643615821a8455a0b2c3e803784b058693d990b11f2dce25a0ca" -[[package]] -name = "dashmap" -version = "5.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" -dependencies = [ - "cfg-if", - "hashbrown 0.14.5", - "lock_api", - "once_cell", - "parking_lot_core", -] - [[package]] name = "dashmap" version = "6.1.0" @@ -2976,16 +2985,17 @@ checksum = "5c297a1c74b71ae29df00c3e22dd9534821d60eb9af5a0192823fa2acea70c2a" [[package]] name = "databend-client" -version = "0.22.2" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd8770a1c49fa21e62a768a0de442cc3f77998a357303d56ddd3485cb7c58d3a" +checksum = "08e0717e89fa2d123f4fa6b0ffee9eb3ed1441b73164308bd1e45be2028389c9" dependencies = [ - "async-trait", + "cookie", "log", "once_cell", "parking_lot", "percent-encoding", "reqwest 0.12.9", + "semver 1.0.26", "serde", "serde_json", "tokio", @@ -3017,11 +3027,25 @@ dependencies = [ "tokio", ] +[[package]] +name = "deadpool" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ed5957ff93768adf7a65ab167a17835c3d2c3c50d084fe305174c112f468e2f" +dependencies = [ + "deadpool-runtime", + "num_cpus", + "tokio", +] + [[package]] name = "deadpool-runtime" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63dfa964fe2a66f3fde91fc70b267fe193d822c7e603e2a675a49a7f46ad3f49" +dependencies = [ + "tokio", +] [[package]] name = "der" @@ -3063,7 +3087,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -3084,7 +3108,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -3094,7 +3118,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b" dependencies = [ "derive_builder_core", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -3128,22 +3152,13 @@ dependencies = [ "subtle", ] -[[package]] -name = "directories" -version = "5.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35" -dependencies = [ - "dirs-sys 0.4.1", -] - [[package]] name = "directories" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16f5094c54661b38d03bd7e50df373292118db60b585c08a411c6d840017fe7d" dependencies = [ - "dirs-sys 0.5.0", + "dirs-sys", ] [[package]] @@ -3156,18 +3171,6 @@ dependencies = [ "dirs-sys-next", ] -[[package]] -name = "dirs-sys" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" -dependencies = [ - "libc", - "option-ext", - "redox_users 0.4.3", - "windows-sys 0.48.0", -] - [[package]] name = "dirs-sys" version = "0.5.0" @@ -3177,7 +3180,7 @@ dependencies = [ "libc", "option-ext", "redox_users 0.5.0", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -3199,7 +3202,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -3210,7 +3213,7 @@ checksum = "e5766087c2235fec47fafa4cfecc81e494ee679d0fd4a59887ea0919bfb0e4fc" dependencies = [ "cfg-if", "libc", - "socket2 0.5.9", + "socket2 0.5.10", "windows-sys 0.48.0", ] @@ -3221,7 +3224,7 @@ dependencies = [ "criterion", "data-encoding", "hickory-proto", - "snafu 0.7.5", + "snafu 0.8.6", ] [[package]] @@ -3238,7 +3241,7 @@ dependencies = [ "paste", "prost 0.12.6", "prost-build 0.12.6", - "snafu 0.7.5", + "snafu 0.8.6", "tracing 0.1.41", "vector-lib", "vrl", @@ -3257,7 +3260,7 @@ dependencies = [ "anyhow", "serde", "serde_json", - "snafu 0.7.5", + "snafu 0.8.6", "tracing 0.1.41", "vector-config", "vector-config-common", @@ -3274,12 +3277,16 @@ dependencies = [ [[package]] name = "domain" -version = "0.10.4" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c84070523f8ba0f9127ff156920f27eb27b302b425efe60bf5f41ec244d1c60" +checksum = "a11dd7f04a6a6d2aea0153c6e31f5ea7af8b2efdf52cdaeea7a9a592c7fefef9" dependencies = [ + "bumpalo", "bytes 1.10.1", + "domain-macros", "futures-util", + "hashbrown 0.14.5", + "log", "moka", "octseq", "rand 0.8.5", @@ -3290,6 +3297,17 @@ dependencies = [ "tracing 0.1.41", ] +[[package]] +name = "domain-macros" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e197fdfd2cdb5fdeb7f8ddcf3aed5d5d04ecde2890d448b14ffb716f7376b70" +dependencies = [ + "proc-macro2 1.0.95", + "quote 1.0.40", + "syn 2.0.104", +] + [[package]] name = "dotenvy" version = "0.15.7" @@ -3316,9 +3334,9 @@ checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" [[package]] name = "dyn-clone" -version = "1.0.19" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c7a8fb8a9fbf66c1f703fe16184d10ca0ee9d23be5b4436400408ba54a95005" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "ecdsa" @@ -3388,6 +3406,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "email_address" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e079f19b08ca6239f47f8ba8509c11cf3ea30095831f7fed61441475edd8c449" +dependencies = [ + "serde", +] + [[package]] name = "ena" version = "0.14.2" @@ -3456,7 +3483,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -3468,27 +3495,27 @@ dependencies = [ "once_cell", "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] name = "enumflags2" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba2f4b465f5318854c6f8dd686ede6c0a9dc67d4b1ac241cf0eb51521a309147" +checksum = "1027f7680c853e056ebcec683615fb6fbbc07dbaa13b4d5d9442b146ded4ecef" dependencies = [ "enumflags2_derive", ] [[package]] name = "enumflags2_derive" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc4caf64a58d7a6d65ab00639b046ff54399a39f5f2554728895ace4b297cd79" +checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -3662,7 +3689,7 @@ version = "0.1.0" dependencies = [ "chrono", "fakedata_generator", - "rand 0.9.1", + "rand 0.9.2", ] [[package]] @@ -3685,9 +3712,21 @@ checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" [[package]] name = "fancy-regex" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298" +checksum = "d6215aee357f8c7c989ebb4b8466ca4d7dc93b3957039f2fc3ea2ade8ea5f279" +dependencies = [ + "bit-set", + "derivative", + "regex-automata 0.4.8", + "regex-syntax 0.8.5", +] + +[[package]] +name = "fancy-regex" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf04c5ec15464ace8355a7b440a33aece288993475556d461154d7a62ad9947c" dependencies = [ "bit-set", "regex-automata 0.4.8", @@ -3734,11 +3773,11 @@ dependencies = [ "chrono", "crc", "criterion", - "dashmap 6.1.0", + "dashmap", "flate2", "futures 0.3.31", "glob", - "indexmap 2.9.0", + "indexmap 2.10.0", "libc", "quickcheck", "scan_fmt", @@ -3753,18 +3792,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "filetime" -version = "0.2.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall 0.3.5", - "windows-sys 0.48.0", -] - [[package]] name = "finl_unicode" version = "1.2.0" @@ -3779,9 +3806,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" +checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" dependencies = [ "crc32fast", "libz-rs-sys", @@ -3803,6 +3830,17 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "fluent-uri" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1918b65d96df47d3591bed19c5cca17e3fa5d0707318e4b5ef2eae01764df7e5" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + [[package]] name = "flume" version = "0.10.14" @@ -3859,6 +3897,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fraction" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b486ab61634f05b11b591c38c71fb25139cb55e22be4fb6ecf649cc3736c074a" +dependencies = [ + "lazy_static", + "num", +] + [[package]] name = "fsevent-sys" version = "4.1.0" @@ -3970,10 +4018,7 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f5edaec856126859abb19ed65f39e90fea3a9574b9707f13539acf4abf7eb532" dependencies = [ - "fastrand 2.3.0", "futures-core", - "futures-io", - "parking", "pin-project-lite", ] @@ -3985,7 +4030,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -4068,8 +4113,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi 0.13.3+wasi-0.2.2", + "wasm-bindgen", "windows-targets 0.52.6", ] @@ -4079,6 +4126,21 @@ version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +[[package]] +name = "git2" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2deb07a133b1520dc1a5690e9bd08950108873d7ed5de38dcc74d3b5ebffa110" +dependencies = [ + "bitflags 2.9.0", + "libc", + "libgit2-sys", + "log", + "openssl-probe", + "openssl-sys", + "url", +] + [[package]] name = "glob" version = "0.3.2" @@ -4112,14 +4174,14 @@ dependencies = [ [[package]] name = "goauth" -version = "0.14.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d351469a584f3b3565e2e740d4da60839bddc4320dadd7d61da8bdd77ffb373b" +checksum = "7b1f1228623a5a37d4834f984573a01086708b109bbf0f7c2ee8d70b0c90d7a5" dependencies = [ "arc-swap", "futures 0.3.31", "log", - "reqwest 0.11.26", + "reqwest 0.12.9", "serde", "serde_derive", "serde_json", @@ -4131,22 +4193,24 @@ dependencies = [ [[package]] name = "governor" -version = "0.7.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0746aa765db78b521451ef74221663b57ba595bf83f75d0ce23cc09447c8139f" +checksum = "3cbe789d04bf14543f03c4b60cd494148aa79438c8440ae7d81a7778147745c3" dependencies = [ "cfg-if", - "dashmap 6.1.0", + "dashmap", "futures-sink", "futures-timer", "futures-util", - "no-std-compat", + "getrandom 0.3.1", + "hashbrown 0.15.2", "nonzero_ext", "parking_lot", "portable-atomic", - "rand 0.8.5", + "rand 0.9.2", "smallvec", "spinning_top", + "web-time", ] [[package]] @@ -4210,7 +4274,7 @@ dependencies = [ [[package]] name = "greptime-proto" version = "0.1.0" -source = "git+https://github.com/GreptimeTeam/greptime-proto.git?tag=v0.7.0#4bc0d17577dbea47396a064c1ccf229a4c9539fa" +source = "git+https://github.com/GreptimeTeam/greptime-proto.git?tag=v0.9.0#396206c2801b5a3ec51bfe8984c66b686da910e6" dependencies = [ "prost 0.12.6", "serde", @@ -4224,9 +4288,9 @@ dependencies = [ [[package]] name = "greptimedb-ingester" version = "0.1.0" -source = "git+https://github.com/GreptimeTeam/greptimedb-ingester-rust?rev=2e6b0c5eb6a5e7549c3100e4d356b07d15cce66d#2e6b0c5eb6a5e7549c3100e4d356b07d15cce66d" +source = "git+https://github.com/GreptimeTeam/greptimedb-ingester-rust?rev=f7243393808640f5123b0d5b7b798da591a4df6e#f7243393808640f5123b0d5b7b798da591a4df6e" dependencies = [ - "dashmap 5.5.3", + "dashmap", "derive_builder", "enum_dispatch", "futures 0.3.31", @@ -4234,20 +4298,20 @@ dependencies = [ "greptime-proto", "parking_lot", "prost 0.12.6", - "rand 0.8.5", - "snafu 0.7.5", + "rand 0.9.2", + "snafu 0.8.6", "tokio", "tokio-stream", "tonic 0.11.0", "tonic-build 0.9.2", - "tower", + "tower 0.4.13", ] [[package]] name = "grok" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "273797968160270573071022613fc4aa28b91fe68f3eef6c96a1b2a1947ddfbd" +checksum = "6c52724b609896f661a3f4641dd3a44dc602958ef615857c12d00756b4e9355b" dependencies = [ "glob", "onig", @@ -4276,7 +4340,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.9", - "indexmap 2.9.0", + "indexmap 2.10.0", "slab", "tokio", "tokio-util", @@ -4285,9 +4349,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.9" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75249d144030531f8dee69fe9cea04d3edf809a017ae445e2abdff6629e86633" +checksum = "17da50a276f1e01e0ba6c029e47b7100754904ee8a278f886546e98575380785" dependencies = [ "atomic-waker", "bytes 1.10.1", @@ -4295,7 +4359,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.1.0", - "indexmap 2.9.0", + "indexmap 2.10.0", "slab", "tokio", "tokio-util", @@ -4314,9 +4378,9 @@ dependencies = [ [[package]] name = "hash_hasher" -version = "2.0.3" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74721d007512d0cb3338cd20f0654ac913920061a4c4d0d8708edb3f2a698c0c" +checksum = "1b4b9ebce26001bad2e6366295f64e381c1e9c479109202149b9e15e154973e9" [[package]] name = "hashbag" @@ -4382,7 +4446,7 @@ dependencies = [ "byteorder", "crossbeam-channel", "flate2", - "nom", + "nom 7.1.3", "num-traits", ] @@ -4442,7 +4506,7 @@ version = "0.1.0-rc.1" source = "git+https://github.com/vectordotdev/heim.git?branch=update-nix#f3537d9b32e69a2a8ab19a0d42a1e6f5577a5a45" dependencies = [ "cfg-if", - "core-foundation", + "core-foundation 0.9.3", "futures-core", "futures-util", "lazy_static", @@ -4479,7 +4543,7 @@ source = "git+https://github.com/vectordotdev/heim.git?branch=update-nix#f3537d9 dependencies = [ "bitflags 1.3.2", "cfg-if", - "core-foundation", + "core-foundation 0.9.3", "heim-common", "heim-runtime", "libc", @@ -4566,15 +4630,13 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hickory-proto" -version = "0.25.1" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d844af74f7b799e41c78221be863bade11c430d46042c3b49ca8ae0c6d27287" +checksum = "f8a6fe56c0038198998a6f217ca4e7ef3a5e51f46163bd6dd60b5c71ca6c6502" dependencies = [ - "async-recursion", "async-trait", "bitflags 2.9.0", "cfg-if", - "critical-section", "data-encoding", "enum-as-inner 0.6.0", "futures-channel", @@ -4583,7 +4645,7 @@ dependencies = [ "idna 1.0.3", "ipnet", "once_cell", - "rand 0.9.1", + "rand 0.9.2", "ring", "rustls-pki-types", "thiserror 2.0.3", @@ -4748,9 +4810,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +checksum = "9b112acc8b3adf4b107a8ec20977da0273a8c386765a3ec0229bd500a1443f9f" [[package]] name = "hyper" @@ -4769,7 +4831,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.10", + "socket2 0.5.10", "tokio", "tower-service", "tracing 0.1.41", @@ -4785,7 +4847,7 @@ dependencies = [ "bytes 1.10.1", "futures-channel", "futures-util", - "h2 0.4.9", + "h2 0.4.11", "http 1.1.0", "http-body 1.0.0", "httparse", @@ -4876,31 +4938,12 @@ dependencies = [ "http 0.2.9", "hyper 0.14.28", "log", - "rustls 0.21.11", + "rustls 0.21.12", "rustls-native-certs 0.6.3", "tokio", "tokio-rustls 0.24.1", ] -[[package]] -name = "hyper-rustls" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" -dependencies = [ - "futures-util", - "http 1.1.0", - "hyper 1.4.1", - "hyper-util", - "log", - "rustls 0.22.4", - "rustls-native-certs 0.7.0", - "rustls-pki-types", - "tokio", - "tokio-rustls 0.25.0", - "tower-service", -] - [[package]] name = "hyper-rustls" version = "0.27.5" @@ -4986,17 +5029,17 @@ dependencies = [ "http-body 1.0.0", "hyper 1.4.1", "pin-project-lite", - "socket2 0.5.9", + "socket2 0.5.10", "tokio", "tower-service", "tracing 0.1.41", ] [[package]] -name = "hyperlocal-next" -version = "0.9.0" +name = "hyperlocal" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acf569d43fa9848e510358c07b80f4adf34084ddc28c6a4a651ee8474c070dcc" +checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" dependencies = [ "hex", "http-body-util", @@ -5145,7 +5188,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -5199,9 +5242,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" +checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661" dependencies = [ "equivalent", "hashbrown 0.15.2", @@ -5210,15 +5253,15 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.17.11" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" +checksum = "70a646d946d06bedbbc4cac4c218acf4bbf2d87757a784857025f4d447e4e1cd" dependencies = [ - "console", - "number_prefix", + "console 0.16.0", "portable-atomic", "unicode-segmentation", "unicode-width 0.2.0", + "unit-prefix", "web-time", ] @@ -5242,7 +5285,7 @@ checksum = "22fa7ee6be451ea0b1912b962c91c8380835e97cf1584a77e18264e908448dcb" dependencies = [ "bytes 1.10.1", "log", - "nom", + "nom 7.1.3", "smallvec", "snafu 0.7.5", ] @@ -5284,7 +5327,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b23a0c8dfe501baac4adf6ebbfa6eddf8f0c07f56b058cc1288017e32397846c" dependencies = [ "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -5316,6 +5359,17 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "io-uring" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4" +dependencies = [ + "bitflags 2.9.0", + "cfg-if", + "libc", +] + [[package]] name = "iovec" version = "0.1.4" @@ -5331,7 +5385,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" dependencies = [ - "socket2 0.5.9", + "socket2 0.5.10", "widestring 1.0.2", "windows-sys 0.48.0", "winreg", @@ -5453,10 +5507,11 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.65" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -5493,11 +5548,37 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c6e529149475ca0b2820835d3dce8fcc41c6b943ca608d32f35b449255e4627" dependencies = [ - "fluent-uri", + "fluent-uri 0.1.4", "serde", "serde_json", ] +[[package]] +name = "jsonschema" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24690c68dfcdde5980d676b0f1820981841016b1f29eecb4c42ad48ab4118681" +dependencies = [ + "ahash 0.8.11", + "base64 0.22.1", + "bytecount", + "email_address", + "fancy-regex 0.16.1", + "fraction", + "idna 1.0.3", + "itoa", + "num-cmp", + "num-traits", + "once_cell", + "percent-encoding", + "referencing", + "regex", + "regex-syntax 0.8.5", + "serde", + "serde_json", + "uuid-simd", +] + [[package]] name = "k8s-e2e-tests" version = "0.1.0" @@ -5507,7 +5588,7 @@ dependencies = [ "indoc", "k8s-openapi 0.16.0", "k8s-test-framework", - "rand 0.9.1", + "rand 0.9.2", "regex", "reqwest 0.11.26", "serde_json", @@ -5564,9 +5645,9 @@ dependencies = [ [[package]] name = "kqueue" -version = "1.0.8" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7447f1ca1b7b563588a205fe93dea8df60fd981423a768bc1c0ded35ed147d0c" +checksum = "eac30106d7dce88daf4a3fcb4879ea939476d5074a9b7ddd0fb97fa4bed5596a" dependencies = [ "kqueue-sys", "libc", @@ -5634,7 +5715,7 @@ dependencies = [ "thiserror 1.0.68", "tokio", "tokio-util", - "tower", + "tower 0.4.13", "tower-http 0.5.2", "tracing 0.1.41", ] @@ -5747,9 +5828,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.172" +version = "0.2.174" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" [[package]] name = "libflate" @@ -5775,6 +5856,20 @@ dependencies = [ "rle-decode-fast", ] +[[package]] +name = "libgit2-sys" +version = "0.18.2+1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c42fe03df2bd3c53a3a9c7317ad91d80c81cd1fb0caec8d7cc4cd2bfa10c222" +dependencies = [ + "cc", + "libc", + "libssh2-sys", + "libz-sys", + "openssl-sys", + "pkg-config", +] + [[package]] name = "libm" version = "0.2.8" @@ -5801,20 +5896,34 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libssh2-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "220e4f05ad4a218192533b300327f5150e809b54c4ec83b5a1d91833601811b9" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", +] + [[package]] name = "libz-rs-sys" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6489ca9bd760fe9642d7644e827b0c9add07df89857b0416ee15c1cc1a3b8c5a" +checksum = "172a788537a2221661b480fee8dc5f96c580eb34fa88764d3205dc356c7e4221" dependencies = [ "zlib-rs", ] [[package]] name = "libz-sys" -version = "1.1.12" +version = "1.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" +checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" dependencies = [ "cc", "libc", @@ -5880,9 +5989,9 @@ checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5" [[package]] name = "lock_api" -version = "0.4.11" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" dependencies = [ "autocfg", "scopeguard", @@ -5921,9 +6030,9 @@ dependencies = [ [[package]] name = "lru" -version = "0.14.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f8cc7106155f10bdf99a6f379688f543ad6596a415375b36a59a054ceda1198" +checksum = "86ea4e65087ff52f3862caff188d489f1fab49a0cb09e01b2e3f1a617b10aaed" [[package]] name = "lru-cache" @@ -5955,19 +6064,18 @@ dependencies = [ [[package]] name = "lz4" -version = "1.24.0" +version = "1.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e9e2dd86df36ce760a60f6ff6ad526f7ba1f14ba0356f8254fb6905e6494df1" +checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" dependencies = [ - "libc", "lz4-sys", ] [[package]] name = "lz4-sys" -version = "1.9.4" +version = "1.11.1+lz4-1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" dependencies = [ "cc", "libc", @@ -5975,11 +6083,11 @@ dependencies = [ [[package]] name = "lz4_flex" -version = "0.11.3" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" +checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" dependencies = [ - "twox-hash 1.6.3", + "twox-hash", ] [[package]] @@ -6069,15 +6177,15 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.4" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" [[package]] name = "memmap2" -version = "0.9.5" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f" +checksum = "483758ad303d734cec05e5c12b41d7e93e6a6390c5e9dae6bdeb7c1259012d28" dependencies = [ "libc", ] @@ -6125,7 +6233,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1ada651cd6bdffe01e5f35067df53491f1fe853d2b154008ca2bd30b3d3fcf6" dependencies = [ - "indexmap 2.9.0", + "indexmap 2.10.0", "itoa", "lockfree-object-pool", "metrics", @@ -6146,7 +6254,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "hashbrown 0.15.2", - "indexmap 2.9.0", + "indexmap 2.10.0", "metrics", "ordered-float 4.6.0", "quanta", @@ -6209,9 +6317,9 @@ dependencies = [ [[package]] name = "mlua" -version = "0.10.3" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3f763c1041eff92ffb5d7169968a327e1ed2ebfe425dac0ee5a35f29082534b" +checksum = "c1f5f8fbebc7db5f671671134b9321c4b9aa9adeafccfd9a8c020ae45c6a35d0" dependencies = [ "bstr 1.12.0", "either", @@ -6220,13 +6328,14 @@ dependencies = [ "num-traits", "parking_lot", "rustc-hash", + "rustversion", ] [[package]] name = "mlua-sys" -version = "0.6.6" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63a11d485edf0f3f04a508615d36c7d50d299cf61a7ee6d3e2530651e0a31771" +checksum = "380c1f7e2099cafcf40e51d3a9f20a346977587aa4d012eae1f043149a728a93" dependencies = [ "cc", "cfg-if", @@ -6247,14 +6356,14 @@ dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", "regex", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] name = "mock_instant" -version = "0.5.3" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e1d4c44418358edcac6e1d9ce59cea7fb38052429c7704033f1196f0c179e6a" +checksum = "dce6dd36094cac388f119d2e9dc82dc730ef91c32a6222170d630e5414b956e6" [[package]] name = "moka" @@ -6305,7 +6414,7 @@ dependencies = [ "percent-encoding", "rand 0.8.5", "rustc_version_runtime", - "rustls 0.21.11", + "rustls 0.21.12", "rustls-pemfile 1.0.4", "serde", "serde_bytes", @@ -6353,18 +6462,17 @@ checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" [[package]] name = "native-tls" -version = "0.2.11" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" dependencies = [ - "lazy_static", "libc", "log", "openssl", "openssl-probe", "openssl-sys", "schannel", - "security-framework", + "security-framework 2.10.0", "security-framework-sys", "tempfile", ] @@ -6390,7 +6498,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17ebbe97acce52d06aebed4cd4a87c0941f4b2519b59b82b4feb5bd0ce003dfd" dependencies = [ - "indexmap 2.9.0", + "indexmap 2.10.0", "itertools 0.13.0", "ndarray", "noisy_float", @@ -6507,9 +6615,9 @@ dependencies = [ [[package]] name = "nix" -version = "0.29.0" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ "bitflags 2.9.0", "cfg-if", @@ -6519,25 +6627,9 @@ dependencies = [ [[package]] name = "nkeys" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aad178aad32087b19042ee36dfd450b73f5f934fbfb058b59b198684dfec4c47" -dependencies = [ - "byteorder", - "data-encoding", - "ed25519", - "ed25519-dalek", - "getrandom 0.2.15", - "log", - "rand 0.8.5", - "signatory", -] - -[[package]] -name = "nkeys" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f49e787f4c61cbd0f9320b31cc26e58719f6aa5068e34697dd3aea361412fe3" +checksum = "879011babc47a1c7fdf5a935ae3cfe94f34645ca0cac1c7f6424b36fc743d1bf" dependencies = [ "data-encoding", "ed25519", @@ -6558,12 +6650,6 @@ dependencies = [ "serde", ] -[[package]] -name = "no-std-compat" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" - [[package]] name = "nohash" version = "0.2.0" @@ -6589,6 +6675,24 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "nom-language" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2de2bc5b451bfedaef92c90b8939a8fff5770bdcc1fafd6239d086aab8fa6b29" +dependencies = [ + "nom 8.0.0", +] + [[package]] name = "nonzero_ext" version = "0.3.0" @@ -6597,12 +6701,11 @@ checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" [[package]] name = "notify" -version = "8.0.0" +version = "8.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fee8403b3d66ac7b26aee6e40a897d85dc5ce26f44da36b8b73e987cc52e943" +checksum = "3163f59cd3fa0e9ef8c32f242966a7b9994fd7378366099593e0e73077cd8c97" dependencies = [ "bitflags 2.9.0", - "filetime", "fsevent-sys", "inotify", "kqueue", @@ -6611,7 +6714,7 @@ dependencies = [ "mio", "notify-types", "walkdir", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -6656,13 +6759,26 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "num" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05180d69e3da0e530ba2a1dae5110317e49e3b7f3d41be227dc5f92e49ee7af" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational 0.4.2", + "num-traits", +] + [[package]] name = "num-bigint" -version = "0.4.4" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ - "autocfg", "num-integer", "num-traits", ] @@ -6684,6 +6800,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-cmp" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa" + [[package]] name = "num-complex" version = "0.4.4" @@ -6712,11 +6834,10 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "autocfg", "num-traits", ] @@ -6742,6 +6863,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -6789,7 +6921,7 @@ dependencies = [ "proc-macro-crate 1.3.1", "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -6801,7 +6933,7 @@ dependencies = [ "proc-macro-crate 3.2.0", "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -6850,13 +6982,23 @@ dependencies = [ [[package]] name = "objc2-core-foundation" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daeaf60f25471d26948a1c2f840e3f7d86f4109e3af4e8e4b5cd70c39690d925" +checksum = "1c10c2894a6fed806ade6027bcd50662746363a9589d3ec9d9bef30a4e4bc166" dependencies = [ "bitflags 2.9.0", ] +[[package]] +name = "objc2-io-kit" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71c1c64d6120e51cd86033f67176b1cb66780c2efe34dec55176f77befd93c0a" +dependencies = [ + "libc", + "objc2-core-foundation", +] + [[package]] name = "object" version = "0.32.1" @@ -6888,9 +7030,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.20.2" +version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" dependencies = [ "critical-section", "portable-atomic", @@ -6898,11 +7040,11 @@ dependencies = [ [[package]] name = "onig" -version = "6.4.0" +version = "6.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f" +checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.0", "libc", "once_cell", "onig_sys", @@ -6910,9 +7052,9 @@ dependencies = [ [[package]] name = "onig_sys" -version = "69.8.1" +version = "69.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7" +checksum = "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc" dependencies = [ "cc", "pkg-config", @@ -6932,12 +7074,11 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "opendal" -version = "0.53.1" +version = "0.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0f407ca2797ca6c010720aa3cedfa0187430e4f70b729c75c33142d44021f59" +checksum = "ffb9838d0575c6dbaf3fcec7255af8d5771996d4af900bbb6fa9a314dec00a1a" dependencies = [ "anyhow", - "async-trait", "backon", "base64 0.22.1", "bytes 1.10.1", @@ -6959,9 +7100,9 @@ dependencies = [ [[package]] name = "openidconnect" -version = "3.4.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62d6050f6a84b81f23c569f5607ad883293e57491036e318fafe6fc4895fadb1" +checksum = "f47e80a9cfae4462dd29c41e987edd228971d6565553fbc14b8a11e666d91590" dependencies = [ "base64 0.13.1", "chrono", @@ -6982,7 +7123,7 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_plain", - "serde_with 3.12.0", + "serde_with 3.14.0", "sha2", "subtle", "thiserror 1.0.68", @@ -6991,9 +7132,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.72" +version = "0.10.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fedfea7d58a1f73118430a55da6a286e7b044961736ce96a16a17068ea25e5da" +checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" dependencies = [ "bitflags 2.9.0", "cfg-if", @@ -7012,7 +7153,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -7023,18 +7164,18 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "openssl-src" -version = "300.5.0+3.5.0" +version = "300.5.1+3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8ce546f549326b0e6052b649198487d91320875da901e7bd11a06d1ee3f9c2f" +checksum = "735230c832b28c000e3bc117119e6466a663ec73506bc0a9907ea4187508e42a" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.107" +version = "0.9.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8288979acd84749c744a9014b4382d42b8f7b2592847b5afb2ed29e5d16ede07" +checksum = "90096e2e47630d78b7d1c20952dc621f957103f8bc2c8359ec81290d75238571" dependencies = [ "cc", "libc", @@ -7049,6 +7190,7 @@ version = "0.1.0" dependencies = [ "bytes 1.10.1", "chrono", + "glob", "hex", "ordered-float 4.6.0", "prost 0.12.6", @@ -7108,9 +7250,9 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] name = "owo-colors" -version = "4.2.0" +version = "4.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1036865bb9422d3300cf723f657c2851d0e9ab12567854b1f4eba3d77decf564" +checksum = "48dd4f4a2c8405440fd0462561f0e5806bd0f77e86f51c761481bdd4018b545e" dependencies = [ "supports-color 2.1.0", "supports-color 3.0.1", @@ -7157,9 +7299,9 @@ checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" [[package]] name = "parking_lot" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" dependencies = [ "lock_api", "parking_lot_core", @@ -7167,15 +7309,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.9" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.4.1", + "redox_syscall 0.5.12", "smallvec", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] @@ -7184,15 +7326,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "487f2ccd1e17ce8c1bfab3a65c89525af41cfad4c8659021a1e9a2aacd73b89b" -[[package]] -name = "parse-zoneinfo" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c705f256449c60da65e11ff6626e0c16a0a0b96aaa348de61376b249bc340f41" -dependencies = [ - "regex", -] - [[package]] name = "passt" version = "0.3.0" @@ -7222,11 +7355,11 @@ checksum = "9e9ed2178b0575fff8e1b83b58ba6f75e727aafac2e1b6c795169ad3b17eb518" [[package]] name = "pem" -version = "3.0.2" +version = "3.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3163d2912b7c3b52d651a055f2c7eec9ba5cd22d26ef75b8dd3a59980b185923" +checksum = "38af38e8470ac9dee3ce1bae1af9c1671fffc44ddfd8bd1d0a3445bf349a8ef3" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "serde", ] @@ -7276,7 +7409,7 @@ dependencies = [ "pest_meta", "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -7297,7 +7430,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 2.9.0", + "indexmap 2.10.0", ] [[package]] @@ -7310,41 +7443,39 @@ dependencies = [ ] [[package]] -name = "phf_codegen" -version = "0.11.2" +name = "phf" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" dependencies = [ - "phf_generator", - "phf_shared 0.11.2", + "phf_shared 0.12.1", ] [[package]] -name = "phf_generator" -version = "0.11.2" +name = "phf_shared" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" +checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" dependencies = [ - "phf_shared 0.11.2", - "rand 0.8.5", + "siphasher 0.3.11", ] [[package]] name = "phf_shared" -version = "0.10.0" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" dependencies = [ - "siphasher", + "siphasher 0.3.11", ] [[package]] name = "phf_shared" -version = "0.11.2" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" dependencies = [ - "siphasher", + "siphasher 1.0.1", ] [[package]] @@ -7364,7 +7495,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -7524,7 +7655,7 @@ dependencies = [ name = "portpicker" version = "1.0.0" dependencies = [ - "rand 0.9.1", + "rand 0.9.2", ] [[package]] @@ -7552,7 +7683,7 @@ dependencies = [ "hmac", "md-5", "memchr", - "rand 0.9.1", + "rand 0.9.2", "sha2", "stringprep", ] @@ -7642,7 +7773,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2 1.0.95", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -7683,7 +7814,7 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" dependencies = [ - "toml_edit 0.22.26", + "toml_edit 0.22.27", ] [[package]] @@ -7705,7 +7836,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -7742,28 +7873,28 @@ dependencies = [ name = "prometheus-parser" version = "0.1.0" dependencies = [ - "indexmap 2.9.0", - "nom", + "indexmap 2.10.0", + "nom 8.0.0", "prost 0.12.6", "prost-build 0.12.6", "prost-types 0.12.6", - "snafu 0.7.5", + "snafu 0.8.6", "vector-common", ] [[package]] name = "proptest" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14cae93065090804185d3b75f0bf93b8eeda30c7a9b4a33d3bdb3988d6229e50" +checksum = "6fcdab19deb5195a31cf7726a210015ff1496ba1464fd42cb4f537b8b01b471f" dependencies = [ "bit-set", - "bit-vec 0.8.0", + "bit-vec", "bitflags 2.9.0", "lazy_static", "num-traits", - "rand 0.8.5", - "rand_chacha 0.3.1", + "rand 0.9.2", + "rand_chacha 0.9.0", "rand_xorshift", "regex-syntax 0.8.5", "rusty-fork", @@ -7773,13 +7904,13 @@ dependencies = [ [[package]] name = "proptest-derive" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ee1c9ac207483d5e7db4940700de86a9aae46ef90c48b57f99fe7edb8345e49" +checksum = "095a99f75c69734802359b682be8daaf8980296731f6470434ea2c652af1dd30" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -7804,12 +7935,12 @@ dependencies = [ [[package]] name = "prost" -version = "0.13.3" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" dependencies = [ "bytes 1.10.1", - "prost-derive 0.13.3", + "prost-derive 0.13.5", ] [[package]] @@ -7851,7 +7982,27 @@ dependencies = [ "prost 0.12.6", "prost-types 0.12.6", "regex", - "syn 2.0.101", + "syn 2.0.104", + "tempfile", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck 0.5.0", + "itertools 0.14.0", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease 0.2.15", + "prost 0.13.5", + "prost-types 0.13.5", + "regex", + "syn 2.0.104", "tempfile", ] @@ -7878,20 +8029,20 @@ dependencies = [ "itertools 0.12.1", "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] name = "prost-derive" -version = "0.13.3" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.14.0", "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -7902,8 +8053,8 @@ checksum = "7b5edd582b62f5cde844716e66d92565d7faf7ab1445c8cebce6e00fba83ddb2" dependencies = [ "base64 0.22.1", "once_cell", - "prost 0.13.3", - "prost-types 0.13.3", + "prost 0.13.5", + "prost-types 0.13.5", "serde", "serde-value", ] @@ -7928,11 +8079,11 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.13.3" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4759aa0d3a6232fb8dbdb97b61de2c20047c68aca932c7ed76da9d788508d670" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" dependencies = [ - "prost 0.13.3", + "prost 0.13.5", ] [[package]] @@ -7982,30 +8133,28 @@ dependencies = [ [[package]] name = "pulsar" -version = "6.3.0" +version = "6.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7f3541ff84e39da334979ac4bf171e0f277f4f782603aeae65bf5795dc7275a" +checksum = "6cee616af00383c461f9ceb0067d15dee68e7d313ae47dbd7f8543236aed7ee9" dependencies = [ + "async-channel 2.3.1", "async-trait", - "bit-vec 0.6.3", "bytes 1.10.1", "chrono", "crc", "data-url", "flate2", "futures 0.3.31", - "futures-io", - "futures-timer", "log", "lz4", "native-tls", - "nom", + "nom 7.1.3", "oauth2", "openidconnect", "pem", - "prost 0.11.9", - "prost-build 0.11.9", - "prost-derive 0.11.9", + "prost 0.13.5", + "prost-build 0.13.5", + "prost-derive 0.13.5", "rand 0.8.5", "regex", "serde", @@ -8016,7 +8165,7 @@ dependencies = [ "tokio-util", "url", "uuid", - "zstd 0.12.4", + "zstd 0.13.2", ] [[package]] @@ -8027,9 +8176,9 @@ checksum = "658fa1faf7a4cc5f057c9ee5ef560f717ad9d8dc66d975267f709624d6e1ab88" [[package]] name = "quanta" -version = "0.12.5" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bd1fe6824cea6538803de3ff1bc0cf3949024db3d43c9643024bfb33a807c0e" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" dependencies = [ "crossbeam-utils", "libc", @@ -8053,7 +8202,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ed1a693391a16317257103ad06a88c6529ac640846021da7c435a06fffdacd7" dependencies = [ "chrono", - "indexmap 2.9.0", + "indexmap 2.10.0", "newtype-uuid", "quick-xml 0.37.4", "strip-ansi-escapes", @@ -8094,13 +8243,13 @@ dependencies = [ [[package]] name = "quickcheck_macros" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b22a693222d716a9587786f37ac3f6b4faedb5b80c23914e7303ff5a1d8016e9" +checksum = "f71ee38b42f8459a88d3362be6f9b841ad2d5421844f61eb1c59c11bff3ac14a" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 1.0.109", + "syn 2.0.104", ] [[package]] @@ -8115,7 +8264,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls 0.23.23", - "socket2 0.5.9", + "socket2 0.5.10", "thiserror 2.0.3", "tokio", "tracing 0.1.41", @@ -8150,7 +8299,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.9", + "socket2 0.5.10", "tracing 0.1.41", "windows-sys 0.59.0", ] @@ -8221,9 +8370,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.0", @@ -8294,7 +8443,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" dependencies = [ "num-traits", - "rand 0.9.1", + "rand 0.9.2", ] [[package]] @@ -8308,11 +8457,11 @@ dependencies = [ [[package]] name = "rand_xorshift" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" dependencies = [ - "rand_core 0.6.4", + "rand_core 0.9.0", ] [[package]] @@ -8425,24 +8574,27 @@ dependencies = [ [[package]] name = "redis" -version = "0.24.0" +version = "0.32.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c580d9cbbe1d1b479e8d67cf9daf6a62c957e6846048408b80b43ac3f6af84cd" +checksum = "e1f66bf4cac9733a23bcdf1e0e01effbaaad208567beba68be8f67e5f4af3ee1" dependencies = [ "arc-swap", - "async-trait", + "backon", "bytes 1.10.1", + "cfg-if", "combine 4.6.6", - "futures 0.3.31", + "futures-channel", "futures-util", "itoa", "native-tls", + "num-bigint", "percent-encoding", "pin-project-lite", + "rand 0.9.2", "ryu", + "socket2 0.6.0", "tokio", "tokio-native-tls", - "tokio-retry", "tokio-util", "url", ] @@ -8458,20 +8610,20 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.3.5" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ "bitflags 1.3.2", ] [[package]] name = "redox_syscall" -version = "0.4.1" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +checksum = "928fca9cf2aa042393a8325b9ead81d2f0df4cb12e1e24cef072922ccd99c5af" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.0", ] [[package]] @@ -8496,6 +8648,40 @@ dependencies = [ "thiserror 2.0.3", ] +[[package]] +name = "ref-cast" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a0ae411dbe946a674d89546582cea4ba2bb8defac896622d6496f14c23ba5cf" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" +dependencies = [ + "proc-macro2 1.0.95", + "quote 1.0.40", + "syn 2.0.104", +] + +[[package]] +name = "referencing" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a3d769362109497b240e66462606bc28af68116436c8669bac17069533b908e" +dependencies = [ + "ahash 0.8.11", + "fluent-uri 0.3.2", + "once_cell", + "parking_lot", + "percent-encoding", + "serde_json", +] + [[package]] name = "regex" version = "1.11.1" @@ -8535,7 +8721,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c11639076bf147be211b90e47790db89f4c22b6c8a9ca6e960833869da67166" dependencies = [ "aho-corasick", - "indexmap 2.9.0", + "indexmap 2.10.0", "itertools 0.13.0", "nohash", "regex", @@ -8600,7 +8786,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.11", + "rustls 0.21.12", "rustls-pemfile 1.0.4", "serde", "serde_json", @@ -8610,10 +8796,12 @@ dependencies = [ "tokio", "tokio-native-tls", "tokio-rustls 0.24.1", + "tokio-util", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", "webpki-roots 0.25.2", "winreg", @@ -8627,6 +8815,9 @@ checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" dependencies = [ "base64 0.22.1", "bytes 1.10.1", + "cookie", + "cookie_store", + "futures-channel", "futures-core", "futures-util", "http 1.1.0", @@ -8689,9 +8880,9 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.12" +version = "0.17.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed9b823fa29b721a59671b41d6b06e66b29e0628e207e8b1c3ceeda701ec928d" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", @@ -8772,9 +8963,9 @@ dependencies = [ [[package]] name = "roaring" -version = "0.10.12" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e8d2cfa184d94d0726d650a9f4a1be7f9b76ac9fdb954219878dc00c1c1e7b" +checksum = "f08d6a905edb32d74a5d5737a0c9d7e950c312f3c46cb0ca0a2ca09ea11878a0" dependencies = [ "bytemuck", "byteorder", @@ -8808,21 +8999,20 @@ dependencies = [ [[package]] name = "rstest" -version = "0.25.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fc39292f8613e913f7df8fa892b8944ceb47c247b78e1b1ae2f09e019be789d" +checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49" dependencies = [ "futures-timer", "futures-util", "rstest_macros", - "rustc_version 0.4.1", ] [[package]] name = "rstest_macros" -version = "0.25.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f168d99749d307be9de54d23fd226628d99768225ef08f6ffb52e0182a27746" +checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" dependencies = [ "cfg-if", "glob", @@ -8832,7 +9022,7 @@ dependencies = [ "regex", "relative-path", "rustc_version 0.4.1", - "syn 2.0.101", + "syn 2.0.104", "unicode-ident", ] @@ -8946,9 +9136,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.11" +version = "0.21.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fecbfb7b1444f477b345853b1fce097a2c6fb637b2bfb87e6bc5db0f043fae4" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", "ring", @@ -8993,7 +9183,7 @@ dependencies = [ "openssl-probe", "rustls-pemfile 1.0.4", "schannel", - "security-framework", + "security-framework 2.10.0", ] [[package]] @@ -9006,7 +9196,19 @@ dependencies = [ "rustls-pemfile 2.1.0", "rustls-pki-types", "schannel", - "security-framework", + "security-framework 2.10.0", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework 3.2.0", ] [[package]] @@ -9078,9 +9280,9 @@ dependencies = [ [[package]] name = "rustyline" -version = "15.0.0" +version = "16.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ee1e066dc922e513bda599c6ccb5f3bb2b0ea5870a579448f2622993f0a9a2f" +checksum = "62fd9ca5ebc709e8535e8ef7c658eb51457987e48c98ead2be482172accc408d" dependencies = [ "bitflags 2.9.0", "cfg-if", @@ -9088,7 +9290,7 @@ dependencies = [ "libc", "log", "memchr", - "nix 0.29.0", + "nix 0.30.1", "unicode-segmentation", "unicode-width 0.2.0", "utf8parse", @@ -9150,6 +9352,30 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1375ba8ef45a6f15d83fa8748f1079428295d403d6ea991d09ab100155fbc06d" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + [[package]] name = "scoped-tls" version = "1.0.1" @@ -9209,7 +9435,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "770452e37cad93e0a50d5abc3990d2bc351c36d0328f86cefec2f2fb206eaef6" dependencies = [ "bitflags 1.3.2", - "core-foundation", + "core-foundation 0.9.3", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" +dependencies = [ + "bitflags 2.9.0", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -9217,9 +9456,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.10.0" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f3cc463c0ef97e11c3461a9d3787412d30e8e7eb907c79180c4a57bf7c04ef" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" dependencies = [ "core-foundation-sys", "libc", @@ -9260,11 +9499,11 @@ dependencies = [ [[package]] name = "serde-toml-merge" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5817202d670278fb4dded71a70bae5181e00112543b1313463b02d43fc2d9243" +checksum = "0fc44799282f511a5d403d72a4ff028dc2c87f7fe6830abe3c33bb2fa6dfccec" dependencies = [ - "toml", + "toml 0.9.4", ] [[package]] @@ -9294,7 +9533,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -9305,16 +9544,16 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] name = "serde_json" -version = "1.0.140" +version = "1.0.142" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +checksum = "030fedb782600dcbd6f02d479bf0d817ac3bb40d644745b769d6a96bc3afc5a7" dependencies = [ - "indexmap 2.9.0", + "indexmap 2.10.0", "itoa", "memchr", "ryu", @@ -9368,14 +9607,23 @@ checksum = "3081f5ffbb02284dda55132aa26daecedd7372a42417bbbab6f14ab7d6bb9145" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", ] [[package]] name = "serde_spanned" -version = "0.6.8" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +checksum = "40734c41988f7306bb04f0ecf60ec0f3f1caa34290e4e8ea471dcd3346483b83" dependencies = [ "serde", ] @@ -9404,19 +9652,21 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.12.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa" +checksum = "f2c45cd61fefa9db6f254525d46e392b852e0e61d9a1fd36e5bd183450a556d5" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.9.0", + "indexmap 2.10.0", + "schemars 0.9.0", + "schemars 1.0.3", "serde", "serde_derive", "serde_json", - "serde_with_macros 3.12.0", + "serde_with_macros 3.14.0", "time", ] @@ -9434,14 +9684,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.12.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e" +checksum = "de90945e6565ce0d9a25098082ed4ee4002e047cb59892c318d66821e14bb30f" dependencies = [ "darling 0.20.11", "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -9450,7 +9700,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.9.0", + "indexmap 2.10.0", "itoa", "ryu", "serde", @@ -9599,7 +9849,7 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5b441962c817e33508847a22bd82f03a30cff43642dc2fae8b050566121eb9a" dependencies = [ - "console", + "console 0.15.7", "similar", ] @@ -9615,6 +9865,12 @@ version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" +[[package]] +name = "siphasher" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" + [[package]] name = "sketches-ddsketch" version = "0.3.0" @@ -9632,9 +9888,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.15.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" dependencies = [ "serde", ] @@ -9651,7 +9907,7 @@ dependencies = [ "async-io 1.13.0", "async-lock 2.8.0", "async-net", - "async-process 1.8.1", + "async-process", "blocking", "futures-lite 1.13.0", ] @@ -9679,18 +9935,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6" dependencies = [ "doc-comment", - "futures-core", - "pin-project", "snafu-derive 0.7.5", ] [[package]] name = "snafu" -version = "0.8.0" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d342c51730e54029130d7dc9fd735d28c4cd360f1368c01981d4f03ff207f096" +checksum = "320b01e011bf8d5d7a4a4a4be966d9160968935849c83b918827f6a435e7f627" dependencies = [ - "snafu-derive 0.8.0", + "futures-core", + "pin-project", + "snafu-derive 0.8.6", ] [[package]] @@ -9707,14 +9963,14 @@ dependencies = [ [[package]] name = "snafu-derive" -version = "0.8.0" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "080c44971436b1af15d6f61ddd8b543995cf63ab8e677d46b00cc06f4ef267a0" +checksum = "1961e2ef424c1424204d3a5d6975f934f56b6d50ff5732382d84ebf460e147f7" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -9735,14 +9991,24 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" dependencies = [ "libc", "windows-sys 0.52.0", ] +[[package]] +name = "socket2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "spin" version = "0.5.2" @@ -9779,9 +10045,9 @@ dependencies = [ [[package]] name = "sqlx" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3a85280daca669cfd3bcb68a337882a8bc57ec882f72c5d13a430613a738e" +checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" dependencies = [ "sqlx-core", "sqlx-macros", @@ -9792,9 +10058,9 @@ dependencies = [ [[package]] name = "sqlx-core" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f743f2a3cea30a58cd479013f75550e879009e3a02f616f18ca699335aa248c3" +checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" dependencies = [ "base64 0.22.1", "bytes 1.10.1", @@ -9809,7 +10075,7 @@ dependencies = [ "futures-util", "hashbrown 0.15.2", "hashlink", - "indexmap 2.9.0", + "indexmap 2.10.0", "log", "memchr", "once_cell", @@ -9827,22 +10093,22 @@ dependencies = [ [[package]] name = "sqlx-macros" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f4200e0fde19834956d4252347c12a083bdcb237d7a1a1446bffd8768417dce" +checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", "sqlx-core", "sqlx-macros-core", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] name = "sqlx-macros-core" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "882ceaa29cade31beca7129b6beeb05737f44f82dbe2a9806ecea5a7093d00b7" +checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" dependencies = [ "dotenvy", "either", @@ -9858,17 +10124,16 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn 2.0.101", - "tempfile", + "syn 2.0.104", "tokio", "url", ] [[package]] name = "sqlx-mysql" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0afdd3aa7a629683c2d750c2df343025545087081ab5942593a5288855b1b7a7" +checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" dependencies = [ "atoi", "base64 0.22.1", @@ -9909,9 +10174,9 @@ dependencies = [ [[package]] name = "sqlx-postgres" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0bedbe1bbb5e2615ef347a5e9d8cd7680fb63e77d9dafc0f29be15e53f1ebe6" +checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" dependencies = [ "atoi", "base64 0.22.1", @@ -9947,9 +10212,9 @@ dependencies = [ [[package]] name = "sqlx-sqlite" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c26083e9a520e8eb87a06b12347679b142dc2ea29e6e409f805644a7a979a5bc" +checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" dependencies = [ "atoi", "chrono", @@ -10069,7 +10334,7 @@ dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", "rustversion", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -10082,7 +10347,7 @@ dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", "rustversion", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -10134,9 +10399,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.101" +version = "2.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" +checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", @@ -10166,20 +10431,21 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] name = "sysinfo" -version = "0.34.2" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4b93974b3d3aeaa036504b8eefd4c039dced109171c1ae973f1dc63b2c7e4b2" +checksum = "252800745060e7b9ffb7b2badbd8b31cfa4aa2e61af879d0a3bf2a317c20217d" dependencies = [ "libc", "memchr", "ntapi 0.4.1", "objc2-core-foundation", - "windows 0.57.0", + "objc2-io-kit", + "windows 0.60.0", ] [[package]] @@ -10197,12 +10463,12 @@ dependencies = [ [[package]] name = "syslog_loose" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "161028c00842709450114c39db3b29f44c898055ed8833bb9b535aba7facf30e" +checksum = "d6ec4df26907adce53e94eac201a9ba38744baea3bc97f34ffd591d5646231a6" dependencies = [ "chrono", - "nom", + "nom 8.0.0", ] [[package]] @@ -10212,7 +10478,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags 1.3.2", - "core-foundation", + "core-foundation 0.9.3", "system-configuration-sys", ] @@ -10257,15 +10523,15 @@ dependencies = [ [[package]] name = "temp-dir" -version = "0.1.14" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc1ee6eef34f12f765cb94725905c6312b6610ab2b0940889cfe58dae7bc3c72" +checksum = "83176759e9416cf81ee66cb6508dbfe9c96f20b8b56265a39917551c23c70964" [[package]] name = "tempfile" -version = "3.19.1" +version = "3.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" dependencies = [ "fastrand 2.3.0", "getrandom 0.3.1", @@ -10358,7 +10624,7 @@ checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -10369,17 +10635,16 @@ checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] name = "thread_local" -version = "1.1.8" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ "cfg-if", - "once_cell", ] [[package]] @@ -10473,21 +10738,23 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.44.2" +version = "1.47.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48" +checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" dependencies = [ "backtrace", "bytes 1.10.1", + "io-uring", "libc", "mio", "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.9", + "slab", + "socket2 0.6.0", "tokio-macros", "tracing 0.1.41", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -10519,7 +10786,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -10558,12 +10825,12 @@ dependencies = [ "log", "parking_lot", "percent-encoding", - "phf", + "phf 0.11.2", "pin-project-lite", "postgres-protocol", "postgres-types", - "rand 0.9.1", - "socket2 0.5.9", + "rand 0.9.2", + "socket2 0.5.10", "tokio", "tokio-util", "whoami", @@ -10586,7 +10853,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.11", + "rustls 0.21.12", "tokio", ] @@ -10644,7 +10911,7 @@ checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" dependencies = [ "futures-util", "log", - "rustls 0.21.11", + "rustls 0.21.12", "tokio", "tungstenite 0.20.1", ] @@ -10675,23 +10942,68 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-websockets" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f591660438b3038dd04d16c938271c79e7e06260ad2ea2885a4861bfb238605d" +dependencies = [ + "base64 0.22.1", + "bytes 1.10.1", + "futures-core", + "futures-sink", + "http 1.1.0", + "httparse", + "rand 0.8.5", + "ring", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.2", + "tokio-util", + "webpki-roots 0.26.1", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", + "toml_edit 0.22.27", +] + [[package]] name = "toml" -version = "0.8.22" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ae329d1f08c4d17a59bed7ff5b5a769d062e64a62d34a3261b219e62cd5aae" +checksum = "41ae868b5a0f67631c14589f7e250c1ea2c574ee5ba21c6c8dd4b1485705a5a1" dependencies = [ + "indexmap 2.10.0", "serde", - "serde_spanned", - "toml_datetime", - "toml_edit 0.22.26", + "serde_spanned 1.0.0", + "toml_datetime 0.7.0", + "toml_parser", + "toml_writer", + "winnow 0.7.10", ] [[package]] name = "toml_datetime" -version = "0.6.9" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_datetime" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3da5db5a963e24bc68be8b17b6fa82814bb22ee8660f192bb182771d498f09a3" +checksum = "bade1c3e902f58d73d3f294cd7f20391c1cb2fbcb643b73566bc773971df91e3" dependencies = [ "serde", ] @@ -10702,30 +11014,45 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.9.0", - "toml_datetime", + "indexmap 2.10.0", + "toml_datetime 0.6.11", "winnow 0.5.18", ] [[package]] name = "toml_edit" -version = "0.22.26" +version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "310068873db2c5b3e7659d2cc35d21855dbafa50d1ce336397c666e3cb08137e" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap 2.9.0", + "indexmap 2.10.0", "serde", - "serde_spanned", - "toml_datetime", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", "toml_write", - "winnow 0.7.7", + "winnow 0.7.10", +] + +[[package]] +name = "toml_parser" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97200572db069e74c512a14117b296ba0a80a30123fbbb5aa1f4a348f639ca30" +dependencies = [ + "winnow 0.7.10", ] [[package]] name = "toml_write" -version = "0.1.1" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "toml_writer" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfb942dfe1d8e29a7ee7fcbde5bd2b9a25fb89aa70caea2eba3bee836ff41076" +checksum = "fcc842091f2def52017664b53082ecbbeb5c7731092bad69d2c63050401dfd64" [[package]] name = "tonic" @@ -10753,7 +11080,7 @@ dependencies = [ "tokio", "tokio-rustls 0.25.0", "tokio-stream", - "tower", + "tower 0.4.13", "tower-layer", "tower-service", "tracing 0.1.41", @@ -10771,7 +11098,7 @@ dependencies = [ "axum 0.7.5", "base64 0.22.1", "bytes 1.10.1", - "h2 0.4.9", + "h2 0.4.11", "http 1.1.0", "http-body 1.0.0", "http-body-util", @@ -10780,11 +11107,11 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "prost 0.13.3", - "socket2 0.5.9", + "prost 0.13.5", + "socket2 0.5.10", "tokio", "tokio-stream", - "tower", + "tower 0.4.13", "tower-layer", "tower-service", "tracing 0.1.41", @@ -10813,7 +11140,7 @@ dependencies = [ "proc-macro2 1.0.95", "prost-build 0.12.6", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -10836,6 +11163,25 @@ dependencies = [ "tracing 0.1.41", ] +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 2.10.0", + "pin-project-lite", + "slab", + "sync_wrapper 1.0.1", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing 0.1.41", +] + [[package]] name = "tower-http" version = "0.4.4" @@ -10879,15 +11225,15 @@ dependencies = [ [[package]] name = "tower-layer" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tower-test" @@ -10933,7 +11279,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -10991,7 +11337,7 @@ name = "tracing-limit" version = "0.1.0" dependencies = [ "criterion", - "dashmap 6.1.0", + "dashmap", "mock_instant", "tracing 0.1.41", "tracing-core 0.1.33", @@ -11058,7 +11404,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04659ddb06c87d233c566112c1c9c5b9e98256d9af50ec3bc9c8327f873a7568" dependencies = [ "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -11129,6 +11475,16 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +[[package]] +name = "tryhard" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fe58ebd5edd976e0fe0f8a14d2a04b7c81ef153ea9a54eebc42e67c2c23b4e5" +dependencies = [ + "pin-project-lite", + "tokio", +] + [[package]] name = "tungstenite" version = "0.20.1" @@ -11169,19 +11525,9 @@ dependencies = [ [[package]] name = "twox-hash" -version = "1.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" -dependencies = [ - "cfg-if", - "static_assertions", -] - -[[package]] -name = "twox-hash" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7b17f197b3050ba473acf9181f7b1d3b66d1cf7356c6cc57886662276e65908" +checksum = "8b907da542cbced5261bd3256de1b3a1bf340a3d37f93425a07362a1d687de56" [[package]] name = "typed-builder" @@ -11211,7 +11557,7 @@ checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -11241,7 +11587,16 @@ checksum = "35f5380909ffc31b4de4f4bdf96b877175a016aa2ca98cee39fcfd8c4d53d952" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", +] + +[[package]] +name = "tz-rs" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33851b15c848fad2cf4b105c6bb66eb9512b6f6c44a4b13f57c53c73c707e2b4" +dependencies = [ + "const_fn", ] [[package]] @@ -11337,6 +11692,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +[[package]] +name = "unit-prefix" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "323402cff2dd658f39ca17c789b502021b3f18707c91cdf22e3838e1b4023817" + [[package]] name = "universal-hash" version = "0.5.1" @@ -11374,7 +11735,7 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1ee6bfd0a27bf614353809a035cf6880b74239ec6c5e39a7b2860ca16809137" dependencies = [ - "num-rational", + "num-rational 0.3.2", "num-traits", "typenum", ] @@ -11429,17 +11790,28 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" +checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d" dependencies = [ "getrandom 0.3.1", "js-sys", - "rand 0.9.1", + "rand 0.9.2", "serde", "wasm-bindgen", ] +[[package]] +name = "uuid-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b082222b4f6619906941c17eb2297fff4c2fb96cb60164170522942a200bd8" +dependencies = [ + "outref", + "uuid", + "vsimd", +] + [[package]] name = "valuable" version = "0.1.0" @@ -11462,11 +11834,12 @@ dependencies = [ "clap-verbosity-flag", "clap_complete", "confy", - "directories 6.0.0", + "directories", "dunce", + "git2", "glob", "hex", - "indexmap 2.9.0", + "indexmap 2.10.0", "indicatif", "indoc", "itertools 0.14.0", @@ -11481,12 +11854,12 @@ dependencies = [ "serde_yaml", "sha2", "tempfile", - "toml", + "toml 0.9.4", ] [[package]] name = "vector" -version = "0.47.0" +version = "0.50.0" dependencies = [ "apache-avro", "approx", @@ -11541,6 +11914,7 @@ dependencies = [ "crossterm 0.29.0", "csv", "databend-client", + "deadpool 0.12.2", "derivative", "dirs-next", "dnsmsg-parser", @@ -11559,7 +11933,7 @@ dependencies = [ "goauth", "governor", "greptimedb-ingester", - "h2 0.4.9", + "h2 0.4.11", "hash_hasher", "hashbrown 0.14.5", "headers", @@ -11571,10 +11945,11 @@ dependencies = [ "http 1.1.0", "http-body 0.4.5", "http-serde", + "humantime", "hyper 0.14.28", "hyper-openssl 0.9.2", "hyper-proxy", - "indexmap 2.9.0", + "indexmap 2.10.0", "indoc", "inventory", "ipnet", @@ -11585,7 +11960,7 @@ dependencies = [ "libc", "listenfd", "loki-logproto", - "lru 0.14.0", + "lru 0.16.0", "maxminddb", "md-5", "metrics", @@ -11597,8 +11972,8 @@ dependencies = [ "netlink-packet-utils", "netlink-sys", "nix 0.26.2", - "nkeys 0.4.4", - "nom", + "nkeys", + "nom 8.0.0", "notify", "num-format", "number_prefix", @@ -11621,7 +11996,7 @@ dependencies = [ "pulsar", "quick-junit", "quickcheck", - "rand 0.9.1", + "rand 0.9.2", "rand_distr", "ratatui", "rdkafka", @@ -11639,14 +12014,14 @@ dependencies = [ "serde-toml-merge", "serde_bytes", "serde_json", - "serde_with 3.12.0", + "serde_with 3.14.0", "serde_yaml", "similar-asserts", "smallvec", "smpl_jwt", - "snafu 0.7.5", + "snafu 0.8.6", "snap", - "socket2 0.5.9", + "socket2 0.5.10", "sqlx", "stream-cancel", "strip-ansi-escapes", @@ -11663,10 +12038,10 @@ dependencies = [ "tokio-test", "tokio-tungstenite 0.20.1", "tokio-util", - "toml", + "toml 0.9.4", "tonic 0.11.0", "tonic-build 0.11.0", - "tower", + "tower 0.5.2", "tower-http 0.4.4", "tower-test", "tracing 0.1.41", @@ -11724,6 +12099,7 @@ dependencies = [ "criterion", "crossbeam-queue", "crossbeam-utils", + "dashmap", "derivative", "fslock", "futures 0.3.31", @@ -11733,14 +12109,15 @@ dependencies = [ "metrics-tracing-context", "metrics-util", "num-traits", + "ordered-float 4.6.0", "paste", "proptest", "quickcheck", - "rand 0.9.1", + "rand 0.9.2", "rkyv", "serde", "serde_yaml", - "snafu 0.7.5", + "snafu 0.8.6", "temp-dir", "tokio", "tokio-test", @@ -11762,7 +12139,7 @@ dependencies = [ "crossbeam-utils", "derivative", "futures 0.3.31", - "indexmap 2.9.0", + "indexmap 2.10.0", "metrics", "paste", "pin-project", @@ -11785,15 +12162,15 @@ dependencies = [ "chrono-tz", "encoding_rs", "http 0.2.9", - "indexmap 2.9.0", + "indexmap 2.10.0", "inventory", "no-proxy", "num-traits", "serde", "serde_json", - "serde_with 3.12.0", - "snafu 0.7.5", - "toml", + "serde_with 3.14.0", + "snafu 0.8.6", + "toml 0.9.4", "tracing 0.1.41", "url", "vector-config-common", @@ -11805,13 +12182,13 @@ dependencies = [ name = "vector-config-common" version = "0.1.0" dependencies = [ - "convert_case 0.7.1", + "convert_case 0.8.0", "darling 0.20.11", "proc-macro2 1.0.95", "quote 1.0.40", "serde", "serde_json", - "syn 2.0.101", + "syn 2.0.104", "tracing 0.1.41", ] @@ -11824,7 +12201,7 @@ dependencies = [ "quote 1.0.40", "serde", "serde_derive_internals", - "syn 2.0.101", + "syn 2.0.104", "vector-config", "vector-config-common", ] @@ -11852,7 +12229,7 @@ dependencies = [ "headers", "http 0.2.9", "hyper-proxy", - "indexmap 2.9.0", + "indexmap 2.10.0", "inventory", "ipnet", "metrics", @@ -11874,26 +12251,26 @@ dependencies = [ "quanta", "quickcheck", "quickcheck_macros", - "rand 0.9.1", + "rand 0.9.2", "rand_distr", "regex", "ryu", "schannel", - "security-framework", + "security-framework 3.2.0", "serde", "serde_json", - "serde_with 3.12.0", + "serde_with 3.14.0", "serde_yaml", "similar-asserts", "smallvec", - "snafu 0.7.5", - "socket2 0.5.9", + "snafu 0.8.6", + "socket2 0.5.10", "tokio", "tokio-openssl", "tokio-stream", "tokio-test", "tokio-util", - "toml", + "toml 0.9.4", "tonic 0.11.0", "tracing 0.1.41", "tracing-subscriber", @@ -11949,13 +12326,13 @@ dependencies = [ "futures-util", "pin-project", "proptest", - "rand 0.9.1", + "rand 0.9.2", "rand_distr", "tokio", "tokio-util", - "tower", + "tower 0.5.2", "tracing 0.1.41", - "twox-hash 2.1.0", + "twox-hash", "vector-common", "vector-core", ] @@ -12030,6 +12407,7 @@ dependencies = [ "vector-vrl-functions", "vrl", "wasm-bindgen", + "web-sys", ] [[package]] @@ -12046,8 +12424,8 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" [[package]] name = "vrl" -version = "0.23.0" -source = "git+https://github.com/vectordotdev/vrl?branch=main#7f8ed50af55b936c6da62a19dbbd966a8af903b6" +version = "0.26.0" +source = "git+https://github.com/vectordotdev/vrl.git?branch=main#40d3f6dfa395ef5dd306432e6cfd03af9966da00" dependencies = [ "aes", "aes-siv", @@ -12055,7 +12433,7 @@ dependencies = [ "arbitrary", "base16", "base62", - "base64 0.22.1", + "base64-simd", "bytes 1.10.1", "cbc", "cfb-mode", @@ -12074,14 +12452,13 @@ dependencies = [ "crypto_secretbox", "csv", "ctr", - "data-encoding", "digest", "dns-lookup", "domain", "dyn-clone", "encoding_rs", "exitcode", - "fancy-regex", + "fancy-regex 0.15.0", "flate2", "grok", "hex", @@ -12089,16 +12466,18 @@ dependencies = [ "hostname 0.4.0", "iana-time-zone", "idna 1.0.3", - "indexmap 2.9.0", + "indexmap 2.10.0", "indoc", "influxdb-line-protocol", "itertools 0.14.0", + "jsonschema", "lalrpop", "lalrpop-util", "lz4_flex", "md-5", "mlua", - "nom", + "nom 8.0.0", + "nom-language", "ofb", "onig", "ordered-float 4.6.0", @@ -12111,7 +12490,7 @@ dependencies = [ "prettytable-rs", "proptest", "proptest-derive", - "prost 0.13.3", + "prost 0.13.5", "prost-reflect", "psl", "psl-types", @@ -12131,7 +12510,7 @@ dependencies = [ "sha2", "sha3", "simdutf8", - "snafu 0.8.0", + "snafu 0.8.6", "snap", "strip-ansi-escapes", "syslog_loose", @@ -12275,7 +12654,7 @@ dependencies = [ "log", "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", "wasm-bindgen-shared", ] @@ -12309,7 +12688,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -12338,9 +12717,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.65" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5db499c5f66323272151db0e666cd34f78617522fb0c1604d31a27c50c206a85" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" dependencies = [ "js-sys", "wasm-bindgen", @@ -12362,7 +12741,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b6f804e41d0852e16d2eaee61c7e4f7d3e8ffdb7b8ed85886aeb0791fe9fcd" dependencies = [ - "core-foundation", + "core-foundation 0.9.3", "home", "jni", "log", @@ -12479,12 +12858,24 @@ dependencies = [ [[package]] name = "windows" -version = "0.57.0" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" +checksum = "ddf874e74c7a99773e62b1c671427abf01a425e77c3d3fb9fb1e4883ea934529" dependencies = [ - "windows-core 0.57.0", - "windows-targets 0.52.6", + "windows-collections", + "windows-core 0.60.1", + "windows-future", + "windows-link", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5467f79cc1ba3f52ebb2ed41dbb459b8e7db636cc3429458d9a852e15bc24dec" +dependencies = [ + "windows-core 0.60.1", ] [[package]] @@ -12498,36 +12889,47 @@ dependencies = [ [[package]] name = "windows-core" -version = "0.57.0" +version = "0.60.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" +checksum = "ca21a92a9cae9bf4ccae5cf8368dce0837100ddf6e6d57936749e85f152f6247" dependencies = [ "windows-implement", "windows-interface", - "windows-result 0.1.2", - "windows-targets 0.52.6", + "windows-link", + "windows-result 0.3.1", + "windows-strings 0.3.1", +] + +[[package]] +name = "windows-future" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a787db4595e7eb80239b74ce8babfb1363d8e343ab072f2ffe901400c03349f0" +dependencies = [ + "windows-core 0.60.1", + "windows-link", ] [[package]] name = "windows-implement" -version = "0.57.0" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" +checksum = "83577b051e2f49a058c308f17f273b570a6a758386fc291b5f6a934dd84e48c1" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] name = "windows-interface" -version = "0.57.0" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -12536,6 +12938,16 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dccfd733ce2b1753b03b6d3c65edf020262ea35e20ccdf3e288043e6dd620e3" +[[package]] +name = "windows-numerics" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "005dea54e2f6499f2cee279b8f703b3cf3b5734a2d8d21867c8f44003182eeed" +dependencies = [ + "windows-core 0.60.1", + "windows-link", +] + [[package]] name = "windows-registry" version = "0.2.0" @@ -12543,37 +12955,37 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" dependencies = [ "windows-result 0.2.0", - "windows-strings", + "windows-strings 0.1.0", "windows-targets 0.52.6", ] [[package]] name = "windows-result" -version = "0.1.2" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" dependencies = [ "windows-targets 0.52.6", ] [[package]] name = "windows-result" -version = "0.2.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +checksum = "06374efe858fab7e4f881500e6e86ec8bc28f9462c47e5a9941a0142ad86b189" dependencies = [ - "windows-targets 0.52.6", + "windows-link", ] [[package]] name = "windows-service" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d24d6bcc7f734a4091ecf8d7a64c5f7d7066f45585c1861eba06449909609c8a" +checksum = "193cae8e647981c35bc947fdd57ba7928b1fa0d4a79305f6dd2dc55221ac35ac" dependencies = [ "bitflags 2.9.0", "widestring 1.0.2", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -12586,6 +12998,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-strings" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.45.0" @@ -12622,6 +13043,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.2", +] + [[package]] name = "windows-targets" version = "0.42.2" @@ -12661,13 +13091,29 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef" +dependencies = [ + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -12686,6 +13132,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -12704,6 +13156,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -12722,12 +13180,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -12746,6 +13216,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -12764,6 +13240,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -12782,6 +13264,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -12800,6 +13288,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + [[package]] name = "winnow" version = "0.5.18" @@ -12811,9 +13305,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.7.7" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb8234a863ea0e8cd7284fcdd4f145233eb00fee02bbdd9861aec44e6477bc5" +checksum = "c06928c8748d81b05c9be96aad92e1b6ff01833332f281e8cfca3be4b35fc9ec" dependencies = [ "memchr", ] @@ -12830,14 +13324,14 @@ dependencies = [ [[package]] name = "wiremock" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "101681b74cd87b5899e87bcf5a64e83334dd313fcd3053ea72e6dba18928e301" +checksum = "a2b8b99d4cdbf36b239a9532e31fe4fb8acc38d1897c1761e161550a7dc78e6a" dependencies = [ "assert-json-diff", "async-trait", "base64 0.22.1", - "deadpool", + "deadpool 0.10.0", "futures 0.3.31", "http 1.1.0", "http-body-util", @@ -12918,7 +13412,7 @@ checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", "synstructure", ] @@ -12948,7 +13442,7 @@ checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -12959,7 +13453,7 @@ checksum = "5226bc9a9a9836e7428936cde76bb6b22feea1a8bfdbc0d241136e4d13417e25" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -12979,7 +13473,7 @@ checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", "synstructure", ] @@ -13008,14 +13502,14 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] name = "zlib-rs" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "868b928d7949e09af2f6086dfc1e01936064cc7a819253bce650d4e2a2d63ba8" +checksum = "626bd9fa9734751fc50d6060752170984d7053f5a39061f524cda68023d4db8a" [[package]] name = "zstd" diff --git a/Cargo.toml b/Cargo.toml index fdbcfb5e25eed..aefad578698dc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vector" -version = "0.47.0" +version = "0.50.0" authors = ["Vector Contributors "] edition = "2021" description = "A lightweight and ultra-fast tool for building observability pipelines" @@ -12,7 +12,7 @@ default-run = "vector" autobenches = false # our benchmarks are not runnable on their own either way # Minimum supported rust version # See docs/DEVELOPING.md for policy -rust-version = "1.83" +rust-version = "1.86" [[bin]] name = "vector" @@ -135,46 +135,50 @@ members = [ [workspace.dependencies] anyhow = "1.0.98" -cfg-if = { version = "1.0.0", default-features = false } +cfg-if = { version = "1.0.1", default-features = false } chrono = { version = "0.4.41", default-features = false, features = ["clock", "serde"] } -chrono-tz = { version = "0.10.3", default-features = false, features = ["serde"] } -clap = { version = "4.5.37", default-features = false, features = ["derive", "error-context", "env", "help", "std", "string", "usage", "wrap_help"] } +chrono-tz = { version = "0.10.4", default-features = false, features = ["serde"] } +clap = { version = "4.5.42", default-features = false, features = ["derive", "error-context", "env", "help", "std", "string", "usage", "wrap_help"] } darling = { version = "0.20.11", default-features = false, features = ["suggestions"] } -flate2 = { version = "1.1.1", default-features = false, features = ["zlib-rs"] } +flate2 = { version = "1.1.2", default-features = false, features = ["zlib-rs"] } futures = { version = "0.3.31", default-features = false, features = ["compat", "io-compat", "std"], package = "futures" } glob = { version = "0.3.2", default-features = false } -hickory-proto = { version = "0.25.1", default-features = false, features = ["dnssec-ring"] } -indexmap = { version = "2.9.0", default-features = false, features = ["serde", "std"] } -inventory = { version = "0.3" } +hickory-proto = { version = "0.25.2", default-features = false, features = ["dnssec-ring"] } +humantime = { version = "2.2.0", default-features = false } +indexmap = { version = "2.10.0", default-features = false, features = ["serde", "std"] } indoc = { version = "2.0.6" } +inventory = { version = "0.3" } +itertools = { version = "0.14.0", default-features = false, features = ["use_alloc"] } metrics = "0.24.2" metrics-tracing-context = { version = "0.17.0", default-features = false } metrics-util = { version = "0.18.0", default-features = false, features = ["registry"] } +nom = { version = "8.0.0", default-features = false } paste = { version = "1.0.15" } pin-project = { version = "1.1.10", default-features = false } -proptest = { version = "1.6" } -proptest-derive = { version = "0.5.1" } +proptest = { version = "1.7" } +proptest-derive = { version = "0.6.0" } prost = { version = "0.12", default-features = false, features = ["std"] } prost-build = { version = "0.12", default-features = false } prost-reflect = { version = "0.14", features = ["serde"], default-features = false } prost-types = { version = "0.12", default-features = false } -rand = { version = "0.9.1", default-features = false, features = ["small_rng", "thread_rng"] } +rand = { version = "0.9.2", default-features = false, features = ["small_rng", "thread_rng"] } rand_distr = { version = "0.5.1", default-features = false } semver = { version = "1.0.26", default-features = false, features = ["serde", "std"] } -serde_json = { version = "1.0.140", default-features = false, features = ["raw_value", "std"] } serde = { version = "1.0.219", default-features = false, features = ["alloc", "derive", "rc"] } -snafu = { version = "0.7.5", default-features = false, features = ["futures", "std"] } -tempfile = "3.19.1" -tokio = { version = "1.44.2", default-features = false, features = ["full"] } -toml = { version = "0.8.22", default-features = false, features = ["display", "parse"] } +serde_json = { version = "1.0.142", default-features = false, features = ["raw_value", "std"] } +snafu = { version = "0.8.6", default-features = false, features = ["futures", "std"] } +socket2 = { version = "0.5.10", default-features = false } +tempfile = "3.20.0" +tokio = { version = "1.45.1", default-features = false, features = ["full"] } +toml = { version = "0.9.4", default-features = false, features = ["serde", "display", "parse"] } tonic = { version = "0.11", default-features = false, features = ["transport", "codegen", "prost", "tls", "tls-roots", "gzip"] } tonic-build = { version = "0.11", default-features = false, features = ["transport", "prost"] } -uuid = { version = "1.16.0", features = ["v4", "v7", "serde"] } +uuid = { version = "1.17.0", features = ["v4", "v7", "serde"] } vector-lib = { path = "lib/vector-lib", default-features = false, features = ["vrl"] } vector-config = { path = "lib/vector-config" } vector-config-common = { path = "lib/vector-config-common" } vector-config-macros = { path = "lib/vector-config-macros" } -vrl = { git = "https://github.com/vectordotdev/vrl", branch = "main", features = ["arbitrary", "cli", "test", "test_framework"] } +vrl = { git = "https://github.com/vectordotdev/vrl.git", branch = "main", features = ["arbitrary", "cli", "test", "test_framework"] } [dependencies] cfg-if.workspace = true @@ -207,7 +211,7 @@ loki-logproto = { path = "lib/loki-logproto", optional = true } async-stream = { version = "0.3.6", default-features = false } async-trait = { version = "0.1.88", default-features = false } futures.workspace = true -tokio = { version = "1.44.2", default-features = false, features = ["full"] } +tokio = { version = "1.45.1", default-features = false, features = ["full"] } tokio-openssl = { version = "0.6.5", default-features = false } tokio-stream = { version = "0.1.17", default-features = false, features = ["net", "sync", "time"] } tokio-util = { version = "0.7", default-features = false, features = ["io", "time"] } @@ -225,29 +229,29 @@ metrics.workspace = true metrics-tracing-context.workspace = true # AWS - Official SDK -aws-runtime = { version = "1.5.6", optional = true } +aws-runtime = { version = "1.5.9", optional = true } aws-config = { version = "1.6.1", default-features = false, features = ["behavior-version-latest", "credentials-process", "sso", "rt-tokio"], optional = true } -aws-credential-types = { version = "1.2.3", default-features = false, features = ["hardcoded-credentials"], optional = true } +aws-credential-types = { version = "1.2.4", default-features = false, features = ["hardcoded-credentials"], optional = true } aws-sdk-cloudwatch = { version = "1.70.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } aws-sdk-cloudwatchlogs = { version = "1.76.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } aws-sdk-elasticsearch = { version = "1.67.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } aws-sdk-firehose = { version = "1.71.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } aws-sdk-kinesis = { version = "1.66.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-kms = { version = "1.65.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-kms = { version = "1.75.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } aws-sdk-s3 = { version = "1.15.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-secretsmanager = { version = "1.68.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-sns = { version = "1.65.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-secretsmanager = { version = "1.76.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-sns = { version = "1.73.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } aws-sdk-sqs = { version = "1.64.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-types = { version = "1.3.7", default-features = false, optional = true } +aws-types = { version = "1.3.8", default-features = false, optional = true } # The sts crate is needed despite not being referred to anywhere in the code because we need to set the # `behavior-version-latest` feature. Without this we get a runtime panic when `auth.assume_role` authentication # is configured. -aws-sdk-sts = { version = "1.65.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-sts = { version = "1.73.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } # The `aws-sdk-sts` crate is needed despite not being referred to anywhere in the code because we need to set the # `behavior-version-latest` feature. Without this we get a runtime panic when `auth.assume_role` authentication is configured. -aws-sigv4 = { version = "1.3.1", default-features = false, features = ["sign-http"], optional = true } +aws-sigv4 = { version = "1.3.2", default-features = false, features = ["sign-http"], optional = true } aws-smithy-async = { version = "1.2.5", default-features = false, features = ["rt-tokio"], optional = true } aws-smithy-http = { version = "0.62", default-features = false, features = ["event-stream", "rt-tokio"], optional = true } @@ -256,23 +260,23 @@ aws-smithy-runtime-api = { version = "1.7.3", default-features = false, optional aws-smithy-types = { version = "1.2.11", default-features = false, features = ["rt-tokio"], optional = true } # Azure -azure_core = { version = "0.21", default-features = false, features = ["hmac_rust", "enable_reqwest"], optional = true } -azure_identity = { version = "0.21", default-features = false, features = ["enable_reqwest"], optional = true } -azure_storage = { version = "0.21", default-features = false, optional = true } -azure_storage_blobs = { version = "0.21", default-features = false, optional = true } +azure_core = { version = "0.17", default-features = false, features = ["enable_reqwest"], optional = true } +azure_identity = { version = "0.17", default-features = false, features = ["enable_reqwest"], optional = true } +azure_storage = { version = "0.17", default-features = false, optional = true } +azure_storage_blobs = { version = "0.17", default-features = false, optional = true } # OpenDAL -opendal = { version = "0.53", default-features = false, features = ["services-webhdfs"], optional = true } +opendal = { version = "0.54", default-features = false, features = ["services-webhdfs"], optional = true } # Tower -tower = { version = "0.4.13", default-features = false, features = ["buffer", "limit", "retry", "timeout", "util", "balance", "discover"] } +tower = { version = "0.5.2", default-features = false, features = ["buffer", "limit", "retry", "timeout", "util", "balance", "discover"] } tower-http = { version = "0.4.4", default-features = false, features = ["compression-full", "decompression-gzip", "trace"] } # Serde serde.workspace = true -serde-toml-merge = { version = "0.3.9", default-features = false } +serde-toml-merge = { version = "0.3.11", default-features = false } serde_bytes = { version = "0.11.17", default-features = false, features = ["std"], optional = true } serde_json.workspace = true -serde_with = { version = "3.12.0", default-features = false, features = ["macros", "std"] } +serde_with = { version = "3.14.0", default-features = false, features = ["macros", "std"] } serde_yaml = { version = "0.9.34", default-features = false } # Messagepack @@ -285,15 +289,16 @@ prost-reflect = { workspace = true, optional = true } prost-types = { workspace = true, optional = true } # GCP -goauth = { version = "0.14.0", optional = true } +goauth = { version = "0.16.0", optional = true } smpl_jwt = { version = "0.8.0", default-features = false, optional = true } # AMQP lapin = { version = "2.5.3", default-features = false, features = ["native-tls"], optional = true } +deadpool = { version = "0.12.2", default-features = false, features = ["managed", "rt_tokio_1"], optional = true } # API -async-graphql = { version = "7.0.16", default-features = false, optional = true, features = ["chrono", "playground"] } -async-graphql-warp = { version = "7.0.16", default-features = false, optional = true } +async-graphql = { version = "7.0.17", default-features = false, optional = true, features = ["chrono", "playground"] } +async-graphql-warp = { version = "7.0.17", default-features = false, optional = true } # API client crossterm = { version = "0.29.0", default-features = false, features = ["event-stream", "windows"], optional = true } @@ -306,26 +311,26 @@ ratatui = { version = "0.29.0", optional = true, default-features = false, featu hex = { version = "0.4.3", default-features = false, optional = true } # GreptimeDB -greptimedb-ingester = { git = "https://github.com/GreptimeTeam/greptimedb-ingester-rust", rev = "2e6b0c5eb6a5e7549c3100e4d356b07d15cce66d", optional = true } +greptimedb-ingester = { git = "https://github.com/GreptimeTeam/greptimedb-ingester-rust", rev = "f7243393808640f5123b0d5b7b798da591a4df6e", optional = true } # External libs arc-swap = { version = "1.7", default-features = false, optional = true } -async-compression = { version = "0.4.23", default-features = false, features = ["tokio", "gzip", "zstd"], optional = true } +async-compression = { version = "0.4.27", default-features = false, features = ["tokio", "gzip", "zstd"], optional = true } apache-avro = { version = "0.16.0", default-features = false, optional = true } axum = { version = "0.6.20", default-features = false } base64 = { version = "0.22.1", default-features = false, optional = true } bloomy = { version = "1.2.0", default-features = false, optional = true } -bollard = { version = "0.16.1", default-features = false, features = ["ssl", "chrono"], optional = true } +bollard = { version = "0.19.1", default-features = false, features = ["pipe", "ssl", "chrono"], optional = true } bytes = { version = "1.10.1", default-features = false, features = ["serde"] } bytesize = { version = "2.0.1", default-features = false } chrono.workspace = true chrono-tz.workspace = true colored = { version = "3.0.0", default-features = false } csv = { version = "1.3", default-features = false } -databend-client = { version = "0.22.2", default-features = false, features = ["rustls"], optional = true } +databend-client = { version = "0.28.0", default-features = false, features = ["rustls"], optional = true } derivative = { version = "2.2.0", default-features = false } dirs-next = { version = "2.0.0", default-features = false, optional = true } -dyn-clone = { version = "1.0.19", default-features = false } +dyn-clone = { version = "1.0.20", default-features = false } encoding_rs = { version = "0.8.35", default-features = false, features = ["serde"] } enum_dispatch = { version = "0.3.13", default-features = false } evmap = { version = "10.0.2", default-features = false, optional = true } @@ -334,9 +339,9 @@ exitcode = { version = "1.1.2", default-features = false } flate2.workspace = true futures-util = { version = "0.3.29", default-features = false } glob.workspace = true -governor = { version = "0.7.0", default-features = false, features = ["dashmap", "jitter", "std"], optional = true } -h2 = { version = "0.4.9", default-features = false, optional = true } -hash_hasher = { version = "2.0.0", default-features = false } +governor = { version = "0.10.0", default-features = false, features = ["dashmap", "jitter", "std"], optional = true } +h2 = { version = "0.4.11", default-features = false, optional = true } +hash_hasher = { version = "2.0.4", default-features = false } hashbrown = { version = "0.14.5", default-features = false, optional = true, features = ["ahash"] } headers = { version = "0.3.9", default-features = false } hostname = { version = "0.4.0", default-features = false } @@ -344,43 +349,44 @@ http = { version = "0.2.9", default-features = false } http-1 = { package = "http", version = "1.0", default-features = false, features = ["std"] } http-serde = "1.1.3" http-body = { version = "0.4.5", default-features = false } +humantime.workspace = true hyper = { version = "0.14.28", default-features = false, features = ["client", "runtime", "http1", "http2", "server", "stream"] } hyper-openssl = { version = "0.9.2", default-features = false } hyper-proxy = { version = "0.9.1", default-features = false, features = ["openssl-tls"] } indexmap.workspace = true inventory = { version = "0.3.20", default-features = false } ipnet = { version = "2", default-features = false, optional = true, features = ["serde", "std"] } -itertools = { version = "0.14.0", default-features = false, optional = false, features = ["use_alloc"] } +itertools.workspace = true k8s-openapi = { version = "0.22.0", default-features = false, features = ["v1_26"], optional = true } kube = { version = "0.93.0", default-features = false, features = ["client", "openssl-tls", "runtime"], optional = true } listenfd = { version = "1.0.2", default-features = false, optional = true } -lru = { version = "0.14.0", default-features = false, optional = true } +lru = { version = "0.16.0", default-features = false, optional = true } maxminddb = { version = "0.26.0", default-features = false, optional = true, features = ["simdutf8"] } md-5 = { version = "0.10", default-features = false, optional = true } mongodb = { version = "2.8.2", default-features = false, features = ["tokio-runtime"], optional = true } -async-nats = { version = "0.33.0", default-features = false, optional = true } -nkeys = { version = "0.4.4", default-features = false, optional = true } -nom = { version = "7.1.3", default-features = false, optional = true } -notify = { version = "8.0.0", default-features = false, features = ["macos_fsevent"] } -openssl = { version = "0.10.72", default-features = false, features = ["vendored"] } +async-nats = { version = "0.42.0", default-features = false, optional = true, features = ["ring"] } +nkeys = { version = "0.4.5", default-features = false, optional = true } +nom = { workspace = true, optional = true } +notify = { version = "8.1.0", default-features = false, features = ["macos_fsevent"] } +openssl = { version = "0.10.73", default-features = false, features = ["vendored"] } openssl-probe = { version = "0.1.6", default-features = false } ordered-float = { version = "4.6.0", default-features = false } percent-encoding = { version = "2.3.1", default-features = false } postgres-openssl = { version = "0.5.1", default-features = false, features = ["runtime"], optional = true } -pulsar = { version = "6.3.0", default-features = false, features = ["tokio-runtime", "auth-oauth2", "flate2", "lz4", "snap", "zstd"], optional = true } +pulsar = { version = "6.3.1", default-features = false, features = ["tokio-runtime", "auth-oauth2", "flate2", "lz4", "snap", "zstd"], optional = true } quick-junit = { version = "0.5.1" } rand.workspace = true rand_distr.workspace = true rdkafka = { version = "0.37.0", default-features = false, features = ["curl-static", "tokio", "libz", "ssl", "zstd"], optional = true } -redis = { version = "0.24.0", default-features = false, features = ["connection-manager", "tokio-comp", "tokio-native-tls-comp"], optional = true } +redis = { version = "0.32.4", default-features = false, features = ["connection-manager", "sentinel", "tokio-comp", "tokio-native-tls-comp"], optional = true } regex = { version = "1.11.1", default-features = false, features = ["std", "perf"] } -roaring = { version = "0.10.12", default-features = false, features = ["std"], optional = true } +roaring = { version = "0.11.2", default-features = false, features = ["std"], optional = true } rumqttc = { version = "0.24.0", default-features = false, features = ["use-rustls"], optional = true } seahash = { version = "4.1.0", default-features = false } smallvec = { version = "1", default-features = false, features = ["union", "serde"] } snap = { version = "1.1.1", default-features = false } -socket2 = { version = "0.5.9", default-features = false } -sqlx = { version = "0.8.5", default-features = false, features = ["derive", "postgres", "chrono", "runtime-tokio"], optional=true } +socket2.workspace = true +sqlx = { version = "0.8.6", default-features = false, features = ["derive", "postgres", "chrono", "runtime-tokio"], optional=true } stream-cancel = { version = "0.8.2", default-features = false } strip-ansi-escapes = { version = "0.2.1", default-features = false } syslog = { version = "6.1.1", default-features = false, optional = true } @@ -390,7 +396,7 @@ tokio-tungstenite = { version = "0.20.1", default-features = false, features = [ toml.workspace = true hickory-proto = { workspace = true, optional = true } tonic = { workspace = true, optional = true } -thread_local = { version = "1.1.8", default-features = false, optional = true } +thread_local = { version = "1.1.9", default-features = false, optional = true } typetag = { version = "0.2.20", default-features = false } url = { version = "2.5.4", default-features = false, features = ["serde"] } warp = { version = "0.3.7", default-features = false } @@ -402,12 +408,12 @@ arr_macro = { version = "0.2.1" } heim = { git = "https://github.com/vectordotdev/heim.git", branch = "update-nix", default-features = false, features = ["disk"] } # make sure to update the external docs when the Lua version changes -mlua = { version = "0.10.3", default-features = false, features = ["lua54", "send", "vendored", "macros"], optional = true } -sysinfo = "0.34.2" +mlua = { version = "0.10.5", default-features = false, features = ["lua54", "send", "vendored", "macros"], optional = true } +sysinfo = "0.36.1" byteorder = "1.5.0" [target.'cfg(windows)'.dependencies] -windows-service = "0.7.0" +windows-service = "0.8.0" [target.'cfg(unix)'.dependencies] nix = { version = "0.26.2", default-features = false, features = ["socket", "signal"] } @@ -428,28 +434,28 @@ openssl-src = { version = "300", default-features = false, features = ["force-en approx = "0.5.1" assert_cmd = { version = "2.0.17", default-features = false } aws-smithy-runtime = { version = "1.8.3", default-features = false, features = ["tls-rustls"] } -azure_core = { version = "0.21", default-features = false, features = ["enable_reqwest", "azurite_workaround"] } -azure_identity = { version = "0.21", default-features = false, features = ["enable_reqwest"] } -azure_storage_blobs = { version = "0.21", default-features = false, features = ["azurite_workaround"] } -azure_storage = { version = "0.21", default-features = false } +azure_core = { version = "0.17", default-features = false, features = ["enable_reqwest", "azurite_workaround"] } +azure_identity = { version = "0.17", default-features = false, features = ["enable_reqwest"] } +azure_storage_blobs = { version = "0.17", default-features = false, features = ["azurite_workaround"] } +azure_storage = { version = "0.17", default-features = false } base64 = "0.22.1" -criterion = { version = "0.5.1", features = ["html_reports", "async_tokio"] } -itertools = { version = "0.14.0", default-features = false, features = ["use_alloc"] } -libc = "0.2.172" +criterion = { version = "0.7.0", features = ["html_reports", "async_tokio"] } +itertools.workspace = true +libc = "0.2.174" similar-asserts = "1.7.0" proptest.workspace = true quickcheck = "1.0.3" reqwest = { version = "0.11", features = ["json"] } -rstest = { version = "0.25.0" } +rstest = { version = "0.26.1" } tempfile.workspace = true test-generator = "0.3.1" -tokio = { version = "1.44.2", features = ["test-util"] } +tokio = { version = "1.45.1", features = ["test-util"] } tokio-test = "0.4.4" tower-test = "0.4.0" vector-lib = { workspace = true, features = ["test"] } vrl.workspace = true -wiremock = "0.6.3" +wiremock = "0.6.4" zstd = { version = "0.13.0", default-features = false } [patch.crates-io] @@ -580,6 +586,7 @@ sources-logs = [ "sources-kafka", "sources-kubernetes_logs", "sources-logstash", + "sources-mqtt", "sources-nats", "sources-opentelemetry", "sources-pulsar", @@ -590,6 +597,7 @@ sources-logs = [ "sources-stdin", "sources-syslog", "sources-vector", + "sources-websocket", ] sources-metrics = [ "dep:prost", @@ -605,6 +613,7 @@ sources-metrics = [ "sources-static_metrics", "sources-statsd", "sources-vector", + "sources-websocket", ] sources-amqp = ["lapin"] @@ -635,6 +644,7 @@ sources-kafka = ["dep:rdkafka"] sources-kubernetes_logs = ["vector-lib/file-source", "kubernetes", "transforms-reduce"] sources-logstash = ["sources-utils-net-tcp", "tokio-util/net"] sources-mongodb_metrics = ["dep:mongodb"] +sources-mqtt = ["dep:rumqttc"] sources-nats = ["dep:async-nats", "dep:nkeys"] sources-nginx_metrics = ["dep:nom"] sources-opentelemetry = ["dep:hex", "vector-lib/opentelemetry", "dep:prost", "dep:prost-types", "sources-http_server", "sources-utils-http", "sources-utils-http-headers", "sources-vector"] @@ -662,6 +672,7 @@ sources-utils-net = ["sources-utils-net-tcp", "sources-utils-net-udp", "sources- sources-utils-net-tcp = ["listenfd", "dep:ipnet"] sources-utils-net-udp = ["listenfd"] sources-utils-net-unix = [] +sources-websocket = ["dep:tokio-tungstenite"] sources-vector = ["dep:prost", "dep:tonic", "protobuf-build"] @@ -782,7 +793,7 @@ sinks-metrics = [ "sinks-splunk_hec" ] -sinks-amqp = ["lapin"] +sinks-amqp = ["deadpool", "lapin"] sinks-appsignal = [] sinks-aws_cloudwatch_logs = ["aws-core", "dep:aws-sdk-cloudwatchlogs", "dep:aws-sdk-kms"] sinks-aws_cloudwatch_metrics = ["aws-core", "dep:aws-sdk-cloudwatch"] @@ -934,7 +945,7 @@ kafka-integration-tests = ["sinks-kafka", "sources-kafka"] logstash-integration-tests = ["docker", "sources-logstash"] loki-integration-tests = ["sinks-loki"] mongodb_metrics-integration-tests = ["sources-mongodb_metrics"] -mqtt-integration-tests = ["sinks-mqtt"] +mqtt-integration-tests = ["sinks-mqtt", "sources-mqtt"] nats-integration-tests = ["sinks-nats", "sources-nats"] nginx-integration-tests = ["sources-nginx_metrics"] opentelemetry-integration-tests = ["sources-opentelemetry", "dep:prost"] @@ -953,7 +964,8 @@ test-utils = [] # End-to-End testing-related features all-e2e-tests = [ - "e2e-tests-datadog" + "e2e-tests-datadog", + "e2e-tests-opentelemetry" ] e2e-tests-datadog = [ @@ -963,6 +975,15 @@ e2e-tests-datadog = [ "dep:async-compression" ] +e2e-tests-opentelemetry = [ + "sources-opentelemetry", + "sinks-opentelemetry", + "sources-internal_metrics", + "transforms-remap", + "sinks-console", + "sinks-file" +] + vector-api-tests = [ "sources-demo_logs", "transforms-log_to_metric", diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 7f7cb766ac1b3..567df49b9eba4 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -84,7 +84,6 @@ azure_core,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft Corp. azure_identity,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft Corp. azure_storage,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft Corp. azure_storage_blobs,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft Corp. -azure_svc_blobstorage,https://github.com/azure/azure-sdk-for-rust,MIT,The azure_svc_blobstorage Authors backoff,https://github.com/ihrwein/backoff,MIT OR Apache-2.0,Tibor Benke backon,https://github.com/Xuanwo/backon,Apache-2.0,The backon Authors backtrace,https://github.com/rust-lang/backtrace-rs,MIT OR Apache-2.0,The Rust Project Developers @@ -97,7 +96,6 @@ base64-simd,https://github.com/Nugine/simd,MIT,The base64-simd Authors base64ct,https://github.com/RustCrypto/formats/tree/master/base64ct,Apache-2.0 OR MIT,RustCrypto Developers bit-set,https://github.com/contain-rs/bit-set,Apache-2.0 OR MIT,Alexis Beingessner bit-vec,https://github.com/contain-rs/bit-vec,Apache-2.0 OR MIT,Alexis Beingessner -bit-vec,https://github.com/contain-rs/bit-vec,MIT OR Apache-2.0,Alexis Beingessner bitflags,https://github.com/bitflags/bitflags,MIT OR Apache-2.0,The Rust Project Developers bitmask-enum,https://github.com/Lukas3674/rust-bitmask-enum,MIT OR Apache-2.0,Lukas3674 bitvec,https://github.com/bitvecto-rs/bitvec,MIT,The bitvec Authors @@ -106,12 +104,14 @@ block-padding,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto D blocking,https://github.com/smol-rs/blocking,Apache-2.0 OR MIT,Stjepan Glavina bloomy,https://docs.rs/bloomy/,MIT,"Aleksandr Bezobchuk , Alexis Sellier " bollard,https://github.com/fussybeaver/bollard,Apache-2.0,Bollard contributors +borrow-or-share,https://github.com/yescallop/borrow-or-share,MIT-0,Scallop Ye brotli,https://github.com/dropbox/rust-brotli,BSD-3-Clause AND MIT,"Daniel Reiter Horn , The Brotli Authors" brotli-decompressor,https://github.com/dropbox/rust-brotli-decompressor,BSD-3-Clause OR MIT,"Daniel Reiter Horn , The Brotli Authors" bson,https://github.com/mongodb/bson-rust,MIT,"Y. T. Chung , Kevin Yeh , Saghm Rossi , Patrick Freed , Isabel Atkinson , Abraham Egnor " bstr,https://github.com/BurntSushi/bstr,MIT OR Apache-2.0,Andrew Gallant bumpalo,https://github.com/fitzgen/bumpalo,MIT OR Apache-2.0,Nick Fitzgerald bytecheck,https://github.com/djkoloski/bytecheck,MIT,David Koloski +bytecount,https://github.com/llogiq/bytecount,Apache-2.0 OR MIT,"Andre Bogus , Joshua Landau " bytemuck,https://github.com/Lokathor/bytemuck,Zlib OR Apache-2.0 OR MIT,Lokathor byteorder,https://github.com/BurntSushi/byteorder,Unlicense OR MIT,Andrew Gallant bytes,https://github.com/carllerche/bytes,MIT,Carl Lerche @@ -123,7 +123,7 @@ castaway,https://github.com/sagebind/castaway,MIT,Stephen M. Coakley cfb-mode,https://github.com/RustCrypto/block-modes,MIT OR Apache-2.0,RustCrypto Developers -cfg-if,https://github.com/alexcrichton/cfg-if,MIT OR Apache-2.0,Alex Crichton +cfg-if,https://github.com/rust-lang/cfg-if,MIT OR Apache-2.0,Alex Crichton chacha20,https://github.com/RustCrypto/stream-ciphers,Apache-2.0 OR MIT,RustCrypto Developers chacha20poly1305,https://github.com/RustCrypto/AEADs/tree/master/chacha20poly1305,Apache-2.0 OR MIT,RustCrypto Developers charset,https://github.com/hsivonen/charset,MIT OR Apache-2.0,Henri Sivonen @@ -146,10 +146,14 @@ community-id,https://github.com/traceflight/rs-community-id,MIT OR Apache-2.0,Ju compact_str,https://github.com/ParkMyCar/compact_str,MIT,Parker Timmerman concurrent-queue,https://github.com/smol-rs/concurrent-queue,Apache-2.0 OR MIT,"Stjepan Glavina , Taiki Endo , John Nunley " const-oid,https://github.com/RustCrypto/formats/tree/master/const-oid,Apache-2.0 OR MIT,RustCrypto Developers +const_fn,https://github.com/taiki-e/const_fn,Apache-2.0 OR MIT,The const_fn Authors convert_case,https://github.com/rutrum/convert-case,MIT,David Purdum convert_case,https://github.com/rutrum/convert-case,MIT,rutrum +cookie,https://github.com/SergioBenitez/cookie-rs,MIT OR Apache-2.0,"Sergio Benitez , Alex Crichton " cookie-factory,https://github.com/rust-bakery/cookie-factory,MIT,"Geoffroy Couprie , Pierre Chifflier " +cookie_store,https://github.com/pfernie/cookie_store,MIT OR Apache-2.0,Patrick Fernie core-foundation,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers +core-foundation,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers core2,https://github.com/bbqsrc/core2,Apache-2.0 OR MIT,Brendan Molloy cpufeatures,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers crc,https://github.com/mrhooray/crc-rs,MIT OR Apache-2.0,"Rui Hu , Akhil Velagapudi <4@4khil.com>" @@ -180,6 +184,7 @@ data-encoding,https://github.com/ia0/data-encoding,MIT,Julien Cretin databend-client,https://github.com/databendlabs/bendsql,Apache-2.0,Databend Authors dbl,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers +deadpool,https://github.com/bikeshedder/deadpool,MIT OR Apache-2.0,Michael P. Jung der,https://github.com/RustCrypto/formats/tree/master/der,Apache-2.0 OR MIT,RustCrypto Developers deranged,https://github.com/jhpratt/deranged,MIT OR Apache-2.0,Jacob Pratt derivative,https://github.com/mcarton/rust-derivative,MIT OR Apache-2.0,mcarton @@ -203,6 +208,7 @@ ed25519,https://github.com/RustCrypto/signatures/tree/master/ed25519,Apache-2.0 ed25519-dalek,https://github.com/dalek-cryptography/ed25519-dalek,BSD-3-Clause,"isis lovecruft , Tony Arcieri , Michael Rosenberg " either,https://github.com/bluss/either,MIT OR Apache-2.0,bluss elliptic-curve,https://github.com/RustCrypto/traits/tree/master/elliptic-curve,Apache-2.0 OR MIT,RustCrypto Developers +email_address,https://github.com/johnstonskj/rust-email_address,MIT,Simon Johnston encode_unicode,https://github.com/tormol/encode_unicode,Apache-2.0 OR MIT,Torbjørn Birch Moltu encoding_rs,https://github.com/hsivonen/encoding_rs,(Apache-2.0 OR MIT) AND BSD-3-Clause,Henri Sivonen endian-type,https://github.com/Lolirofle/endian-type,MIT,Lolirofle @@ -225,10 +231,10 @@ exitcode,https://github.com/benwilber/exitcode,Apache-2.0,Ben Wilber fallible-iterator,https://github.com/sfackler/rust-fallible-iterator,MIT OR Apache-2.0,Steven Fackler fancy-regex,https://github.com/fancy-regex/fancy-regex,MIT,"Raph Levien , Robin Stocker " +fancy-regex,https://github.com/fancy-regex/fancy-regex,MIT,"Raph Levien , Robin Stocker , Keith Hall " fastrand,https://github.com/smol-rs/fastrand,Apache-2.0 OR MIT,Stjepan Glavina ff,https://github.com/zkcrypto/ff,MIT OR Apache-2.0,"Sean Bowe , Jack Grigg " fiat-crypto,https://github.com/mit-plv/fiat-crypto,MIT OR Apache-2.0 OR BSD-1-Clause,Fiat Crypto library authors -filetime,https://github.com/alexcrichton/filetime,MIT OR Apache-2.0,Alex Crichton finl_unicode,https://github.com/dahosek/finl_unicode,MIT OR Apache-2.0,The finl_unicode Authors flate2,https://github.com/rust-lang/flate2-rs,MIT OR Apache-2.0,"Alex Crichton , Josh Triplett " float_eq,https://github.com/jtempest/float_eq-rs,MIT OR Apache-2.0,jtempest @@ -237,6 +243,7 @@ flume,https://github.com/zesterer/flume,Apache-2.0 OR MIT,Joshua Barretto foldhash,https://github.com/orlp/foldhash,Zlib,Orson Peters foreign-types,https://github.com/sfackler/foreign-types,MIT OR Apache-2.0,Steven Fackler +fraction,https://github.com/dnsl48/fraction,MIT OR Apache-2.0,dnsl48 fsevent-sys,https://github.com/octplane/fsevent-rust/tree/master/fsevent-sys,MIT,Pierre Baillet fslock,https://github.com/brunoczim/fslock,MIT,The fslock Authors funty,https://github.com/myrrlyn/funty,MIT,myrrlyn @@ -267,11 +274,11 @@ graphql_client_codegen,https://github.com/graphql-rust/graphql-client,Apache-2.0 graphql_query_derive,https://github.com/graphql-rust/graphql-client,Apache-2.0 OR MIT,Tom Houlé greptime-proto,https://github.com/GreptimeTeam/greptime-proto,Apache-2.0,The greptime-proto Authors greptimedb-ingester,https://github.com/GreptimeTeam/greptimedb-ingester-rust,Apache-2.0,The greptimedb-ingester Authors -grok,https://github.com/daschl/grok,Apache-2.0,Michael Nitschinger +grok,https://github.com/mmastrac/grok,Apache-2.0,"Matt Mastracci , Michael Nitschinger " group,https://github.com/zkcrypto/group,MIT OR Apache-2.0,"Sean Bowe , Jack Grigg " h2,https://github.com/hyperium/h2,MIT,"Carl Lerche , Sean McArthur " half,https://github.com/starkat99/half-rs,MIT OR Apache-2.0,Kathryn Long -hash_hasher,https://github.com/Fraser999/Hash-Hasher,Apache-2.0 OR MIT,Fraser Hutchison +hash_hasher,https://github.com/Fraser999/Hash-Hasher,Apache-2.0 OR MIT,Fraser Hutchison hashbag,https://github.com/jonhoo/hashbag,MIT OR Apache-2.0,Jon Gjengset hashbrown,https://github.com/rust-lang/hashbrown,MIT OR Apache-2.0,Amanieu d'Antras hashlink,https://github.com/kyren/hashlink,MIT OR Apache-2.0,kyren @@ -293,6 +300,7 @@ http-serde,https://gitlab.com/kornelski/http-serde,Apache-2.0 OR MIT,Kornel httparse,https://github.com/seanmonstar/httparse,MIT OR Apache-2.0,Sean McArthur httpdate,https://github.com/pyfisch/httpdate,MIT OR Apache-2.0,Pyfisch +humantime,https://github.com/chronotope/humantime,MIT OR Apache-2.0,The humantime Authors hyper,https://github.com/hyperium/hyper,MIT,Sean McArthur hyper-named-pipe,https://github.com/fussybeaver/hyper-named-pipe,Apache-2.0,The hyper-named-pipe Authors hyper-openssl,https://github.com/sfackler/hyper-openssl,MIT OR Apache-2.0,Steven Fackler @@ -301,7 +309,7 @@ hyper-rustls,https://github.com/rustls/hyper-rustls,Apache-2.0 OR ISC OR MIT,The hyper-timeout,https://github.com/hjr3/hyper-timeout,MIT OR Apache-2.0,Herman J. Radtke III hyper-tls,https://github.com/hyperium/hyper-tls,MIT OR Apache-2.0,Sean McArthur hyper-util,https://github.com/hyperium/hyper-util,MIT,Sean McArthur -hyperlocal-next,https://github.com/softprops/hyperlocal,MIT,softprops +hyperlocal,https://github.com/softprops/hyperlocal,MIT,softprops iana-time-zone,https://github.com/strawlab/iana-time-zone,MIT OR Apache-2.0,"Andrew Straw , René Kijewski , Ryan Lopopolo " iana-time-zone-haiku,https://github.com/strawlab/iana-time-zone,MIT OR Apache-2.0,René Kijewski icu_collections,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers @@ -328,6 +336,7 @@ instability,https://github.com/ratatui-org/instability,MIT,"Stephen M. Coakley < instant,https://github.com/sebcrozet/instant,BSD-3-Clause,sebcrozet inventory,https://github.com/dtolnay/inventory,MIT OR Apache-2.0,David Tolnay io-lifetimes,https://github.com/sunfishcode/io-lifetimes,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Dan Gohman +io-uring,https://github.com/tokio-rs/io-uring,MIT OR Apache-2.0,quininer iovec,https://github.com/carllerche/iovec,MIT OR Apache-2.0,Carl Lerche ipconfig,https://github.com/liranringel/ipconfig,MIT OR Apache-2.0,Liran Ringel ipnet,https://github.com/krisprice/ipnet,MIT OR Apache-2.0,Kris Price @@ -342,6 +351,7 @@ js-sys,https://github.com/rustwasm/wasm-bindgen/tree/master/crates/js-sys,MIT OR json-patch,https://github.com/idubrov/json-patch,MIT OR Apache-2.0,Ivan Dubrov jsonpath-rust,https://github.com/besok/jsonpath-rust,MIT,BorisZhguchev jsonptr,https://github.com/chanced/jsonptr,MIT OR Apache-2.0,chance dinkins +jsonschema,https://github.com/Stranger6667/jsonschema,MIT,Dmitry Dygalo k8s-openapi,https://github.com/Arnavion/k8s-openapi,Apache-2.0,Arnav Singh keccak,https://github.com/RustCrypto/sponges/tree/master/keccak,Apache-2.0 OR MIT,RustCrypto Developers kqueue,https://gitlab.com/rust-kqueue/rust-kqueue,MIT,William Orr @@ -389,8 +399,8 @@ minimal-lexical,https://github.com/Alexhuszagh/minimal-lexical,MIT OR Apache-2.0 miniz_oxide,https://github.com/Frommi/miniz_oxide/tree/master/miniz_oxide,MIT OR Zlib OR Apache-2.0,"Frommi , oyvindln " miniz_oxide,https://github.com/Frommi/miniz_oxide/tree/master/miniz_oxide,MIT OR Zlib OR Apache-2.0,"Frommi , oyvindln , Rich Geldreich richgel99@gmail.com" mio,https://github.com/tokio-rs/mio,MIT,"Carl Lerche , Thomas de Zeeuw , Tokio Contributors " -mlua,https://github.com/khvzak/mlua,MIT,"Aleksandr Orlenko , kyren " -mlua-sys,https://github.com/khvzak/mlua,MIT,Aleksandr Orlenko +mlua,https://github.com/mlua-rs/mlua,MIT,"Aleksandr Orlenko , kyren " +mlua-sys,https://github.com/mlua-rs/mlua,MIT,Aleksandr Orlenko mlua_derive,https://github.com/khvzak/mlua,MIT,Aleksandr Orlenko moka,https://github.com/moka-rs/moka,MIT OR Apache-2.0,The moka Authors mongodb,https://github.com/mongodb/mongo-rust-driver,Apache-2.0,"Saghm Rossi , Patrick Freed , Isabel Atkinson , Abraham Egnor , Kaitlin Mahar " @@ -406,34 +416,39 @@ nibble_vec,https://github.com/michaelsproul/rust_nibble_vec,MIT,Michael Sproul < nix,https://github.com/nix-rust/nix,MIT,The nix-rust Project Developers nkeys,https://github.com/wasmcloud/nkeys,Apache-2.0,wasmCloud Team no-proxy,https://github.com/jdrouet/no-proxy,MIT,Jérémie Drouet -no-std-compat,https://gitlab.com/jD91mZM2/no-std-compat,MIT,jD91mZM2 nohash,https://github.com/tetcoin/nohash,Apache-2.0 OR MIT,Parity Technologies nom,https://github.com/Geal/nom,MIT,contact@geoffroycouprie.com +nom,https://github.com/rust-bakery/nom,MIT,contact@geoffroycouprie.com nonzero_ext,https://github.com/antifuchs/nonzero_ext,Apache-2.0,Andreas Fuchs notify,https://github.com/notify-rs/notify,CC0-1.0,"Félix Saparelli , Daniel Faust , Aron Heinecke " notify-types,https://github.com/notify-rs/notify,MIT OR Apache-2.0,Daniel Faust ntapi,https://github.com/MSxDOS/ntapi,Apache-2.0 OR MIT,MSxDOS nu-ansi-term,https://github.com/nushell/nu-ansi-term,MIT,"ogham@bsago.me, Ryan Scheel (Havvy) , Josh Triplett , The Nushell Project Developers" nuid,https://github.com/casualjim/rs-nuid,Apache-2.0,Ivan Porto Carrero +num,https://github.com/rust-num/num,MIT OR Apache-2.0,The Rust Project Developers num-bigint,https://github.com/rust-num/num-bigint,MIT OR Apache-2.0,The Rust Project Developers num-bigint-dig,https://github.com/dignifiedquire/num-bigint,MIT OR Apache-2.0,"dignifiedquire , The Rust Project Developers" +num-cmp,https://github.com/lifthrasiir/num-cmp,MIT OR Apache-2.0,Kang Seonghoon +num-complex,https://github.com/rust-num/num-complex,MIT OR Apache-2.0,The Rust Project Developers num-conv,https://github.com/jhpratt/num-conv,MIT OR Apache-2.0,Jacob Pratt num-format,https://github.com/bcmyers/num-format,MIT OR Apache-2.0,Brian Myers num-integer,https://github.com/rust-num/num-integer,MIT OR Apache-2.0,The Rust Project Developers num-iter,https://github.com/rust-num/num-iter,MIT OR Apache-2.0,The Rust Project Developers num-rational,https://github.com/rust-num/num-rational,MIT OR Apache-2.0,The Rust Project Developers num-traits,https://github.com/rust-num/num-traits,MIT OR Apache-2.0,The Rust Project Developers +num_cpus,https://github.com/seanmonstar/num_cpus,MIT OR Apache-2.0,Sean McArthur num_enum,https://github.com/illicitonion/num_enum,BSD-3-Clause OR MIT OR Apache-2.0,"Daniel Wagner-Hall , Daniel Henry-Mantilla , Vincent Esche " num_threads,https://github.com/jhpratt/num_threads,MIT OR Apache-2.0,Jacob Pratt number_prefix,https://github.com/ogham/rust-number-prefix,MIT,Benjamin Sago oauth2,https://github.com/ramosbugs/oauth2-rs,MIT OR Apache-2.0,"Alex Crichton , Florin Lipan , David A. Ramos " objc,http://github.com/SSheldon/rust-objc,MIT,Steven Sheldon objc2-core-foundation,https://github.com/madsmtm/objc2,Zlib OR Apache-2.0 OR MIT,The objc2-core-foundation Authors +objc2-io-kit,https://github.com/madsmtm/objc2,Zlib OR Apache-2.0 OR MIT,The objc2-io-kit Authors object,https://github.com/gimli-rs/object,Apache-2.0 OR MIT,The object Authors octseq,https://github.com/NLnetLabs/octets/,BSD-3-Clause,NLnet Labs ofb,https://github.com/RustCrypto/block-modes,MIT OR Apache-2.0,RustCrypto Developers once_cell,https://github.com/matklad/once_cell,MIT OR Apache-2.0,Aleksey Kladov -onig,http://github.com/iwillspeak/rust-onig,MIT,"Will Speak , Ivan Ivashchenko " +onig,https://github.com/iwillspeak/rust-onig,MIT,"Will Speak , Ivan Ivashchenko " opaque-debug,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers opendal,https://github.com/apache/opendal,Apache-2.0,Apache OpenDAL openidconnect,https://github.com/ramosbugs/openidconnect-rs,MIT,David A. Ramos @@ -486,9 +501,7 @@ proc-macro-error2,https://github.com/GnomedDev/proc-macro-error-2,MIT OR Apache- proc-macro-hack,https://github.com/dtolnay/proc-macro-hack,MIT OR Apache-2.0,David Tolnay proc-macro2,https://github.com/dtolnay/proc-macro2,MIT OR Apache-2.0,"David Tolnay , Alex Crichton " proptest,https://github.com/proptest-rs/proptest,MIT OR Apache-2.0,Jason Lingle -prost,https://github.com/tokio-rs/prost,Apache-2.0,"Dan Burkert , Lucio Franco " prost,https://github.com/tokio-rs/prost,Apache-2.0,"Dan Burkert , Lucio Franco , Casper Meijn , Tokio Contributors " -prost-derive,https://github.com/tokio-rs/prost,Apache-2.0,"Dan Burkert , Lucio Franco , Tokio Contributors " prost-reflect,https://github.com/andrewhickman/prost-reflect,MIT OR Apache-2.0,Andrew Hickman psl,https://github.com/addr-rs/psl,MIT OR Apache-2.0,rushmorem psl-types,https://github.com/addr-rs/psl-types,MIT OR Apache-2.0,rushmorem @@ -520,6 +533,7 @@ rdkafka,https://github.com/fede1024/rust-rdkafka,MIT,Federico Giraud redox_users,https://gitlab.redox-os.org/redox-os/users,MIT,"Jose Narvaez , Wesley Hershberger " +ref-cast,https://github.com/dtolnay/ref-cast,MIT OR Apache-2.0,David Tolnay regex,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " regex-automata,https://github.com/BurntSushi/regex-automata,Unlicense OR MIT,Andrew Gallant regex-automata,https://github.com/rust-lang/regex/tree/master/regex-automata,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " @@ -531,7 +545,7 @@ rend,https://github.com/djkoloski/rend,MIT,David Koloski reqwest,https://github.com/seanmonstar/reqwest,MIT OR Apache-2.0,Sean McArthur resolv-conf,http://github.com/tailhook/resolv-conf,MIT OR Apache-2.0,paul@colomiets.name rfc6979,https://github.com/RustCrypto/signatures/tree/master/rfc6979,Apache-2.0 OR MIT,RustCrypto Developers -ring,https://github.com/ctz/ring,Apache-2.0 AND ISC,The ring Authors +ring,https://github.com/briansmith/ring,Apache-2.0 AND ISC,The ring Authors rkyv,https://github.com/rkyv/rkyv,MIT,David Koloski rle-decode-fast,https://github.com/WanzenBug/rle-decode-helper,MIT OR Apache-2.0,Moritz Wanzenböck rmp,https://github.com/3Hren/msgpack-rust,MIT,Evgeny Safronov @@ -562,6 +576,7 @@ same-file,https://github.com/BurntSushi/same-file,Unlicense OR MIT,Andrew Gallan sasl2-sys,https://github.com/MaterializeInc/rust-sasl,Apache-2.0,"Materialize, Inc." scan_fmt,https://github.com/wlentz/scan_fmt,MIT,wlentz schannel,https://github.com/steffengy/schannel-rs,MIT,"Steven Fackler , Steffen Butzer " +schemars,https://github.com/GREsau/schemars,MIT,Graham Esau scoped-tls,https://github.com/alexcrichton/scoped-tls,MIT OR Apache-2.0,Alex Crichton scopeguard,https://github.com/bluss/scopeguard,MIT OR Apache-2.0,bluss sct,https://github.com/rustls/sct.rs,Apache-2.0 OR ISC OR MIT,Joseph Birr-Pixton @@ -655,9 +670,14 @@ tokio-postgres,https://github.com/sfackler/rust-postgres,MIT OR Apache-2.0,Steve tokio-retry,https://github.com/srijs/rust-tokio-retry,MIT,Sam Rijs tokio-rustls,https://github.com/rustls/tokio-rustls,MIT OR Apache-2.0,The tokio-rustls Authors tokio-tungstenite,https://github.com/snapview/tokio-tungstenite,MIT,"Daniel Abramov , Alexey Galakhov " -toml,https://github.com/toml-rs/toml,MIT OR Apache-2.0,Alex Crichton +tokio-websockets,https://github.com/Gelbpunkt/tokio-websockets,MIT,The tokio-websockets Authors +toml,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml Authors +toml_datetime,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_datetime Authors toml_edit,https://github.com/toml-rs/toml,MIT OR Apache-2.0,"Andronik Ordian , Ed Page " +toml_edit,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_edit Authors +toml_parser,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_parser Authors toml_write,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_write Authors +toml_writer,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_writer Authors tonic,https://github.com/hyperium/tonic,MIT,Lucio Franco tower,https://github.com/tower-rs/tower,MIT,Tower Maintainers tower-http,https://github.com/tower-rs/tower-http,MIT,Tower Maintainers @@ -672,11 +692,13 @@ triomphe,https://github.com/Manishearth/triomphe,MIT OR Apache-2.0,"Manish Goreg trust-dns-proto,https://github.com/bluejekyll/trust-dns,MIT OR Apache-2.0,Benjamin Fry trust-dns-resolver,https://github.com/bluejekyll/trust-dns,MIT OR Apache-2.0,Benjamin Fry try-lock,https://github.com/seanmonstar/try-lock,MIT,Sean McArthur +tryhard,https://github.com/EmbarkStudios/tryhard,MIT OR Apache-2.0,Embark tungstenite,https://github.com/snapview/tungstenite-rs,MIT OR Apache-2.0,"Alexey Galakhov, Daniel Abramov" twox-hash,https://github.com/shepmaster/twox-hash,MIT,Jake Goulding typed-builder,https://github.com/idanarye/rust-typed-builder,MIT OR Apache-2.0,"IdanArye , Chris Morgan " typenum,https://github.com/paholg/typenum,MIT OR Apache-2.0,"Paho Lurie-Gregg , Andre Bogus " typetag,https://github.com/dtolnay/typetag,MIT OR Apache-2.0,David Tolnay +tz-rs,https://github.com/x-hgg-x/tz-rs,MIT OR Apache-2.0,x-hgg-x ua-parser,https://github.com/ua-parser/uap-rust,Apache-2.0,The ua-parser Authors ucd-trie,https://github.com/BurntSushi/ucd-generate,MIT OR Apache-2.0,Andrew Gallant unarray,https://github.com/cameron1024/unarray,MIT OR Apache-2.0,The unarray Authors @@ -699,6 +721,7 @@ utf16_iter,https://github.com/hsivonen/utf16_iter,Apache-2.0 OR MIT,Henri Sivone utf8-width,https://github.com/magiclen/utf8-width,MIT,Magic Len utf8_iter,https://github.com/hsivonen/utf8_iter,Apache-2.0 OR MIT,Henri Sivonen uuid,https://github.com/uuid-rs/uuid,Apache-2.0 OR MIT,"Ashley Mannix, Dylan DPC, Hunar Roop Kahlon" +uuid-simd,https://github.com/Nugine/simd,MIT,The uuid-simd Authors valuable,https://github.com/tokio-rs/valuable,MIT,The valuable Authors void,https://github.com/reem/rust-void,MIT,Jonathan Reem vrl,https://github.com/vectordotdev/vrl,MPL-2.0,Vector Contributors @@ -729,6 +752,9 @@ widestring,https://github.com/starkat99/widestring-rs,MIT OR Apache-2.0,The wide winapi,https://github.com/retep998/winapi-rs,MIT OR Apache-2.0,Peter Atashian winapi-util,https://github.com/BurntSushi/winapi-util,Unlicense OR MIT,Andrew Gallant windows,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows-collections,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-collections Authors +windows-future,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-future Authors +windows-numerics,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-numerics Authors windows-service,https://github.com/mullvad/windows-service-rs,MIT OR Apache-2.0,Mullvad VPN winnow,https://github.com/winnow-rs/winnow,MIT,The winnow Authors winreg,https://github.com/gentoo90/winreg-rs,MIT,Igor Shaula diff --git a/Makefile b/Makefile index afe737044b878..a77dcda4ad750 100644 --- a/Makefile +++ b/Makefile @@ -119,6 +119,9 @@ define MAYBE_ENVIRONMENT_COPY_ARTIFACTS endef endif +# docker container id file needs to live in the host machine and is later mounted into the container +CIDFILE := $(shell mktemp -u /tmp/vector-environment-docker-cid.XXXXXX) + # We use a volume here as non-Linux hosts are extremely slow to share disks, and Linux hosts tend to get permissions clobbered. define ENVIRONMENT_EXEC ${ENVIRONMENT_PREPARE} @@ -134,11 +137,13 @@ define ENVIRONMENT_EXEC $(if $(ENVIRONMENT_NETWORK),--network $(ENVIRONMENT_NETWORK),) \ --mount type=bind,source=${CURRENT_DIR},target=/git/vectordotdev/vector \ $(if $(findstring docker,$(CONTAINER_TOOL)),--mount type=bind$(COMMA)source=/var/run/docker.sock$(COMMA)target=/var/run/docker.sock,) \ + $(if $(findstring docker,$(CONTAINER_TOOL)),--cidfile $(CIDFILE),) \ + $(if $(findstring docker,$(CONTAINER_TOOL)),--mount type=bind$(COMMA)source=$(CIDFILE)$(COMMA)target=/.docker-container-id,) \ --mount type=volume,source=vector-target,target=/git/vectordotdev/vector/target \ --mount type=volume,source=vector-cargo-cache,target=/root/.cargo \ --mount type=volume,source=vector-rustup-cache,target=/root/.rustup \ $(foreach publish,$(ENVIRONMENT_PUBLISH),--publish $(publish)) \ - $(ENVIRONMENT_UPSTREAM) + $(ENVIRONMENT_UPSTREAM); rm -f $(CIDFILE) endef @@ -369,7 +374,7 @@ test-integration: test-integration-databend test-integration-docker-logs test-in test-integration: test-integration-eventstoredb test-integration-fluent test-integration-gcp test-integration-greptimedb test-integration-humio test-integration-http-client test-integration-influxdb test-integration: test-integration-kafka test-integration-logstash test-integration-loki test-integration-mongodb test-integration-nats test-integration: test-integration-nginx test-integration-opentelemetry test-integration-postgres test-integration-prometheus test-integration-pulsar -test-integration: test-integration-redis test-integration-splunk test-integration-dnstap test-integration-datadog-agent test-integration-datadog-logs test-integration-e2e-datadog-logs +test-integration: test-integration-redis test-integration-splunk test-integration-dnstap test-integration-datadog-agent test-integration-datadog-logs test-integration-e2e-datadog-logs test-integration-e2e-opentelemetry-logs test-integration: test-integration-datadog-traces test-integration-shutdown test-integration-%-cleanup: diff --git a/README.md b/README.md index 117c284a72d2a..bf4db29273e00 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,11 @@ +[![Nightly](https://github.com/vectordotdev/vector/actions/workflows/nightly.yml/badge.svg)](https://github.com/vectordotdev/vector/actions/workflows/nightly.yml) +[![E2E Test Suite](https://github.com/vectordotdev/vector/actions/workflows/e2e.yml/badge.svg)](https://github.com/vectordotdev/vector/actions/workflows/e2e.yml) +[![Component Features](https://github.com/vectordotdev/vector/actions/workflows/component_features.yml/badge.svg)](https://github.com/vectordotdev/vector/actions/workflows/component_features.yml) + +

+ Vector +

+

Quickstart  •   @@ -9,9 +17,6 @@ Rust Crate Docs

-

- Vector -

## What is Vector? @@ -27,6 +32,8 @@ faster than every alternative in the space. To get started, follow our [**quickstart guide**][docs.quickstart] or [**install Vector**][docs.installation]. +Vector is maintained by Datadog's [Community Open Source Engineering team](https://opensource.datadoghq.com/about/#the-community-open-source-engineering-team). + ### Principles * **Reliable** - Built in [Rust][urls.rust], Vector's primary design goal is reliability. @@ -48,50 +55,23 @@ Vector**][docs.installation]. **Tuple**, **Douban**, **Visa**, **Mambu**, **Blockfi**, **Claranet**, **Instacart**, **Forcepoint**, and [many more][urls.production_users]. * Vector is **downloaded over 100,000 times per day**. -* Vector's largest user **processes over 30TB daily**. -* Vector has **over 100 contributors** and growing. +* Vector's largest user **processes over 500TB daily**. +* Vector has **over 500 contributors** and growing. -## [Documentation](https://vector.dev/docs/) +## Documentation -### About +All user documentation is available at **[vector.dev/docs](https://vector.dev/docs)**. -* [**Concepts**][docs.about.concepts] -* [**Under the hood**][docs.about.under-the-hood] - * [**Architecture**][docs.under-the-hood.architecture] - [data model][docs.architecture.data-model] ([log][docs.data-model.log], [metric][docs.data-model.metric]), [pipeline model][docs.architecture.pipeline-model], [concurrency model][docs.architecture.concurrency-model], [runtime model][docs.architecture.runtime-model] - * [**Networking**][docs.under-the-hood.networking] - [ARC][docs.networking.adaptive-request-concurrency] - * [**Guarantees**][docs.under-the-hood.guarantees] +Other Resources: -### Setup - -* [**Quickstart**][docs.setup.quickstart] -* [**Installation**][docs.setup.installation] - [operating systems][docs.installation.operating_systems], [package managers][docs.installation.package_managers], [platforms][docs.installation.platforms] ([Kubernetes][docs.platforms.kubernetes]), [manual][docs.installation.manual] -* [**Deployment**][docs.deployment] - [roles][docs.deployment.roles], [topologies][docs.deployment.topologies] - -### Reference - -* **Configuration** - * [**Sources**][docs.configuration.sources] - [docker_logs][docs.sources.docker_logs], [file][docs.sources.file], [http][docs.sources.http], [journald][docs.sources.journald], [kafka][docs.sources.kafka], [socket][docs.sources.socket], and [many more...][docs.sources] - * [**Transforms**][docs.configuration.transforms] - [dedupe][docs.transforms.dedupe], [filter][docs.transforms.filter], [log_to_metric][docs.transforms.log_to_metric], [lua][docs.transforms.lua], [remap][docs.transforms.remap], - and [many more...][docs.transforms] - * [**Sinks**][docs.configuration.sinks] - [aws_cloudwatch_logs][docs.sinks.aws_cloudwatch_logs], [aws_s3][docs.sinks.aws_s3], [clickhouse][docs.sinks.clickhouse], [elasticsearch][docs.sinks.elasticsearch], [gcp_cloud_storage][docs.sinks.gcp_cloud_storage], and [many more...][docs.sinks] - * [**Enrichment Tables**][docs.configuration.enrichment_tables] - * [**Unit tests**][docs.configuration.tests] -* [**Remap Language**][docs.reference.vrl] -* [**API**][docs.reference.api] -* [**CLI**][docs.reference.cli] - -### Administration - -* [**Management**][docs.administration.management] -* [**Monitoring & observing**][docs.administration.monitoring] -* [**Upgrading**][docs.administration.upgrading] -* [**Validating**][docs.administration.validating] - -### Resources - -* [**Community**][urls.vector_community] - [chat][urls.vector_chat], [calendar][urls.vector_calendar], [@vectordotdev][urls.vector_twitter] -* [**Releases**][urls.vector_releases] -* **Policies** - [Code of Conduct][urls.vector_code_of_conduct], [Privacy][urls.vector_privacy_policy], [Releases][urls.vector_releases_policy], [Security][urls.vector_security_policy], [Versioning][urls.vector_versioning_policy] +* [**Vector Calendar**][urls.vector_calendar] +* **Policies**: + * [**Code of Conduct**][urls.vector_code_of_conduct] + * [**Contributing**][urls.vector_contributing_policy] + * [**Privacy**][urls.vector_privacy_policy] + * [**Releases**][urls.vector_releases_policy] + * [**Versioning**][urls.vector_versioning_policy] + * [**Security**][urls.vector_security_policy] ## Comparisons @@ -156,23 +136,23 @@ Vector is an end-to-end, unified, open data platform. Developed with ❤️ by Datadog - Security Policy - Privacy Policy

-[docs.about.concepts]: https://vector.dev/docs/about/concepts/ -[docs.about.under-the-hood]: https://vector.dev/docs/about/under-the-hood/ +[docs.about.concepts]: https://vector.dev/docs/introduction/concepts/ +[docs.about.introduction]: https://vector.dev/docs/introduction/ [docs.administration.monitoring]: https://vector.dev/docs/administration/monitoring/ [docs.administration.management]: https://vector.dev/docs/administration/management/ [docs.administration.upgrading]: https://vector.dev/docs/administration/upgrading/ [docs.administration.validating]: https://vector.dev/docs/administration/validating/ -[docs.architecture.concurrency-model]: https://vector.dev/docs/about/under-the-hood/architecture/concurrency-model/ -[docs.architecture.data-model]: https://vector.dev/docs/about/under-the-hood/architecture/data-model/ -[docs.architecture.pipeline-model]: https://vector.dev/docs/about/under-the-hood/architecture/pipeline-model/ -[docs.architecture.runtime-model]: https://vector.dev/docs/about/under-the-hood/architecture/runtime-model/ +[docs.architecture.concurrency-model]: https://vector.dev/docs/architecture/concurrency-model/ +[docs.architecture.data-model]: https://vector.dev/docs/architecture/data-model/ +[docs.architecture.pipeline-model]: https://vector.dev/docs/architecture/pipeline-model/ +[docs.architecture.runtime-model]: https://vector.dev/docs/architecture/runtime-model/ [docs.configuration.sinks]: https://vector.dev/docs/reference/configuration/sinks/ [docs.configuration.sources]: https://vector.dev/docs/reference/configuration/sources/ [docs.configuration.tests]: https://vector.dev/docs/reference/configuration/tests/ [docs.configuration.transforms]: https://vector.dev/docs/reference/configuration/transforms/ [docs.configuration.enrichment_tables]: https://vector.dev/docs/reference/configuration/global-options/#enrichment_tables -[docs.data-model.log]: https://vector.dev/docs/about/under-the-hood/architecture/data-model/log/ -[docs.data-model.metric]: https://vector.dev/docs/about/under-the-hood/architecture/data-model/metric/ +[docs.data-model.log]: https://vector.dev/docs/architecture/data-model/log/ +[docs.data-model.metric]: https://vector.dev/docs/architecture/data-model/metric/ [docs.deployment.roles]: https://vector.dev/docs/setup/deployment/roles/ [docs.deployment.topologies]: https://vector.dev/docs/setup/deployment/topologies/ [docs.deployment]: https://vector.dev/docs/setup/deployment/ @@ -181,7 +161,7 @@ Vector is an end-to-end, unified, open data platform. [docs.installation.package_managers]: https://vector.dev/docs/setup/installation/package-managers/ [docs.installation.platforms]: https://vector.dev/docs/setup/installation/platforms/ [docs.installation]: https://vector.dev/docs/setup/installation/ -[docs.networking.adaptive-request-concurrency]: https://vector.dev/docs/about/under-the-hood/networking/arc/ +[docs.architecture.adaptive-request-concurrency]: https://vector.dev/docs/architecture/arc/ [docs.platforms.kubernetes]: https://vector.dev/docs/setup/installation/platforms/kubernetes/ [docs.quickstart]: https://vector.dev/docs/setup/quickstart/ [docs.reference.api]: https://vector.dev/docs/reference/api/ @@ -210,14 +190,15 @@ Vector is an end-to-end, unified, open data platform. [docs.transforms.lua]: https://vector.dev/docs/reference/configuration/transforms/lua/ [docs.transforms.remap]: https://vector.dev/docs/reference/configuration/transforms/remap/ [docs.transforms]: https://vector.dev/docs/reference/configuration/transforms/ -[docs.under-the-hood.architecture]: https://vector.dev/docs/about/under-the-hood/architecture/ -[docs.under-the-hood.guarantees]: https://vector.dev/docs/about/under-the-hood/guarantees/ -[docs.under-the-hood.networking]: https://vector.dev/docs/about/under-the-hood/networking/ +[docs.introduction.architecture]: https://vector.dev/docs/architecture/ +[docs.introduction.guarantees]: https://vector.dev/docs/introduction/guarantees/ +[docs.introduction.architecture]: https://vector.dev/docs/architecture/ [urls.production_users]: https://github.com/vectordotdev/vector/issues/790 [urls.rust]: https://www.rust-lang.org/ [urls.vector_calendar]: https://calendar.vector.dev [urls.vector_chat]: https://chat.vector.dev [urls.vector_code_of_conduct]: https://github.com/vectordotdev/vector/blob/master/CODE_OF_CONDUCT.md +[urls.vector_contributing_policy]: https://github.com/vectordotdev/vector/blob/master/CONTRIBUTING.md [urls.vector_community]: https://vector.dev/community/ [urls.vector_privacy_policy]: https://github.com/vectordotdev/vector/blob/master/PRIVACY.md [urls.vector_release_policy]: https://github.com/vectordotdev/vector/blob/master/RELEASING.md @@ -228,4 +209,3 @@ Vector is an end-to-end, unified, open data platform. [urls.vector_twitter]: https://twitter.com/vectordotdev [urls.vector_versioning_policy]: https://github.com/vectordotdev/vector/blob/master/VERSIONING.md [urls.vote_feature]: https://github.com/vectordotdev/vector/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc+label%3A%22type%3A+feature%22 - diff --git a/VERSIONING.md b/VERSIONING.md index 33d37b8aa7277..a84b608f6ad80 100644 --- a/VERSIONING.md +++ b/VERSIONING.md @@ -122,7 +122,7 @@ here. Each minor release bump will include an upgrade guide in the [CLI]: https://vector.dev/docs/reference/cli/ [configuration schema]: https://vector.dev/docs/reference/configuration/ [data directory]: https://vector.dev/docs/reference/configuration/global-options/#data_dir -[data model]: https://vector.dev/docs/about/under-the-hood/architecture/data-model/ +[data model]: https://vector.dev/docs/architecture/data-model/ [GitHub repository]: https://github.com/vectordotdev/vector [GraphQL API]: https://vector.dev/docs/reference/api/ [Installation workflows]: https://vector.dev/docs/setup/installation/ diff --git a/aqua/aqua.yaml b/aqua/aqua.yaml index e72564f079e00..7a1d785e753bc 100644 --- a/aqua/aqua.yaml +++ b/aqua/aqua.yaml @@ -9,6 +9,6 @@ packages: - name: crates.io/cargo-deb@2.9.1 - name: cross-rs/cross@v0.2.5 - name: nextest-rs/nextest/cargo-nextest@cargo-nextest-0.9.47 - - name: EmbarkStudios/cargo-deny@0.16.1 + - name: EmbarkStudios/cargo-deny@0.16.2 - name: foresterre/cargo-msrv@v0.15.1 - name: crates.io/dd-rust-license-tool@1.0.1 diff --git a/benches/batch.rs b/benches/batch.rs index 02e9fc071aead..6734c8da8ae9c 100644 --- a/benches/batch.rs +++ b/benches/batch.rs @@ -38,7 +38,7 @@ fn benchmark_batch(c: &mut Criterion) { .collect(); for (compression, batch_size) in cases.iter() { - group.bench_function(format!("partitioned/{}_{}", compression, batch_size), |b| { + group.bench_function(format!("partitioned/{compression}_{batch_size}"), |b| { b.iter_batched( || { let rt = runtime(); @@ -68,35 +68,32 @@ fn benchmark_batch(c: &mut Criterion) { ) }); - group.bench_function( - format!("unpartitioned/{}_{}", compression, batch_size), - |b| { - b.iter_batched( - || { - let rt = runtime(); - let mut batch = BatchSettings::default(); - batch.size.bytes = *batch_size; - batch.size.events = num_events; - - let batch_sink = BatchSink::new( - tower::service_fn(|_| future::ok::<_, Infallible>(())), - Buffer::new(batch.size, *compression), - Duration::from_secs(1), - ) - .sink_map_err(|error| panic!("{}", error)); - - ( - rt, - stream::iter(input.clone()) - .map(|item| Ok(EncodedEvent::new(item, 0, JsonSize::zero()))), - batch_sink, - ) - }, - |(rt, input, batch_sink)| rt.block_on(input.forward(batch_sink)).unwrap(), - criterion::BatchSize::LargeInput, - ) - }, - ); + group.bench_function(format!("unpartitioned/{compression}_{batch_size}"), |b| { + b.iter_batched( + || { + let rt = runtime(); + let mut batch = BatchSettings::default(); + batch.size.bytes = *batch_size; + batch.size.events = num_events; + + let batch_sink = BatchSink::new( + tower::service_fn(|_| future::ok::<_, Infallible>(())), + Buffer::new(batch.size, *compression), + Duration::from_secs(1), + ) + .sink_map_err(|error| panic!("{}", error)); + + ( + rt, + stream::iter(input.clone()) + .map(|item| Ok(EncodedEvent::new(item, 0, JsonSize::zero()))), + batch_sink, + ) + }, + |(rt, input, batch_sink)| rt.block_on(input.forward(batch_sink)).unwrap(), + criterion::BatchSize::LargeInput, + ) + }); } } diff --git a/benches/codecs/encoder.rs b/benches/codecs/encoder.rs index 47f9d9ae25110..63091ec46ee5f 100644 --- a/benches/codecs/encoder.rs +++ b/benches/codecs/encoder.rs @@ -92,7 +92,7 @@ fn encoder(c: &mut Criterion) { b.iter_batched( || { vector::codecs::Encoder::::new( - NewlineDelimitedEncoder::new().into(), + NewlineDelimitedEncoder::default().into(), JsonSerializerConfig::default().build().into(), ) }, diff --git a/benches/codecs/newline_bytes.rs b/benches/codecs/newline_bytes.rs index 95d8962036a60..20e1d586b6850 100644 --- a/benches/codecs/newline_bytes.rs +++ b/benches/codecs/newline_bytes.rs @@ -50,8 +50,8 @@ fn decoding(c: &mut Criterion) { let framer = Framer::NewlineDelimited( param .max_length - .map(|ml| NewlineDelimitedDecoder::new_with_max_length(ml)) - .unwrap_or(NewlineDelimitedDecoder::new()), + .map(NewlineDelimitedDecoder::new_with_max_length) + .unwrap_or_default(), ); let deserializer = Deserializer::Bytes(BytesDeserializer); let decoder = vector::codecs::Decoder::new(framer, deserializer); diff --git a/benches/distribution_statistic.rs b/benches/distribution_statistic.rs index 088c653067510..891a9c34f9907 100644 --- a/benches/distribution_statistic.rs +++ b/benches/distribution_statistic.rs @@ -1,13 +1,12 @@ use criterion::{criterion_group, criterion_main, BatchSize, Criterion}; -use rand::{ - distr::{Distribution, Uniform}, - seq::SliceRandom, -}; +use rand::distr::Distribution; +use rand::{distr::Uniform, seq::SliceRandom}; use vector::{event::metric::Sample, sinks::util::statistic::DistributionStatistic}; fn generate_samples(mut size: u32, max_bin_count: u32) -> Vec { let mut rng = rand::rng(); - let range = Uniform::from(1..=max_bin_count); + // Fix Uniform usage for inclusive range + let range = Uniform::new_inclusive(1, max_bin_count).unwrap(); let mut value = 1.0; let mut samples = Vec::new(); while size > 0 { @@ -28,7 +27,7 @@ fn bench_statistic(c: &mut Criterion) { let sizes = [5, 10, 50, 100, 200, 500, 1000]; for &size in &sizes { - group.bench_function(format!("small-bin-{}", size), |b| { + group.bench_function(format!("small-bin-{size}"), |b| { b.iter_batched( move || generate_samples(size, 3), |samples| { @@ -41,7 +40,7 @@ fn bench_statistic(c: &mut Criterion) { let sizes = [50, 100, 200, 500, 1000]; for &size in &sizes { - group.bench_function(format!("large-bin-{}", size), |b| { + group.bench_function(format!("large-bin-{size}"), |b| { b.iter_batched( move || generate_samples(size, 20), |samples| { diff --git a/benches/dnstap/mod.rs b/benches/dnstap/mod.rs index 37839b41ace06..2110941f337ef 100644 --- a/benches/dnstap/mod.rs +++ b/benches/dnstap/mod.rs @@ -1,21 +1,31 @@ +use base64::engine::general_purpose::STANDARD; +use base64::Engine; use bytes::Bytes; use criterion::{criterion_group, criterion_main, BatchSize, Criterion, Throughput}; +use dnsmsg_parser::dns_message_parser::DnsParserOptions; +use dnstap_parser::parser::DnstapParser; use vector::event::LogEvent; -use vector::sources::dnstap::parser::DnstapParser; fn benchmark_query_parsing(c: &mut Criterion) { let mut event = LogEvent::default(); let raw_dnstap_data = "ChVqYW1lcy1WaXJ0dWFsLU1hY2hpbmUSC0JJTkQgOS4xNi4zcnoIAxACGAEiEAAAAAAAAA\ AAAAAAAAAAAAAqECABBQJwlAAAAAAAAAAAADAw8+0CODVA7+zq9wVNMU3WNlI2kwIAAAABAAAAAAABCWZhY2Vib29rMQNjb\ 20AAAEAAQAAKQIAAACAAAAMAAoACOxjCAG9zVgzWgUDY29tAHgB"; - let dnstap_data = base64::decode(raw_dnstap_data).unwrap(); + let dnstap_data = STANDARD.decode(raw_dnstap_data).unwrap(); let mut group = c.benchmark_group("dnstap"); group.throughput(Throughput::Bytes(dnstap_data.len() as u64)); group.bench_function("dns_query_parsing", |b| { b.iter_batched( || dnstap_data.clone(), - |dnstap_data| DnstapParser::parse(&mut event, Bytes::from(dnstap_data)).unwrap(), + |dnstap_data| { + DnstapParser::parse( + &mut event, + Bytes::from(dnstap_data), + DnsParserOptions::default(), + ) + .unwrap() + }, BatchSize::SmallInput, ) }); @@ -28,14 +38,21 @@ fn benchmark_update_parsing(c: &mut Criterion) { let raw_dnstap_data = "ChVqYW1lcy1WaXJ0dWFsLU1hY2hpbmUSC0JJTkQgOS4xNi4zcmsIDhABGAEiBH8AAA\ EqBH8AAAEwrG44AEC+iu73BU14gfofUh1wi6gAAAEAAAAAAAAHZXhhbXBsZQNjb20AAAYAAWC+iu73BW0agDwvch1wi6gAA\ AEAAAAAAAAHZXhhbXBsZQNjb20AAAYAAXgB"; - let dnstap_data = base64::decode(raw_dnstap_data).unwrap(); + let dnstap_data = STANDARD.decode(raw_dnstap_data).unwrap(); let mut group = c.benchmark_group("dnstap"); group.throughput(Throughput::Bytes(dnstap_data.len() as u64)); group.bench_function("dns_update_parsing", |b| { b.iter_batched( || dnstap_data.clone(), - |dnstap_data| DnstapParser::parse(&mut event, Bytes::from(dnstap_data)).unwrap(), + |dnstap_data| { + DnstapParser::parse( + &mut event, + Bytes::from(dnstap_data), + DnsParserOptions::default(), + ) + .unwrap() + }, BatchSize::SmallInput, ) }); diff --git a/benches/enrichment_tables.rs b/benches/enrichment_tables.rs index 5c9a11f157a62..b56f4f794627e 100644 --- a/benches/enrichment_tables.rs +++ b/benches/enrichment_tables.rs @@ -2,6 +2,7 @@ use std::time::SystemTime; use chrono::prelude::*; use criterion::{criterion_group, criterion_main, BatchSize, Criterion}; +use vector::enrichment_tables::file::FileData; use vector::enrichment_tables::{ file::File, geoip::{Geoip, GeoipConfig}, @@ -27,12 +28,11 @@ fn column(col: usize, row: usize) -> Value { // And a final column with a date, each of the above duplicated row should have // a unique date. Value::Timestamp( - Utc.ymd(2013, row as u32 % 10 + 1, 15) - .and_hms_opt(0, 0, 0) - .expect("invalid timestamp"), + Utc.with_ymd_and_hms(2013, row as u32 % 10 + 1, 15, 0, 0, 0) + .unwrap(), ) } else { - Value::from(format!("data-{}-{}", col, row)) + Value::from(format!("data-{col}-{row}")) } } @@ -49,12 +49,13 @@ fn benchmark_enrichment_tables_file(c: &mut Criterion) { let mut file = File::new( Default::default(), - SystemTime::now(), - data, - // Headers. - (0..10) - .map(|header| format!("field-{}", header)) - .collect::>(), + FileData { + data, + headers: (0..10) + .map(|header| format!("field-{header}")) + .collect::>(), + modified: SystemTime::now(), + }, ); let (condition, index, result_offset) = if date_range { @@ -67,14 +68,8 @@ fn benchmark_enrichment_tables_file(c: &mut Criterion) { }, Condition::BetweenDates { field: "field-1", - from: Utc - .ymd(2013, 6, 1) - .and_hms_opt(0, 0, 0) - .expect("invalid timestamp"), - to: Utc - .ymd(2013, 7, 1) - .and_hms_opt(0, 0, 0) - .expect("invalid timestamp"), + from: Utc.with_ymd_and_hms(2013, 6, 1, 0, 0, 0).unwrap(), + to: Utc.with_ymd_and_hms(2013, 7, 1, 0, 0, 0).unwrap(), }, ], file.add_index(case, &["field-0"]).unwrap(), @@ -100,7 +95,7 @@ fn benchmark_enrichment_tables_file(c: &mut Criterion) { let result = (0..10) .map(|idx| { ( - format!("field-{}", idx).into(), + format!("field-{idx}").into(), column(idx, size - result_offset), ) }) @@ -112,11 +107,11 @@ fn benchmark_enrichment_tables_file(c: &mut Criterion) { group.bench_function("enrichment_tables/file_date_10", |b| { let (file, index, condition, expected) = setup(10, true, Case::Sensitive); b.iter_batched( - || (&file, &condition, expected.clone()), - |(file, condition, expected)| { + || (&file, &condition, expected.clone(), Some(index)), + |(file, condition, expected, index)| { assert_eq!( Ok(expected), - file.find_table_row(Case::Sensitive, condition, None, Some(index)) + file.find_table_row(Case::Sensitive, condition, None, None, index) ) }, BatchSize::SmallInput, @@ -126,11 +121,11 @@ fn benchmark_enrichment_tables_file(c: &mut Criterion) { group.bench_function("enrichment_tables/file_hashindex_sensitive_10", |b| { let (file, index, condition, expected) = setup(10, false, Case::Sensitive); b.iter_batched( - || (&file, index, &condition, expected.clone()), - |(file, index, condition, expected)| { + || (&file, &condition, expected.clone(), Some(index)), + |(file, condition, expected, index)| { assert_eq!( Ok(expected), - file.find_table_row(Case::Sensitive, condition, None, Some(index)) + file.find_table_row(Case::Sensitive, condition, None, None, index) ) }, BatchSize::SmallInput, @@ -140,11 +135,11 @@ fn benchmark_enrichment_tables_file(c: &mut Criterion) { group.bench_function("enrichment_tables/file_hashindex_insensitive_10", |b| { let (file, index, condition, expected) = setup(10, false, Case::Insensitive); b.iter_batched( - || (&file, index, &condition, expected.clone()), - |(file, index, condition, expected)| { + || (&file, &condition, expected.clone(), Some(index)), + |(file, condition, expected, index)| { assert_eq!( Ok(expected), - file.find_table_row(Case::Insensitive, condition, None, Some(index)) + file.find_table_row(Case::Insensitive, condition, None, None, index) ) }, BatchSize::SmallInput, @@ -154,11 +149,11 @@ fn benchmark_enrichment_tables_file(c: &mut Criterion) { group.bench_function("enrichment_tables/file_date_1_000", |b| { let (file, index, condition, expected) = setup(1_000, true, Case::Sensitive); b.iter_batched( - || (&file, &condition, expected.clone()), - |(file, condition, expected)| { + || (&file, &condition, expected.clone(), Some(index)), + |(file, condition, expected, index)| { assert_eq!( Ok(expected), - file.find_table_row(Case::Sensitive, condition, None, Some(index)) + file.find_table_row(Case::Sensitive, condition, None, None, index) ) }, BatchSize::SmallInput, @@ -168,11 +163,11 @@ fn benchmark_enrichment_tables_file(c: &mut Criterion) { group.bench_function("enrichment_tables/file_hashindex_sensitive_1_000", |b| { let (file, index, condition, expected) = setup(1_000, false, Case::Sensitive); b.iter_batched( - || (&file, index, &condition, expected.clone()), - |(file, index, condition, expected)| { + || (&file, &condition, expected.clone(), Some(index)), + |(file, condition, expected, index)| { assert_eq!( Ok(expected), - file.find_table_row(Case::Sensitive, condition, None, Some(index)) + file.find_table_row(Case::Sensitive, condition, None, None, index) ) }, BatchSize::SmallInput, @@ -182,11 +177,11 @@ fn benchmark_enrichment_tables_file(c: &mut Criterion) { group.bench_function("enrichment_tables/file_hashindex_insensitive_1_000", |b| { let (file, index, condition, expected) = setup(1_000, false, Case::Insensitive); b.iter_batched( - || (&file, index, &condition, expected.clone()), - |(file, index, condition, expected)| { + || (&file, &condition, expected.clone(), Some(index)), + |(file, condition, expected, index)| { assert_eq!( Ok(expected), - file.find_table_row(Case::Insensitive, condition, None, Some(index)) + file.find_table_row(Case::Insensitive, condition, None, None, index) ) }, BatchSize::SmallInput, @@ -196,11 +191,11 @@ fn benchmark_enrichment_tables_file(c: &mut Criterion) { group.bench_function("enrichment_tables/file_date_1_000_000", |b| { let (file, index, condition, expected) = setup(1_000_000, true, Case::Sensitive); b.iter_batched( - || (&file, &condition, expected.clone()), - |(file, condition, expected)| { + || (&file, &condition, expected.clone(), Some(index)), + |(file, condition, expected, index)| { assert_eq!( Ok(expected), - file.find_table_row(Case::Sensitive, condition, None, Some(index)) + file.find_table_row(Case::Sensitive, condition, None, None, index) ) }, BatchSize::SmallInput, @@ -212,11 +207,11 @@ fn benchmark_enrichment_tables_file(c: &mut Criterion) { |b| { let (file, index, condition, expected) = setup(1_000_000, false, Case::Sensitive); b.iter_batched( - || (&file, index, &condition, expected.clone()), - |(file, index, condition, expected)| { + || (&file, &condition, expected.clone(), Some(index)), + |(file, condition, expected, index)| { assert_eq!( Ok(expected), - file.find_table_row(Case::Sensitive, condition, None, Some(index)) + file.find_table_row(Case::Sensitive, condition, None, None, index) ) }, BatchSize::SmallInput, @@ -229,11 +224,11 @@ fn benchmark_enrichment_tables_file(c: &mut Criterion) { |b| { let (file, index, condition, expected) = setup(1_000_000, false, Case::Insensitive); b.iter_batched( - || (&file, index, &condition, expected.clone()), - |(file, index, condition, expected)| { + || (&file, &condition, expected.clone(), Some(index)), + |(file, condition, expected, index)| { assert_eq!( Ok(expected), - file.find_table_row(Case::Insensitive, condition, None, Some(index)) + file.find_table_row(Case::Insensitive, condition, None, None, index) ) }, BatchSize::SmallInput, @@ -268,18 +263,17 @@ fn benchmark_enrichment_tables_geoip(c: &mut Criterion) { || (&table, ip, &expected), |(table, ip, expected)| { assert_eq!( - Ok(expected), - table - .find_table_row( - Case::Insensitive, - &[Condition::Equals { - field: "ip", - value: ip.into(), - }], - None, - None, - ) - .as_ref() + Ok(expected.clone()), + table.find_table_row( + Case::Insensitive, + &[Condition::Equals { + field: "ip", + value: ip.into(), + }], + None, + None, + None, + ) ) }, BatchSize::SmallInput, @@ -306,18 +300,17 @@ fn benchmark_enrichment_tables_geoip(c: &mut Criterion) { || (&table, ip, &expected), |(table, ip, expected)| { assert_eq!( - Ok(expected), - table - .find_table_row( - Case::Insensitive, - &[Condition::Equals { - field: "ip", - value: ip.into(), - }], - None, - None, - ) - .as_ref() + Ok(expected.clone()), + table.find_table_row( + Case::Insensitive, + &[Condition::Equals { + field: "ip", + value: ip.into(), + }], + None, + None, + None, + ) ) }, BatchSize::SmallInput, @@ -350,18 +343,17 @@ fn benchmark_enrichment_tables_mmdb(c: &mut Criterion) { || (&table, ip, &expected), |(table, ip, expected)| { assert_eq!( - Ok(expected), - table - .find_table_row( - Case::Insensitive, - &[Condition::Equals { - field: "ip", - value: ip.into(), - }], - None, - None, - ) - .as_ref() + Ok(expected.clone()), + table.find_table_row( + Case::Insensitive, + &[Condition::Equals { + field: "ip", + value: ip.into(), + }], + None, + None, + None, + ) ) }, BatchSize::SmallInput, @@ -385,21 +377,20 @@ fn benchmark_enrichment_tables_mmdb(c: &mut Criterion) { || (&table, ip, &expected), |(table, ip, expected)| { assert_eq!( - Ok(expected), - table - .find_table_row( - Case::Insensitive, - &[Condition::Equals { - field: "ip", - value: ip.into(), - }], - Some(&[ - "location.latitude".to_string(), - "location.longitude".to_string(), - ]), - None, - ) - .as_ref() + Ok(expected.clone()), + table.find_table_row( + Case::Insensitive, + &[Condition::Equals { + field: "ip", + value: ip.into(), + }], + Some(&[ + "location.latitude".to_string(), + "location.longitude".to_string(), + ]), + None, + None, + ) ) }, BatchSize::SmallInput, diff --git a/benches/http.rs b/benches/http.rs index b60b25a49556f..84d97eb1a6be1 100644 --- a/benches/http.rs +++ b/benches/http.rs @@ -8,9 +8,13 @@ use hyper::{ }; use tokio::runtime::Runtime; use vector::{ - config, sinks, - sinks::util::{BatchConfig, Compression}, + config, + sinks::{ + self, + util::{BatchConfig, Compression}, + }, sources, + template::Template, test_util::{next_addr, random_lines, runtime, send_lines, start_topology, wait_for_tcp}, Error, }; @@ -48,7 +52,7 @@ fn benchmark_http(c: &mut Criterion) { "out", &["in"], sinks::http::config::HttpSinkConfig { - uri: out_addr.to_string().parse::().unwrap().into(), + uri: Template::try_from(out_addr.to_string()).unwrap(), compression: *compression, method: Default::default(), auth: Default::default(), diff --git a/benches/languages.rs b/benches/languages.rs index 2c386f786fd4f..6e0b0417055ad 100644 --- a/benches/languages.rs +++ b/benches/languages.rs @@ -256,11 +256,9 @@ fn benchmark_configs( let in_addr = next_addr(); let out_addr = next_addr(); - let lines: Vec<_> = ::std::iter::repeat(input.to_string()) - .take(num_lines) - .collect(); + let lines: Vec<_> = std::iter::repeat_n(input.to_string(), num_lines).collect(); - let mut group = criterion.benchmark_group(format!("languages/{}", benchmark_name)); + let mut group = criterion.benchmark_group(format!("languages/{benchmark_name}")); group.sampling_mode(SamplingMode::Flat); let source_config = format!( @@ -286,15 +284,15 @@ fn benchmark_configs( for (name, transform_config) in configs.into_iter() { group.throughput(Throughput::Elements(num_lines as u64)); - group.bench_function(name.clone(), |b| { + group.bench_function(name, |b| { b.iter_batched( || { let mut config = source_config.clone(); - config.push_str(&transform_config); + config.push_str(transform_config); config.push_str(&sink_config); let config = config::load_from_str(&config, config::Format::Toml) - .expect(&format!("invalid TOML configuration: {}", &config)); + .unwrap_or_else(|_| panic!("invalid TOML configuration: {}", &config)); let rt = runtime(); let (output_lines, topology) = rt.block_on(async move { let output_lines = CountReceiver::receive_lines(out_addr); @@ -322,7 +320,7 @@ fn benchmark_configs( // avoids asserting the actual == expected as the socket transform // adds dynamic keys like timestamp for (key, value) in output.iter() { - assert_eq!(Some(value), actual.get(key), "for key {}", key,); + assert_eq!(Some(value), actual.get(key), "for key {key}",); } } } diff --git a/benches/lua.rs b/benches/lua.rs index 0d8a409d8acef..8bf99aaf9f0d4 100644 --- a/benches/lua.rs +++ b/benches/lua.rs @@ -22,7 +22,7 @@ fn bench_add_fields(c: &mut Criterion) { let benchmarks: Vec<(&str, Transform)> = vec![ ("v1", { - let source = format!("event['{}'] = '{}'", key, value); + let source = format!("event['{key}'] = '{value}'"); Transform::event_task(transforms::lua::v1::Lua::new(source, vec![]).unwrap()) }), @@ -148,9 +148,8 @@ fn bench_field_filter(c: &mut Criterion) { b.iter_batched( || (tx.clone(), events.clone()), |(mut tx, events)| { - let _ = - futures::executor::block_on(tx.send_all(&mut stream::iter(events).map(Ok))) - .unwrap(); + futures::executor::block_on(tx.send_all(&mut stream::iter(events).map(Ok))) + .unwrap(); let output = futures::executor::block_on(collect_ready(&mut rx)); diff --git a/benches/metrics_snapshot.rs b/benches/metrics_snapshot.rs index bb59ccf65433c..637335967826c 100644 --- a/benches/metrics_snapshot.rs +++ b/benches/metrics_snapshot.rs @@ -23,7 +23,7 @@ fn prepare_metrics(cardinality: usize) -> &'static vector::metrics::Controller { controller.reset(); for idx in 0..cardinality { - metrics::counter!("test", 1, "idx" => idx.to_string()); + metrics::counter!("test", "idx" => idx.to_string()).increment(1); } controller diff --git a/benches/remap.rs b/benches/remap.rs index 5eedd24445e84..2cf0829ba2f4a 100644 --- a/benches/remap.rs +++ b/benches/remap.rs @@ -27,7 +27,7 @@ fn benchmark_remap(c: &mut Criterion) { let add_fields_runner = |tform: &mut Box, event: Event| { let mut outputs = TransformOutputsBuf::new_with_capacity( - vec![TransformOutput::new(DataType::all(), HashMap::new())], + vec![TransformOutput::new(DataType::all_bits(), HashMap::new())], 1, ); tform.transform(event, &mut outputs); @@ -90,7 +90,7 @@ fn benchmark_remap(c: &mut Criterion) { let json_parser_runner = |tform: &mut Box, event: Event| { let mut outputs = TransformOutputsBuf::new_with_capacity( - vec![TransformOutput::new(DataType::all(), HashMap::new())], + vec![TransformOutput::new(DataType::all_bits(), HashMap::new())], 1, ); tform.transform(event, &mut outputs); @@ -144,7 +144,7 @@ fn benchmark_remap(c: &mut Criterion) { let coerce_runner = |tform: &mut Box, event: Event, timestamp: DateTime| { let mut outputs = TransformOutputsBuf::new_with_capacity( - vec![TransformOutput::new(DataType::all(), HashMap::new())], + vec![TransformOutput::new(DataType::all_bits(), HashMap::new())], 1, ); tform.transform(event, &mut outputs); diff --git a/benches/transform/common.rs b/benches/transform/common.rs index 1760c5e6a5537..481e65e2d7541 100644 --- a/benches/transform/common.rs +++ b/benches/transform/common.rs @@ -38,7 +38,7 @@ impl FixedLogStream { let mut events = Vec::with_capacity(total.get()); let mut cycle = 0; for _ in 0..total.get() { - events.push(Event::Log(LogEvent::from(format!("event{}", cycle)))); + events.push(Event::Log(LogEvent::from(format!("event{cycle}")))); cycle = (cycle + 1) % cycle_size; } Self::new_from_vec(events) diff --git a/benches/transform/dedupe.rs b/benches/transform/dedupe.rs index 5df5a5cf96ce4..2e95f5d28a9f7 100644 --- a/benches/transform/dedupe.rs +++ b/benches/transform/dedupe.rs @@ -1,15 +1,16 @@ use core::fmt; use std::{num::NonZeroUsize, time::Duration}; +use crate::common::{consume, FixedLogStream}; use criterion::{ criterion_group, measurement::WallTime, BatchSize, BenchmarkGroup, BenchmarkId, Criterion, SamplingMode, Throughput, }; -use vector::transforms::dedupe::{CacheConfig, Dedupe, DedupeConfig, FieldMatchConfig}; +use vector::transforms::dedupe::common::{CacheConfig, FieldMatchConfig, TimedCacheConfig}; +use vector::transforms::dedupe::config::DedupeConfig; +use vector::transforms::dedupe::transform::Dedupe; use vector_lib::transform::Transform; -use crate::common::{consume, FixedLogStream}; - #[derive(Debug)] struct Param { slug: &'static str, @@ -44,6 +45,7 @@ fn dedupe(c: &mut Criterion) { dedupe_config: DedupeConfig { fields: Some(FieldMatchConfig::IgnoreFields(vec!["message".into()])), cache: cache.clone(), + time_settings: None, }, }, // Modification of previous where field "message" is matched. @@ -53,6 +55,33 @@ fn dedupe(c: &mut Criterion) { dedupe_config: DedupeConfig { fields: Some(FieldMatchConfig::MatchFields(vec!["message".into()])), cache: cache.clone(), + time_settings: None, + }, + }, + // Modification of previous where deduplication with max age is used. + Param { + slug: "field_match_message_timed", + input: fixed_stream.clone(), + dedupe_config: DedupeConfig { + fields: Some(FieldMatchConfig::MatchFields(vec!["message".into()])), + cache: cache.clone(), + time_settings: Some(TimedCacheConfig { + max_age_ms: Duration::from_secs(5), + refresh_on_drop: false, + }), + }, + }, + // Modification of previous where refresh on drop is enabled. + Param { + slug: "field_match_message_timed_refresh_on_drop", + input: fixed_stream.clone(), + dedupe_config: DedupeConfig { + fields: Some(FieldMatchConfig::MatchFields(vec!["message".into()])), + cache: cache.clone(), + time_settings: Some(TimedCacheConfig { + max_age_ms: Duration::from_secs(5), + refresh_on_drop: true, + }), }, }, // Measurement where ignore fields do not exist in the event. @@ -68,6 +97,7 @@ fn dedupe(c: &mut Criterion) { "cdeab".into(), "bcdea".into(), ])), + time_settings: None, }, }, // Modification of previous where match fields do not exist in the @@ -84,6 +114,7 @@ fn dedupe(c: &mut Criterion) { "cdeab".into(), "bcdea".into(), ])), + time_settings: None, }, }, ] { @@ -91,8 +122,12 @@ fn dedupe(c: &mut Criterion) { group.bench_with_input(BenchmarkId::new("transform", param), ¶m, |b, param| { b.iter_batched( || { - let dedupe = - Transform::event_task(Dedupe::new(param.dedupe_config.clone())).into_task(); + let config = param.dedupe_config.clone(); + let dedupe = Transform::event_task(Dedupe::new( + config.cache.num_events, + config.fields.unwrap(), + )) + .into_task(); (Box::new(dedupe), Box::pin(param.input.clone())) }, |(dedupe, input)| { diff --git a/benches/transform/reduce.rs b/benches/transform/reduce.rs index 9f0e4ed60fdd2..fd3f54275766a 100644 --- a/benches/transform/reduce.rs +++ b/benches/transform/reduce.rs @@ -1,16 +1,16 @@ use core::fmt; use std::{num::NonZeroUsize, time::Duration}; +use crate::common::{consume, FixedLogStream}; use criterion::{ criterion_group, measurement::WallTime, BatchSize, BenchmarkGroup, BenchmarkId, Criterion, SamplingMode, Throughput, }; use indexmap::IndexMap; -use vector::transforms::reduce::{Reduce, ReduceConfig}; +use vector::transforms::reduce::config::ReduceConfig; +use vector::transforms::reduce::transform::Reduce; use vector_lib::transform::Transform; -use crate::common::{consume, FixedLogStream}; - #[derive(Debug)] struct Param { slug: &'static str, @@ -33,17 +33,13 @@ fn reduce(c: &mut Criterion) { NonZeroUsize::new(128).unwrap(), NonZeroUsize::new(2).unwrap(), ); - for param in &[ - // The `Reduce` transform has a high configuration surface. For now we - // only benchmark the "proof of concept" configuration, demonstrating - // that the benchmark does minimally work. Once we have soak tests with - // reduces in them we should extend this array to include those - // configurations. - Param { + { + let param = &Param { slug: "proof_of_concept", input: fixed_stream.clone(), reduce_config: ReduceConfig { expire_after_ms: Duration::from_secs(30), + end_every_period_ms: None, flush_period_ms: Duration::from_secs(1), group_by: vec![String::from("message")], merge_strategies: IndexMap::default(), @@ -51,8 +47,7 @@ fn reduce(c: &mut Criterion) { starts_when: None, max_events: None, }, - }, - ] { + }; group.throughput(Throughput::Elements(param.input.len() as u64)); group.bench_with_input(BenchmarkId::new("transform", param), ¶m, |b, param| { b.to_async(tokio::runtime::Runtime::new().unwrap()) diff --git a/benches/transform/route.rs b/benches/transform/route.rs index 0ef6efa30b179..c49ce47f13404 100644 --- a/benches/transform/route.rs +++ b/benches/transform/route.rs @@ -3,8 +3,8 @@ use std::time::Duration; use bytes::Bytes; use criterion::{ - black_box, criterion_group, measurement::WallTime, BatchSize, BenchmarkGroup, BenchmarkId, - Criterion, SamplingMode, Throughput, + criterion_group, measurement::WallTime, BatchSize, BenchmarkGroup, BenchmarkId, Criterion, + SamplingMode, Throughput, }; use vector::config::TransformContext; use vector::transforms::{ @@ -159,7 +159,8 @@ fn route(c: &mut Criterion) { (route, param.input.clone(), param.output_buffer.clone()) }, |(mut route, input, mut output_buffer)| { - black_box(route.transform(input, &mut output_buffer)); + route.transform(input, &mut output_buffer); + std::hint::black_box(()); }, BatchSize::SmallInput, ) diff --git a/build.rs b/build.rs index b603074f90bcc..cf35846cbe61c 100644 --- a/build.rs +++ b/build.rs @@ -20,7 +20,7 @@ impl TrackedEnv { pub fn emit_rerun_stanzas(&self) { for env_var in &self.tracked { - println!("cargo:rerun-if-env-changed={}", env_var); + println!("cargo:rerun-if-env-changed={env_var}"); } } } @@ -33,9 +33,9 @@ enum ConstantValue { impl ConstantValue { pub fn as_parts(&self) -> (&'static str, String) { match &self { - ConstantValue::Required(value) => ("&str", format!("\"{}\"", value)), + ConstantValue::Required(value) => ("&str", format!("\"{value}\"")), ConstantValue::Optional(value) => match value { - Some(value) => ("Option<&str>", format!("Some(\"{}\")", value)), + Some(value) => ("Option<&str>", format!("Some(\"{value}\")")), None => ("Option<&str>", "None".to_string()), }, } @@ -79,10 +79,8 @@ impl BuildConstants { for (name, desc, value) in self.values { let (const_type, const_val) = value.as_parts(); - let full = format!( - "#[doc=r#\"{}\"#]\npub const {}: {} = {};\n", - desc, name, const_type, const_val - ); + let full = + format!("#[doc=r#\"{desc}\"#]\npub const {name}: {const_type} = {const_val};\n"); output_file.write_all(full.as_ref())?; } @@ -202,10 +200,7 @@ fn main() { .map_err(|e| { #[allow(clippy::print_stderr)] { - eprintln!( - "Unable to determine git short hash from rev-parse command: {}", - e - ); + eprintln!("Unable to determine git short hash from rev-parse command: {e}"); } }) .expect("git hash detection failed"); diff --git a/changelog.d/14082_add_mermaid_support.feature.md b/changelog.d/14082_add_mermaid_support.feature.md deleted file mode 100644 index 912652f9bc0d8..0000000000000 --- a/changelog.d/14082_add_mermaid_support.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Add support for rendering to Mermaid format in `vector graph` - -authors: Firehed diff --git a/changelog.d/17294_allocation_group_id_groups_max_limit_increased.fix.md b/changelog.d/17294_allocation_group_id_groups_max_limit_increased.fix.md deleted file mode 100644 index 58791882556ba..0000000000000 --- a/changelog.d/17294_allocation_group_id_groups_max_limit_increased.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fix a Vector crash that occurred when the internal metrics generated a lot of groups by increasing groups max limit from 128 to 256. - -authors: triggerhappy17 diff --git a/changelog.d/22125.feature.md b/changelog.d/22125.feature.md deleted file mode 100644 index a2fb312ef23ed..0000000000000 --- a/changelog.d/22125.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Allows users to specify AWS authentication and the AWS service name for HTTP sinks to support AWS API endpoints that require SigV4. - -authors: johannesfloriangeiger diff --git a/changelog.d/22212_fluent_unix_socket.feature.md b/changelog.d/22212_fluent_unix_socket.feature.md deleted file mode 100644 index be21ffb0c042a..0000000000000 --- a/changelog.d/22212_fluent_unix_socket.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Add support for fluentd forwarding over a unix socket - -authors: tustvold diff --git a/changelog.d/22533_zlib-rs.enhancement.md b/changelog.d/22533_zlib-rs.enhancement.md deleted file mode 100644 index cbc922fcc52b5..0000000000000 --- a/changelog.d/22533_zlib-rs.enhancement.md +++ /dev/null @@ -1,3 +0,0 @@ -Replaced miniz_oxide backend for zlib compression with zlib-rs for significantly improved compression performance. zlib-rs provides 2-3x performance improvements for compression, and ~20% improvements for decompression than miniz_oxide. - -authors: JakubOnderka diff --git a/changelog.d/22540_websocket_server_buffering_ack_support.feature.md b/changelog.d/22540_websocket_server_buffering_ack_support.feature.md deleted file mode 100644 index 0ce4cadf6740c..0000000000000 --- a/changelog.d/22540_websocket_server_buffering_ack_support.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Add ACK support to message buffering feature of `websocket_server` sink, allowing this component to cache latest received messages per client. - -authors: esensar Quad9DNS diff --git a/changelog.d/22651_greptimedb_logs_headers.breaking.md b/changelog.d/22651_greptimedb_logs_headers.breaking.md deleted file mode 100644 index 674bf5d69bacb..0000000000000 --- a/changelog.d/22651_greptimedb_logs_headers.breaking.md +++ /dev/null @@ -1,32 +0,0 @@ -Added a new `extra_headers` option to `greptimedb_logs` sink configuration to set additional headers for outgoing requests. - -change `greptimedb_logs` sink default content type to `application/x-ndjson` to match the default content type of `greptimedb` sink -if you use the greptimedb version v0.12 or earlier, you need to set the content type to `application/json` in the sink configuration - -Example: - -```yaml -sinks: - greptime_logs: - type: greptimedb_logs - inputs: ["my_source_id"] - endpoint: "http://localhost:4000" - table: "demo_logs" - dbname: "public" - extra_headers: - x-source: vector -``` - -```toml -[sinks.greptime_logs] -type = "greptimedb_logs" -inputs = ["my_source_id"] -endpoint = "http://localhost:4000" -table = "demo_logs" -dbname = "public" - -[sinks.greptime_logs.extra_headers] -x-source = "vector" -``` - -authors: greptimedb diff --git a/changelog.d/22656_statsd_timer_conversion.feature.md b/changelog.d/22656_statsd_timer_conversion.feature.md deleted file mode 100644 index fa55207ebd3b6..0000000000000 --- a/changelog.d/22656_statsd_timer_conversion.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -This change introduces a new configuration option in the StatsD source: `convert_to` of type `ConversionUnit`. By default, timing values in milliseconds (`ms`) are converted to seconds (`s`). Users can set `convert_to` to `"milliseconds"` to preserve the original millisecond values. - -authors: devkoriel diff --git a/changelog.d/22786_remove_empty_file_after_secs.fix.md b/changelog.d/22786_remove_empty_file_after_secs.fix.md deleted file mode 100644 index 70b9d5177e956..0000000000000 --- a/changelog.d/22786_remove_empty_file_after_secs.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fixed `file` source bug where known small files were not deleted after the specified `remove_after_secs`. - -authors: linw1995 diff --git a/changelog.d/22827_cloudwatch_auth_missing_region.fix.md b/changelog.d/22827_cloudwatch_auth_missing_region.fix.md deleted file mode 100644 index 7cfacdc2191c1..0000000000000 --- a/changelog.d/22827_cloudwatch_auth_missing_region.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fixed a `AwsAuthentication::File` bug where `region` was missing from the `STS` authentication endpoint. - -authors: cahartma diff --git a/changelog.d/22831_increase_cloudwatch_log_event_max_size.fix.md b/changelog.d/22831_increase_cloudwatch_log_event_max_size.fix.md deleted file mode 100644 index dddae5c17b134..0000000000000 --- a/changelog.d/22831_increase_cloudwatch_log_event_max_size.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Increase the max event size for `aws_cloudwatch_logs` sink based on newly increased limit from `AWS` - -authors: cahartma diff --git a/changelog.d/22850_custom_auth_address_access.feature.md b/changelog.d/22850_custom_auth_address_access.feature.md deleted file mode 100644 index a955cec1b0aa5..0000000000000 --- a/changelog.d/22850_custom_auth_address_access.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Enabled IP address access in VRL scripts of custom auth strategy for server components. - -authors: esensar Quad9DNS diff --git a/changelog.d/22854_websocket_server_subprotocol_support.feature.md b/changelog.d/22854_websocket_server_subprotocol_support.feature.md deleted file mode 100644 index 322aa36a8b4d9..0000000000000 --- a/changelog.d/22854_websocket_server_subprotocol_support.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Added support for the `Sec-WebSocket-Protocol` header in the `websocket_server` sink to better accommodate clients that require it. - -authors: esensar Quad9DNS diff --git a/changelog.d/22857_junit_reports_for_unit_tests.feature.md b/changelog.d/22857_junit_reports_for_unit_tests.feature.md deleted file mode 100644 index 1c72c4919937e..0000000000000 --- a/changelog.d/22857_junit_reports_for_unit_tests.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Added the ability to generate JUnit reports for unit tests. - -authors: simplepad diff --git a/changelog.d/22865_framing_performance.enhancement.md b/changelog.d/22865_framing_performance.enhancement.md deleted file mode 100644 index 00a8479f662dd..0000000000000 --- a/changelog.d/22865_framing_performance.enhancement.md +++ /dev/null @@ -1,3 +0,0 @@ -Reduced unnecessary buffer reallocation when using `framing.method = length_delimited` in sinks for significantly improved performance with large (more than 10MB) batches. - -authors: Ilmarii diff --git a/changelog.d/22909_redis_sink_supports_encoding_input.feature.md b/changelog.d/22909_redis_sink_supports_encoding_input.feature.md deleted file mode 100644 index 52f5514516570..0000000000000 --- a/changelog.d/22909_redis_sink_supports_encoding_input.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -The redis sink now supports any input event type that the configured encoding supports. It previously only supported log events. - -authors: ynachi diff --git a/changelog.d/22921_remove_limit_if_use_apiserver_cache.fix.md b/changelog.d/22921_remove_limit_if_use_apiserver_cache.fix.md deleted file mode 100644 index e0f68868b5ce1..0000000000000 --- a/changelog.d/22921_remove_limit_if_use_apiserver_cache.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fixed a `kubernetes source` bug where `use_apiserver_cache=true` but there is no `resourceVersion=0` parameters in list request. As [the issue](https://github.com/kube-rs/kube/issues/1743) mentioned, when there are `resourceVersion =0` and `!page_size.is_none` in the `ListParams` fields, the parameter `resourceVersion=0` will be ignored by `kube-rs` sdk. If no parameter `resourceVersion` passed to the apiserver, the apiserver will list pods from ETCD instead of in memory cache. - -authors: xiaozongyang diff --git a/changelog.d/22922_healthcheck_timeout_config.feature.md b/changelog.d/22922_healthcheck_timeout_config.feature.md deleted file mode 100644 index 4865e0f047876..0000000000000 --- a/changelog.d/22922_healthcheck_timeout_config.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Added `timeout` config option to the `healthcheck` sink configuration. Previously it was hardcoded to 10 seconds across all components, but now it can be configured per component. - -authors: esensar Quad9DNS diff --git a/changelog.d/22990_allow_not_watching_k8s_ns.enhancement.md b/changelog.d/22990_allow_not_watching_k8s_ns.enhancement.md new file mode 100644 index 0000000000000..50e2ce6d92f66 --- /dev/null +++ b/changelog.d/22990_allow_not_watching_k8s_ns.enhancement.md @@ -0,0 +1,4 @@ +Allow disabling listwatching of Kubernetes namespaces to reduce resource usage in clusters with +large numbers of namespaces. + +authors: imbstack diff --git a/changelog.d/add_priority_amqp_sink.enhancement.md b/changelog.d/add_priority_amqp_sink.enhancement.md deleted file mode 100644 index 6b665ebef64b1..0000000000000 --- a/changelog.d/add_priority_amqp_sink.enhancement.md +++ /dev/null @@ -1,3 +0,0 @@ -The `amqp` sink now supports setting the `priority` for messages. The priority value can be templated to an integer between 0 and 255 (inclusive). Also, this new field is template-able and Vector templates can now render numerical values. - -authors: aramperes diff --git a/changelog.d/datadog_agent_http_header.fix.md b/changelog.d/datadog_agent_http_header.fix.md deleted file mode 100644 index a075a4976e157..0000000000000 --- a/changelog.d/datadog_agent_http_header.fix.md +++ /dev/null @@ -1,31 +0,0 @@ -Adding an option in the `datadog_logs` sink to allow Vector to mutate the record to conform to the -protocol used by the Datadog Agent itself. To enable use the `conforms_as_agent` option or have the -appropriate agent header (`DD-PROTOCOL: agent-json`) within the additional HTTP Headers list. - -The log will be mutated so that any data that is a reserved keyword that exists at the root will be -moved under a new object keyed at the field 'message'. One will be created if it already does not -exist. As an example: - -```json -{ - "key1": "value1", - "key2": { "key2-1" : "value2" }, - "message" : "Hello world", - ... rest of reserved fields -} -``` - -will be modified to: - -```json -{ - "message" : { - "message" : "Hello world", - "key1": "value1", - "key2": { "key2-1" : "value2" } - }, - ... rest of reserved fields -} -``` - -authors: graphcareful diff --git a/changelog.d/datadog_logs_sink_logs_namespace.fix.md b/changelog.d/datadog_logs_sink_logs_namespace.fix.md deleted file mode 100644 index c430a21b4f448..0000000000000 --- a/changelog.d/datadog_logs_sink_logs_namespace.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fix bug in the `datadog_logs` sink where the content of the log message is dropped when logs namespacing is enabled. - -authors: graphcareful diff --git a/changelog.d/opentelemetry_metrics.feature.md b/changelog.d/opentelemetry_metrics.feature.md deleted file mode 100644 index 824383d14747b..0000000000000 --- a/changelog.d/opentelemetry_metrics.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -The `opentelemetry` source now supports metrics ingestion. - -authors: cmcmacs diff --git a/changelog.d/s3-source-defer-or-delete.enhancement.md b/changelog.d/s3-source-defer-or-delete.enhancement.md deleted file mode 100644 index c4d0b54ccce4a..0000000000000 --- a/changelog.d/s3-source-defer-or-delete.enhancement.md +++ /dev/null @@ -1,3 +0,0 @@ -Add deferred processing options (`deferred.max_age_secs`, `deferred.queue_url`) to automatically route older event notifications to a separate queue, enabling prioritized processing of recent files - -authors: akutta diff --git a/changelog.d/window-transform.feature.md b/changelog.d/window-transform.feature.md deleted file mode 100644 index b3f366108c625..0000000000000 --- a/changelog.d/window-transform.feature.md +++ /dev/null @@ -1,5 +0,0 @@ -Add a new `window` transform, a variant of ring buffer or backtrace logging implemented as a sliding window. -Allows for reduction of log volume by filtering out logs when the system is healthy, but preserving detailed -logs when they are most relevant. - -authors: ilinas diff --git a/clippy.toml b/clippy.toml index 25937ac64122a..20274f646b1d7 100644 --- a/clippy.toml +++ b/clippy.toml @@ -1,4 +1,4 @@ -cognitive-complexity-threshold = 75 +large-error-threshold = 256 # in bytes # for `disallowed_method`: # https://rust-lang.github.io/rust-clippy/master/index.html#disallowed_method diff --git a/deny.toml b/deny.toml index 7d5b3a5a7168d..7e200969f6e01 100644 --- a/deny.toml +++ b/deny.toml @@ -8,6 +8,7 @@ allow = [ "CC0-1.0", "ISC", "MIT", + "MIT-0", "OpenSSL", "Unicode-3.0", "Unicode-DFS-2016", diff --git a/distribution/docker/alpine/Dockerfile b/distribution/docker/alpine/Dockerfile index a33ef65479b15..45ae3194b264f 100644 --- a/distribution/docker/alpine/Dockerfile +++ b/distribution/docker/alpine/Dockerfile @@ -12,7 +12,7 @@ RUN ARCH=$(if [ "$TARGETPLATFORM" = "linux/arm/v6" ]; then echo "arm"; else cat RUN mkdir -p /var/lib/vector -FROM docker.io/alpine:3.21 +FROM docker.io/alpine:3.22 # https://github.com/opencontainers/image-spec/blob/main/annotations.md LABEL org.opencontainers.image.url="https://vector.dev" diff --git a/distribution/install.sh b/distribution/install.sh index 13d8f847b6e06..ceaad6bae3645 100755 --- a/distribution/install.sh +++ b/distribution/install.sh @@ -13,7 +13,7 @@ set -u # If PACKAGE_ROOT is unset or empty, default it. PACKAGE_ROOT="${PACKAGE_ROOT:-"https://packages.timber.io/vector"}" # If VECTOR_VERSION is unset or empty, default it. -VECTOR_VERSION="${VECTOR_VERSION:-"0.46.1"}" +VECTOR_VERSION="${VECTOR_VERSION:-"0.49.0"}" _divider="--------------------------------------------------------------------------------" _prompt=">>>" _indent=" " diff --git a/distribution/kubernetes/vector-agent/README.md b/distribution/kubernetes/vector-agent/README.md index 27da769cff90c..4c0cfbf8fc128 100644 --- a/distribution/kubernetes/vector-agent/README.md +++ b/distribution/kubernetes/vector-agent/README.md @@ -1,6 +1,6 @@ The kubernetes manifests found in this directory have been automatically generated from the [helm chart `vector/vector`](https://github.com/vectordotdev/helm-charts/tree/master/charts/vector) -version 0.42.1 with the following `values.yaml`: +version 0.45.0 with the following `values.yaml`: ```yaml role: Agent diff --git a/distribution/kubernetes/vector-agent/configmap.yaml b/distribution/kubernetes/vector-agent/configmap.yaml index 6db2a2ea95a6b..1e0f0d561d3d5 100644 --- a/distribution/kubernetes/vector-agent/configmap.yaml +++ b/distribution/kubernetes/vector-agent/configmap.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" data: agent.yaml: | data_dir: /vector-data-dir diff --git a/distribution/kubernetes/vector-agent/daemonset.yaml b/distribution/kubernetes/vector-agent/daemonset.yaml index 41d64e003a753..910d1b042851b 100644 --- a/distribution/kubernetes/vector-agent/daemonset.yaml +++ b/distribution/kubernetes/vector-agent/daemonset.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" spec: selector: matchLabels: @@ -30,7 +30,7 @@ spec: dnsPolicy: ClusterFirst containers: - name: vector - image: "timberio/vector:0.46.1-distroless-libc" + image: "timberio/vector:0.49.0-distroless-libc" imagePullPolicy: IfNotPresent args: - --config-dir diff --git a/distribution/kubernetes/vector-agent/rbac.yaml b/distribution/kubernetes/vector-agent/rbac.yaml index 4c776a65970e3..d1bdf5b57f8de 100644 --- a/distribution/kubernetes/vector-agent/rbac.yaml +++ b/distribution/kubernetes/vector-agent/rbac.yaml @@ -10,7 +10,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" rules: - apiGroups: - "" @@ -31,7 +31,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole diff --git a/distribution/kubernetes/vector-agent/service-headless.yaml b/distribution/kubernetes/vector-agent/service-headless.yaml index 45fc274219f22..f9c1ed731f311 100644 --- a/distribution/kubernetes/vector-agent/service-headless.yaml +++ b/distribution/kubernetes/vector-agent/service-headless.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" annotations: spec: clusterIP: None diff --git a/distribution/kubernetes/vector-agent/serviceaccount.yaml b/distribution/kubernetes/vector-agent/serviceaccount.yaml index 6582dfe20ca82..feccbcb04dbb1 100644 --- a/distribution/kubernetes/vector-agent/serviceaccount.yaml +++ b/distribution/kubernetes/vector-agent/serviceaccount.yaml @@ -9,5 +9,5 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" automountServiceAccountToken: true diff --git a/distribution/kubernetes/vector-aggregator/README.md b/distribution/kubernetes/vector-aggregator/README.md index f50f97fa32989..cbecb3df87692 100644 --- a/distribution/kubernetes/vector-aggregator/README.md +++ b/distribution/kubernetes/vector-aggregator/README.md @@ -1,6 +1,6 @@ The kubernetes manifests found in this directory have been automatically generated from the [helm chart `vector/vector`](https://github.com/vectordotdev/helm-charts/tree/master/charts/vector) -version 0.42.1 with the following `values.yaml`: +version 0.45.0 with the following `values.yaml`: ```yaml diff --git a/distribution/kubernetes/vector-aggregator/configmap.yaml b/distribution/kubernetes/vector-aggregator/configmap.yaml index c3b6e0ac25ada..020bc89eed6fa 100644 --- a/distribution/kubernetes/vector-aggregator/configmap.yaml +++ b/distribution/kubernetes/vector-aggregator/configmap.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" data: aggregator.yaml: | data_dir: /vector-data-dir diff --git a/distribution/kubernetes/vector-aggregator/service-headless.yaml b/distribution/kubernetes/vector-aggregator/service-headless.yaml index c88a67f2b310b..554ced0faafc9 100644 --- a/distribution/kubernetes/vector-aggregator/service-headless.yaml +++ b/distribution/kubernetes/vector-aggregator/service-headless.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" annotations: spec: clusterIP: None diff --git a/distribution/kubernetes/vector-aggregator/service.yaml b/distribution/kubernetes/vector-aggregator/service.yaml index f85a74c2acdab..8daa6a12b421d 100644 --- a/distribution/kubernetes/vector-aggregator/service.yaml +++ b/distribution/kubernetes/vector-aggregator/service.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" annotations: spec: ports: diff --git a/distribution/kubernetes/vector-aggregator/serviceaccount.yaml b/distribution/kubernetes/vector-aggregator/serviceaccount.yaml index ca52b73bb4fca..bfac6910540f9 100644 --- a/distribution/kubernetes/vector-aggregator/serviceaccount.yaml +++ b/distribution/kubernetes/vector-aggregator/serviceaccount.yaml @@ -9,5 +9,5 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" automountServiceAccountToken: true diff --git a/distribution/kubernetes/vector-aggregator/statefulset.yaml b/distribution/kubernetes/vector-aggregator/statefulset.yaml index 03a554ba2568c..ac25b64588011 100644 --- a/distribution/kubernetes/vector-aggregator/statefulset.yaml +++ b/distribution/kubernetes/vector-aggregator/statefulset.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" annotations: {} spec: replicas: 1 @@ -34,7 +34,7 @@ spec: dnsPolicy: ClusterFirst containers: - name: vector - image: "timberio/vector:0.46.1-distroless-libc" + image: "timberio/vector:0.49.0-distroless-libc" imagePullPolicy: IfNotPresent args: - --config-dir diff --git a/distribution/kubernetes/vector-stateless-aggregator/README.md b/distribution/kubernetes/vector-stateless-aggregator/README.md index 773c7190c08e7..a2e9e8fe81c46 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/README.md +++ b/distribution/kubernetes/vector-stateless-aggregator/README.md @@ -1,6 +1,6 @@ The kubernetes manifests found in this directory have been automatically generated from the [helm chart `vector/vector`](https://github.com/vectordotdev/helm-charts/tree/master/charts/vector) -version 0.42.1 with the following `values.yaml`: +version 0.45.0 with the following `values.yaml`: ```yaml role: Stateless-Aggregator diff --git a/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml b/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml index 6e2277f4b0b8d..07174c2028a1b 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" data: aggregator.yaml: | data_dir: /vector-data-dir diff --git a/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml b/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml index abd9830d1e9a2..4805960f163a6 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" annotations: {} spec: replicas: 1 @@ -32,7 +32,7 @@ spec: dnsPolicy: ClusterFirst containers: - name: vector - image: "timberio/vector:0.46.1-distroless-libc" + image: "timberio/vector:0.49.0-distroless-libc" imagePullPolicy: IfNotPresent args: - --config-dir diff --git a/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml b/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml index ce28b3f972de6..39b4aee2d0993 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" annotations: spec: clusterIP: None diff --git a/distribution/kubernetes/vector-stateless-aggregator/service.yaml b/distribution/kubernetes/vector-stateless-aggregator/service.yaml index 0a5a0cb3eea98..b3ea61a5f1366 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/service.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/service.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" annotations: spec: ports: diff --git a/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml b/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml index 9c6e57f0f5155..9ff057a9ab32f 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml @@ -9,5 +9,5 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" automountServiceAccountToken: true diff --git a/docs/DEVELOPING.md b/docs/DEVELOPING.md index 7b9116a2a806b..356f780e54cb6 100644 --- a/docs/DEVELOPING.md +++ b/docs/DEVELOPING.md @@ -126,7 +126,7 @@ Loosely, you'll need the following: - **To run integration tests:** Have `docker` available, or a real live version of that service. (Use `AUTOSPAWN=false`) - **To run `make check-component-features`:** Have `remarshal` installed. - **To run `make check-licenses` or `cargo vdev build licenses`:** Have `dd-rust-license-tool` [installed](https://github.com/DataDog/rust-license-tool). -- **To run `cargo vdev build component-docs`:** Have `cue` [installed](https://cuelang.org/docs/install/). +- **To run `make generate-component-docs`:** Have `cue` [installed](https://cuelang.org/docs/install/). If you find yourself needing to run something inside the Docker environment described above, that's totally fine, they won't collide or hurt each other. In this case, you'd just run `make environment-generate`. @@ -161,7 +161,7 @@ cargo bench transforms::example make fmt cargo fmt # Build component documentation for the website -cargo vdev build component-docs +make generate-component-docs ``` If you run `make` you'll see a full list of all our tasks. Some of these will start Docker containers, sign commits, or even make releases. These are not common development commands and your mileage may vary. diff --git a/docs/REVIEWING.md b/docs/REVIEWING.md index 9075b74556a89..d40d311c63a91 100644 --- a/docs/REVIEWING.md +++ b/docs/REVIEWING.md @@ -26,7 +26,7 @@ should be used for all pull requests: - [ ] Is backward compatibility broken? If so, can it be avoided or deprecated? (see [Backward compatibility](#backward-compatibility)) - [ ] Have dependencies changed? (see [Dependencies](#dependencies)) - [ ] Has the code been explicitly reviewed for security issues? Dependencies included. (see [Security](#security)) -- [ ] Is there a risk of performance regressions? If so, have run the [Vector test harness](https://github.com/vectordotdev/vector-test-harness)? (see [Performance Testing](#performance-testing)) +- [ ] Is there a risk of performance regressions? (see [Performance Testing](#performance-testing)) - [ ] Should documentation be adjusted to reflect any of these changes? (see [Documentation](#documentation)) For component changes, especially pull requests introducing new components, the diff --git a/docs/tutorials/sinks/1_basic_sink.md b/docs/tutorials/sinks/1_basic_sink.md index 65bb7b4078880..a099a5530602d 100644 --- a/docs/tutorials/sinks/1_basic_sink.md +++ b/docs/tutorials/sinks/1_basic_sink.md @@ -348,7 +348,7 @@ Our sink works! [event_streams_tracking]: https://github.com/vectordotdev/vector/issues/9261 [vdev_install]: https://github.com/vectordotdev/vector/tree/master/vdev#installation -[acknowledgements]: https://vector.dev/docs/about/under-the-hood/architecture/end-to-end-acknowledgements/ +[acknowledgements]: https://vector.dev/docs/architecture/end-to-end-acknowledgements/ [configurable_component]: https://rust-doc.vector.dev/vector_config/attr.configurable_component.html [generate_config]: https://rust-doc.vector.dev/vector/config/trait.generateconfig [sink_config]: https://rust-doc.vector.dev/vector/config/trait.sinkconfig diff --git a/docs/tutorials/sinks/2_http_sink.md b/docs/tutorials/sinks/2_http_sink.md index ca25791136ecb..4c100fcb8e4bc 100644 --- a/docs/tutorials/sinks/2_http_sink.md +++ b/docs/tutorials/sinks/2_http_sink.md @@ -150,7 +150,7 @@ generated by our `BasicEncoder`. *finalizers* - [`EventFinalizers`][event_finalizers] is a collection of `EventFinalizer`s. An [`EventFinalizer`][event_finalizer] is used to track the status of a given event and is used to support [end to end acknowledgements] -(https://vector.dev/docs/about/under-the-hood/guarantees/#acknowledgement- +(https://vector.dev/docs/architecture/guarantees/#acknowledgement- guarantees). *metadata* - the metadata contains additional data that is used to emit various metrics when diff --git a/lib/codecs/Cargo.toml b/lib/codecs/Cargo.toml index 01112bc2e6dd7..3a26a46eb3fa7 100644 --- a/lib/codecs/Cargo.toml +++ b/lib/codecs/Cargo.toml @@ -26,11 +26,11 @@ prost-reflect.workspace = true rand.workspace = true regex = { version = "1.11.1", default-features = false, features = ["std", "perf"] } serde.workspace = true -serde_with = { version = "3.12.0", default-features = false, features = ["std", "macros", "chrono_0_4"] } +serde_with = { version = "3.14.0", default-features = false, features = ["std", "macros", "chrono_0_4"] } serde_json.workspace = true smallvec = { version = "1", default-features = false, features = ["union"] } snafu.workspace = true -syslog_loose = { version = "0.21", default-features = false, optional = true } +syslog_loose = { version = "0.22", default-features = false, optional = true } tokio-util = { version = "0.7", default-features = false, features = ["codec"] } tokio.workspace = true tracing = { version = "0.1", default-features = false } @@ -46,7 +46,7 @@ indoc.workspace = true tokio = { version = "1", features = ["test-util"] } similar-asserts = "1.7.0" vector-core = { path = "../vector-core", default-features = false, features = ["vrl", "test"] } -rstest = "0.25.0" +rstest = "0.26.1" tracing-test = "0.2.5" uuid.workspace = true vrl.workspace = true diff --git a/lib/codecs/src/decoding/format/avro.rs b/lib/codecs/src/decoding/format/avro.rs index 7a49763179891..53646a11bcd58 100644 --- a/lib/codecs/src/decoding/format/avro.rs +++ b/lib/codecs/src/decoding/format/avro.rs @@ -41,7 +41,7 @@ impl AvroDeserializerConfig { /// Build the `AvroDeserializer` from this configuration. pub fn build(&self) -> AvroDeserializer { let schema = apache_avro::Schema::parse_str(&self.avro_options.schema) - .map_err(|error| format!("Failed building Avro serializer: {}", error)) + .map_err(|error| format!("Failed building Avro serializer: {error}")) .unwrap(); AvroDeserializer { schema, @@ -90,7 +90,7 @@ impl From<&AvroDeserializerOptions> for AvroSerializerOptions { #[derive(Clone, Debug)] pub struct AvroDeserializerOptions { /// The Avro schema definition. - /// Please note that the following [`apache_avro::types::Value`] variants are currently *not* supported: + /// **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: /// * `Date` /// * `Decimal` /// * `Duration` @@ -103,7 +103,7 @@ pub struct AvroDeserializerOptions { ))] pub schema: String, - /// For Avro datum encoded in Kafka messages, the bytes are prefixed with the schema ID. Set this to true to strip the schema ID prefix. + /// For Avro datum encoded in Kafka messages, the bytes are prefixed with the schema ID. Set this to `true` to strip the schema ID prefix. /// According to [Confluent Kafka's document](https://docs.confluent.io/platform/current/schema-registry/fundamentals/serdes-develop/index.html#wire-format). pub strip_schema_id_prefix: bool, } diff --git a/lib/codecs/src/decoding/format/gelf.rs b/lib/codecs/src/decoding/format/gelf.rs index c37924072ffa6..aff349ccb7645 100644 --- a/lib/codecs/src/decoding/format/gelf.rs +++ b/lib/codecs/src/decoding/format/gelf.rs @@ -84,7 +84,7 @@ impl GelfDeserializerConfig { #[derive(Debug, Clone, PartialEq, Eq, Derivative)] #[derivative(Default)] pub struct GelfDeserializerOptions { - /// Determines whether or not to replace invalid UTF-8 sequences instead of failing. + /// Determines whether to replace invalid UTF-8 sequences instead of failing. /// /// When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. /// @@ -118,11 +118,9 @@ impl GelfDeserializer { // GELF spec defines the version as 1.1 which has not changed since 2013 if parsed.version != GELF_VERSION { - return Err(format!( - "{} does not match GELF spec version ({})", - VERSION, GELF_VERSION - ) - .into()); + return Err( + format!("{VERSION} does not match GELF spec version ({GELF_VERSION})").into(), + ); } log.insert(&GELF_TARGET_PATHS.version, parsed.version.to_string()); @@ -166,16 +164,15 @@ impl GelfDeserializer { // per GELF spec, Additional field names must be prefixed with an underscore if !key.starts_with('_') { return Err(format!( - "'{}' field is invalid. \ - Additional field names must be prefixed with an underscore.", - key + "'{key}' field is invalid. \ + Additional field names must be prefixed with an underscore." ) .into()); } // per GELF spec, Additional field names must be characters dashes or dots if !VALID_FIELD_REGEX.is_match(key) { - return Err(format!("'{}' field contains invalid characters. Field names may \ - contain only letters, numbers, underscores, dashes and dots.", key).into()); + return Err(format!("'{key}' field contains invalid characters. Field names may \ + contain only letters, numbers, underscores, dashes and dots.").into()); } // per GELF spec, Additional field values must be either strings or numbers @@ -191,8 +188,8 @@ impl GelfDeserializer { serde_json::Value::Array(_) => "array", serde_json::Value::Object(_) => "object", }; - return Err(format!("The value type for field {} is an invalid type ({}). Additional field values \ - should be either strings or numbers.", key, type_).into()); + return Err(format!("The value type for field {key} is an invalid type ({type_}). Additional field values \ + should be either strings or numbers.").into()); } } } diff --git a/lib/codecs/src/decoding/format/influxdb.rs b/lib/codecs/src/decoding/format/influxdb.rs index a4cb5cbfe7827..5ca8352dd9ba6 100644 --- a/lib/codecs/src/decoding/format/influxdb.rs +++ b/lib/codecs/src/decoding/format/influxdb.rs @@ -56,7 +56,7 @@ impl InfluxdbDeserializerConfig { #[derive(Debug, Clone, PartialEq, Eq, Derivative)] #[derivative(Default)] pub struct InfluxdbDeserializerOptions { - /// Determines whether or not to replace invalid UTF-8 sequences instead of failing. + /// Determines whether to replace invalid UTF-8 sequences instead of failing. /// /// When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. /// diff --git a/lib/codecs/src/decoding/format/json.rs b/lib/codecs/src/decoding/format/json.rs index a017cba3c8fa6..016cf5918ca3f 100644 --- a/lib/codecs/src/decoding/format/json.rs +++ b/lib/codecs/src/decoding/format/json.rs @@ -67,7 +67,7 @@ impl JsonDeserializerConfig { #[derive(Debug, Clone, PartialEq, Eq, Derivative)] #[derivative(Default)] pub struct JsonDeserializerOptions { - /// Determines whether or not to replace invalid UTF-8 sequences instead of failing. + /// Determines whether to replace invalid UTF-8 sequences instead of failing. /// /// When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. /// @@ -111,7 +111,7 @@ impl Deserializer for JsonDeserializer { true => serde_json::from_str(&String::from_utf8_lossy(&bytes)), false => serde_json::from_slice(&bytes), } - .map_err(|error| format!("Error parsing JSON: {:?}", error))?; + .map_err(|error| format!("Error parsing JSON: {error:?}"))?; // If the root is an Array, split it into multiple events let mut events = match json { diff --git a/lib/codecs/src/decoding/format/native_json.rs b/lib/codecs/src/decoding/format/native_json.rs index eac15b30529ea..3bda6848d528e 100644 --- a/lib/codecs/src/decoding/format/native_json.rs +++ b/lib/codecs/src/decoding/format/native_json.rs @@ -57,7 +57,7 @@ impl NativeJsonDeserializerConfig { #[derive(Debug, Clone, PartialEq, Eq, Derivative)] #[derivative(Default)] pub struct NativeJsonDeserializerOptions { - /// Determines whether or not to replace invalid UTF-8 sequences instead of failing. + /// Determines whether to replace invalid UTF-8 sequences instead of failing. /// /// When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. /// @@ -97,7 +97,7 @@ impl Deserializer for NativeJsonDeserializer { true => serde_json::from_str(&String::from_utf8_lossy(&bytes)), false => serde_json::from_slice(&bytes), } - .map_err(|error| format!("Error parsing JSON: {:?}", error))?; + .map_err(|error| format!("Error parsing JSON: {error:?}"))?; let events = match json { serde_json::Value::Array(values) => values diff --git a/lib/codecs/src/decoding/format/protobuf.rs b/lib/codecs/src/decoding/format/protobuf.rs index 0a4552b076424..cacd7912b9cbc 100644 --- a/lib/codecs/src/decoding/format/protobuf.rs +++ b/lib/codecs/src/decoding/format/protobuf.rs @@ -33,7 +33,7 @@ impl ProtobufDeserializerConfig { /// Return the type of event build by this deserializer. pub fn output_type(&self) -> DataType { - DataType::Log + DataType::all_bits() } /// The schema produced by the deserializer. @@ -98,7 +98,7 @@ impl Deserializer for ProtobufDeserializer { log_namespace: LogNamespace, ) -> vector_common::Result> { let dynamic_message = DynamicMessage::decode(self.message_descriptor.clone(), bytes) - .map_err(|error| format!("Error parsing protobuf: {:?}", error))?; + .map_err(|error| format!("Error parsing protobuf: {error:?}"))?; let proto_vrl = vrl::protobuf::proto_to_value(&prost_reflect::Value::Message(dynamic_message), None)?; diff --git a/lib/codecs/src/decoding/format/syslog.rs b/lib/codecs/src/decoding/format/syslog.rs index 6a5d7e135e841..993de798aee14 100644 --- a/lib/codecs/src/decoding/format/syslog.rs +++ b/lib/codecs/src/decoding/format/syslog.rs @@ -241,7 +241,7 @@ impl SyslogDeserializerConfig { #[derive(Debug, Clone, PartialEq, Eq, Derivative)] #[derivative(Default)] pub struct SyslogDeserializerOptions { - /// Determines whether or not to replace invalid UTF-8 sequences instead of failing. + /// Determines whether to replace invalid UTF-8 sequences instead of failing. /// /// When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. /// diff --git a/lib/codecs/src/decoding/format/vrl.rs b/lib/codecs/src/decoding/format/vrl.rs index 63a127955ee87..c11149d0fa842 100644 --- a/lib/codecs/src/decoding/format/vrl.rs +++ b/lib/codecs/src/decoding/format/vrl.rs @@ -37,7 +37,7 @@ pub struct VrlDeserializerOptions { /// time zone. The time zone name may be any name in the [TZ database][tz_database], or `local` /// to indicate system local time. /// - /// If not set, `local` will be used. + /// If not set, `local` is used. /// /// [tz_database]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones #[serde(default)] diff --git a/lib/codecs/src/decoding/framing/chunked_gelf.rs b/lib/codecs/src/decoding/framing/chunked_gelf.rs index b5a67b7aec5ba..9c34e002cce43 100644 --- a/lib/codecs/src/decoding/framing/chunked_gelf.rs +++ b/lib/codecs/src/decoding/framing/chunked_gelf.rs @@ -67,7 +67,7 @@ pub struct ChunkedGelfDecoderOptions { /// be dropped. If this option is not set, the decoder does not limit the length of messages and /// the per-message memory is unbounded. /// - /// Note that a message can be composed of multiple chunks and this limit is applied to the whole + /// **Note**: A message can be composed of multiple chunks and this limit is applied to the whole /// message, not to individual chunks. /// /// This limit takes only into account the message's payload and the GELF header bytes are excluded from the calculation. diff --git a/lib/codecs/src/decoding/framing/octet_counting.rs b/lib/codecs/src/decoding/framing/octet_counting.rs index b5248b26b6c36..e90173f5a28cf 100644 --- a/lib/codecs/src/decoding/framing/octet_counting.rs +++ b/lib/codecs/src/decoding/framing/octet_counting.rs @@ -97,8 +97,7 @@ impl OctetCountingDecoder { // There are enough chars in this frame to discard src.advance(chars); self.octet_decoding = None; - Err(LinesCodecError::Io(io::Error::new( - io::ErrorKind::Other, + Err(LinesCodecError::Io(io::Error::other( "Frame length limit exceeded", ))) } @@ -117,8 +116,7 @@ impl OctetCountingDecoder { // When discarding we keep discarding to the next newline. src.advance(offset + 1); self.octet_decoding = None; - Err(LinesCodecError::Io(io::Error::new( - io::ErrorKind::Other, + Err(LinesCodecError::Io(io::Error::other( "Frame length limit exceeded", ))) } @@ -203,8 +201,7 @@ impl OctetCountingDecoder { (State::NotDiscarding, Some(newline_pos), _) => { // Beyond maximum length, advance to the newline. src.advance(newline_pos + 1); - Err(LinesCodecError::Io(io::Error::new( - io::ErrorKind::Other, + Err(LinesCodecError::Io(io::Error::other( "Frame length limit exceeded", ))) } @@ -392,7 +389,7 @@ mod tests { buffer.put(&b"defghijklmnopqrstuvwxyzand here we are"[..]); let result = decoder.decode(&mut buffer); - println!("{:?}", result); + println!("{result:?}"); assert!(result.is_err()); assert_eq!(b"and here we are"[..], buffer); } diff --git a/lib/codecs/src/decoding/mod.rs b/lib/codecs/src/decoding/mod.rs index 0e14722dd3430..cf5f479d3c600 100644 --- a/lib/codecs/src/decoding/mod.rs +++ b/lib/codecs/src/decoding/mod.rs @@ -51,8 +51,8 @@ pub enum Error { impl std::fmt::Display for Error { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Self::FramingError(error) => write!(formatter, "FramingError({})", error), - Self::ParsingError(error) => write!(formatter, "ParsingError({})", error), + Self::FramingError(error) => write!(formatter, "FramingError({error})"), + Self::ParsingError(error) => write!(formatter, "ParsingError({error})"), } } } @@ -205,11 +205,11 @@ impl tokio_util::codec::Decoder for Framer { } } -/// Deserializer configuration. +/// Configures how events are decoded from raw bytes. Note some decoders can also determine the event output +/// type (log, metric, trace). #[configurable_component] #[derive(Clone, Debug)] #[serde(tag = "codec", rename_all = "snake_case")] -#[configurable(description = "Configures how events are decoded from raw bytes.")] #[configurable(metadata(docs::enum_tag_description = "The codec to use for decoding events."))] pub enum DeserializerConfig { /// Uses the raw bytes as-is. @@ -237,6 +237,8 @@ pub enum DeserializerConfig { /// Decodes the raw bytes as [native Protocol Buffers format][vector_native_protobuf]. /// + /// This decoder can output all types of events (logs, metrics, traces). + /// /// This codec is **[experimental][experimental]**. /// /// [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto @@ -245,6 +247,8 @@ pub enum DeserializerConfig { /// Decodes the raw bytes as [native JSON format][vector_native_json]. /// + /// This decoder can output all types of events (logs, metrics, traces). + /// /// This codec is **[experimental][experimental]**. /// /// [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue @@ -256,7 +260,7 @@ pub enum DeserializerConfig { /// This codec is experimental for the following reason: /// /// The GELF specification is more strict than the actual Graylog receiver. - /// Vector's decoder currently adheres more strictly to the GELF spec, with + /// Vector's decoder adheres more strictly to the GELF spec, with /// the exception that some characters such as `@` are allowed in field names. /// /// Other GELF codecs such as Loki's, use a [Go SDK][implementation] that is maintained @@ -464,6 +468,7 @@ impl DeserializerConfig { } /// Parse structured events from bytes. +#[allow(clippy::large_enum_variant)] #[derive(Clone)] pub enum Deserializer { /// Uses a `AvroDeserializer` for deserialization. diff --git a/lib/codecs/src/encoding/format/avro.rs b/lib/codecs/src/encoding/format/avro.rs index bfac175f18c21..c06bdee2294a2 100644 --- a/lib/codecs/src/encoding/format/avro.rs +++ b/lib/codecs/src/encoding/format/avro.rs @@ -23,7 +23,7 @@ impl AvroSerializerConfig { /// Build the `AvroSerializer` from this configuration. pub fn build(&self) -> Result { let schema = apache_avro::Schema::parse_str(&self.avro.schema) - .map_err(|error| format!("Failed building Avro serializer: {}", error))?; + .map_err(|error| format!("Failed building Avro serializer: {error}"))?; Ok(AvroSerializer { schema }) } diff --git a/lib/codecs/src/encoding/format/cef.rs b/lib/codecs/src/encoding/format/cef.rs index a22ba102251f9..78056e03c5889 100644 --- a/lib/codecs/src/encoding/format/cef.rs +++ b/lib/codecs/src/encoding/format/cef.rs @@ -321,7 +321,7 @@ impl Encoder for CefSerializer { continue; } let value = escape_extension(&value); - formatted_extensions.push(format!("{}={}", extension, value)); + formatted_extensions.push(format!("{extension}={value}")); } buffer.write_fmt(format_args!( @@ -368,7 +368,7 @@ fn escape_extension(s: &str) -> String { fn escape_special_chars(s: &str, extra_char: char) -> String { s.replace('\\', r#"\\"#) - .replace(extra_char, &format!(r#"\{}"#, extra_char)) + .replace(extra_char, &format!(r#"\{extra_char}"#)) } fn validate_length(field: &str, field_name: &str, max_length: usize) -> Result { diff --git a/lib/codecs/src/encoding/format/gelf.rs b/lib/codecs/src/encoding/format/gelf.rs index be5c861548822..ddd08e7d151f4 100644 --- a/lib/codecs/src/encoding/format/gelf.rs +++ b/lib/codecs/src/encoding/format/gelf.rs @@ -25,6 +25,12 @@ use vector_core::{ pub enum GelfSerializerError { #[snafu(display(r#"LogEvent does not contain required field: "{}""#, field))] MissingField { field: KeyString }, + #[snafu(display( + r#"LogEvent contains field with invalid name not matching pattern '{}': "{}""#, + pattern, + field, + ))] + InvalidField { field: KeyString, pattern: String }, #[snafu(display( r#"LogEvent contains a value with an invalid type. field = "{}" type = "{}" expected type = "{}""#, field, @@ -190,8 +196,9 @@ fn coerce_field_names_and_values( _ => { // additional fields must be only word chars, dashes and periods. if !VALID_FIELD_REGEX.is_match(field) { - return MissingFieldSnafu { + return InvalidFieldSnafu { field: field.clone(), + pattern: VALID_FIELD_REGEX.to_string(), } .fail() .map_err(|e| e.to_string().into()); diff --git a/lib/codecs/src/encoding/format/protobuf.rs b/lib/codecs/src/encoding/format/protobuf.rs index c014a529455e1..9d65ede9f7f60 100644 --- a/lib/codecs/src/encoding/format/protobuf.rs +++ b/lib/codecs/src/encoding/format/protobuf.rs @@ -30,7 +30,7 @@ impl ProtobufSerializerConfig { /// The data type of events that are accepted by `ProtobufSerializer`. pub fn input_type(&self) -> DataType { - DataType::Log + DataType::all_bits() } /// The schema required by the serializer. diff --git a/lib/codecs/src/encoding/mod.rs b/lib/codecs/src/encoding/mod.rs index 411f3f692515f..b1c900336aaa8 100644 --- a/lib/codecs/src/encoding/mod.rs +++ b/lib/codecs/src/encoding/mod.rs @@ -39,8 +39,8 @@ pub enum Error { impl std::fmt::Display for Error { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Self::FramingError(error) => write!(formatter, "FramingError({})", error), - Self::SerializingError(error) => write!(formatter, "SerializingError({})", error), + Self::FramingError(error) => write!(formatter, "FramingError({error})"), + Self::SerializingError(error) => write!(formatter, "SerializingError({error})"), } } } @@ -397,7 +397,7 @@ impl SerializerConfig { } SerializerConfig::Cef(config) => config.input_type(), SerializerConfig::Csv(config) => config.input_type(), - SerializerConfig::Gelf { .. } => GelfSerializerConfig::input_type(), + SerializerConfig::Gelf => GelfSerializerConfig::input_type(), SerializerConfig::Json(config) => config.input_type(), SerializerConfig::Logfmt => LogfmtSerializerConfig.input_type(), SerializerConfig::Native => NativeSerializerConfig.input_type(), @@ -416,7 +416,7 @@ impl SerializerConfig { } SerializerConfig::Cef(config) => config.schema_requirement(), SerializerConfig::Csv(config) => config.schema_requirement(), - SerializerConfig::Gelf { .. } => GelfSerializerConfig::schema_requirement(), + SerializerConfig::Gelf => GelfSerializerConfig::schema_requirement(), SerializerConfig::Json(config) => config.schema_requirement(), SerializerConfig::Logfmt => LogfmtSerializerConfig.schema_requirement(), SerializerConfig::Native => NativeSerializerConfig.schema_requirement(), diff --git a/lib/codecs/tests/native.rs b/lib/codecs/tests/native.rs index c39a02ea5fe95..e145508bbe1f4 100644 --- a/lib/codecs/tests/native.rs +++ b/lib/codecs/tests/native.rs @@ -314,7 +314,7 @@ fn rebuild_fixtures(proto: &str, deserializer: &dyn Deserializer, serializer: &m .into_iter() .collect(); let mut out = File::create(&new_path).unwrap_or_else(|error| { - panic!("Could not create rebuilt file {:?}: {:?}", new_path, error) + panic!("Could not create rebuilt file {new_path:?}: {error:?}") }); out.write_all(&buf).expect("Could not write rebuilt data"); out.flush().expect("Could not write rebuilt data"); diff --git a/lib/dnsmsg-parser/Cargo.toml b/lib/dnsmsg-parser/Cargo.toml index 4722e929c13cc..bc32ecfe35fd0 100644 --- a/lib/dnsmsg-parser/Cargo.toml +++ b/lib/dnsmsg-parser/Cargo.toml @@ -12,7 +12,7 @@ hickory-proto.workspace = true snafu.workspace = true [dev-dependencies] -criterion = "0.5" +criterion = "0.7" [lib] bench = false diff --git a/lib/dnsmsg-parser/src/dns_message_parser.rs b/lib/dnsmsg-parser/src/dns_message_parser.rs index f7d6ba033dc45..34d6a740d6de8 100644 --- a/lib/dnsmsg-parser/src/dns_message_parser.rs +++ b/lib/dnsmsg-parser/src/dns_message_parser.rs @@ -5,11 +5,11 @@ use std::str::Utf8Error; use data_encoding::{BASE32HEX_NOPAD, BASE64, HEXUPPER}; use hickory_proto::dnssec::rdata::{DNSSECRData, CDNSKEY, CDS, DNSKEY, DS}; use hickory_proto::dnssec::SupportedAlgorithms; +use hickory_proto::rr::rdata::caa::Property; use hickory_proto::{ op::{message::Message as TrustDnsMessage, Query}, rr::{ rdata::{ - caa::Value, opt::{EdnsCode, EdnsOption}, A, AAAA, NULL, OPT, SVCB, }, @@ -241,8 +241,7 @@ impl DnsMessageParser { } for _i in 0..8 { if current_byte & 0b1000_0000 == 0b1000_0000 { - write!(port_string, "{} ", current_bit) - .expect("can always write to String"); + write!(port_string, "{current_bit} ").expect("can always write to String"); } current_byte <<= 1; current_bit += 1; @@ -263,6 +262,11 @@ impl DnsMessageParser { let mut decoder = BinDecoder::new(raw_rdata); let prefix = parse_u8(&mut decoder)?; let ipv6_address = { + if prefix > 128 { + return Err(DnsMessageParserError::SimpleError { + cause: String::from("IPV6 prefix can't be greater than 128."), + }); + } let address_length = (128 - prefix) / 8; let mut address_vec = parse_vec(&mut decoder, address_length)?; if address_vec.len() < 16 { @@ -275,10 +279,7 @@ impl DnsMessageParser { parse_ipv6_address(&mut dec)? }; let domain_name = Self::parse_domain_name(&mut decoder, &self.options)?; - Ok(( - Some(format!("{} {} {}", prefix, ipv6_address, domain_name)), - None, - )) + Ok((Some(format!("{prefix} {ipv6_address} {domain_name}")), None)) } fn parse_loc_rdata( @@ -325,8 +326,7 @@ impl DnsMessageParser { Ok(( Some(format!( - "{} {} {:.2}m {}m {}m {}m", - latitude, longitude, altitude, size, horizontal_precision, vertical_precision + "{latitude} {longitude} {altitude:.2}m {size}m {horizontal_precision}m {vertical_precision}m" )), None, )) @@ -362,12 +362,8 @@ impl DnsMessageParser { let mut dec = BinDecoder::new(&address_vec); parse_ipv6_address(&mut dec)? }; - write!( - apl_rdata, - "{}{}:{}/{}", - negation, address_family, address, prefix - ) - .expect("can always write to String"); + write!(apl_rdata, "{negation}{address_family}:{address}/{prefix}") + .expect("can always write to String"); apl_rdata.push(' '); } Ok((Some(apl_rdata.trim_end().to_string()), None)) @@ -407,7 +403,7 @@ impl DnsMessageParser { let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); let rmailbx = Self::parse_domain_name(&mut decoder, &options)?; let emailbx = Self::parse_domain_name(&mut decoder, &options)?; - Ok((Some(format!("{} {}", rmailbx, emailbx)), None)) + Ok((Some(format!("{rmailbx} {emailbx}")), None)) } dns_message::RTYPE_RP => { @@ -415,7 +411,7 @@ impl DnsMessageParser { let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); let mbox = Self::parse_domain_name(&mut decoder, &options)?; let txt = Self::parse_domain_name(&mut decoder, &options)?; - Ok((Some(format!("{} {}", mbox, txt)), None)) + Ok((Some(format!("{mbox} {txt}")), None)) } dns_message::RTYPE_AFSDB => { @@ -423,7 +419,7 @@ impl DnsMessageParser { let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); let subtype = parse_u16(&mut decoder)?; let hostname = Self::parse_domain_name(&mut decoder, &options)?; - Ok((Some(format!("{} {}", subtype, hostname)), None)) + Ok((Some(format!("{subtype} {hostname}")), None)) } dns_message::RTYPE_X25 => { @@ -467,7 +463,7 @@ impl DnsMessageParser { let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); let preference = parse_u16(&mut decoder)?; let intermediate_host = Self::parse_domain_name(&mut decoder, &options)?; - Ok((Some(format!("{} {}", preference, intermediate_host)), None)) + Ok((Some(format!("{preference} {intermediate_host}")), None)) } dns_message::RTYPE_NSAP => { @@ -475,7 +471,7 @@ impl DnsMessageParser { let mut decoder = BinDecoder::new(raw_rdata); let rdata_len = raw_rdata.len() as u16; let nsap_rdata = HEXUPPER.encode(&parse_vec_with_u16_len(&mut decoder, rdata_len)?); - Ok((Some(format!("0x{}", nsap_rdata)), None)) + Ok((Some(format!("0x{nsap_rdata}")), None)) } dns_message::RTYPE_PX => { @@ -484,7 +480,7 @@ impl DnsMessageParser { let preference = parse_u16(&mut decoder)?; let map822 = Self::parse_domain_name(&mut decoder, &options)?; let mapx400 = Self::parse_domain_name(&mut decoder, &options)?; - Ok((Some(format!("{} {} {}", preference, map822, mapx400)), None)) + Ok((Some(format!("{preference} {map822} {mapx400}")), None)) } dns_message::RTYPE_LOC => self.parse_loc_rdata(rdata.anything()), @@ -494,7 +490,7 @@ impl DnsMessageParser { let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); let preference = parse_u16(&mut decoder)?; let exchanger = Self::parse_domain_name(&mut decoder, &options)?; - Ok((Some(format!("{} {}", preference, exchanger)), None)) + Ok((Some(format!("{preference} {exchanger}")), None)) } dns_message::RTYPE_A6 => self.parse_a6_rdata(rdata.anything()), @@ -508,10 +504,7 @@ impl DnsMessageParser { let data_len = raw_rdata.len() as u16 - 3; let data = BASE64.encode(&parse_vec_with_u16_len(&mut decoder, data_len)?); - Ok(( - Some(format!("{} {} {} {}", meaning, coding, subcoding, data)), - None, - )) + Ok((Some(format!("{meaning} {coding} {subcoding} {data}")), None)) } dns_message::RTYPE_APL => self.parse_apl_rdata(rdata.anything()), @@ -561,7 +554,7 @@ impl DnsMessageParser { RData::CSYNC(csync) => { // Using CSYNC's formatter since not all data is exposed otherwise - let csync_rdata = format!("{}", csync); + let csync_rdata = format!("{csync}"); Ok((Some(csync_rdata), None)) } RData::MX(mx) => { @@ -628,11 +621,19 @@ impl DnsMessageParser { "{} {} \"{}\"", caa.issuer_critical() as u8, caa.tag().as_str(), - match caa.value() { - Value::Url(url) => { + match caa.tag() { + Property::Iodef => { + let url = caa.value_as_iodef().map_err(|source| { + DnsMessageParserError::TrustDnsError { source } + })?; url.as_str().to_string() } - Value::Issuer(option_name, vec_keyvalue) => { + Property::Issue | Property::IssueWild => { + let (option_name, vec_keyvalue) = + caa.value_as_issue().map_err(|source| { + DnsMessageParserError::TrustDnsError { source } + })?; + let mut final_issuer = String::new(); if let Some(name) = option_name { final_issuer.push_str(&name.to_string_with_options(&self.options)); @@ -645,9 +646,14 @@ impl DnsMessageParser { } final_issuer.trim_end().to_string() } - Value::Unknown(unknown) => std::str::from_utf8(unknown) - .map_err(|source| DnsMessageParserError::Utf8ParsingError { source })? - .to_string(), + Property::Unknown(_) => { + let unknown = caa.raw_value(); + std::str::from_utf8(unknown) + .map_err(|source| DnsMessageParserError::Utf8ParsingError { + source, + })? + .to_string() + } } ); Ok((Some(caa_rdata), None)) @@ -833,11 +839,11 @@ impl DnsMessageParser { Ok((None, Some(rdata.anything().to_vec()))) } _ => Err(DnsMessageParserError::SimpleError { - cause: format!("Unsupported rdata {:?}", rdata), + cause: format!("Unsupported rdata {rdata:?}"), }), }, _ => Err(DnsMessageParserError::SimpleError { - cause: format!("Unsupported rdata {:?}", rdata), + cause: format!("Unsupported rdata {rdata:?}"), }), } } @@ -940,8 +946,8 @@ fn parse_response_code(rcode: u16) -> Option<&'static str> { 9 => Some("NotAuth"), // 9 NotAuth Server Not Authoritative for zone [RFC2136] 10 => Some("NotZone"), // 10 NotZone Name not contained in zone [RFC2136] // backwards compat for 4 bit ResponseCodes so far. - // 16 BADVERS Bad OPT Version [RFC6891] - 16 => Some("BADSIG"), // 16 BADSIG TSIG Signature Failure [RFC2845] + 16 => Some("BADVERS"), // 16 BADVERS Bad OPT Version [RFC6891] + // 16 BADSIG TSIG Signature Failure [RFC2845] 17 => Some("BADKEY"), // 17 BADKEY Key not recognized [RFC2845] 18 => Some("BADTIME"), // 18 BADTIME Signature out of time window [RFC2845] 19 => Some("BADMODE"), // 19 BADMODE Bad TKEY Mode [RFC2930] @@ -1037,7 +1043,7 @@ fn parse_edns_opt_dnssec_algorithms( let algorithm_names: Vec = algorithms.iter().map(|alg| alg.to_string()).collect(); EdnsOptionEntry { opt_code: Into::::into(opt_code), - opt_name: format!("{:?}", opt_code), + opt_name: format!("{opt_code:?}"), opt_data: algorithm_names.join(" "), } } @@ -1045,7 +1051,7 @@ fn parse_edns_opt_dnssec_algorithms( fn parse_edns_opt(opt_code: EdnsCode, opt_data: &[u8]) -> EdnsOptionEntry { EdnsOptionEntry { opt_code: Into::::into(opt_code), - opt_name: format!("{:?}", opt_code), + opt_name: format!("{opt_code:?}"), opt_data: BASE64.encode(opt_data), } } @@ -1054,17 +1060,14 @@ fn parse_loc_rdata_size(data: u8) -> DnsParserResult { let base = (data & 0xF0) >> 4; if base > 9 { return Err(DnsMessageParserError::SimpleError { - cause: format!("The base shouldn't be greater than 9. Base: {}", base), + cause: format!("The base shouldn't be greater than 9. Base: {base}"), }); } let exponent = data & 0x0F; if exponent > 9 { return Err(DnsMessageParserError::SimpleError { - cause: format!( - "The exponent shouldn't be greater than 9. Exponent: {}", - exponent - ), + cause: format!("The exponent shouldn't be greater than 9. Exponent: {exponent}"), }); } @@ -1280,7 +1283,7 @@ fn parse_unknown_record_type(rtype: u16) -> Option { fn format_bytes_as_hex_string(bytes: &[u8]) -> String { bytes .iter() - .map(|e| format!("{:02X}", e)) + .map(|e| format!("{e:02X}")) .collect::>() .join(".") } @@ -1440,7 +1443,7 @@ mod tests { DnsMessageParserError::SimpleError { cause: e } => { panic!("Expected TrustDnsError, got {}.", &e) } - _ => panic!("{}.", err), + _ => panic!("{err}."), } } @@ -1534,6 +1537,18 @@ mod tests { .is_err()); } + #[test] + fn test_parse_bad_prefix_value() { + // this testcase have prefix value of 160, + let raw_dns_message = "oAAAMgABAAAAAAABAAABAAAAACYAAC8BAAAAAaAAAAAAAA=="; + let raw_query_message = BASE64 + .decode(raw_dns_message.as_bytes()) + .expect("Invalid base64 encoded data."); + assert!(DnsMessageParser::new(raw_query_message) + .parse_as_query_message() + .is_err()); + } + #[test] fn test_parse_as_update_message() { let raw_dns_message = "xjUoAAABAAAAAQAAB2V4YW1wbGUDY29tAAAGAAECaDXADAD/AP8AAAAAAAA="; diff --git a/lib/dnstap-parser/src/parser.rs b/lib/dnstap-parser/src/parser.rs index 6a950c1a152b7..ce0895651a14c 100644 --- a/lib/dnstap-parser/src/parser.rs +++ b/lib/dnstap-parser/src/parser.rs @@ -165,7 +165,7 @@ impl DnstapParser { } } else { emit!(DnstapParseWarning { - error: format!("Unknown dnstap data type: {}", dnstap_data_type_id) + error: format!("Unknown dnstap data type: {dnstap_data_type_id}") }); need_raw_data = true; } @@ -236,7 +236,7 @@ impl DnstapParser { dnstap_message_type_id, dnstap_message.query_message.as_ref(), &DNSTAP_MESSAGE_REQUEST_TYPE_IDS, - ); + )?; } if let Some(response_time_sec) = dnstap_message.response_time_sec { @@ -248,7 +248,7 @@ impl DnstapParser { dnstap_message_type_id, dnstap_message.response_message.as_ref(), &DNSTAP_MESSAGE_RESPONSE_TYPE_IDS, - ); + )?; } DnstapParser::parse_dnstap_message_type( @@ -366,19 +366,37 @@ impl DnstapParser { dnstap_message_type_id: i32, message: Option<&Vec>, type_ids: &HashSet, - ) { + ) -> Result<()> { + if time_sec > i64::MAX as u64 { + return Err(Error::from("Cannot parse timestamp")); + } + let (time_in_nanosec, query_time_nsec) = match time_nsec { - Some(nsec) => (time_sec as i64 * 1_000_000_000_i64 + nsec as i64, nsec), - None => (time_sec as i64 * 1_000_000_000_i64, 0), + Some(nsec) => { + if let Some(time_in_ns) = (time_sec as i64) + .checked_mul(1_000_000_000) + .and_then(|v| v.checked_add(nsec as i64)) + { + (time_in_ns, nsec) + } else { + return Err(Error::from("Cannot parse timestamp")); + } + } + None => { + if let Some(time_in_ns) = (time_sec as i64).checked_mul(1_000_000_000) { + (time_in_ns, 0) + } else { + return Err(Error::from("Cannot parse timestamp")); + } + } }; if type_ids.contains(&dnstap_message_type_id) { DnstapParser::log_time(event, prefix.clone(), time_in_nanosec, "ns"); - let timestamp = Utc .timestamp_opt(time_sec.try_into().unwrap(), query_time_nsec) .single() - .expect("invalid timestamp"); + .ok_or("Invalid timestamp")?; if let Some(timestamp_key) = log_schema().timestamp_key() { DnstapParser::insert(event, prefix.clone(), timestamp_key, timestamp); } @@ -392,6 +410,7 @@ impl DnstapParser { "ns", ); } + Ok(()) } fn parse_dnstap_message_socket_family<'a>( @@ -418,9 +437,15 @@ impl DnstapParser { if let Some(query_address) = dnstap_message.query_address.as_ref() { let source_address = if socket_family == 1 { + if query_address.len() < 4 { + return Err(Error::from("Cannot parse query_address")); + } let address_buffer: [u8; 4] = query_address[0..4].try_into()?; IpAddr::V4(Ipv4Addr::from(address_buffer)) } else { + if query_address.len() < 16 { + return Err(Error::from("Cannot parse query_address")); + } let address_buffer: [u8; 16] = query_address[0..16].try_into()?; IpAddr::V6(Ipv6Addr::from(address_buffer)) }; @@ -444,9 +469,15 @@ impl DnstapParser { if let Some(response_address) = dnstap_message.response_address.as_ref() { let response_addr = if socket_family == 1 { + if response_address.len() < 4 { + return Err(Error::from("Cannot parse response_address")); + } let address_buffer: [u8; 4] = response_address[0..4].try_into()?; IpAddr::V4(Ipv4Addr::from(address_buffer)) } else { + if response_address.len() < 16 { + return Err(Error::from("Cannot parse response_address")); + } let address_buffer: [u8; 16] = response_address[0..16].try_into()?; IpAddr::V6(Ipv6Addr::from(address_buffer)) }; @@ -959,8 +990,7 @@ fn to_socket_family_name(socket_family: i32) -> Result<&'static str> { Ok("INET6") } else { Err(Error::from(format!( - "Unknown socket family: {}", - socket_family + "Unknown socket family: {socket_family}" ))) } } @@ -980,8 +1010,7 @@ fn to_socket_protocol_name(socket_protocol: i32) -> Result<&'static str> { Ok("DNSCryptTCP") } else { Err(Error::from(format!( - "Unknown socket protocol: {}", - socket_protocol + "Unknown socket protocol: {socket_protocol}" ))) } } @@ -1009,7 +1038,7 @@ fn to_dnstap_message_type(type_id: i32) -> String { 12 => String::from("ToolResponse"), 13 => String::from("UpdateQuery"), 14 => String::from("UpdateResponse"), - _ => format!("Unknown dnstap message type: {}", type_id), + _ => format!("Unknown dnstap message type: {type_id}"), } } @@ -1018,7 +1047,7 @@ mod tests { use super::*; use chrono::DateTime; use dnsmsg_parser::dns_message_parser::DnsParserOptions; - use std::collections::BTreeMap; + use std::{collections::BTreeMap, vec}; #[test] fn test_parse_dnstap_data_with_query_message() { @@ -1321,6 +1350,72 @@ mod tests { assert!(e.to_string().contains("Protobuf message")); } + #[test] + fn test_parse_dnstap_data_with_invalid_timestamp() { + fn test_one_timestamp_parse(time_sec: u64, time_nsec: Option) -> Result<()> { + let mut event = LogEvent::default(); + let root = owned_value_path!(); + let type_ids = HashSet::from([1]); + DnstapParser::parse_dnstap_message_time( + &mut event, &root, time_sec, time_nsec, 1, None, &type_ids, + ) + } + // okay case + assert!(test_one_timestamp_parse(1337, Some(42)).is_ok()); + // overflow in cast (u64 -> i64) + assert!(test_one_timestamp_parse(u64::MAX, Some(42)).is_err()); + assert!(test_one_timestamp_parse(u64::MAX, None).is_err()); + // overflow in multiplication + assert!(test_one_timestamp_parse(i64::MAX as u64, Some(42)).is_err()); + assert!(test_one_timestamp_parse(i64::MAX as u64, None).is_err()); + // overflow in add + assert!( + test_one_timestamp_parse((i64::MAX / 1_000_000_000) as u64, Some(u32::MAX)).is_err() + ); + // cannot be parsed by timestamp_opt + assert!(test_one_timestamp_parse(96, Some(1616928816)).is_err()); + } + + #[test] + fn test_parse_dnstap_message_socket_family_bad_addr() { + // while parsing address is optional, but in this function assume otherwise + fn test_one_input(socket_family: i32, msg: DnstapMessage) -> Result<()> { + let mut event = LogEvent::default(); + let root = owned_value_path!(); + DnstapParser::parse_dnstap_message_socket_family(&mut event, &root, socket_family, &msg) + } + // all bad cases which can panic + { + let message = DnstapMessage { + query_address: Some(vec![]), + ..Default::default() + }; + assert!(test_one_input(1, message).is_err()); + } + { + let message = DnstapMessage { + query_address: Some(vec![]), + ..Default::default() + }; + assert!(test_one_input(2, message).is_err()); + } + + { + let message = DnstapMessage { + response_address: Some(vec![]), + ..Default::default() + }; + assert!(test_one_input(1, message).is_err()); + } + { + let message = DnstapMessage { + response_address: Some(vec![]), + ..Default::default() + }; + assert!(test_one_input(2, message).is_err()); + } + } + #[test] fn test_get_socket_family_name() { assert_eq!("INET", to_socket_family_name(1).unwrap()); diff --git a/lib/dnstap-parser/src/vrl_functions/parse_dnstap.rs b/lib/dnstap-parser/src/vrl_functions/parse_dnstap.rs index ee6c5d52bc8b8..3b397478d00b8 100644 --- a/lib/dnstap-parser/src/vrl_functions/parse_dnstap.rs +++ b/lib/dnstap-parser/src/vrl_functions/parse_dnstap.rs @@ -120,7 +120,7 @@ impl Function for ParseDnstap { "questionTypeId": 6 } ], - "rcodeName": "BADSIG" + "rcodeName": "BADVERS" }, "responseAddress": "2001:502:7094::30", "responsePort": 53, @@ -295,7 +295,7 @@ mod tests { questionTypeId: 6, } ], - rcodeName: "BADSIG", + rcodeName: "BADVERS", }, responseAddress: "2001:502:7094::30", responsePort: 53, diff --git a/lib/docs-renderer/src/main.rs b/lib/docs-renderer/src/main.rs index dfd9c829d858c..ac7e8132828e4 100644 --- a/lib/docs-renderer/src/main.rs +++ b/lib/docs-renderer/src/main.rs @@ -68,8 +68,7 @@ fn main() -> Result<()> { let component_name = component_schema.component_name().to_string(); let component_schema_renderer = SchemaRenderer::new(&querier, component_schema); let rendered_component_schema = component_schema_renderer.render().context(format!( - "Failed to render the '{}' component schema.", - component_name + "Failed to render the '{component_name}' component schema." ))?; rendered_component_schemas.insert( format!("{}s/base/{}", base_component_type.as_str(), component_name), diff --git a/lib/docs-renderer/src/renderer.rs b/lib/docs-renderer/src/renderer.rs index 49065c761ae67..d95b76bf98114 100644 --- a/lib/docs-renderer/src/renderer.rs +++ b/lib/docs-renderer/src/renderer.rs @@ -300,7 +300,7 @@ fn render_bare_schema( // All we need to do is figure out the rendered type for the constant value, so we can // generate the right type path and stick the constant value in it. let rendered_const_type = get_rendered_value_type(&schema, const_value)?; - let const_type_path = format!("/type/{}/const", rendered_const_type); + let const_type_path = format!("/type/{rendered_const_type}/const"); data.write(const_type_path.as_str(), const_value.clone()); } SchemaType::Enum(enum_values) => { @@ -417,7 +417,7 @@ fn render_schema_description(schema: T) -> Result Ok(None), (None, Some(description)) => Ok(Some(description.trim().to_string())), (Some(title), Some(description)) => { - let concatenated = format!("{}\n\n{}", title, description); + let concatenated = format!("{title}\n\n{description}"); Ok(Some(concatenated.trim().to_string())) } } diff --git a/lib/enrichment/Cargo.toml b/lib/enrichment/Cargo.toml index d03d71e607046..d6939de55ee95 100644 --- a/lib/enrichment/Cargo.toml +++ b/lib/enrichment/Cargo.toml @@ -8,5 +8,5 @@ publish = false [dependencies] arc-swap = { version = "1.7.1", default-features = false } chrono.workspace = true -dyn-clone = { version = "1.0.19", default-features = false } +dyn-clone = { version = "1.0.20", default-features = false } vrl.workspace = true diff --git a/lib/enrichment/src/find_enrichment_table_records.rs b/lib/enrichment/src/find_enrichment_table_records.rs index f2ec4313b60d5..03540d8384137 100644 --- a/lib/enrichment/src/find_enrichment_table_records.rs +++ b/lib/enrichment/src/find_enrichment_table_records.rs @@ -12,6 +12,7 @@ fn find_enrichment_table_records( enrichment_tables: &TableSearch, table: &str, case_sensitive: Case, + wildcard: Option, condition: &[Condition], index: Option, ) -> Resolved { @@ -34,6 +35,7 @@ fn find_enrichment_table_records( case_sensitive, condition, select.as_ref().map(|select| select.as_ref()), + wildcard.as_ref(), index, )? .into_iter() @@ -71,6 +73,11 @@ impl Function for FindEnrichmentTableRecords { kind: kind::BOOLEAN, required: false, }, + Parameter { + keyword: "wildcard", + kind: kind::BYTES, + required: false, + }, ] } @@ -112,6 +119,7 @@ impl Function for FindEnrichmentTableRecords { let select = arguments.optional("select"); let case_sensitive = is_case_sensitive(&arguments, state)?; + let wildcard = arguments.optional("wildcard"); let index = Some( add_index(registry, &table, case_sensitive, &condition) .map_err(|err| Box::new(err) as Box<_>)?, @@ -123,6 +131,7 @@ impl Function for FindEnrichmentTableRecords { index, select, case_sensitive, + wildcard, enrichment_tables: registry.as_readonly(), } .as_expr()) @@ -136,6 +145,7 @@ pub struct FindEnrichmentTableRecordsFn { index: Option, select: Option>, case_sensitive: Case, + wildcard: Option>, enrichment_tables: TableSearch, } @@ -158,6 +168,11 @@ impl FunctionExpression for FindEnrichmentTableRecordsFn { let table = &self.table; let case_sensitive = self.case_sensitive; + let wildcard = self + .wildcard + .as_ref() + .map(|array| array.resolve(ctx)) + .transpose()?; let index = self.index; let enrichment_tables = &self.enrichment_tables; @@ -166,6 +181,7 @@ impl FunctionExpression for FindEnrichmentTableRecordsFn { enrichment_tables, table, case_sensitive, + wildcard, &condition, index, ) @@ -199,6 +215,7 @@ mod tests { index: Some(IndexHandle(999)), select: None, case_sensitive: Case::Sensitive, + wildcard: None, enrichment_tables: registry.as_readonly(), }; diff --git a/lib/enrichment/src/get_enrichment_table_record.rs b/lib/enrichment/src/get_enrichment_table_record.rs index ddd99c99c804b..fea1d77be9ed2 100644 --- a/lib/enrichment/src/get_enrichment_table_record.rs +++ b/lib/enrichment/src/get_enrichment_table_record.rs @@ -12,6 +12,7 @@ fn get_enrichment_table_record( enrichment_tables: &TableSearch, table: &str, case_sensitive: Case, + wildcard: Option, condition: &[Condition], index: Option, ) -> Resolved { @@ -27,11 +28,13 @@ fn get_enrichment_table_record( }), }) .transpose()?; + let data = enrichment_tables.find_table_row( table, case_sensitive, condition, select.as_ref().map(|select| select.as_ref()), + wildcard.as_ref(), index, )?; @@ -67,6 +70,11 @@ impl Function for GetEnrichmentTableRecord { kind: kind::BOOLEAN, required: false, }, + Parameter { + keyword: "wildcard", + kind: kind::BYTES, + required: false, + }, ] } @@ -104,6 +112,7 @@ impl Function for GetEnrichmentTableRecord { let select = arguments.optional("select"); let case_sensitive = is_case_sensitive(&arguments, state)?; + let wildcard = arguments.optional("wildcard"); let index = Some( add_index(registry, &table, case_sensitive, &condition) .map_err(|err| Box::new(err) as Box<_>)?, @@ -115,6 +124,7 @@ impl Function for GetEnrichmentTableRecord { index, select, case_sensitive, + wildcard, enrichment_tables: registry.as_readonly(), } .as_expr()) @@ -127,6 +137,7 @@ pub struct GetEnrichmentTableRecordFn { condition: BTreeMap, index: Option, select: Option>, + wildcard: Option>, case_sensitive: Case, enrichment_tables: TableSearch, } @@ -150,6 +161,11 @@ impl FunctionExpression for GetEnrichmentTableRecordFn { let table = &self.table; let case_sensitive = self.case_sensitive; + let wildcard = self + .wildcard + .as_ref() + .map(|array| array.resolve(ctx)) + .transpose()?; let index = self.index; let enrichment_tables = &self.enrichment_tables; @@ -158,6 +174,7 @@ impl FunctionExpression for GetEnrichmentTableRecordFn { enrichment_tables, table, case_sensitive, + wildcard, &condition, index, ) @@ -191,6 +208,7 @@ mod tests { index: Some(IndexHandle(999)), select: None, case_sensitive: Case::Sensitive, + wildcard: None, enrichment_tables: registry.as_readonly(), }; diff --git a/lib/enrichment/src/lib.rs b/lib/enrichment/src/lib.rs index 52f2a547f50f1..ce02e5f8f8436 100644 --- a/lib/enrichment/src/lib.rs +++ b/lib/enrichment/src/lib.rs @@ -26,6 +26,16 @@ pub enum Condition<'a> { from: chrono::DateTime, to: chrono::DateTime, }, + /// The date in the field is greater than or equal to `from`. + FromDate { + field: &'a str, + from: chrono::DateTime, + }, + /// The date in the field is less than or equal to `to`. + ToDate { + field: &'a str, + to: chrono::DateTime, + }, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -47,6 +57,7 @@ pub trait Table: DynClone { case: Case, condition: &'a [Condition<'a>], select: Option<&[String]>, + wildcard: Option<&Value>, index: Option, ) -> Result; @@ -58,6 +69,7 @@ pub trait Table: DynClone { case: Case, condition: &'a [Condition<'a>], select: Option<&[String]>, + wildcard: Option<&Value>, index: Option, ) -> Result, String>; diff --git a/lib/enrichment/src/tables.rs b/lib/enrichment/src/tables.rs index 522d77c1c2b78..d4ccb4b0f99bc 100644 --- a/lib/enrichment/src/tables.rs +++ b/lib/enrichment/src/tables.rs @@ -35,7 +35,7 @@ use std::{ }; use arc_swap::ArcSwap; -use vrl::value::ObjectMap; +use vrl::value::{ObjectMap, Value}; use super::{Condition, IndexHandle, Table}; use crate::Case; @@ -134,7 +134,7 @@ impl TableRegistry { pub fn table_ids(&self) -> Vec { let locked = self.loading.lock().unwrap(); match *locked { - Some(ref tables) => tables.iter().map(|(key, _)| key.clone()).collect(), + Some(ref tables) => tables.keys().cloned().collect(), None => Vec::new(), } } @@ -157,7 +157,7 @@ impl TableRegistry { match *locked { None => Err("finish_load has been called".to_string()), Some(ref mut tables) => match tables.get_mut(table) { - None => Err(format!("table '{}' not loaded", table)), + None => Err(format!("table '{table}' not loaded")), Some(table) => table.add_index(case, fields), }, } @@ -216,13 +216,14 @@ impl TableSearch { case: Case, condition: &'a [Condition<'a>], select: Option<&[String]>, + wildcard: Option<&Value>, index: Option, ) -> Result { let tables = self.0.load(); if let Some(ref tables) = **tables { match tables.get(table) { - None => Err(format!("table {} not loaded", table)), - Some(table) => table.find_table_row(case, condition, select, index), + None => Err(format!("table {table} not loaded")), + Some(table) => table.find_table_row(case, condition, select, wildcard, index), } } else { Err("finish_load not called".to_string()) @@ -238,13 +239,14 @@ impl TableSearch { case: Case, condition: &'a [Condition<'a>], select: Option<&[String]>, + wildcard: Option<&Value>, index: Option, ) -> Result, String> { let tables = self.0.load(); if let Some(ref tables) = **tables { match tables.get(table) { - None => Err(format!("table {} not loaded", table)), - Some(table) => table.find_table_rows(case, condition, select, index), + None => Err(format!("table {table} not loaded")), + Some(table) => table.find_table_rows(case, condition, select, wildcard, index), } } else { Err("finish_load not called".to_string()) @@ -276,9 +278,9 @@ fn fmt_enrichment_table( tables.truncate(std::cmp::max(tables.len(), 0)); tables.push(')'); - write!(f, "{} {}", name, tables) + write!(f, "{name} {tables}") } - None => write!(f, "{} loading", name), + None => write!(f, "{name} loading"), } } @@ -337,6 +339,7 @@ mod tests { value: Value::from("thang"), }], None, + None, None ) ); @@ -378,6 +381,7 @@ mod tests { value: Value::from("thang"), }], None, + None, None ) ); @@ -442,7 +446,7 @@ mod tests { tables .get("dummy1") .unwrap() - .find_table_row(Case::Sensitive, &Vec::new(), None, None) + .find_table_row(Case::Sensitive, &Vec::new(), None, None, None) .unwrap() .get("field") .cloned() @@ -455,7 +459,7 @@ mod tests { tables .get("dummy2") .unwrap() - .find_table_row(Case::Sensitive, &Vec::new(), None, None) + .find_table_row(Case::Sensitive, &Vec::new(), None, None, None) .unwrap() .get("thing") .cloned() diff --git a/lib/enrichment/src/test_util.rs b/lib/enrichment/src/test_util.rs index 1eed09e200ca6..f56b3342ddaf4 100644 --- a/lib/enrichment/src/test_util.rs +++ b/lib/enrichment/src/test_util.rs @@ -39,6 +39,7 @@ impl Table for DummyEnrichmentTable { _case: Case, _condition: &[Condition], _select: Option<&[String]>, + _wildcard: Option<&Value>, _index: Option, ) -> Result { Ok(self.data.clone()) @@ -49,6 +50,7 @@ impl Table for DummyEnrichmentTable { _case: Case, _condition: &[Condition], _select: Option<&[String]>, + _wildcard: Option<&Value>, _index: Option, ) -> Result, String> { Ok(vec![self.data.clone()]) diff --git a/lib/enrichment/src/vrl_util.rs b/lib/enrichment/src/vrl_util.rs index 61e7044c53c99..0beb948cc435b 100644 --- a/lib/enrichment/src/vrl_util.rs +++ b/lib/enrichment/src/vrl_util.rs @@ -55,6 +55,22 @@ pub(crate) fn evaluate_condition(key: &str, value: Value) -> ExpressionResult Condition::FromDate { + field: key, + from: *map + .get("from") + .expect("should contain from") + .as_timestamp() + .ok_or("from in condition must be a timestamp")?, + }, + Value::Object(map) if map.contains_key("to") => Condition::ToDate { + field: key, + to: *map + .get("to") + .expect("should contain to") + .as_timestamp() + .ok_or("to in condition must be a timestamp")?, + }, _ => Condition::Equals { field: key, value }, }) } @@ -71,7 +87,12 @@ pub(crate) fn add_index( .filter_map(|(field, value)| match value { expression::Expr::Container(expression::Container { variant: expression::Variant::Object(map), - }) if map.contains_key("from") && map.contains_key("to") => None, + }) if (map.contains_key("from") && map.contains_key("to")) + || map.contains_key("from") + || map.contains_key("to") => + { + None + } _ => Some(field.as_ref()), }) .collect::>(); @@ -80,6 +101,7 @@ pub(crate) fn add_index( Ok(index) } +#[allow(clippy::result_large_err)] pub(crate) fn is_case_sensitive( arguments: &ArgumentList, state: &TypeState, diff --git a/lib/file-source/Cargo.toml b/lib/file-source/Cargo.toml index f9c5c178d5651..36c1addc6cb2f 100644 --- a/lib/file-source/Cargo.toml +++ b/lib/file-source/Cargo.toml @@ -11,7 +11,7 @@ libc = "0.2" winapi = { version = "0.3", features = ["winioctl"] } [dependencies] -crc = "3.2.1" +crc = "3.3.0" glob.workspace = true scan_fmt = "0.2.6" vector-common = { path = "../vector-common", default-features = false } @@ -38,12 +38,12 @@ default-features = false features = [] [dependencies.indexmap] -version = "2.9.0" +version = "2.10.0" default-features = false features = ["serde"] [dependencies.flate2] -version = "1.0" +version = "1.1" default-features = false features = ["rust_backend"] @@ -68,12 +68,12 @@ default-features = false features = [] [dependencies.tokio] -version = "1.44.2" +version = "1.45.1" default-features = false features = ["full"] [dev-dependencies] -criterion = "0.5" +criterion = "0.7" quickcheck = "1" tempfile.workspace = true similar-asserts = "1.7.0" diff --git a/lib/file-source/src/buffer.rs b/lib/file-source/src/buffer.rs index c8dbe1f400905..55dd481334e1d 100644 --- a/lib/file-source/src/buffer.rs +++ b/lib/file-source/src/buffer.rs @@ -1,11 +1,18 @@ -use std::io::{self, BufRead}; +use std::{ + cmp::min, + io::{self, BufRead}, +}; use bstr::Finder; use bytes::BytesMut; -use tracing::warn; use crate::FilePosition; +pub struct ReadResult { + pub successfully_read: Option, + pub discarded_for_size_and_truncated: Vec, +} + /// Read up to `max_size` bytes from `reader`, splitting by `delim` /// /// The function reads up to `max_size` bytes from `reader`, splitting the input @@ -29,17 +36,18 @@ use crate::FilePosition; /// Benchmarks indicate that this function processes in the high single-digit /// GiB/s range for buffers of length 1KiB. For buffers any smaller than this /// the overhead of setup dominates our benchmarks. -pub fn read_until_with_max_size( - reader: &mut R, - position: &mut FilePosition, - delim: &[u8], - buf: &mut BytesMut, +pub fn read_until_with_max_size<'a, R: BufRead + ?Sized>( + reader: &'a mut R, + position: &'a mut FilePosition, + delim: &'a [u8], + buf: &'a mut BytesMut, max_size: usize, -) -> io::Result> { +) -> io::Result { let mut total_read = 0; let mut discarding = false; let delim_finder = Finder::new(delim); let delim_len = delim.len(); + let mut discarded_for_size_and_truncated = Vec::new(); loop { let available: &[u8] = match reader.fill_buf() { Ok(n) => n, @@ -68,16 +76,20 @@ pub fn read_until_with_max_size( total_read += used; if !discarding && buf.len() > max_size { - warn!( - message = "Found line that exceeds max_line_bytes; discarding.", - internal_log_rate_limit = true - ); + // keep only the first <1k bytes to make sure we can actually emit a usable error + let length_to_keep = min(1000, max_size); + let mut truncated: BytesMut = BytesMut::zeroed(length_to_keep); + truncated.copy_from_slice(&buf[0..length_to_keep]); + discarded_for_size_and_truncated.push(truncated); discarding = true; } if done { if !discarding { - return Ok(Some(total_read)); + return Ok(ReadResult { + successfully_read: Some(total_read), + discarded_for_size_and_truncated, + }); } else { discarding = false; buf.clear(); @@ -87,7 +99,10 @@ pub fn read_until_with_max_size( // us to observe an incomplete write. We return None here and let the loop continue // next time the method is called. This is safe because the buffer is specific to this // FileWatcher. - return Ok(None); + return Ok(ReadResult { + successfully_read: None, + discarded_for_size_and_truncated, + }); } } } @@ -99,6 +114,8 @@ mod test { use bytes::{BufMut, BytesMut}; use quickcheck::{QuickCheck, TestResult}; + use crate::buffer::ReadResult; + use super::read_until_with_max_size; fn qc_inner(chunks: Vec>, delim: u8, max_size: NonZeroU8) -> TestResult { @@ -181,7 +198,10 @@ mod test { ) .unwrap() { - None => { + ReadResult { + successfully_read: None, + discarded_for_size_and_truncated: _, + } => { // Subject only returns None if this is the last chunk _and_ // the chunk did not contain a delimiter _or_ the delimiter // was outside the max_size range _or_ the current chunk is empty. @@ -190,7 +210,10 @@ mod test { .any(|details| ((details.chunk_index == idx) && details.within_max_size)); assert!(chunk.is_empty() || !has_valid_delimiter) } - Some(total_read) => { + ReadResult { + successfully_read: Some(total_read), + discarded_for_size_and_truncated: _, + } => { // Now that the function has returned we confirm that the // returned details match our `first_delim` and also that // the `buffer` is populated correctly. diff --git a/lib/file-source/src/checkpointer.rs b/lib/file-source/src/checkpointer.rs index a7eaea307aacf..3f569a55b7423 100644 --- a/lib/file-source/src/checkpointer.rs +++ b/lib/file-source/src/checkpointer.rs @@ -229,10 +229,10 @@ impl Checkpointer { use FileFingerprint::*; let path = match fng { - BytesChecksum(c) => format!("g{:x}.{}", c, pos), - FirstLinesChecksum(c) => format!("h{:x}.{}", c, pos), - DevInode(dev, ino) => format!("i{:x}.{:x}.{}", dev, ino, pos), - Unknown(x) => format!("{:x}.{}", x, pos), + BytesChecksum(c) => format!("g{c:x}.{pos}"), + FirstLinesChecksum(c) => format!("h{c:x}.{pos}"), + DevInode(dev, ino) => format!("i{dev:x}.{ino:x}.{pos}"), + Unknown(x) => format!("{x:x}.{pos}"), }; self.directory.join(path) } diff --git a/lib/file-source/src/file_server.rs b/lib/file-source/src/file_server.rs index 1dfc6ebd08004..ebb1867339242 100644 --- a/lib/file-source/src/file_server.rs +++ b/lib/file-source/src/file_server.rs @@ -19,7 +19,7 @@ use tracing::{debug, error, info, trace}; use crate::{ checkpointer::{Checkpointer, CheckpointsView}, - file_watcher::FileWatcher, + file_watcher::{FileWatcher, RawLineResult}, fingerprinter::{FileFingerprint, Fingerprinter}, paths_provider::PathsProvider, FileSourceInternalEvents, ReadFrom, @@ -263,7 +263,19 @@ where let start = time::Instant::now(); let mut bytes_read: usize = 0; - while let Ok(Some(line)) = watcher.read_line() { + while let Ok(RawLineResult { + raw_line: Some(line), + discarded_for_size_and_truncated, + }) = watcher.read_line() + { + discarded_for_size_and_truncated.iter().for_each(|buf| { + self.emitter.emit_file_line_too_long( + &buf.clone(), + self.max_line_bytes, + buf.len(), + ) + }); + let sz = line.bytes.len(); trace!( message = "Read bytes.", diff --git a/lib/file-source/src/file_watcher/mod.rs b/lib/file-source/src/file_watcher/mod.rs index 80db2b9bc876c..a8d4fd0f81641 100644 --- a/lib/file-source/src/file_watcher/mod.rs +++ b/lib/file-source/src/file_watcher/mod.rs @@ -12,7 +12,9 @@ use tracing::debug; use vector_common::constants::GZIP_MAGIC; use crate::{ - buffer::read_until_with_max_size, metadata_ext::PortableFileExt, FilePosition, ReadFrom, + buffer::{read_until_with_max_size, ReadResult}, + metadata_ext::PortableFileExt, + FilePosition, ReadFrom, }; #[cfg(test)] mod tests; @@ -28,6 +30,12 @@ pub(super) struct RawLine { pub bytes: Bytes, } +#[derive(Debug)] +pub struct RawLineResult { + pub raw_line: Option, + pub discarded_for_size_and_truncated: Vec, +} + /// The `FileWatcher` struct defines the polling based state machine which reads /// from a file path, transparently updating the underlying file descriptor when /// the file has been rolled over, as is common for logs. @@ -207,7 +215,7 @@ impl FileWatcher { /// This function will attempt to read a new line from its file, blocking, /// up to some maximum but unspecified amount of time. `read_line` will open /// a new file handler as needed, transparently to the caller. - pub(super) fn read_line(&mut self) -> io::Result> { + pub(super) fn read_line(&mut self) -> io::Result { self.track_read_attempt(); let reader = &mut self.reader; @@ -220,14 +228,23 @@ impl FileWatcher { &mut self.buf, self.max_line_bytes, ) { - Ok(Some(_)) => { + Ok(ReadResult { + successfully_read: Some(_), + discarded_for_size_and_truncated, + }) => { self.track_read_success(); - Ok(Some(RawLine { - offset: initial_position, - bytes: self.buf.split().freeze(), - })) + Ok(RawLineResult { + raw_line: Some(RawLine { + offset: initial_position, + bytes: self.buf.split().freeze(), + }), + discarded_for_size_and_truncated, + }) } - Ok(None) => { + Ok(ReadResult { + successfully_read: None, + discarded_for_size_and_truncated, + }) => { if !self.file_findable() { self.set_dead(); // File has been deleted, so return what we have in the buffer, even though it @@ -237,16 +254,25 @@ impl FileWatcher { if buf.is_empty() { // EOF self.reached_eof = true; - Ok(None) + Ok(RawLineResult { + raw_line: None, + discarded_for_size_and_truncated, + }) } else { - Ok(Some(RawLine { - offset: initial_position, - bytes: buf, - })) + Ok(RawLineResult { + raw_line: Some(RawLine { + offset: initial_position, + bytes: buf, + }), + discarded_for_size_and_truncated, + }) } } else { self.reached_eof = true; - Ok(None) + Ok(RawLineResult { + raw_line: None, + discarded_for_size_and_truncated, + }) } } Err(e) => { diff --git a/lib/file-source/src/file_watcher/tests/experiment.rs b/lib/file-source/src/file_watcher/tests/experiment.rs index decdbdab98240..17bd3f5f9539d 100644 --- a/lib/file-source/src/file_watcher/tests/experiment.rs +++ b/lib/file-source/src/file_watcher/tests/experiment.rs @@ -8,7 +8,7 @@ use bytes::Bytes; use quickcheck::{QuickCheck, TestResult}; use crate::{ - file_watcher::{tests::*, FileWatcher}, + file_watcher::{tests::*, FileWatcher, RawLineResult}, ReadFrom, }; @@ -82,7 +82,7 @@ fn experiment(actions: Vec) { } FileWatcherAction::RotateFile => { let mut new_path = path.clone(); - new_path.set_extension(format!("log.{}", rotation_count)); + new_path.set_extension(format!("log.{rotation_count}")); rotation_count += 1; fs::rename(&path, &new_path).expect("could not rename"); fp = fs::File::create(&path).expect("could not create"); @@ -96,11 +96,14 @@ fn experiment(actions: Vec) { Err(_) => { unreachable!(); } - Ok(Some(line)) if line.bytes.is_empty() => { + Ok(RawLineResult { + raw_line: Some(line), + .. + }) if line.bytes.is_empty() => { attempts -= 1; continue; } - Ok(None) => { + Ok(RawLineResult { raw_line: None, .. }) => { attempts -= 1; continue; } @@ -129,7 +132,7 @@ fn file_watcher_with_truncation() { TestResult::passed() } QuickCheck::new() - .tests(10000) - .max_tests(100000) + .tests(5000) + .max_tests(50000) .quickcheck(inner as fn(Vec) -> TestResult); } diff --git a/lib/file-source/src/file_watcher/tests/experiment_no_truncations.rs b/lib/file-source/src/file_watcher/tests/experiment_no_truncations.rs index ee8a24a9f95bf..363fb6f761b9a 100644 --- a/lib/file-source/src/file_watcher/tests/experiment_no_truncations.rs +++ b/lib/file-source/src/file_watcher/tests/experiment_no_truncations.rs @@ -4,7 +4,7 @@ use bytes::Bytes; use quickcheck::{QuickCheck, TestResult}; use crate::{ - file_watcher::{tests::*, FileWatcher}, + file_watcher::{tests::*, FileWatcher, RawLineResult}, ReadFrom, }; @@ -49,7 +49,7 @@ fn experiment_no_truncations(actions: Vec) { } FileWatcherAction::RotateFile => { let mut new_path = path.clone(); - new_path.set_extension(format!("log.{}", rotation_count)); + new_path.set_extension(format!("log.{rotation_count}")); rotation_count += 1; fs::rename(&path, &new_path).expect("could not rename"); fp = fs::File::create(&path).expect("could not create"); @@ -63,17 +63,23 @@ fn experiment_no_truncations(actions: Vec) { Err(_) => { unreachable!(); } - Ok(Some(line)) if line.bytes.is_empty() => { + Ok(RawLineResult { + raw_line: Some(line), + .. + }) if line.bytes.is_empty() => { attempts -= 1; assert!(fwfiles[read_index].read_line().is_none()); continue; } - Ok(None) => { + Ok(RawLineResult { raw_line: None, .. }) => { attempts -= 1; assert!(fwfiles[read_index].read_line().is_none()); continue; } - Ok(Some(line)) => { + Ok(RawLineResult { + raw_line: Some(line), + .. + }) => { let exp = fwfiles[read_index].read_line().expect("could not readline"); assert_eq!(exp.into_bytes(), line.bytes); // assert_eq!(sz, buf.len() + 1); @@ -93,7 +99,7 @@ fn file_watcher_no_truncation() { TestResult::passed() } QuickCheck::new() - .tests(10000) - .max_tests(100000) + .tests(5000) + .max_tests(50000) .quickcheck(inner as fn(Vec) -> TestResult); } diff --git a/lib/file-source/src/fingerprinter.rs b/lib/file-source/src/fingerprinter.rs index b176f6e9964ac..a7c339d249760 100644 --- a/lib/file-source/src/fingerprinter.rs +++ b/lib/file-source/src/fingerprinter.rs @@ -392,6 +392,7 @@ mod test { time::Duration, }; + use bytes::BytesMut; use flate2::write::GzEncoder; use tempfile::{tempdir, TempDir}; @@ -813,5 +814,7 @@ mod test { fn emit_files_open(&self, _: usize) {} fn emit_path_globbing_failed(&self, _: &Path, _: &Error) {} + + fn emit_file_line_too_long(&self, _: &BytesMut, _: usize, _: usize) {} } } diff --git a/lib/file-source/src/internal_events.rs b/lib/file-source/src/internal_events.rs index 9eb60e65397a1..3077a51d8cab9 100644 --- a/lib/file-source/src/internal_events.rs +++ b/lib/file-source/src/internal_events.rs @@ -1,5 +1,7 @@ use std::{io::Error, path::Path, time::Duration}; +use bytes::BytesMut; + /// Every internal event in this crate has a corresponding /// method in this trait which should emit the event. pub trait FileSourceInternalEvents: Send + Sync + Clone + 'static { @@ -26,4 +28,11 @@ pub trait FileSourceInternalEvents: Send + Sync + Clone + 'static { fn emit_files_open(&self, count: usize); fn emit_path_globbing_failed(&self, path: &Path, error: &Error); + + fn emit_file_line_too_long( + &self, + truncated_bytes: &BytesMut, + configured_limit: usize, + encountered_size_so_far: usize, + ); } diff --git a/lib/k8s-e2e-tests/Cargo.toml b/lib/k8s-e2e-tests/Cargo.toml index 954675e9b3b76..fc5339e054e79 100644 --- a/lib/k8s-e2e-tests/Cargo.toml +++ b/lib/k8s-e2e-tests/Cargo.toml @@ -14,7 +14,7 @@ k8s-test-framework = { version = "0.1", path = "../k8s-test-framework" } regex = "1" reqwest = { version = "0.11.26", features = ["json"] } serde_json.workspace = true -tokio = { version = "1.44.2", features = ["full"] } +tokio = { version = "1.45.1", features = ["full"] } indoc.workspace = true env_logger = "0.11" tracing = { version = "0.1", features = ["log"] } diff --git a/lib/k8s-e2e-tests/src/lib.rs b/lib/k8s-e2e-tests/src/lib.rs index fc319adee0fcb..4c0ddad797676 100644 --- a/lib/k8s-e2e-tests/src/lib.rs +++ b/lib/k8s-e2e-tests/src/lib.rs @@ -27,17 +27,17 @@ pub fn get_namespace() -> String { // There is a 36 ^ 5 chance of a name collision, which is likely to be an acceptable risk. let id = Alphanumeric.sample_string(&mut rng(), 5).to_lowercase(); - format!("vector-{}", id) + format!("vector-{id}") } pub fn get_namespace_appended(namespace: &str, suffix: &str) -> String { - format!("{}-{}", namespace, suffix) + format!("{namespace}-{suffix}") } /// Gets a name we can use for roles to prevent them conflicting with other tests. /// Uses the provided namespace as the root. pub fn get_override_name(namespace: &str, suffix: &str) -> String { - format!("{}-{}", namespace, suffix) + format!("{namespace}-{suffix}") } /// Is the MULTINODE environment variable set? @@ -49,7 +49,7 @@ pub fn is_multinode() -> bool { /// to be run against the same cluster without the role names clashing. pub fn config_override_name(name: &str, cleanup: bool) -> String { let vectordir = if is_multinode() { - format!("{}-vector", name) + format!("{name}-vector") } else { "vector".to_string() }; @@ -236,9 +236,7 @@ pub async fn smoke_check_first_line(log_reader: &mut Reader) { let expected_pat = "INFO vector::app:"; assert!( first_line.contains(expected_pat), - "Expected a line ending with {:?} but got {:?}; vector might be malfunctioning", - expected_pat, - first_line + "Expected a line ending with {expected_pat:?} but got {first_line:?}; vector might be malfunctioning" ); } diff --git a/lib/k8s-e2e-tests/src/metrics.rs b/lib/k8s-e2e-tests/src/metrics.rs index d3fd61548efd9..a815acac35de3 100644 --- a/lib/k8s-e2e-tests/src/metrics.rs +++ b/lib/k8s-e2e-tests/src/metrics.rs @@ -64,7 +64,7 @@ pub async fn get_component_sent_events_total(url: &str) -> Result Result<(), Box> { let metrics = load(url).await?; if !extract_vector_started(&metrics) { - return Err(format!("`vector_started`-ish metric was not found:\n{}", metrics).into()); + return Err(format!("`vector_started`-ish metric was not found:\n{metrics}").into()); } Ok(()) } @@ -125,7 +125,7 @@ pub async fn assert_metrics_present( required_metrics.remove(metric_name); } if !required_metrics.is_empty() { - return Err(format!("Some host metrics were not found:\n{:?}", required_metrics).into()); + return Err(format!("Some host metrics were not found:\n{required_metrics:?}").into()); } Ok(()) } @@ -217,7 +217,7 @@ mod tests { for (input, expected_value) in cases { let input = input.join("\n"); let actual_value = extract_vector_started(&input); - assert_eq!(expected_value, actual_value, "input: {}", input); + assert_eq!(expected_value, actual_value, "input: {input}"); } } } diff --git a/lib/k8s-test-framework/Cargo.toml b/lib/k8s-test-framework/Cargo.toml index fa7b47e3bf5f5..c5a4a213500a3 100644 --- a/lib/k8s-test-framework/Cargo.toml +++ b/lib/k8s-test-framework/Cargo.toml @@ -11,5 +11,5 @@ license = "MPL-2.0" k8s-openapi = { version = "0.16.0", default-features = false, features = ["v1_19"] } serde_json.workspace = true tempfile.workspace = true -tokio = { version = "1.44.2", features = ["full"] } +tokio = { version = "1.45.1", features = ["full"] } log = "0.4" diff --git a/lib/k8s-test-framework/src/helm_values_file.rs b/lib/k8s-test-framework/src/helm_values_file.rs index a539d608bf5ba..e57e8a23958a8 100644 --- a/lib/k8s-test-framework/src/helm_values_file.rs +++ b/lib/k8s-test-framework/src/helm_values_file.rs @@ -9,7 +9,7 @@ pub struct HelmValuesFile(TempFile); impl HelmValuesFile { pub fn new(data: &str) -> std::io::Result { - info!("Using values \n {}", data); + info!("Using values \n {data}"); Ok(Self(TempFile::new("values.yml", data)?)) } diff --git a/lib/k8s-test-framework/src/port_forward.rs b/lib/k8s-test-framework/src/port_forward.rs index b2acefa71f79b..457bce8169990 100644 --- a/lib/k8s-test-framework/src/port_forward.rs +++ b/lib/k8s-test-framework/src/port_forward.rs @@ -33,7 +33,7 @@ pub fn port_forward( command.arg("port-forward"); command.arg("-n").arg(namespace); command.arg(resource); - command.arg(format!("{}:{}", local_port, resource_port)); + command.arg(format!("{local_port}:{resource_port}")); command.kill_on_drop(true); diff --git a/lib/k8s-test-framework/src/reader.rs b/lib/k8s-test-framework/src/reader.rs index 0b83d5885afcd..e6af49066a8b2 100644 --- a/lib/k8s-test-framework/src/reader.rs +++ b/lib/k8s-test-framework/src/reader.rs @@ -98,7 +98,7 @@ mod tests { let mut expected_num = 0; while let Some(line) = reader.read_line().await { // Assert we're getting expected lines. - assert_eq!(line, format!("Line {}\n", expected_num)); + assert_eq!(line, format!("Line {expected_num}\n")); // On line 100 issue a `kill` to stop the infinite stream. if expected_num == 100 { diff --git a/lib/k8s-test-framework/src/temp_file.rs b/lib/k8s-test-framework/src/temp_file.rs index d1535b6b5a658..567de01031037 100644 --- a/lib/k8s-test-framework/src/temp_file.rs +++ b/lib/k8s-test-framework/src/temp_file.rs @@ -9,7 +9,7 @@ pub struct TempFile { impl TempFile { pub fn new(file_name: &str, data: &str) -> std::io::Result { - let dir = tempdir()?.into_path(); + let dir = tempdir()?.keep(); let path = dir.join(file_name); std::fs::write(&path, data)?; Ok(Self { path }) diff --git a/lib/k8s-test-framework/src/util.rs b/lib/k8s-test-framework/src/util.rs index a0499ba6fc875..67e8849809ee5 100644 --- a/lib/k8s-test-framework/src/util.rs +++ b/lib/k8s-test-framework/src/util.rs @@ -3,28 +3,28 @@ use log::info; use crate::Result; pub async fn run_command(mut command: tokio::process::Command) -> Result<()> { - info!("Running command `{:?}`", command); + info!("Running command `{command:?}`"); let exit_status = command.spawn()?.wait().await?; if !exit_status.success() { - return Err(format!("exec failed: {:?}", command).into()); + return Err(format!("exec failed: {command:?}").into()); } Ok(()) } pub fn run_command_blocking(mut command: std::process::Command) -> Result<()> { - info!("Running command blocking `{:?}`", command); + info!("Running command blocking `{command:?}`"); let exit_status = command.spawn()?.wait()?; if !exit_status.success() { - return Err(format!("exec failed: {:?}", command).into()); + return Err(format!("exec failed: {command:?}").into()); } Ok(()) } pub async fn run_command_output(mut command: tokio::process::Command) -> Result { - info!("Fetching command `{:?}`", command); + info!("Fetching command `{command:?}`"); let output = command.spawn()?.wait_with_output().await?; if !output.status.success() { - return Err(format!("exec failed: {:?}", command).into()); + return Err(format!("exec failed: {command:?}").into()); } let output = String::from_utf8(output.stdout)?; diff --git a/lib/k8s-test-framework/src/wait_for_resource.rs b/lib/k8s-test-framework/src/wait_for_resource.rs index d3cba0fd56d45..7beb0d7721efc 100644 --- a/lib/k8s-test-framework/src/wait_for_resource.rs +++ b/lib/k8s-test-framework/src/wait_for_resource.rs @@ -86,7 +86,7 @@ where command.arg("--for"); match wait_for { WaitFor::Delete => command.arg("delete"), - WaitFor::Condition(cond) => command.arg(format!("condition={}", cond)), + WaitFor::Condition(cond) => command.arg(format!("condition={cond}")), }; command.args(extra); diff --git a/lib/loki-logproto/src/lib.rs b/lib/loki-logproto/src/lib.rs index 81821dad7da9d..9aae27cf7925d 100644 --- a/lib/loki-logproto/src/lib.rs +++ b/lib/loki-logproto/src/lib.rs @@ -83,7 +83,7 @@ pub mod util { let mut labels: Vec = labels .iter() .filter(|(k, _)| !RESERVED_LABELS.contains(&k.as_str())) - .map(|(k, v)| format!("{}=\"{}\"", k, v)) + .map(|(k, v)| format!("{k}=\"{v}\"")) .collect(); labels.sort(); format!("{{{}}}", labels.join(", ")) diff --git a/lib/opentelemetry-proto/Cargo.toml b/lib/opentelemetry-proto/Cargo.toml index 76c7f9b98aeea..147d17b778067 100644 --- a/lib/opentelemetry-proto/Cargo.toml +++ b/lib/opentelemetry-proto/Cargo.toml @@ -8,6 +8,7 @@ publish = false [build-dependencies] prost-build.workspace = true tonic-build.workspace = true +glob.workspace = true [dependencies] bytes = { version = "1.10.1", default-features = false, features = ["serde"] } diff --git a/lib/opentelemetry-proto/build.rs b/lib/opentelemetry-proto/build.rs index 4e642ce8ab283..19fc3d533bbc7 100644 --- a/lib/opentelemetry-proto/build.rs +++ b/lib/opentelemetry-proto/build.rs @@ -1,22 +1,28 @@ -use std::io::Error; +use glob::glob; +use std::{env, io::Result, path::PathBuf}; + +fn main() -> Result<()> { + let proto_root = PathBuf::from("src/proto/opentelemetry-proto"); + let include_path = proto_root.clone(); + + let proto_paths: Vec<_> = glob(&format!("{}/**/*.proto", proto_root.display())) + .expect("Failed to read glob pattern") + .filter_map(|result| result.ok()) + .collect(); + + // Set up re-run triggers + for proto in &proto_paths { + println!("cargo:rerun-if-changed={}", proto.display()); + } + + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + let descriptor_path = out_dir.join("opentelemetry-proto.desc"); -fn main() -> Result<(), Error> { tonic_build::configure() .build_client(true) .build_server(true) - .compile( - &[ - "src/proto/opentelemetry-proto/opentelemetry/proto/common/v1/common.proto", - "src/proto/opentelemetry-proto/opentelemetry/proto/resource/v1/resource.proto", - "src/proto/opentelemetry-proto/opentelemetry/proto/logs/v1/logs.proto", - "src/proto/opentelemetry-proto/opentelemetry/proto/metrics/v1/metrics.proto", - "src/proto/opentelemetry-proto/opentelemetry/proto/trace/v1/trace.proto", - "src/proto/opentelemetry-proto/opentelemetry/proto/collector/trace/v1/trace_service.proto", - "src/proto/opentelemetry-proto/opentelemetry/proto/collector/logs/v1/logs_service.proto", - "src/proto/opentelemetry-proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto", - ], - &["src/proto/opentelemetry-proto"], - )?; + .file_descriptor_set_path(&descriptor_path) + .compile(&proto_paths, &[include_path])?; Ok(()) } diff --git a/lib/opentelemetry-proto/src/common.rs b/lib/opentelemetry-proto/src/common.rs index d3c061a3d7e35..d369c4e860f12 100644 --- a/lib/opentelemetry-proto/src/common.rs +++ b/lib/opentelemetry-proto/src/common.rs @@ -10,7 +10,7 @@ impl From for Value { PBValue::StringValue(v) => Value::Bytes(Bytes::from(v)), PBValue::BoolValue(v) => Value::Boolean(v), PBValue::IntValue(v) => Value::Integer(v), - PBValue::DoubleValue(v) => Value::Float(NotNan::new(v).unwrap()), + PBValue::DoubleValue(v) => NotNan::new(v).map(Value::Float).unwrap_or(Value::Null), PBValue::BytesValue(v) => Value::Bytes(Bytes::from(v)), PBValue::ArrayValue(arr) => Value::Array( arr.values @@ -57,3 +57,38 @@ pub fn to_hex(d: &[u8]) -> String { } hex::encode(d) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_pb_double_value_nan_handling() { + // Test that NaN values are converted to Value::Null instead of panicking + let nan_value = PBValue::DoubleValue(f64::NAN); + let result = Value::from(nan_value); + assert_eq!(result, Value::Null); + } + + #[test] + fn test_pb_double_value_infinity() { + // Test that infinity values work correctly + let inf_value = PBValue::DoubleValue(f64::INFINITY); + let result = Value::from(inf_value); + match result { + Value::Float(f) => { + assert!(f.into_inner().is_infinite() && f.into_inner().is_sign_positive()) + } + _ => panic!("Expected Float value, got {result:?}"), + } + + let neg_inf_value = PBValue::DoubleValue(f64::NEG_INFINITY); + let result = Value::from(neg_inf_value); + match result { + Value::Float(f) => { + assert!(f.into_inner().is_infinite() && f.into_inner().is_sign_negative()) + } + _ => panic!("Expected Float value, got {result:?}"), + } + } +} diff --git a/lib/prometheus-parser/Cargo.toml b/lib/prometheus-parser/Cargo.toml index 4e1c261b094f2..6f5b5708b3277 100644 --- a/lib/prometheus-parser/Cargo.toml +++ b/lib/prometheus-parser/Cargo.toml @@ -10,7 +10,7 @@ license = "MPL-2.0" [dependencies] indexmap.workspace = true -nom = "7.1.3" +nom.workspace = true prost.workspace = true prost-types.workspace = true snafu.workspace = true diff --git a/lib/prometheus-parser/src/line.rs b/lib/prometheus-parser/src/line.rs index aeac1e81105c6..d7c36fdfa90fd 100644 --- a/lib/prometheus-parser/src/line.rs +++ b/lib/prometheus-parser/src/line.rs @@ -6,11 +6,12 @@ use nom::{ branch::alt, bytes::complete::{is_not, tag, take_while, take_while1}, character::complete::{char, digit1}, - combinator::{map, opt, recognize, value}, + combinator::{map, map_res, opt, recognize, value}, error::ParseError, multi::fold_many0, number::complete::double, - sequence::{delimited, pair, preceded, tuple}, + sequence::{delimited, pair, preceded}, + Parser, }; /// We try to catch all nom's `ErrorKind` with our own `ErrorKind`, @@ -140,7 +141,8 @@ impl Metric { // This shouldn't be necessary if that issue is remedied. value(f64::NAN, tag("NaN")), double, - ))(input) + )) + .parse(input) .map_err(|_: NomError| { ErrorKind::ParseFloatError { input: input.to_owned(), @@ -151,16 +153,25 @@ impl Metric { fn parse_timestamp(input: &str) -> IResult> { let input = trim_space(input); - opt(map(recognize(pair(opt(char('-')), digit1)), |s: &str| { - s.parse().unwrap() - }))(input) + opt(map_res( + recognize(pair(opt(char('-')), digit1)), + |s: &str| s.parse(), + )) + .parse(input) + .map_err(|_: NomError| { + ErrorKind::ParseTimestampError { + input: input.to_owned(), + } + .into() + }) } fn parse_name_value(input: &str) -> IResult<(String, String)> { map( - tuple((parse_name, match_char('='), Self::parse_escaped_string)), + (parse_name, match_char('='), Self::parse_escaped_string), |(name, _, value)| (name, value), - )(input) + ) + .parse(input) } // Return: @@ -217,7 +228,7 @@ impl Metric { fn parse_labels(input: &str) -> IResult> { let input = trim_space(input); - match opt(char('{'))(input) { + match opt(char('{')).parse(input) { Ok((input, None)) => Ok((input, BTreeMap::new())), Ok((input, Some(_))) => Self::parse_labels_inner(input), Err(failure) => Err(failure), @@ -273,7 +284,7 @@ impl Metric { }) } - delimited(match_quote, build_string, match_quote)(input) + delimited(match_quote, build_string, match_quote).parse(input) } } @@ -310,7 +321,8 @@ impl Header { value(MetricKind::Summary, tag("summary")), value(MetricKind::Histogram, tag("histogram")), value(MetricKind::Untyped, tag("untyped")), - ))(input) + )) + .parse(input) .map_err(|_: NomError| ErrorKind::InvalidMetricKind { input: input.to_owned(), })?; @@ -349,7 +361,7 @@ impl Line { }; if let Ok((input, _)) = char::<_, NomErrorType>('#')(input) { - if tuple::<_, _, NomErrorType, _>((sp, tag("TYPE")))(input).is_ok() { + if (sp, tag::<_, _, NomErrorType>("TYPE")).parse(input).is_ok() { return Err(header_error); } Ok(None) @@ -365,7 +377,8 @@ fn parse_name(input: &str) -> IResult { let (input, (a, b)) = pair( take_while1(|c: char| c.is_alphabetic() || c == '_'), take_while(|c: char| c.is_alphanumeric() || c == '_' || c == ':'), - )(input) + ) + .parse(input) .map_err(|_: NomError| ErrorKind::ParseNameError { input: input.to_owned(), })?; @@ -382,7 +395,7 @@ fn sp<'a, E: ParseError<&'a str>>(i: &'a str) -> nom::IResult<&'a str, &'a str, fn match_char(c: char) -> impl Fn(&str) -> IResult { move |input| { - preceded(sp, char(c))(input).map_err(|_: NomError| { + preceded(sp, char(c)).parse(input).map_err(|_: NomError| { ErrorKind::ExpectedChar { expected: c, input: input.to_owned(), @@ -401,7 +414,7 @@ mod test { #[test] fn test_parse_escaped_string() { fn wrap(s: &str) -> String { - format!(" \t \"{}\" .", s) + format!(" \t \"{s}\" .") } // parser should not consume more that it needed @@ -441,7 +454,7 @@ mod test { #[test] fn test_parse_name() { fn wrap(s: &str) -> String { - format!(" \t {} .", s) + format!(" \t {s} .") } let tail = " ."; @@ -467,7 +480,7 @@ mod test { #[test] fn test_parse_header() { fn wrap(s: &str) -> String { - format!(" \t {} .", s) + format!(" \t {s} .") } let tail = " ."; @@ -541,7 +554,7 @@ mod test { #[test] fn test_parse_value() { fn wrap(s: &str) -> String { - format!(" \t {} .", s) + format!(" \t {s} .") } let tail = " ."; @@ -609,7 +622,7 @@ mod test { #[test] fn test_parse_labels() { fn wrap(s: &str) -> String { - format!(" \t {} .", s) + format!(" \t {s} .") } let tail = " ."; @@ -667,6 +680,24 @@ mod test { assert_eq!(Metric::parse_timestamp(""), Ok(("", None))); assert_eq!(Metric::parse_timestamp("123"), Ok(("", Some(123)))); assert_eq!(Metric::parse_timestamp(" -23"), Ok(("", Some(-23)))); + // Edge cases + assert_eq!( + Metric::parse_timestamp("9223372036854775807"), + Ok(("", Some(9223372036854775807i64))) + ); + assert_eq!( + Metric::parse_timestamp("-9223372036854775808"), + Ok(("", Some(-9223372036854775808i64))) + ); + // overflow + assert_eq!( + Metric::parse_timestamp("9223372036854775809"), + Ok(("9223372036854775809", None)) + ); + assert_eq!( + Metric::parse_timestamp("-9223372036854775809"), + Ok(("-9223372036854775809", None)) + ); } #[test] diff --git a/lib/tracing-limit/Cargo.toml b/lib/tracing-limit/Cargo.toml index 8fb0fb62090e1..2cf44e8873b9f 100644 --- a/lib/tracing-limit/Cargo.toml +++ b/lib/tracing-limit/Cargo.toml @@ -12,9 +12,9 @@ tracing-subscriber = { version = "0.3", default-features = false, features = ["r dashmap = { version = "6.1.0", default-features = false } [dev-dependencies] -criterion = "0.5" +criterion = "0.7" tracing = "0.1.34" -mock_instant = { version = "0.5" } +mock_instant = { version = "0.6" } tracing-subscriber = { version = "0.3.19", default-features = false, features = ["env-filter", "fmt"] } [[bench]] diff --git a/lib/tracing-limit/benches/limit.rs b/lib/tracing-limit/benches/limit.rs index f7214525e9819..79ed62b6de3d4 100644 --- a/lib/tracing-limit/benches/limit.rs +++ b/lib/tracing-limit/benches/limit.rs @@ -1,15 +1,14 @@ -#[macro_use] -extern crate tracing; - #[macro_use] extern crate criterion; +#[macro_use] +extern crate tracing; +use criterion::{BenchmarkId, Criterion}; +use std::hint::black_box; use std::{ fmt, sync::{Mutex, MutexGuard}, }; - -use criterion::{black_box, BenchmarkId, Criterion}; use tracing::{field, span, subscriber::Interest, Event, Metadata, Subscriber}; use tracing_limit::RateLimitedLayer; use tracing_subscriber::layer::{Context, Layer, SubscriberExt}; @@ -97,7 +96,7 @@ struct Visitor<'a>(MutexGuard<'a, String>); impl field::Visit for Visitor<'_> { fn record_debug(&mut self, _field: &field::Field, value: &dyn fmt::Debug) { use std::fmt::Write; - _ = write!(&mut *self.0, "{:?}", value); + _ = write!(&mut *self.0, "{value:?}"); } } diff --git a/lib/tracing-limit/src/lib.rs b/lib/tracing-limit/src/lib.rs index bd5952bf39760..7367f8c8c7612 100644 --- a/lib/tracing-limit/src/lib.rs +++ b/lib/tracing-limit/src/lib.rs @@ -389,7 +389,7 @@ impl Visit for RateLimitedSpanKeys { } fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) { - self.record(field, format!("{:?}", value).into()); + self.record(field, format!("{value:?}").into()); } } @@ -437,7 +437,7 @@ impl Visit for MessageVisitor { fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) { if self.message.is_none() && field.name() == MESSAGE_FIELD { - self.message = Some(format!("{:?}", value)); + self.message = Some(format!("{value:?}")); } } } @@ -577,8 +577,7 @@ mod test { info_span!("span", component_id = &key, vrl_position = &line_number); let _enter = span.enter(); info!( - message = - format!("Hello {} on line_number {}!", key, line_number).as_str(), + message = format!("Hello {key} on line_number {line_number}!").as_str(), internal_log_rate_limit = true ); } @@ -639,8 +638,7 @@ mod test { for key in &["foo", "bar"] { for line_number in &[1, 2] { info!( - message = - format!("Hello {} on line_number {}!", key, line_number).as_str(), + message = format!("Hello {key} on line_number {line_number}!").as_str(), internal_log_rate_limit = true, component_id = &key, vrl_position = &line_number diff --git a/lib/vector-api-client/Cargo.toml b/lib/vector-api-client/Cargo.toml index be614b8ccbb7f..30a7c9d80f990 100644 --- a/lib/vector-api-client/Cargo.toml +++ b/lib/vector-api-client/Cargo.toml @@ -17,7 +17,7 @@ anyhow = { version = "1.0.98", default-features = false, features = ["std"] } # Tokio / Futures futures.workspace = true -tokio = { version = "1.44.2", default-features = false, features = ["macros", "rt", "sync"] } +tokio = { version = "1.45.1", default-features = false, features = ["macros", "rt", "sync"] } tokio-stream = { version = "0.1.17", default-features = false, features = ["sync"] } # GraphQL diff --git a/lib/vector-api-client/src/gql/components.rs b/lib/vector-api-client/src/gql/components.rs index a2bfde4cdf487..d278b30c317c8 100644 --- a/lib/vector-api-client/src/gql/components.rs +++ b/lib/vector-api-client/src/gql/components.rs @@ -181,7 +181,7 @@ impl fmt::Display for components_query::ComponentsQueryComponentsEdgesNodeOn { components_query::ComponentsQueryComponentsEdgesNodeOn::Sink(_) => "sink", }; - write!(f, "{}", res) + write!(f, "{res}") } } @@ -199,7 +199,7 @@ impl fmt::Display for component_added_subscription::ComponentAddedSubscriptionCo } }; - write!(f, "{}", res) + write!(f, "{res}") } } @@ -219,6 +219,6 @@ impl fmt::Display } }; - write!(f, "{}", res) + write!(f, "{res}") } } diff --git a/lib/vector-buffers/Cargo.toml b/lib/vector-buffers/Cargo.toml index 24e6c90ff0862..3af8bfa27e229 100644 --- a/lib/vector-buffers/Cargo.toml +++ b/lib/vector-buffers/Cargo.toml @@ -14,13 +14,13 @@ async-stream = "0.3.6" async-trait = { version = "0.1", default-features = false } bytecheck = { version = "0.6.9", default-features = false, features = ["std"] } bytes = { version = "1.10.1", default-features = false } -crc32fast = { version = "1.4.2", default-features = false } +crc32fast = { version = "1.5.0", default-features = false } crossbeam-queue = { version = "0.3.12", default-features = false, features = ["std"] } crossbeam-utils = { version = "0.8.21", default-features = false } derivative = { version = "2.2.0", default-features = false } fslock = { version = "0.2.1", default-features = false, features = ["std"] } futures.workspace = true -memmap2 = { version = "0.9.5", default-features = false } +memmap2 = { version = "0.9.7", default-features = false } metrics.workspace = true num-traits = { version = "0.2.19", default-features = false } paste.workspace = true @@ -28,23 +28,25 @@ rkyv = { version = "0.7.45", default-features = false, features = ["size_32", "s serde.workspace = true snafu.workspace = true tokio-util = { version = "0.7.0", default-features = false } -tokio = { version = "1.44.2", default-features = false, features = ["rt", "macros", "rt-multi-thread", "sync", "fs", "io-util", "time"] } +tokio = { version = "1.45.1", default-features = false, features = ["rt", "macros", "rt-multi-thread", "sync", "fs", "io-util", "time"] } tracing = { version = "0.1.34", default-features = false, features = ["attributes"] } vector-config = { path = "../vector-config", default-features = false } vector-common = { path = "../vector-common", default-features = false, features = ["byte_size_of"] } +dashmap = { version = "6.1", default-features = false } +ordered-float = { version = "4.6.0", default-features = false } [dev-dependencies] clap.workspace = true -criterion = { version = "0.5", features = ["html_reports", "async_tokio"] } +criterion = { version = "0.7", features = ["html_reports", "async_tokio"] } crossbeam-queue = "0.3.12" hdrhistogram = "7.5.4" metrics-tracing-context.workspace = true metrics-util = { workspace = true, features = ["debugging"] } -proptest = "1.6" +proptest = "1.7" quickcheck = "1.0" rand.workspace = true serde_yaml = { version = "0.9", default-features = false } -temp-dir = "0.1.14" +temp-dir = "0.1.16" tokio-test = "0.4.4" tracing-fluent-assertions = { version = "0.3" } tracing-subscriber = { version = "0.3.19", default-features = false, features = ["env-filter", "fmt", "registry", "std", "ansi"] } diff --git a/lib/vector-buffers/benches/common.rs b/lib/vector-buffers/benches/common.rs index cb2f1ec402116..04c4582dbd487 100644 --- a/lib/vector-buffers/benches/common.rs +++ b/lib/vector-buffers/benches/common.rs @@ -17,9 +17,12 @@ use vector_buffers::{ use vector_common::byte_size_of::ByteSizeOf; use vector_common::finalization::{AddBatchNotifier, BatchNotifier, EventFinalizers, Finalizable}; -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Debug)] pub struct Message { id: u64, + // Purpose of `_heap_allocated` is to simulate memory pressure in the buffer benchmarks when the + // max_size option is selected. + _heap_allocated: Box<[u64; N]>, _padding: [u64; N], } @@ -27,6 +30,7 @@ impl Message { fn new(id: u64) -> Self { Message { id, + _heap_allocated: Box::new([0; N]), _padding: [0; N], } } @@ -40,7 +44,7 @@ impl AddBatchNotifier for Message { impl ByteSizeOf for Message { fn allocated_bytes(&self) -> usize { - 0 + N * std::mem::size_of::() } } @@ -61,7 +65,7 @@ pub struct EncodeError; impl fmt::Display for EncodeError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self) + write!(f, "{self:?}") } } @@ -72,7 +76,7 @@ pub struct DecodeError; impl fmt::Display for DecodeError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self) + write!(f, "{self:?}") } } @@ -88,8 +92,8 @@ impl FixedEncodable for Message { Self: Sized, { buffer.put_u64(self.id); - for _ in 0..N { - // this covers self._padding + for _ in 0..(N * 2) { + // this covers self._padding and self.heap_allocated buffer.put_u64(0); } Ok(()) @@ -101,8 +105,8 @@ impl FixedEncodable for Message { Self: Sized, { let id = buffer.get_u64(); - for _ in 0..N { - // this covers self._padding + for _ in 0..(N * 2) { + // this covers self._padding and self.heap_allocated _ = buffer.get_u64(); } Ok(Message::new(id)) diff --git a/lib/vector-buffers/benches/sized_records.rs b/lib/vector-buffers/benches/sized_records.rs index 80ffd4275d198..5fe31a0e3d68b 100644 --- a/lib/vector-buffers/benches/sized_records.rs +++ b/lib/vector-buffers/benches/sized_records.rs @@ -10,7 +10,7 @@ use criterion::{ Criterion, SamplingMode, Throughput, }; use tokio::runtime::{Handle, Runtime}; -use vector_buffers::{BufferType, WhenFull}; +use vector_buffers::{BufferType, MemoryBufferSize, WhenFull}; use crate::common::{init_instrumentation, war_measurement, wtr_measurement}; @@ -37,6 +37,11 @@ struct PathGuard { inner: PathBuf, } +enum BoundBy { + Bytes, + NumberEvents, +} + impl DataDir { fn new(name: &str) -> Self { let mut base_dir = PathBuf::new(); @@ -79,9 +84,15 @@ fn create_disk_v2_variant(_max_events: usize, max_size: u64) -> BufferType { } } -fn create_in_memory_variant(max_events: usize, _max_size: u64) -> BufferType { +fn create_in_memory_variant(bound_by: BoundBy, max_events: usize, max_size: u64) -> BufferType { + let size = match bound_by { + BoundBy::Bytes => MemoryBufferSize::MaxSize(NonZeroUsize::new(max_size as usize).unwrap()), + BoundBy::NumberEvents => { + MemoryBufferSize::MaxEvents(NonZeroUsize::new(max_events).unwrap()) + } + }; BufferType::Memory { - max_events: NonZeroUsize::new(max_events).unwrap(), + size, when_full: WhenFull::DropNewest, } } @@ -146,13 +157,24 @@ fn write_then_read(c: &mut Criterion) { create_disk_v2_variant ); + let f = |a, b| create_in_memory_variant(BoundBy::NumberEvents, a, b); experiment!( c, [32, 64, 128, 256, 512, 1024], "buffer-in-memory-v2", "write-then-read", wtr_measurement, - create_in_memory_variant + f + ); + + let f = |a, b| create_in_memory_variant(BoundBy::Bytes, a, b); + experiment!( + c, + [32, 64, 128, 256, 512, 1024], + "buffer-in-memory-bytes-v2", + "write-then-read", + wtr_measurement, + f ); } @@ -167,13 +189,24 @@ fn write_and_read(c: &mut Criterion) { create_disk_v2_variant ); + let f = |a, b| create_in_memory_variant(BoundBy::NumberEvents, a, b); experiment!( c, [32, 64, 128, 256, 512, 1024], "buffer-in-memory-v2", "write-and-read", war_measurement, - create_in_memory_variant + f + ); + + let f = |a, b| create_in_memory_variant(BoundBy::Bytes, a, b); + experiment!( + c, + [32, 64, 128, 256, 512, 1024], + "buffer-in-memory-bytes-v2", + "write-and-read", + war_measurement, + f ); } diff --git a/lib/vector-buffers/examples/buffer_perf.rs b/lib/vector-buffers/examples/buffer_perf.rs index 67d781bd7cf0a..141cff65aef3f 100644 --- a/lib/vector-buffers/examples/buffer_perf.rs +++ b/lib/vector-buffers/examples/buffer_perf.rs @@ -21,7 +21,7 @@ use vector_buffers::{ builder::TopologyBuilder, channel::{BufferReceiver, BufferSender}, }, - BufferType, Bufferable, EventCount, WhenFull, + BufferType, Bufferable, EventCount, MemoryBufferSize, WhenFull, }; use vector_common::byte_size_of::ByteSizeOf; use vector_common::finalization::{ @@ -113,7 +113,7 @@ pub struct EncodeError; impl fmt::Display for EncodeError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self) + write!(f, "{self:?}") } } @@ -124,7 +124,7 @@ pub struct DecodeError; impl fmt::Display for DecodeError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self) + write!(f, "{self:?}") } } @@ -247,7 +247,7 @@ where T: Bufferable + Clone + Finalizable, { let data_dir = PathBuf::from("/tmp/vector"); - let id = format!("{}-buffer-perf-testing", buffer_type); + let id = format!("{buffer_type}-buffer-perf-testing"); let max_size_events = std::num::NonZeroUsize::new(500).unwrap(); let max_size_bytes = std::num::NonZeroU64::new(32 * 1024 * 1024 * 1024).unwrap(); let when_full = WhenFull::Block; @@ -261,7 +261,7 @@ where max_size_events ); BufferType::Memory { - max_events: max_size_events, + size: MemoryBufferSize::MaxEvents(max_size_events), when_full, } } @@ -276,8 +276,7 @@ where } } s => panic!( - "unknown buffer type '{}' requested; valid types are in-memory, disk-v1, and disk-v2", - s + "unknown buffer type '{s}' requested; valid types are in-memory, disk-v1, and disk-v2" ), }; diff --git a/lib/vector-buffers/src/buffer_usage_data.rs b/lib/vector-buffers/src/buffer_usage_data.rs index ac06d2757da93..90277e8c6328e 100644 --- a/lib/vector-buffers/src/buffer_usage_data.rs +++ b/lib/vector-buffers/src/buffer_usage_data.rs @@ -242,8 +242,10 @@ impl BufferUsage { /// The `buffer_id` should be a unique name -- ideally the `component_id` of the sink using this buffer -- but is /// not used for anything other than reporting, and so has no _requirement_ to be unique. pub fn install(self, buffer_id: &str) { + let buffer_id = buffer_id.to_string(); let span = self.span; let stages = self.stages; + let task_name = format!("buffer usage reporter ({buffer_id})"); let task = async move { let mut interval = interval(Duration::from_secs(2)); @@ -264,6 +266,7 @@ impl BufferUsage { let received = stage.received.consume(); if received.has_updates() { emit(BufferEventsReceived { + buffer_id: buffer_id.clone(), idx: stage.idx, count: received.event_count, byte_size: received.event_byte_size, @@ -273,6 +276,7 @@ impl BufferUsage { let sent = stage.sent.consume(); if sent.has_updates() { emit(BufferEventsSent { + buffer_id: buffer_id.clone(), idx: stage.idx, count: sent.event_count, byte_size: sent.event_byte_size, @@ -282,6 +286,7 @@ impl BufferUsage { let dropped = stage.dropped.consume(); if dropped.has_updates() { emit(BufferEventsDropped { + buffer_id: buffer_id.clone(), idx: stage.idx, intentional: false, reason: "corrupted_events", @@ -293,6 +298,7 @@ impl BufferUsage { let dropped_intentional = stage.dropped_intentional.consume(); if dropped_intentional.has_updates() { emit(BufferEventsDropped { + buffer_id: buffer_id.clone(), idx: stage.idx, intentional: true, reason: "drop_newest", @@ -304,7 +310,6 @@ impl BufferUsage { } }; - let task_name = format!("buffer usage reporter ({buffer_id})"); spawn_named(task.instrument(span.or_current()), task_name.as_str()); } } diff --git a/lib/vector-buffers/src/cast_utils.rs b/lib/vector-buffers/src/cast_utils.rs new file mode 100644 index 0000000000000..f9161c5bff680 --- /dev/null +++ b/lib/vector-buffers/src/cast_utils.rs @@ -0,0 +1,19 @@ +// Maximum i64 value that can be represented exactly in f64 (2^53). +pub const F64_SAFE_INT_MAX: i64 = 1_i64 << 53; + +pub fn i64_to_f64_safe(value: i64) -> f64 { + let capped = value.clamp(0, F64_SAFE_INT_MAX); + debug_assert!(capped <= F64_SAFE_INT_MAX); + #[allow(clippy::cast_precision_loss)] + { + capped as f64 + } +} + +pub fn u64_to_f64_safe(value: u64) -> f64 { + let capped = value.min(F64_SAFE_INT_MAX as u64); + #[allow(clippy::cast_precision_loss)] + { + capped as f64 + } +} diff --git a/lib/vector-buffers/src/config.rs b/lib/vector-buffers/src/config.rs index 14826b6c204c0..11a97a85cddf4 100644 --- a/lib/vector-buffers/src/config.rs +++ b/lib/vector-buffers/src/config.rs @@ -86,16 +86,32 @@ impl BufferTypeVisitor { let when_full = when_full.unwrap_or_default(); match kind { BufferTypeKind::Memory => { - if max_size.is_some() { - return Err(de::Error::unknown_field( - "max_size", - &["type", "max_events", "when_full"], - )); - } - Ok(BufferType::Memory { - max_events: max_events.unwrap_or_else(memory_buffer_default_max_events), - when_full, - }) + let size = match (max_events, max_size) { + (Some(_), Some(_)) => { + return Err(de::Error::unknown_field( + "max_events", + &["type", "max_size", "when_full"], + )); + } + (_, Some(max_size)) => { + if let Ok(bounded_max_bytes) = usize::try_from(max_size.get()) { + MemoryBufferSize::MaxSize(NonZeroUsize::new(bounded_max_bytes).unwrap()) + } else { + return Err(de::Error::invalid_value( + de::Unexpected::Unsigned(max_size.into()), + &format!( + "Value for max_bytes must be a positive integer <= {}", + usize::MAX + ) + .as_str(), + )); + } + } + _ => MemoryBufferSize::MaxEvents( + max_events.unwrap_or_else(memory_buffer_default_max_events), + ), + }; + Ok(BufferType::Memory { size, when_full }) } BufferTypeKind::DiskV2 => { if max_events.is_some() { @@ -175,6 +191,23 @@ impl DiskUsage { } } +/// Enumeration to define exactly what terms the bounds of the buffer is expressed in: length, or +/// `byte_size`. +#[configurable_component(no_deser)] +#[serde(rename_all = "snake_case")] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MemoryBufferSize { + /// The maximum number of events allowed in the buffer. + MaxEvents(#[serde(default = "memory_buffer_default_max_events")] NonZeroUsize), + + // Doc string is duplicated here as a workaround due to a name collision with the `max_size` + // field with the DiskV2 variant of `BufferType`. + /// The maximum allowed amount of allocated memory the buffer can hold. + /// + /// If `type = "disk"` then must be at least ~256 megabytes (268435488 bytes). + MaxSize(#[configurable(metadata(docs::type_unit = "bytes"))] NonZeroUsize), +} + /// A specific type of buffer stage. #[configurable_component(no_deser)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -186,11 +219,10 @@ pub enum BufferType { /// This is more performant, but less durable. Data will be lost if Vector is restarted /// forcefully or crashes. #[configurable(title = "Events are buffered in memory.")] - #[serde(rename = "memory")] Memory { - /// The maximum number of events allowed in the buffer. - #[serde(default = "memory_buffer_default_max_events")] - max_events: NonZeroUsize, + /// The terms around how to express buffering limits, can be in size or bytes_size. + #[serde(flatten)] + size: MemoryBufferSize, #[configurable(derived)] #[serde(default)] @@ -277,11 +309,8 @@ impl BufferType { T: Bufferable + Clone + Finalizable, { match *self { - BufferType::Memory { - when_full, - max_events, - } => { - builder.stage(MemoryBuffer::new(max_events), when_full); + BufferType::Memory { size, when_full } => { + builder.stage(MemoryBuffer::new(size), when_full); } BufferType::DiskV2 { when_full, @@ -290,7 +319,7 @@ impl BufferType { let data_dir = data_dir.ok_or(BufferBuildError::RequiresDataDir)?; builder.stage(DiskV2Buffer::new(id, data_dir, max_size), when_full); } - }; + } Ok(()) } @@ -322,7 +351,7 @@ impl BufferType { description = r#"More information about the individual buffer types, and buffer behavior, can be found in the [Buffering Model][buffering_model] section. -[buffering_model]: /docs/about/under-the-hood/architecture/buffering-model/"# +[buffering_model]: /docs/architecture/buffering-model/"# )] pub enum BufferConfig { /// A single stage buffer topology. @@ -335,7 +364,7 @@ pub enum BufferConfig { impl Default for BufferConfig { fn default() -> Self { Self::Single(BufferType::Memory { - max_events: memory_buffer_default_max_events(), + size: MemoryBufferSize::MaxEvents(memory_buffer_default_max_events()), when_full: WhenFull::default(), }) } @@ -389,7 +418,7 @@ impl BufferConfig { mod test { use std::num::{NonZeroU64, NonZeroUsize}; - use crate::{BufferConfig, BufferType, WhenFull}; + use crate::{BufferConfig, BufferType, MemoryBufferSize, WhenFull}; fn check_single_stage(source: &str, expected: BufferType) { let config: BufferConfig = serde_yaml::from_str(source).unwrap(); @@ -439,7 +468,20 @@ max_events: 42 max_events: 100 ", BufferType::Memory { - max_events: NonZeroUsize::new(100).unwrap(), + size: MemoryBufferSize::MaxEvents(NonZeroUsize::new(100).unwrap()), + when_full: WhenFull::Block, + }, + ); + } + + #[test] + fn parse_memory_with_byte_size_option() { + check_single_stage( + r" + max_size: 4096 + ", + BufferType::Memory { + size: MemoryBufferSize::MaxSize(NonZeroUsize::new(4096).unwrap()), when_full: WhenFull::Block, }, ); @@ -455,11 +497,11 @@ max_events: 42 ", &[ BufferType::Memory { - max_events: NonZeroUsize::new(42).unwrap(), + size: MemoryBufferSize::MaxEvents(NonZeroUsize::new(42).unwrap()), when_full: WhenFull::Block, }, BufferType::Memory { - max_events: NonZeroUsize::new(100).unwrap(), + size: MemoryBufferSize::MaxEvents(NonZeroUsize::new(100).unwrap()), when_full: WhenFull::DropNewest, }, ], @@ -473,7 +515,7 @@ max_events: 42 type: memory ", BufferType::Memory { - max_events: NonZeroUsize::new(500).unwrap(), + size: MemoryBufferSize::MaxEvents(NonZeroUsize::new(500).unwrap()), when_full: WhenFull::Block, }, ); @@ -484,7 +526,7 @@ max_events: 42 max_events: 100 ", BufferType::Memory { - max_events: NonZeroUsize::new(100).unwrap(), + size: MemoryBufferSize::MaxEvents(NonZeroUsize::new(100).unwrap()), when_full: WhenFull::Block, }, ); @@ -495,7 +537,7 @@ max_events: 42 when_full: drop_newest ", BufferType::Memory { - max_events: NonZeroUsize::new(500).unwrap(), + size: MemoryBufferSize::MaxEvents(NonZeroUsize::new(500).unwrap()), when_full: WhenFull::DropNewest, }, ); @@ -506,7 +548,7 @@ max_events: 42 when_full: overflow ", BufferType::Memory { - max_events: NonZeroUsize::new(500).unwrap(), + size: MemoryBufferSize::MaxEvents(NonZeroUsize::new(500).unwrap()), when_full: WhenFull::Overflow, }, ); diff --git a/lib/vector-buffers/src/internal_events.rs b/lib/vector-buffers/src/internal_events.rs index 0cda5beec46f9..659622415847b 100644 --- a/lib/vector-buffers/src/internal_events.rs +++ b/lib/vector-buffers/src/internal_events.rs @@ -1,11 +1,51 @@ +use dashmap::DashMap; +use std::sync::atomic::{AtomicI64, Ordering}; +use std::sync::LazyLock; use std::time::Duration; +use crate::cast_utils::{i64_to_f64_safe, u64_to_f64_safe}; use metrics::{counter, gauge, histogram, Histogram}; use vector_common::{ internal_event::{error_type, InternalEvent}, registered_event, }; +static BUFFER_COUNTERS: LazyLock> = + LazyLock::new(DashMap::new); + +fn update_and_get(counter: &AtomicI64, delta: i64) -> i64 { + let mut new_val = 0; + counter + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |current| { + let updated = current.saturating_add(delta).clamp(0, i64::MAX); + new_val = updated; + Some(updated) + }) + .ok(); + new_val +} + +fn update_buffer_gauge(buffer_id: &str, stage: usize, events_delta: i64, bytes_delta: i64) { + let counters = BUFFER_COUNTERS + .entry((buffer_id.to_string(), stage)) + .or_insert_with(|| (AtomicI64::new(0), AtomicI64::new(0))); + + let new_events = update_and_get(&counters.0, events_delta); + let new_bytes = update_and_get(&counters.1, bytes_delta); + + gauge!("buffer_events", + "buffer_id" => buffer_id.to_string(), + "stage" => stage.to_string() + ) + .set(i64_to_f64_safe(new_events)); + + gauge!("buffer_byte_size", + "buffer_id" => buffer_id.to_string(), + "stage" => stage.to_string() + ) + .set(i64_to_f64_safe(new_bytes)); +} + pub struct BufferCreated { pub idx: usize, pub max_size_events: usize, @@ -13,57 +53,73 @@ pub struct BufferCreated { } impl InternalEvent for BufferCreated { - #[allow(clippy::cast_precision_loss)] fn emit(self) { if self.max_size_events != 0 { gauge!("buffer_max_event_size", "stage" => self.idx.to_string()) - .set(self.max_size_events as f64); + .set(u64_to_f64_safe(self.max_size_events as u64)); } if self.max_size_bytes != 0 { gauge!("buffer_max_byte_size", "stage" => self.idx.to_string()) - .set(self.max_size_bytes as f64); + .set(u64_to_f64_safe(self.max_size_bytes)); } } } pub struct BufferEventsReceived { + pub buffer_id: String, pub idx: usize, pub count: u64, pub byte_size: u64, } impl InternalEvent for BufferEventsReceived { - #[allow(clippy::cast_precision_loss)] fn emit(self) { - counter!("buffer_received_events_total", "stage" => self.idx.to_string()) - .increment(self.count); - counter!("buffer_received_bytes_total", "stage" => self.idx.to_string()) - .increment(self.byte_size); - gauge!("buffer_events", "stage" => self.idx.to_string()).increment(self.count as f64); - gauge!("buffer_byte_size", "stage" => self.idx.to_string()) - .increment(self.byte_size as f64); + counter!("buffer_received_events_total", + "buffer_id" => self.buffer_id.clone(), + "stage" => self.idx.to_string() + ) + .increment(self.count); + + counter!("buffer_received_bytes_total", + "buffer_id" => self.buffer_id.clone(), + "stage" => self.idx.to_string() + ) + .increment(self.byte_size); + + let count_delta = i64::try_from(self.count).unwrap_or(i64::MAX); + let bytes_delta = i64::try_from(self.byte_size).unwrap_or(i64::MAX); + update_buffer_gauge(&self.buffer_id, self.idx, count_delta, bytes_delta); } } pub struct BufferEventsSent { + pub buffer_id: String, pub idx: usize, pub count: u64, pub byte_size: u64, } impl InternalEvent for BufferEventsSent { - #[allow(clippy::cast_precision_loss)] fn emit(self) { - counter!("buffer_sent_events_total", "stage" => self.idx.to_string()).increment(self.count); - counter!("buffer_sent_bytes_total", "stage" => self.idx.to_string()) - .increment(self.byte_size); - gauge!("buffer_events", "stage" => self.idx.to_string()).decrement(self.count as f64); - gauge!("buffer_byte_size", "stage" => self.idx.to_string()) - .decrement(self.byte_size as f64); + counter!("buffer_sent_events_total", + "buffer_id" => self.buffer_id.clone(), + "stage" => self.idx.to_string() + ) + .increment(self.count); + + counter!("buffer_sent_bytes_total", + "buffer_id" => self.buffer_id.clone(), + "stage" => self.idx.to_string()) + .increment(self.byte_size); + + let count_delta = i64::try_from(self.count).unwrap_or(i64::MAX); + let bytes_delta = i64::try_from(self.byte_size).unwrap_or(i64::MAX); + update_buffer_gauge(&self.buffer_id, self.idx, -count_delta, -bytes_delta); } } pub struct BufferEventsDropped { + pub buffer_id: String, pub idx: usize, pub count: u64, pub byte_size: u64, @@ -72,7 +128,6 @@ pub struct BufferEventsDropped { } impl InternalEvent for BufferEventsDropped { - #[allow(clippy::cast_precision_loss)] fn emit(self) { let intentional_str = if self.intentional { "true" } else { "false" }; if self.intentional { @@ -81,6 +136,7 @@ impl InternalEvent for BufferEventsDropped { count = %self.count, intentional = %intentional_str, reason = %self.reason, + buffer_id = %self.buffer_id, stage = %self.idx, ); } else { @@ -89,16 +145,22 @@ impl InternalEvent for BufferEventsDropped { count = %self.count, intentional = %intentional_str, reason = %self.reason, + buffer_id = %self.buffer_id, stage = %self.idx, ); } + counter!( - "buffer_discarded_events_total", "intentional" => intentional_str, + "buffer_discarded_events_total", + "buffer_id" => self.buffer_id.clone(), + "intentional" => intentional_str, ) .increment(self.count); - gauge!("buffer_events", "stage" => self.idx.to_string()).decrement(self.count as f64); - gauge!("buffer_byte_size", "stage" => self.idx.to_string()) - .decrement(self.byte_size as f64); + + let count_delta = i64::try_from(self.count).unwrap_or(i64::MAX); + let bytes_delta = i64::try_from(self.byte_size).unwrap_or(i64::MAX); + + update_buffer_gauge(&self.buffer_id, self.idx, -count_delta, -bytes_delta); } } @@ -137,3 +199,234 @@ registered_event! { self.send_duration.record(duration); } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::cast_utils::F64_SAFE_INT_MAX; + use metrics::{Key, Label}; + use metrics_util::debugging::{DebugValue, DebuggingRecorder}; + use metrics_util::{CompositeKey, MetricKind}; + use ordered_float::OrderedFloat; + use std::borrow::Cow; + use std::sync::Mutex; + use std::thread; + + static TEST_LOCK: LazyLock> = LazyLock::new(|| Mutex::new(())); + + fn reset_counters() { + BUFFER_COUNTERS.clear(); + } + + fn get_counter_values(buffer_id: &str, stage: usize) -> (i64, i64) { + match BUFFER_COUNTERS.get(&(buffer_id.to_string(), stage)) { + Some(counters) => { + let events = counters.0.load(Ordering::Relaxed); + let bytes = counters.1.load(Ordering::Relaxed); + (events, bytes) + } + None => (0, 0), + } + } + + fn assert_gauge_state( + buffer_id: &str, + stage: usize, + updates: &[(i64, i64)], + expected_events: f64, + expected_bytes: f64, + ) { + let _guard = TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + + reset_counters(); + + let recorder = DebuggingRecorder::default(); + let snapshotter = recorder.snapshotter(); + + metrics::with_local_recorder(&recorder, move || { + for (events_delta, bytes_delta) in updates { + update_buffer_gauge(buffer_id, stage, *events_delta, *bytes_delta); + } + + let metrics = snapshotter.snapshot().into_vec(); + + let buffer_id_cow: Cow<'static, str> = Cow::Owned(buffer_id.to_string()); + let buffer_id_label = Label::new("buffer_id", buffer_id_cow); + + let stage_label = Label::new("stage", stage.to_string()); + + let expected_metrics = vec![ + ( + CompositeKey::new( + MetricKind::Gauge, + Key::from_parts( + "buffer_events", + vec![buffer_id_label.clone(), stage_label.clone()], + ), + ), + None, + None, + DebugValue::Gauge(OrderedFloat(expected_events)), + ), + ( + CompositeKey::new( + MetricKind::Gauge, + Key::from_parts( + "buffer_byte_size", + vec![buffer_id_label.clone(), stage_label], + ), + ), + None, + None, + DebugValue::Gauge(OrderedFloat(expected_bytes)), + ), + ]; + + // Compare metrics without needing to sort if order doesn't matter + assert_eq!(metrics.len(), expected_metrics.len()); + for expected in &expected_metrics { + assert!( + metrics.contains(expected), + "Missing expected metric: {expected:?}" + ); + } + }); + } + + #[test] + fn test_increment() { + let _guard = TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + + reset_counters(); + + update_buffer_gauge("test_buffer", 0, 10, 1024); + let (events, bytes) = get_counter_values("test_buffer", 0); + assert_eq!(events, 10); + assert_eq!(bytes, 1024); + } + + #[test] + fn test_increment_and_decrement() { + let _guard = TEST_LOCK.lock().unwrap(); + reset_counters(); + + update_buffer_gauge("test_buffer", 1, 100, 2048); + update_buffer_gauge("test_buffer", 1, -50, -1024); + let (events, bytes) = get_counter_values("test_buffer", 1); + assert_eq!(events, 50); + assert_eq!(bytes, 1024); + } + + #[test] + fn test_decrement_below_zero_clamped_to_zero() { + let _guard = TEST_LOCK.lock().unwrap(); + reset_counters(); + + update_buffer_gauge("test_buffer", 2, 5, 100); + update_buffer_gauge("test_buffer", 2, -10, -200); + let (events, bytes) = get_counter_values("test_buffer", 2); + + assert_eq!(events, 0); + assert_eq!(bytes, 0); + } + + #[test] + fn test_multiple_stages_are_independent() { + let _guard = TEST_LOCK.lock().unwrap(); + reset_counters(); + + update_buffer_gauge("test_buffer", 0, 10, 100); + update_buffer_gauge("test_buffer", 1, 20, 200); + let (events0, bytes0) = get_counter_values("test_buffer", 0); + let (events1, bytes1) = get_counter_values("test_buffer", 1); + assert_eq!(events0, 10); + assert_eq!(bytes0, 100); + assert_eq!(events1, 20); + assert_eq!(bytes1, 200); + } + + #[test] + fn test_multithreaded_updates_are_correct() { + let _guard = TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + + reset_counters(); + + let num_threads = 10; + let increments_per_thread = 1000; + let mut handles = vec![]; + + for _ in 0..num_threads { + let handle = thread::spawn(move || { + for _ in 0..increments_per_thread { + update_buffer_gauge("test_buffer", 0, 1, 10); + } + }); + handles.push(handle); + } + + for handle in handles { + handle.join().unwrap(); + } + + let (final_events, final_bytes) = get_counter_values("test_buffer", 0); + let expected_events = i64::from(num_threads * increments_per_thread); + let expected_bytes = i64::from(num_threads * increments_per_thread * 10); + + assert_eq!(final_events, expected_events); + assert_eq!(final_bytes, expected_bytes); + } + + #[test] + fn test_large_values_capped_to_f64_safe_max() { + let _guard = TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + + reset_counters(); + + update_buffer_gauge("test_buffer", 3, F64_SAFE_INT_MAX * 2, F64_SAFE_INT_MAX * 2); + + let (events, bytes) = get_counter_values("test_buffer", 3); + + assert!(events > F64_SAFE_INT_MAX); + assert!(bytes > F64_SAFE_INT_MAX); + + let capped_events = events.min(F64_SAFE_INT_MAX); + let capped_bytes = bytes.min(F64_SAFE_INT_MAX); + + assert_eq!(capped_events, F64_SAFE_INT_MAX); + assert_eq!(capped_bytes, F64_SAFE_INT_MAX); + } + + #[test] + fn test_increment_with_recorder() { + assert_gauge_state("test_buffer", 0, &[(100, 2048), (200, 1024)], 300.0, 3072.0); + } + + #[test] + fn test_should_not_be_negative_with_recorder() { + assert_gauge_state("test_buffer", 1, &[(100, 1024), (-200, -4096)], 0.0, 0.0); + } + + #[test] + fn test_increment_with_custom_buffer_id() { + assert_gauge_state( + "buffer_alpha", + 0, + &[(100, 2048), (200, 1024)], + 300.0, + 3072.0, + ); + } + + #[test] + fn test_increment_with_another_buffer_id() { + assert_gauge_state("buffer_beta", 0, &[(10, 100), (5, 50)], 15.0, 150.0); + } +} diff --git a/lib/vector-buffers/src/lib.rs b/lib/vector-buffers/src/lib.rs index 84e88b659169c..6f89cf565e45f 100644 --- a/lib/vector-buffers/src/lib.rs +++ b/lib/vector-buffers/src/lib.rs @@ -17,7 +17,7 @@ extern crate tracing; mod buffer_usage_data; pub mod config; -pub use config::{BufferConfig, BufferType}; +pub use config::{BufferConfig, BufferType, MemoryBufferSize}; use encoding::Encodable; use vector_config::configurable_component; @@ -31,6 +31,7 @@ mod internal_events; pub mod test; pub mod topology; +mod cast_utils; pub(crate) mod variants; use std::fmt::Debug; diff --git a/lib/vector-buffers/src/test/messages.rs b/lib/vector-buffers/src/test/messages.rs index 6d73d020937eb..c86d386d2d09b 100644 --- a/lib/vector-buffers/src/test/messages.rs +++ b/lib/vector-buffers/src/test/messages.rs @@ -160,14 +160,11 @@ impl FixedEncodable for SizedRecord { { let minimum_len = self.encoded_len(); if buffer.remaining_mut() < minimum_len { - return Err(io::Error::new( - io::ErrorKind::Other, - format!( - "not enough capacity to encode record: need {}, only have {}", - minimum_len, - buffer.remaining_mut() - ), - )); + return Err(io::Error::other(format!( + "not enough capacity to encode record: need {}, only have {}", + minimum_len, + buffer.remaining_mut() + ))); } buffer.put_u32(self.0); @@ -219,10 +216,7 @@ impl FixedEncodable for UndecodableRecord { B: BufMut, { if buffer.remaining_mut() < 4 { - return Err(io::Error::new( - io::ErrorKind::Other, - "not enough capacity to encode record", - )); + return Err(io::Error::other("not enough capacity to encode record")); } buffer.put_u32(42); @@ -233,7 +227,7 @@ impl FixedEncodable for UndecodableRecord { where B: Buf, { - Err(io::Error::new(io::ErrorKind::Other, "failed to decode")) + Err(io::Error::other("failed to decode")) } } @@ -254,10 +248,7 @@ impl FixedEncodable for MultiEventRecord { B: BufMut, { if buffer.remaining_mut() < self.encoded_size() { - return Err(io::Error::new( - io::ErrorKind::Other, - "not enough capacity to encode record", - )); + return Err(io::Error::other("not enough capacity to encode record")); } buffer.put_u32(self.0); @@ -292,10 +283,7 @@ impl FixedEncodable for PoisonPillMultiEventRecord { B: BufMut, { if buffer.remaining_mut() < self.encoded_size() { - return Err(io::Error::new( - io::ErrorKind::Other, - "not enough capacity to encode record", - )); + return Err(io::Error::other("not enough capacity to encode record")); } buffer.put_u32(self.0); @@ -309,7 +297,7 @@ impl FixedEncodable for PoisonPillMultiEventRecord { { let event_count = buffer.get_u32(); if event_count == 42 { - return Err(io::Error::new(io::ErrorKind::Other, "failed to decode")); + return Err(io::Error::other("failed to decode")); } buffer.advance(usize::try_from(event_count).unwrap_or(usize::MAX)); diff --git a/lib/vector-buffers/src/test/variant.rs b/lib/vector-buffers/src/test/variant.rs index a14995d86b436..64283e3b17d7c 100644 --- a/lib/vector-buffers/src/test/variant.rs +++ b/lib/vector-buffers/src/test/variant.rs @@ -1,5 +1,5 @@ use std::{ - num::{NonZeroU16, NonZeroU64, NonZeroUsize}, + num::{NonZeroU16, NonZeroU64}, path::PathBuf, }; @@ -14,7 +14,7 @@ use crate::{ channel::{BufferReceiver, BufferSender}, }, variants::{DiskV2Buffer, MemoryBuffer}, - Bufferable, WhenFull, + Bufferable, MemoryBufferSize, WhenFull, }; #[cfg(test)] @@ -31,7 +31,7 @@ const ALPHABET: [&str; 27] = [ #[derive(Debug, Clone)] pub enum Variant { Memory { - max_events: NonZeroUsize, + size: MemoryBufferSize, when_full: WhenFull, }, DiskV2 { @@ -50,11 +50,9 @@ impl Variant { let mut builder = TopologyBuilder::default(); match self { Variant::Memory { - max_events, - when_full, - .. + size, when_full, .. } => { - builder.stage(MemoryBuffer::new(*max_events), *when_full); + builder.stage(MemoryBuffer::new(*size), *when_full); } Variant::DiskV2 { max_size, @@ -67,7 +65,7 @@ impl Variant { *when_full, ); } - }; + } let (sender, receiver) = builder .build(String::from("benches"), Span::none()) @@ -105,14 +103,12 @@ impl Arbitrary for Variant { // Using a u16 ensures we avoid any allocation errors for our holding buffers, etc. let max_events = NonZeroU16::arbitrary(g).into(); let max_size = NonZeroU16::arbitrary(g).into(); + let size = MemoryBufferSize::MaxEvents(max_events); let when_full = WhenFull::arbitrary(g); if use_memory_buffer { - Variant::Memory { - max_events, - when_full, - } + Variant::Memory { size, when_full } } else { Variant::DiskV2 { max_size, @@ -126,15 +122,23 @@ impl Arbitrary for Variant { fn shrink(&self) -> Box> { match self { Variant::Memory { - max_events, - when_full, - .. + size, when_full, .. } => { let when_full = *when_full; - Box::new(max_events.shrink().map(move |me| Variant::Memory { - max_events: me, - when_full, - })) + match size { + MemoryBufferSize::MaxEvents(max_events) => { + Box::new(max_events.shrink().map(move |me| Variant::Memory { + size: MemoryBufferSize::MaxEvents(me), + when_full, + })) + } + MemoryBufferSize::MaxSize(max_size) => { + Box::new(max_size.shrink().map(move |me| Variant::Memory { + size: MemoryBufferSize::MaxSize(me), + when_full, + })) + } + } } Variant::DiskV2 { max_size, diff --git a/lib/vector-buffers/src/topology/builder.rs b/lib/vector-buffers/src/topology/builder.rs index 51f52900eca2f..7bea4217b0d7f 100644 --- a/lib/vector-buffers/src/topology/builder.rs +++ b/lib/vector-buffers/src/topology/builder.rs @@ -129,7 +129,7 @@ impl TopologyBuilder { return Err(TopologyError::NextStageNotUsed { stage_idx }); } } - }; + } // Create the buffer usage handle for this stage and initialize it as we create the // sender/receiver. This is slightly awkward since we just end up actually giving @@ -193,7 +193,7 @@ impl TopologyBuilder { ) -> (BufferSender, BufferReceiver) { let usage_handle = BufferUsageHandle::noop(); - let memory_buffer = Box::new(MemoryBuffer::new(max_events)); + let memory_buffer = Box::new(MemoryBuffer::with_max_events(max_events)); let (sender, receiver) = memory_buffer .into_buffer_parts(usage_handle.clone()) .await @@ -229,7 +229,7 @@ impl TopologyBuilder { when_full: WhenFull, usage_handle: BufferUsageHandle, ) -> (BufferSender, BufferReceiver) { - let memory_buffer = Box::new(MemoryBuffer::new(max_events)); + let memory_buffer = Box::new(MemoryBuffer::with_max_events(max_events)); let (sender, receiver) = memory_buffer .into_buffer_parts(usage_handle.clone()) .await @@ -263,8 +263,10 @@ mod tests { use super::TopologyBuilder; use crate::{ - topology::builder::TopologyError, - topology::test_util::{assert_current_send_capacity, Sample}, + topology::{ + builder::TopologyError, + test_util::{assert_current_send_capacity, Sample}, + }, variants::MemoryBuffer, WhenFull, }; @@ -273,7 +275,7 @@ mod tests { async fn single_stage_topology_block() { let mut builder = TopologyBuilder::::default(); builder.stage( - MemoryBuffer::new(NonZeroUsize::new(1).unwrap()), + MemoryBuffer::with_max_events(NonZeroUsize::new(1).unwrap()), WhenFull::Block, ); let result = builder.build(String::from("test"), Span::none()).await; @@ -287,7 +289,7 @@ mod tests { async fn single_stage_topology_drop_newest() { let mut builder = TopologyBuilder::::default(); builder.stage( - MemoryBuffer::new(NonZeroUsize::new(1).unwrap()), + MemoryBuffer::with_max_events(NonZeroUsize::new(1).unwrap()), WhenFull::DropNewest, ); let result = builder.build(String::from("test"), Span::none()).await; @@ -301,7 +303,7 @@ mod tests { async fn single_stage_topology_overflow() { let mut builder = TopologyBuilder::::default(); builder.stage( - MemoryBuffer::new(NonZeroUsize::new(1).unwrap()), + MemoryBuffer::with_max_events(NonZeroUsize::new(1).unwrap()), WhenFull::Overflow, ); let result = builder.build(String::from("test"), Span::none()).await; @@ -315,11 +317,11 @@ mod tests { async fn two_stage_topology_block() { let mut builder = TopologyBuilder::::default(); builder.stage( - MemoryBuffer::new(NonZeroUsize::new(1).unwrap()), + MemoryBuffer::with_max_events(NonZeroUsize::new(1).unwrap()), WhenFull::Block, ); builder.stage( - MemoryBuffer::new(NonZeroUsize::new(1).unwrap()), + MemoryBuffer::with_max_events(NonZeroUsize::new(1).unwrap()), WhenFull::Block, ); let result = builder.build(String::from("test"), Span::none()).await; @@ -333,11 +335,11 @@ mod tests { async fn two_stage_topology_drop_newest() { let mut builder = TopologyBuilder::::default(); builder.stage( - MemoryBuffer::new(NonZeroUsize::new(1).unwrap()), + MemoryBuffer::with_max_events(NonZeroUsize::new(1).unwrap()), WhenFull::DropNewest, ); builder.stage( - MemoryBuffer::new(NonZeroUsize::new(1).unwrap()), + MemoryBuffer::with_max_events(NonZeroUsize::new(1).unwrap()), WhenFull::Block, ); let result = builder.build(String::from("test"), Span::none()).await; @@ -351,11 +353,11 @@ mod tests { async fn two_stage_topology_overflow() { let mut builder = TopologyBuilder::::default(); builder.stage( - MemoryBuffer::new(NonZeroUsize::new(1).unwrap()), + MemoryBuffer::with_max_events(NonZeroUsize::new(1).unwrap()), WhenFull::Overflow, ); builder.stage( - MemoryBuffer::new(NonZeroUsize::new(1).unwrap()), + MemoryBuffer::with_max_events(NonZeroUsize::new(1).unwrap()), WhenFull::Block, ); diff --git a/lib/vector-buffers/src/topology/channel/limited_queue.rs b/lib/vector-buffers/src/topology/channel/limited_queue.rs index 887ce2639ada0..2dd0da905cc87 100644 --- a/lib/vector-buffers/src/topology/channel/limited_queue.rs +++ b/lib/vector-buffers/src/topology/channel/limited_queue.rs @@ -1,5 +1,6 @@ use std::{ cmp, fmt, + fmt::Debug, pin::Pin, sync::{ atomic::{AtomicUsize, Ordering}, @@ -8,11 +9,11 @@ use std::{ }; use async_stream::stream; -use crossbeam_queue::ArrayQueue; +use crossbeam_queue::{ArrayQueue, SegQueue}; use futures::Stream; use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore, TryAcquireError}; -use crate::InMemoryBufferable; +use crate::{config::MemoryBufferSize, InMemoryBufferable}; /// Error returned by `LimitedSender::send` when the receiver has disconnected. #[derive(Debug, PartialEq, Eq)] @@ -54,10 +55,43 @@ impl fmt::Display for TrySendError { impl std::error::Error for TrySendError {} +// Trait over common queue operations so implementation can be chosen at initialization phase +trait QueueImpl: Send + Sync + fmt::Debug { + fn push(&self, item: T); + fn pop(&self) -> Option; +} + +impl QueueImpl for ArrayQueue +where + T: Send + Sync + fmt::Debug, +{ + fn push(&self, item: T) { + self.push(item) + .unwrap_or_else(|_| unreachable!("acquired permits but channel reported being full.")); + } + + fn pop(&self) -> Option { + self.pop() + } +} + +impl QueueImpl for SegQueue +where + T: Send + Sync + fmt::Debug, +{ + fn push(&self, item: T) { + self.push(item); + } + + fn pop(&self) -> Option { + self.pop() + } +} + #[derive(Debug)] struct Inner { - data: Arc>, - limit: usize, + data: Arc>, + limit: MemoryBufferSize, limiter: Arc, read_waker: Arc, } @@ -85,7 +119,11 @@ impl LimitedSender { // We have to limit the number of permits we ask for to the overall limit since we're always // willing to store more items than the limit if the queue is entirely empty, because // otherwise we might deadlock ourselves by not being able to send a single item. - cmp::min(self.inner.limit, item.event_count()) as u32 + let (limit, value) = match self.inner.limit { + MemoryBufferSize::MaxSize(max_size) => (max_size, item.allocated_bytes()), + MemoryBufferSize::MaxEvents(max_events) => (max_events, item.event_count()), + }; + cmp::min(limit.get(), value) as u32 } /// Gets the number of items that this channel could accept. @@ -112,10 +150,7 @@ impl LimitedSender { return Err(SendError(item)); }; - self.inner - .data - .push((permits, item)) - .unwrap_or_else(|_| unreachable!("acquired permits but channel reported being full")); + self.inner.data.push((permits, item)); self.inner.read_waker.notify_one(); trace!("Sent item."); @@ -153,10 +188,7 @@ impl LimitedSender { } }; - self.inner - .data - .push((permits, item)) - .unwrap_or_else(|_| unreachable!("acquired permits but channel reported being full")); + self.inner.data.push((permits, item)); self.inner.read_waker.notify_one(); trace!("Attempt to send item succeeded."); @@ -235,12 +267,22 @@ impl Drop for LimitedReceiver { } } -pub fn limited(limit: usize) -> (LimitedSender, LimitedReceiver) { - let inner = Inner { - data: Arc::new(ArrayQueue::new(limit)), - limit, - limiter: Arc::new(Semaphore::new(limit)), - read_waker: Arc::new(Notify::new()), +pub fn limited( + limit: MemoryBufferSize, +) -> (LimitedSender, LimitedReceiver) { + let inner = match limit { + MemoryBufferSize::MaxEvents(max_events) => Inner { + data: Arc::new(ArrayQueue::new(max_events.get())), + limit, + limiter: Arc::new(Semaphore::new(max_events.get())), + read_waker: Arc::new(Notify::new()), + }, + MemoryBufferSize::MaxSize(max_size) => Inner { + data: Arc::new(SegQueue::new()), + limit, + limiter: Arc::new(Semaphore::new(max_size.get())), + read_waker: Arc::new(Notify::new()), + }, }; let sender = LimitedSender { @@ -254,21 +296,25 @@ pub fn limited(limit: usize) -> (LimitedSender, LimitedReceiver) { #[cfg(test)] mod tests { + use std::num::NonZeroUsize; + use tokio_test::{assert_pending, assert_ready, task::spawn}; + use vector_common::byte_size_of::ByteSizeOf; use super::limited; use crate::{ - test::MultiEventRecord, topology::channel::limited_queue::SendError, - topology::test_util::Sample, + test::MultiEventRecord, + topology::{channel::limited_queue::SendError, test_util::Sample}, + MemoryBufferSize, }; #[tokio::test] async fn send_receive() { - let (mut tx, mut rx) = limited(2); + let (mut tx, mut rx) = limited(MemoryBufferSize::MaxEvents(NonZeroUsize::new(2).unwrap())); assert_eq!(2, tx.available_capacity()); - let msg = Sample(42); + let msg = Sample::new(42); // Create our send and receive futures. let mut send = spawn(async { tx.send(msg).await }); @@ -287,17 +333,58 @@ mod tests { // Now our receive should have been woken up, and should immediately be ready. assert!(recv.is_woken()); - assert_eq!(Some(msg), assert_ready!(recv.poll())); + assert_eq!(Some(Sample::new(42)), assert_ready!(recv.poll())); + } + + #[test] + fn test_limiting_by_byte_size() { + let max_elements = 10; + let msg = Sample::new_with_heap_allocated_values(50); + let msg_size = msg.allocated_bytes(); + let max_allowed_bytes = msg_size * max_elements; + + // With this configuration a maximum of exactly 10 messages can fit in the channel + let (mut tx, mut rx) = limited(MemoryBufferSize::MaxSize( + NonZeroUsize::new(max_allowed_bytes).unwrap(), + )); + + assert_eq!(max_allowed_bytes, tx.available_capacity()); + + // Send 10 messages into the channel, filling it + for _ in 0..10 { + let msg_clone = msg.clone(); + let mut f = spawn(async { tx.send(msg_clone).await }); + assert_eq!(Ok(()), assert_ready!(f.poll())); + } + // With the 10th message in the channel no space should be left + assert_eq!(0, tx.available_capacity()); + + // Attemting to produce one more then the max capacity should block + let mut send_final = spawn({ + let msg_clone = msg.clone(); + async { tx.send(msg_clone).await } + }); + assert_pending!(send_final.poll()); + + // Read all data from the channel, assert final states are as expected + for _ in 0..10 { + let mut f = spawn(async { rx.next().await }); + let value = assert_ready!(f.poll()); + assert_eq!(value.allocated_bytes(), msg_size); + } + // Channel should have no more data + let mut recv = spawn(async { rx.next().await }); + assert_pending!(recv.poll()); } #[test] fn sender_waits_for_more_capacity_when_none_available() { - let (mut tx, mut rx) = limited(1); + let (mut tx, mut rx) = limited(MemoryBufferSize::MaxEvents(NonZeroUsize::new(1).unwrap())); assert_eq!(1, tx.available_capacity()); - let msg1 = Sample(42); - let msg2 = Sample(43); + let msg1 = Sample::new(42); + let msg2 = Sample::new(43); // Create our send and receive futures. let mut send1 = spawn(async { tx.send(msg1).await }); @@ -328,7 +415,7 @@ mod tests { assert_pending!(send2.poll()); // Now if we receive the item, our second send should be woken up and be able to send in. - assert_eq!(Some(msg1), assert_ready!(recv1.poll())); + assert_eq!(Some(Sample::new(42)), assert_ready!(recv1.poll())); drop(recv1); // Since the second send was already waiting for permits, the semaphore returns them @@ -346,14 +433,14 @@ mod tests { // And the final receive to get our second send: assert!(recv2.is_woken()); - assert_eq!(Some(msg2), assert_ready!(recv2.poll())); + assert_eq!(Some(Sample::new(43)), assert_ready!(recv2.poll())); assert_eq!(1, tx.available_capacity()); } #[test] fn sender_waits_for_more_capacity_when_partial_available() { - let (mut tx, mut rx) = limited(7); + let (mut tx, mut rx) = limited(MemoryBufferSize::MaxEvents(NonZeroUsize::new(7).unwrap())); assert_eq!(7, tx.available_capacity()); @@ -441,12 +528,12 @@ mod tests { #[test] fn empty_receiver_returns_none_when_last_sender_drops() { - let (mut tx, mut rx) = limited(1); + let (mut tx, mut rx) = limited(MemoryBufferSize::MaxEvents(NonZeroUsize::new(1).unwrap())); assert_eq!(1, tx.available_capacity()); let tx2 = tx.clone(); - let msg = Sample(42); + let msg = Sample::new(42); // Create our send and receive futures. let mut send = spawn(async { tx.send(msg).await }); @@ -473,7 +560,7 @@ mod tests { drop(tx); assert!(recv.is_woken()); - assert_eq!(Some(msg), assert_ready!(recv.poll())); + assert_eq!(Some(Sample::new(42)), assert_ready!(recv.poll())); drop(recv); let mut recv2 = spawn(async { rx.next().await }); @@ -483,7 +570,8 @@ mod tests { #[test] fn receiver_returns_none_once_empty_when_last_sender_drops() { - let (tx, mut rx) = limited::(1); + let (tx, mut rx) = + limited::(MemoryBufferSize::MaxEvents(NonZeroUsize::new(1).unwrap())); assert_eq!(1, tx.available_capacity()); @@ -512,7 +600,7 @@ mod tests { #[test] fn oversized_send_allowed_when_empty() { - let (mut tx, mut rx) = limited(1); + let (mut tx, mut rx) = limited(MemoryBufferSize::MaxEvents(NonZeroUsize::new(1).unwrap())); assert_eq!(1, tx.available_capacity()); @@ -544,7 +632,7 @@ mod tests { #[test] fn oversized_send_allowed_when_partial_capacity() { - let (mut tx, mut rx) = limited(2); + let (mut tx, mut rx) = limited(MemoryBufferSize::MaxEvents(NonZeroUsize::new(2).unwrap())); assert_eq!(2, tx.available_capacity()); diff --git a/lib/vector-buffers/src/topology/channel/receiver.rs b/lib/vector-buffers/src/topology/channel/receiver.rs index d21aa1ed67e4f..1b51ec26f7ee3 100644 --- a/lib/vector-buffers/src/topology/channel/receiver.rs +++ b/lib/vector-buffers/src/topology/channel/receiver.rs @@ -18,6 +18,7 @@ use crate::{ }; /// Adapter for papering over various receiver backends. +#[allow(clippy::large_enum_variant)] #[derive(Debug)] pub enum ReceiverAdapter { /// The in-memory channel buffer. @@ -54,7 +55,6 @@ where // If we've hit a recoverable error, we'll emit an event to indicate as much but we'll still // keep trying to read the next available record. emit(re); - continue; } None => panic!("Reader encountered unrecoverable error: {e:?}"), }, @@ -157,6 +157,7 @@ impl BufferReceiver { } } +#[allow(clippy::large_enum_variant)] enum StreamState { Idle(BufferReceiver), Polling, diff --git a/lib/vector-buffers/src/topology/channel/sender.rs b/lib/vector-buffers/src/topology/channel/sender.rs index 15191ec02eead..7e4b09dad2753 100644 --- a/lib/vector-buffers/src/topology/channel/sender.rs +++ b/lib/vector-buffers/src/topology/channel/sender.rs @@ -221,7 +221,7 @@ impl BufferSender { .await?; } } - }; + } if sent_to_base || was_dropped { if let (Some(send_duration), Some(send_reference)) = diff --git a/lib/vector-buffers/src/topology/test_util.rs b/lib/vector-buffers/src/topology/test_util.rs index ec0acac7416d2..b1c3a27f48c6b 100644 --- a/lib/vector-buffers/src/topology/test_util.rs +++ b/lib/vector-buffers/src/topology/test_util.rs @@ -12,18 +12,39 @@ use crate::{ Bufferable, EventCount, WhenFull, }; -#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] -pub(crate) struct Sample(pub u64); +const SINGLE_VALUE_FLAG: u8 = 0; +const HEAP_ALLOCATED_VALUES_FLAG: u8 = 1; + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub(crate) enum Sample { + SingleValue(u64), + HeapAllocatedValues(Vec), +} impl From for Sample { fn from(v: u64) -> Self { - Self(v) + Self::SingleValue(v) } } impl From for u64 { - fn from(s: Sample) -> Self { - s.0 + fn from(v: Sample) -> Self { + match v { + Sample::SingleValue(sv) => sv, + Sample::HeapAllocatedValues(_) => { + panic!("Cannot use this API with other enum states of this type.") + } + } + } +} + +impl Sample { + pub fn new(value: u64) -> Self { + Self::SingleValue(value) + } + + pub fn new_with_heap_allocated_values(n: usize) -> Self { + Self::HeapAllocatedValues(vec![0; n]) } } @@ -35,7 +56,10 @@ impl AddBatchNotifier for Sample { impl ByteSizeOf for Sample { fn allocated_bytes(&self) -> usize { - 0 + match self { + Self::SingleValue(_) => 0, + Self::HeapAllocatedValues(uints) => uints.len() * 8, + } } } @@ -44,12 +68,29 @@ impl FixedEncodable for Sample { type EncodeError = BasicError; type DecodeError = BasicError; + // Serialization format: + // - Encode type flag + // - if single flag encode int value + // - otherwise encode array length and encode array contents fn encode(self, buffer: &mut B) -> Result<(), Self::EncodeError> where B: BufMut, Self: Sized, { - buffer.put_u64(self.0); + match self { + Self::SingleValue(uint) => { + buffer.put_u8(SINGLE_VALUE_FLAG); + buffer.put_u64(uint); + } + Self::HeapAllocatedValues(uints) => { + buffer.put_u8(HEAP_ALLOCATED_VALUES_FLAG); + // Prepend with array size + buffer.put_u32(u32::try_from(uints.len()).unwrap()); + for v in uints { + buffer.put_u64(v); + } + } + } Ok(()) } @@ -57,10 +98,16 @@ impl FixedEncodable for Sample { where B: Buf, { - if buffer.remaining() >= 8 { - Ok(Self(buffer.get_u64())) - } else { - Err(BasicError("need 8 bytes minimum".to_string())) + match buffer.get_u8() { + SINGLE_VALUE_FLAG => Ok(Self::SingleValue(buffer.get_u64())), + HEAP_ALLOCATED_VALUES_FLAG => { + let length = buffer.get_u32(); + let values = (0..length).map(|_| buffer.get_u64()).collect(); + Ok(Self::HeapAllocatedValues(values)) + } + _ => Err(BasicError( + "Unknown serialization flag observed".to_string(), + )), } } } diff --git a/lib/vector-buffers/src/variants/disk_v2/reader.rs b/lib/vector-buffers/src/variants/disk_v2/reader.rs index 8aea89645ca64..a584af5d38d6f 100644 --- a/lib/vector-buffers/src/variants/disk_v2/reader.rs +++ b/lib/vector-buffers/src/variants/disk_v2/reader.rs @@ -1035,7 +1035,7 @@ where return Err(e); } - }; + } // Fundamentally, when `try_read_record` returns `None`, there's three possible // scenarios: diff --git a/lib/vector-buffers/src/variants/disk_v2/tests/basic.rs b/lib/vector-buffers/src/variants/disk_v2/tests/basic.rs index 781e30ad5b217..80759529f4d7c 100644 --- a/lib/vector-buffers/src/variants/disk_v2/tests/basic.rs +++ b/lib/vector-buffers/src/variants/disk_v2/tests/basic.rs @@ -1,7 +1,8 @@ -use std::io::Cursor; +use std::{io::Cursor, time::Duration}; use futures::{stream, StreamExt}; -use tokio_test::{assert_pending, assert_ready, task::spawn}; +use tokio::{select, time::sleep}; +use tokio_test::{assert_pending, task::spawn}; use tracing::Instrument; use vector_common::finalization::Finalizable; @@ -66,6 +67,7 @@ async fn basic_read_write_loop() { .await; } +#[ignore = "flaky. See https://github.com/vectordotdev/vector/issues/23456"] #[tokio::test] async fn reader_exits_cleanly_when_writer_done_and_in_flight_acks() { let assertion_registry = install_tracing_helpers(); @@ -127,7 +129,15 @@ async fn reader_exits_cleanly_when_writer_done_and_in_flight_acks() { // albeit with a return value of `None`... because the writer is closed, and we read all // the records, so nothing is left. :) assert!(blocked_read.is_woken()); - let second_read = assert_ready!(blocked_read.poll()); + + let second_read = select! { + // if the reader task finishes in time, extract its output + res = blocked_read => res, + // otherwise panics after 1s + () = sleep(Duration::from_secs(1)) => { + panic!("Reader not ready after 1s"); + } + }; assert_eq!(second_read.expect("read should not fail"), None); // All records should be consumed at this point. diff --git a/lib/vector-buffers/src/variants/disk_v2/tests/size_limits.rs b/lib/vector-buffers/src/variants/disk_v2/tests/size_limits.rs index 1902fb4c4c589..414a10fb4f21e 100644 --- a/lib/vector-buffers/src/variants/disk_v2/tests/size_limits.rs +++ b/lib/vector-buffers/src/variants/disk_v2/tests/size_limits.rs @@ -66,7 +66,7 @@ async fn writer_error_when_record_is_over_the_limit() { } #[tokio::test] -#[ignore] +#[ignore = "Needs investigation"] async fn writer_waits_when_buffer_is_full() { let assertion_registry = install_tracing_helpers(); let fut = with_temp_dir(|dir| { diff --git a/lib/vector-buffers/src/variants/disk_v2/writer.rs b/lib/vector-buffers/src/variants/disk_v2/writer.rs index fc8f60672e1af..426b061ea53bb 100644 --- a/lib/vector-buffers/src/variants/disk_v2/writer.rs +++ b/lib/vector-buffers/src/variants/disk_v2/writer.rs @@ -648,7 +648,7 @@ where /// `InconsistentState`, as being unable to immediately deserialize and decode a record we just serialized and /// encoded implies a fatal, and unrecoverable, error with the buffer implementation as a whole. #[instrument(skip(self), level = "trace")] - pub fn recover_archived_record(&mut self, token: WriteToken) -> Result> { + pub fn recover_archived_record(&mut self, token: &WriteToken) -> Result> { // Make sure the write token we've been given matches whatever the last call to `archive_record` generated. let serialized_len = token.serialized_len(); debug_assert_eq!( @@ -1161,12 +1161,7 @@ where /// If an error occurred while writing the record, an error variant will be returned describing /// the error. pub async fn try_write_record(&mut self, record: T) -> Result, WriterError> { - self.try_write_record_inner(record) - .await - .map(|result| match result { - Ok(_) => None, - Err(record) => Some(record), - }) + self.try_write_record_inner(record).await.map(Result::err) } #[instrument(skip_all, level = "debug")] @@ -1219,8 +1214,6 @@ where last_attempted_write_size = serialized_len, "Current data file reached maximum size. Rolling to the next data file." ); - - continue; } e => return Err(e), }, @@ -1247,7 +1240,7 @@ where // writer and hand it back. This looks a little weird because we want to surface deserialize/decoding // errors if we encounter them, but if we recover the record successfully, we're returning // `Ok(Err(record))` to signal that our attempt failed but the record is able to be retried again later. - return Ok(Err(writer.recover_archived_record(token)?)); + return Ok(Err(writer.recover_archived_record(&token)?)); }; // Track our write since things appear to have succeeded. This only updates our internal @@ -1294,7 +1287,6 @@ where Err(old_record) => { record = old_record; self.ledger.wait_for_reader().await; - continue; } } } diff --git a/lib/vector-buffers/src/variants/in_memory.rs b/lib/vector-buffers/src/variants/in_memory.rs index fedcf35c2d9f1..9dbb79ebf5c18 100644 --- a/lib/vector-buffers/src/variants/in_memory.rs +++ b/lib/vector-buffers/src/variants/in_memory.rs @@ -4,6 +4,7 @@ use async_trait::async_trait; use crate::{ buffer_usage_data::BufferUsageHandle, + config::MemoryBufferSize, topology::{ builder::IntoBuffer, channel::{limited, ReceiverAdapter, SenderAdapter}, @@ -12,13 +13,19 @@ use crate::{ }; pub struct MemoryBuffer { - capacity: NonZeroUsize, + capacity: MemoryBufferSize, } impl MemoryBuffer { - pub fn new(capacity: NonZeroUsize) -> Self { + pub fn new(capacity: MemoryBufferSize) -> Self { MemoryBuffer { capacity } } + + pub fn with_max_events(n: NonZeroUsize) -> Self { + Self { + capacity: MemoryBufferSize::MaxEvents(n), + } + } } #[async_trait] @@ -30,9 +37,14 @@ where self: Box, usage_handle: BufferUsageHandle, ) -> Result<(SenderAdapter, ReceiverAdapter), Box> { - usage_handle.set_buffer_limits(None, Some(self.capacity.get())); + let (max_bytes, max_size) = match self.capacity { + MemoryBufferSize::MaxEvents(max_events) => (None, Some(max_events.get())), + MemoryBufferSize::MaxSize(max_size) => (None, Some(max_size.get())), + }; + + usage_handle.set_buffer_limits(max_bytes, max_size); - let (tx, rx) = limited(self.capacity.get()); + let (tx, rx) = limited(self.capacity); Ok((tx.into(), rx.into())) } } diff --git a/lib/vector-common/Cargo.toml b/lib/vector-common/Cargo.toml index 17ad79cdc73f1..07d09e7d45a48 100644 --- a/lib/vector-common/Cargo.toml +++ b/lib/vector-common/Cargo.toml @@ -48,11 +48,11 @@ serde.workspace = true serde_json.workspace = true smallvec = { version = "1", default-features = false } stream-cancel = { version = "0.8.2", default-features = false } -tokio = { version = "1.44.2", default-features = false, features = ["macros", "time"] } +tokio = { version = "1.45.1", default-features = false, features = ["macros", "time"] } tracing = { version = "0.1.34", default-features = false } vrl.workspace = true vector-config = { path = "../vector-config" } [dev-dependencies] futures = { version = "0.3.31", default-features = false, features = ["async-await"] } -tokio = { version = "1.44.2", default-features = false, features = ["rt", "time"] } +tokio = { version = "1.45.1", default-features = false, features = ["rt", "time"] } diff --git a/lib/vector-common/src/finalization.rs b/lib/vector-common/src/finalization.rs index 48001a2bf8727..56310a37d31b4 100644 --- a/lib/vector-common/src/finalization.rs +++ b/lib/vector-common/src/finalization.rs @@ -462,7 +462,7 @@ mod tests { assert_eq!(receiver2.try_recv(), Ok(BatchStatus::Delivered)); } - #[ignore] // The current implementation does not deduplicate finalizers + #[ignore = "The current implementation does not deduplicate finalizers"] #[test] fn clone_and_merge_events() { let (mut fin1, mut receiver) = make_finalizer(); diff --git a/lib/vector-common/src/internal_event/prelude.rs b/lib/vector-common/src/internal_event/prelude.rs index e15a6b52b4503..92aa160fe509e 100644 --- a/lib/vector-common/src/internal_event/prelude.rs +++ b/lib/vector-common/src/internal_event/prelude.rs @@ -1,5 +1,6 @@ // Set of `stage` tags to use when emitting error events. pub mod error_stage { + pub const INITIALIZING: &str = "initializing"; pub const RECEIVING: &str = "receiving"; pub const PROCESSING: &str = "processing"; pub const SENDING: &str = "sending"; diff --git a/lib/vector-common/src/request_metadata.rs b/lib/vector-common/src/request_metadata.rs index 12a164d23e437..d38321a476e40 100644 --- a/lib/vector-common/src/request_metadata.rs +++ b/lib/vector-common/src/request_metadata.rs @@ -196,7 +196,7 @@ impl AddAssign for GroupedCountByteSize { } } (Self::Untagged { .. }, Self::Tagged { .. }) => unreachable!(), - }; + } } } diff --git a/lib/vector-common/src/shutdown.rs b/lib/vector-common/src/shutdown.rs index d1b35be5c6fb7..741a5b4839d9e 100644 --- a/lib/vector-common/src/shutdown.rs +++ b/lib/vector-common/src/shutdown.rs @@ -111,9 +111,9 @@ type IsInternal = bool; #[derive(Debug, Default)] pub struct SourceShutdownCoordinator { - shutdown_begun_triggers: HashMap, - shutdown_force_triggers: HashMap, - shutdown_complete_tripwires: HashMap, + begun_triggers: HashMap, + force_triggers: HashMap, + complete_tripwires: HashMap, } impl SourceShutdownCoordinator { @@ -129,11 +129,11 @@ impl SourceShutdownCoordinator { let (force_shutdown_trigger, force_shutdown_tripwire) = Tripwire::new(); let (shutdown_complete_trigger, shutdown_complete_tripwire) = Tripwire::new(); - self.shutdown_begun_triggers + self.begun_triggers .insert(id.clone(), (internal, shutdown_begun_trigger)); - self.shutdown_force_triggers + self.force_triggers .insert(id.clone(), force_shutdown_trigger); - self.shutdown_complete_tripwires + self.complete_tripwires .insert(id.clone(), shutdown_complete_tripwire); let shutdown_signal = @@ -151,9 +151,9 @@ impl SourceShutdownCoordinator { /// /// Panics if the other coordinator already had its triggers removed. pub fn takeover_source(&mut self, id: &ComponentKey, other: &mut Self) { - let existing = self.shutdown_begun_triggers.insert( + let existing = self.begun_triggers.insert( id.clone(), - other.shutdown_begun_triggers.remove(id).unwrap_or_else(|| { + other.begun_triggers.remove(id).unwrap_or_else(|| { panic!( "Other ShutdownCoordinator didn't have a shutdown_begun_trigger for \"{id}\"" ) @@ -164,9 +164,9 @@ impl SourceShutdownCoordinator { "ShutdownCoordinator already has a shutdown_begin_trigger for source \"{id}\"" ); - let existing = self.shutdown_force_triggers.insert( + let existing = self.force_triggers.insert( id.clone(), - other.shutdown_force_triggers.remove(id).unwrap_or_else(|| { + other.force_triggers.remove(id).unwrap_or_else(|| { panic!( "Other ShutdownCoordinator didn't have a shutdown_force_trigger for \"{id}\"" ) @@ -177,10 +177,10 @@ impl SourceShutdownCoordinator { "ShutdownCoordinator already has a shutdown_force_trigger for source \"{id}\"" ); - let existing = self.shutdown_complete_tripwires.insert( + let existing = self.complete_tripwires.insert( id.clone(), other - .shutdown_complete_tripwires + .complete_tripwires .remove(id) .unwrap_or_else(|| { panic!( @@ -207,9 +207,9 @@ impl SourceShutdownCoordinator { let mut internal_sources_complete_futures = Vec::new(); let mut external_sources_complete_futures = Vec::new(); - let shutdown_begun_triggers = self.shutdown_begun_triggers; - let mut shutdown_complete_tripwires = self.shutdown_complete_tripwires; - let mut shutdown_force_triggers = self.shutdown_force_triggers; + let shutdown_begun_triggers = self.begun_triggers; + let mut shutdown_complete_tripwires = self.complete_tripwires; + let mut shutdown_force_triggers = self.force_triggers; for (id, (internal, trigger)) in shutdown_begun_triggers { trigger.cancel(); @@ -260,24 +260,23 @@ impl SourceShutdownCoordinator { id: &ComponentKey, deadline: Instant, ) -> impl Future { - let (_, begin_shutdown_trigger) = - self.shutdown_begun_triggers.remove(id).unwrap_or_else(|| { - panic!( + let (_, begin_shutdown_trigger) = self.begun_triggers.remove(id).unwrap_or_else(|| { + panic!( "shutdown_begun_trigger for source \"{id}\" not found in the ShutdownCoordinator" ) - }); + }); // This is what actually triggers the source to begin shutting down. begin_shutdown_trigger.cancel(); let shutdown_complete_tripwire = self - .shutdown_complete_tripwires + .complete_tripwires .remove(id) .unwrap_or_else(|| { panic!( "shutdown_complete_tripwire for source \"{id}\" not found in the ShutdownCoordinator" ) }); - let shutdown_force_trigger = self.shutdown_force_triggers.remove(id).unwrap_or_else(|| { + let shutdown_force_trigger = self.force_triggers.remove(id).unwrap_or_else(|| { panic!( "shutdown_force_trigger for source \"{id}\" not found in the ShutdownCoordinator" ) @@ -294,7 +293,7 @@ impl SourceShutdownCoordinator { #[must_use] pub fn shutdown_tripwire(&self) -> future::BoxFuture<'static, ()> { let futures = self - .shutdown_complete_tripwires + .complete_tripwires .values() .cloned() .map(|tripwire| tripwire.then(tripwire_handler).boxed()); diff --git a/lib/vector-config-common/Cargo.toml b/lib/vector-config-common/Cargo.toml index 5a057e977f656..46d93242ce01a 100644 --- a/lib/vector-config-common/Cargo.toml +++ b/lib/vector-config-common/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" license = "MPL-2.0" [dependencies] -convert_case = { version = "0.7", default-features = false } +convert_case = { version = "0.8", default-features = false } darling.workspace = true proc-macro2 = { version = "1.0", default-features = false } serde.workspace = true diff --git a/lib/vector-config-common/src/schema/json_schema.rs b/lib/vector-config-common/src/schema/json_schema.rs index 3c4a6c99410a8..38fd13b2f8f93 100644 --- a/lib/vector-config-common/src/schema/json_schema.rs +++ b/lib/vector-config-common/src/schema/json_schema.rs @@ -615,8 +615,7 @@ pub fn get_cleaned_schema_reference(schema_ref: &str) -> &str { cleaned } else { panic!( - "Tried to clean schema reference that does not start with the definition prefix: {}", - schema_ref + "Tried to clean schema reference that does not start with the definition prefix: {schema_ref}" ); } } diff --git a/lib/vector-config-macros/src/component_name.rs b/lib/vector-config-macros/src/component_name.rs index 5faf393249173..431e382d712e2 100644 --- a/lib/vector-config-macros/src/component_name.rs +++ b/lib/vector-config-macros/src/component_name.rs @@ -134,8 +134,7 @@ fn attr_to_component_name(attr: &Attribute) -> Result, Error> { return Err(Error::new( attr.span(), format!( - "{}s must have a name specified (e.g. `{}(\"my_component\")`)", - component_type, component_type_attr + "{component_type}s must have a name specified (e.g. `{component_type_attr}(\"my_component\")`)" ), )); } @@ -147,8 +146,7 @@ fn attr_to_component_name(attr: &Attribute) -> Result, Error> { Error::new( attr.span(), format!( - "expected a string literal for the {} name (i.e. `{}(\"...\")`)", - component_type, component_type_attr + "expected a string literal for the {component_type} name (i.e. `{component_type_attr}(\"...\")`)" ), ) }) @@ -185,6 +183,6 @@ fn check_component_name_validity(component_name: &str) -> Result<(), String> { if component_name == component_name_converted { Ok(()) } else { - Err(format!("component names must be lowercase, and contain only letters, numbers, and underscores (e.g. \"{}\")", component_name_converted)) + Err(format!("component names must be lowercase, and contain only letters, numbers, and underscores (e.g. \"{component_name_converted}\")")) } } diff --git a/lib/vector-config-macros/src/configurable.rs b/lib/vector-config-macros/src/configurable.rs index fa27431a6c4c2..cc506c7869552 100644 --- a/lib/vector-config-macros/src/configurable.rs +++ b/lib/vector-config-macros/src/configurable.rs @@ -250,8 +250,7 @@ fn generate_named_struct_field( .expect("named struct fields must always have an ident"); let field_schema_ty = get_field_schema_ty(field); let field_already_contained = format!( - "schema properties already contained entry for `{}`, this should not occur", - field_name + "schema properties already contained entry for `{field_name}`, this should not occur" ); let field_key = field.name(); @@ -675,8 +674,7 @@ fn generate_named_enum_field(field: &Field<'_>) -> proc_macro2::TokenStream { let field_name = field.ident().expect("field should be named"); let field_ty = field.ty(); let field_already_contained = format!( - "schema properties already contained entry for `{}`, this should not occur", - field_name + "schema properties already contained entry for `{field_name}`, this should not occur" ); let field_key = field.name().to_string(); @@ -700,15 +698,24 @@ fn generate_named_enum_field(field: &Field<'_>) -> proc_macro2::TokenStream { None }; - quote! { - { - #field_schema - - if let Some(_) = properties.insert(#field_key.to_string(), subschema) { - panic!(#field_already_contained); + if field.flatten() { + quote! { + { + #field_schema + flattened_subschemas.push(subschema); } + } + } else { + quote! { + { + #field_schema - #maybe_field_required + if let Some(_) = properties.insert(#field_key.to_string(), subschema) { + panic!(#field_already_contained); + } + + #maybe_field_required + } } } } @@ -733,6 +740,7 @@ fn generate_enum_struct_named_variant_schema( { let mut properties = ::vector_config::indexmap::IndexMap::new(); let mut required = ::std::collections::BTreeSet::new(); + let mut flattened_subschemas = ::std::vec::Vec::new(); #(#mapped_fields)* @@ -740,11 +748,18 @@ fn generate_enum_struct_named_variant_schema( #maybe_fill_discriminant_map - ::vector_config::schema::generate_struct_schema( + let mut schema = ::vector_config::schema::generate_struct_schema( properties, required, None - ) + ); + + // If we have any flattened subschemas, deal with them now. + if !flattened_subschemas.is_empty() { + ::vector_config::schema::convert_to_flattened_schema(&mut schema, flattened_subschemas); + } + + schema } } } @@ -841,7 +856,7 @@ fn generate_enum_variant_schema( // { "field_using_enum": { "": "VariantName" } } Tagging::Internal { tag } => match variant.style() { Style::Struct => { - let tag_already_contained = format!("enum tag `{}` already contained as a field in variant; tag cannot overlap with any fields in any variant", tag); + let tag_already_contained = format!("enum tag `{tag}` already contained as a field in variant; tag cannot overlap with any fields in any variant"); // Just generate the tag field directly and pass it along to be included in the // struct schema. diff --git a/lib/vector-config/Cargo.toml b/lib/vector-config/Cargo.toml index 3098fe2488280..fbbc7a249c02e 100644 --- a/lib/vector-config/Cargo.toml +++ b/lib/vector-config/Cargo.toml @@ -20,7 +20,7 @@ no-proxy = { version = "0.3.6", default-features = false, features = ["serialize num-traits = { version = "0.2.19", default-features = false } serde.workspace = true serde_json.workspace = true -serde_with = { version = "3.12.0", default-features = false, features = ["std"] } +serde_with = { version = "3.14.0", default-features = false, features = ["std"] } snafu.workspace = true toml.workspace = true tracing = { version = "0.1.34", default-features = false } @@ -32,4 +32,4 @@ vector-config-macros = { path = "../vector-config-macros" } [dev-dependencies] assert-json-diff = { version = "2", default-features = false } -serde_with = { version = "3.12.0", default-features = false, features = ["std", "macros"] } +serde_with = { version = "3.14.0", default-features = false, features = ["std", "macros"] } diff --git a/lib/vector-config/src/schema/helpers.rs b/lib/vector-config/src/schema/helpers.rs index 9900f32998f91..cfb79d9221c66 100644 --- a/lib/vector-config/src/schema/helpers.rs +++ b/lib/vector-config/src/schema/helpers.rs @@ -670,7 +670,7 @@ pub(crate) fn assert_string_schema_for_map( } } -/// Determines whether or not an enum schema is ambiguous based on discriminants of its variants. +/// Determines whether an enum schema is ambiguous based on discriminants of its variants. /// /// A discriminant is the set of the named fields which are required, which may be an empty set. pub fn has_ambiguous_discriminants( diff --git a/lib/vector-config/src/schema/parser/component.rs b/lib/vector-config/src/schema/parser/component.rs index 37997e023dbab..da86f25ced59d 100644 --- a/lib/vector-config/src/schema/parser/component.rs +++ b/lib/vector-config/src/schema/parser/component.rs @@ -133,7 +133,7 @@ fn get_component_metadata_kv_str<'a>( Value::String(name) => Ok(name), _ => Err(SchemaError::invalid_component_schema( key, - format!("`{}` must be a string", key), + format!("`{key}` must be a string"), )), }) } diff --git a/lib/vector-config/src/schema/visitors/inline_single.rs b/lib/vector-config/src/schema/visitors/inline_single.rs index 4aee9499b2398..92e8576711faf 100644 --- a/lib/vector-config/src/schema/visitors/inline_single.rs +++ b/lib/vector-config/src/schema/visitors/inline_single.rs @@ -41,7 +41,7 @@ impl Visitor for InlineSingleUseReferencesVisitor { // entire schema, by visiting the root schema in a recursive fashion, using a helper visitor. let mut occurrence_visitor = OccurrenceVisitor::default(); occurrence_visitor.visit_root_schema(root); - let occurrence_map = occurrence_visitor.into_occurrences(); + let occurrence_map = occurrence_visitor.occurrence_map; self.eligible_to_inline = occurrence_map .into_iter() @@ -141,16 +141,7 @@ fn is_inlineable_schema(definition_name: &str, schema: &SchemaObject) -> bool { #[derive(Debug, Default)] struct OccurrenceVisitor { scope_stack: SchemaScopeStack, - occurrence_map: HashMap>, -} - -impl OccurrenceVisitor { - fn into_occurrences(self) -> HashMap { - self.occurrence_map - .into_iter() - .map(|(k, v)| (k, v.len())) - .collect() - } + occurrence_map: HashMap, } impl Visitor for OccurrenceVisitor { @@ -162,23 +153,11 @@ impl Visitor for OccurrenceVisitor { visit_schema_object_scoped(self, definitions, schema); if let Some(current_schema_ref) = schema.reference.as_ref() { - // Track the named "parent" schema for the schema we're currently visiting so that if we - // visit this schema again, we don't double count any schema references that it has. The - // "parent" schema is simply the closest ancestor schema that was itself a schema - // reference, or "Root", which represents the oldest schema ancestor in the document. - // - // This lets us couple with scenarios where schema A references schema B, and is the - // only actual direct schema reference to schema B, but due to multiple schemas - // referencing schema A, would otherwise lead to both A and B being visited multiple - // times. - let current_parent_schema_ref = self.get_current_schema_scope().clone(); let current_schema_ref = get_cleaned_schema_reference(current_schema_ref); - - let occurrences = self + *self .occurrence_map .entry(current_schema_ref.into()) - .or_default(); - occurrences.insert(current_parent_schema_ref); + .or_default() += 1; } } } @@ -257,17 +236,7 @@ mod tests { assert_schemas_eq(expected_schema, actual_schema); } - // TODO(tobz): These two tests are ignored because the inliner currently works off of schema - // reference scopes, so two object properties within the same schema don't count as two - // instances of a schema being referenced. - // - // We need to refactor schema scopes to be piecemeal extensible more in the way of how - // `jsonschema` does it rather than an actual stack.... the current approach is good enough for - // now, but not optimal in the way that it could be. - // - // These are here for when I improve the situation after getting this merged. #[test] - #[ignore] fn single_ref_multiple_usages() { let mut actual_schema = as_schema(json!({ "definitions": { @@ -294,7 +263,6 @@ mod tests { } #[test] - #[ignore] fn multiple_refs_mixed_usages() { let mut actual_schema = as_schema(json!({ "definitions": { @@ -346,4 +314,131 @@ mod tests { assert_schemas_eq(expected_schema, actual_schema); } + + #[test] + fn reference_in_multiple_arrays() { + let mut actual_schema = as_schema(json!({ + "definitions": { + "item": { + "type": "object", + "properties": { + "x": { "type": "string" } + } + } + }, + "type": "object", + "properties": { + "arr1": { "type": "array", "items": { "$ref": "#/definitions/item" } }, + "arr2": { "type": "array", "items": { "$ref": "#/definitions/item" } } + } + })); + + let expected_schema = actual_schema.clone(); + + let mut visitor = InlineSingleUseReferencesVisitor::default(); + visitor.visit_root_schema(&mut actual_schema); + + assert_schemas_eq(expected_schema, actual_schema); + } + + #[test] + fn reference_in_oneof_anyof_allof() { + let mut actual_schema = as_schema(json!({ + "definitions": { + "shared": { + "type": "object", + "properties": { + "y": { "type": "string" } + } + } + }, + "type": "object", + "properties": { + "choice": { + "oneOf": [ + { "$ref": "#/definitions/shared" }, + { "$ref": "#/definitions/shared" } + ], + "anyOf": [ + { "$ref": "#/definitions/shared" }, + { "type": "null" } + ], + "allOf": [ + { "$ref": "#/definitions/shared" }, + { "type": "object" } + ] + } + } + })); + + let expected_schema = actual_schema.clone(); + + let mut visitor = InlineSingleUseReferencesVisitor::default(); + visitor.visit_root_schema(&mut actual_schema); + + assert_schemas_eq(expected_schema, actual_schema); + } + + #[test] + fn reference_in_additional_properties() { + let mut actual_schema = as_schema(json!({ + "definitions": { + "val": { + "type": "object", + "properties": { + "z": { "type": "string" } + } + } + }, + "type": "object", + "properties": { + "obj1": { + "type": "object", + "additionalProperties": { "$ref": "#/definitions/val" } + }, + "obj2": { + "type": "object", + "additionalProperties": { "$ref": "#/definitions/val" } + } + } + })); + + let expected_schema = actual_schema.clone(); + + let mut visitor = InlineSingleUseReferencesVisitor::default(); + visitor.visit_root_schema(&mut actual_schema); + + assert_schemas_eq(expected_schema, actual_schema); + } + + #[test] + fn reference_in_pattern_properties() { + let mut actual_schema = as_schema(json!({ + "definitions": { + "pat": { + "type": "object", + "properties": { + "w": { "type": "string" } + } + } + }, + "type": "object", + "properties": { + "obj": { + "type": "object", + "patternProperties": { + "^foo$": { "$ref": "#/definitions/pat" }, + "^bar$": { "$ref": "#/definitions/pat" } + } + } + } + })); + + let expected_schema = actual_schema.clone(); + + let mut visitor = InlineSingleUseReferencesVisitor::default(); + visitor.visit_root_schema(&mut actual_schema); + + assert_schemas_eq(expected_schema, actual_schema); + } } diff --git a/lib/vector-config/src/schema/visitors/merge.rs b/lib/vector-config/src/schema/visitors/merge.rs index 28b724d602c5c..ef7d6f8f01d99 100644 --- a/lib/vector-config/src/schema/visitors/merge.rs +++ b/lib/vector-config/src/schema/visitors/merge.rs @@ -50,7 +50,7 @@ impl Mergeable for Value { // We _may_ need to relax this in practice/in the future, but it's a solid invariant to // enforce for the time being. if discriminant(self) != discriminant(other) { - panic!("Tried to merge two `Value` types together with differing types!\n\nSelf: {:?}\n\nOther: {:?}", self, other); + panic!("Tried to merge two `Value` types together with differing types!\n\nSelf: {self:?}\n\nOther: {other:?}"); } match (self, other) { diff --git a/lib/vector-config/src/schema/visitors/scoped_visit.rs b/lib/vector-config/src/schema/visitors/scoped_visit.rs index 937a4ee4920dd..189bbbcfec9f4 100644 --- a/lib/vector-config/src/schema/visitors/scoped_visit.rs +++ b/lib/vector-config/src/schema/visitors/scoped_visit.rs @@ -15,7 +15,7 @@ pub enum SchemaReference { impl std::fmt::Display for SchemaReference { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Self::Definition(name) => write!(f, "{}", name), + Self::Definition(name) => write!(f, "{name}"), Self::Root => write!(f, ""), } } diff --git a/lib/vector-config/src/stdlib.rs b/lib/vector-config/src/stdlib.rs index ad3ae47aac32d..add05f87fd9ea 100644 --- a/lib/vector-config/src/stdlib.rs +++ b/lib/vector-config/src/stdlib.rs @@ -2,7 +2,7 @@ use std::{ cell::RefCell, collections::{BTreeMap, BTreeSet, HashMap, HashSet}, hash::Hash, - net::SocketAddr, + net::{Ipv4Addr, SocketAddr}, num::{ NonZeroI16, NonZeroI32, NonZeroI64, NonZeroI8, NonZeroU16, NonZeroU32, NonZeroU64, NonZeroU8, NonZeroUsize, @@ -402,6 +402,31 @@ impl ToValue for SocketAddr { } } +impl Configurable for Ipv4Addr { + fn referenceable_name() -> Option<&'static str> { + Some("stdlib::Ipv4Addr") + } + + fn metadata() -> Metadata { + let mut metadata = Metadata::default(); + metadata.set_description("An IPv4 address."); + metadata + } + + fn generate_schema(_: &RefCell) -> Result { + // TODO: We don't need anything other than a string schema to (de)serialize a `Ipv4Addr`, + // but we eventually should have validation since the format for the possible permutations + // is well-known and can be easily codified. + Ok(generate_string_schema()) + } +} + +impl ToValue for Ipv4Addr { + fn to_value(&self) -> Value { + Value::String(self.to_string()) + } +} + impl Configurable for PathBuf { fn referenceable_name() -> Option<&'static str> { Some("stdlib::PathBuf") diff --git a/lib/vector-config/tests/integration/smoke.rs b/lib/vector-config/tests/integration/smoke.rs index 8d9d1fbb8e4df..2b99bc1f9f891 100644 --- a/lib/vector-config/tests/integration/smoke.rs +++ b/lib/vector-config/tests/integration/smoke.rs @@ -312,7 +312,7 @@ impl From for String { if fd == 0 { "systemd".to_owned() } else { - format!("systemd#{}", fd) + format!("systemd#{fd}") } } } @@ -633,8 +633,8 @@ fn generate_semi_real_schema() { let json = serde_json::to_string_pretty(&schema) .expect("rendering root schema to JSON should not fail"); - println!("{}", json); + println!("{json}"); } - Err(e) => eprintln!("error while generating schema: {:?}", e), + Err(e) => eprintln!("error while generating schema: {e:?}"), } } diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index af3c2839e8dfe..a9b120ef660f0 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -16,8 +16,8 @@ chrono.workspace = true chrono-tz.workspace = true crossbeam-utils = { version = "0.8.21", default-features = false } derivative = { version = "2.2.0", default-features = false } -dyn-clone = { version = "1.0.19", default-features = false } -enumflags2 = { version = "0.7.11", default-features = false } +dyn-clone = { version = "1.0.20", default-features = false } +enumflags2 = { version = "0.7.12", default-features = false } float_eq = { version = "1.0", default-features = false } futures.workspace = true futures-util = { version = "0.3.29", default-features = false, features = ["std"] } @@ -31,25 +31,25 @@ lookup = { package = "vector-lookup", path = "../vector-lookup" } metrics.workspace = true metrics-tracing-context.workspace = true metrics-util.workspace = true -mlua = { version = "0.10.3", default-features = false, features = ["lua54", "send", "vendored"], optional = true } +mlua = { version = "0.10.5", default-features = false, features = ["lua54", "send", "vendored"], optional = true } no-proxy = { version = "0.3.6", default-features = false, features = ["serialize"] } ordered-float = { version = "4.6.0", default-features = false } -openssl = { version = "0.10.72", default-features = false, features = ["vendored"] } -parking_lot = { version = "0.12.3", default-features = false } +openssl = { version = "0.10.73", default-features = false, features = ["vendored"] } +parking_lot = { version = "0.12.4", default-features = false } pin-project.workspace = true -proptest = { version = "1.6", optional = true } +proptest = { version = "1.7", optional = true } prost-types.workspace = true prost .workspace = true -quanta = { version = "0.12.5", default-features = false } +quanta = { version = "0.12.6", default-features = false } regex = { version = "1.11.1", default-features = false, features = ["std", "perf"] } ryu = { version = "1", default-features = false } serde.workspace = true serde_json.workspace = true -serde_with = { version = "3.12.0", default-features = false, features = ["std", "macros"] } +serde_with = { version = "3.14.0", default-features = false, features = ["std", "macros"] } smallvec = { version = "1", default-features = false, features = ["serde", "const_generics"] } snafu.workspace = true -socket2 = { version = "0.5.9", default-features = false } -tokio = { version = "1.44.2", default-features = false, features = ["net"] } +socket2.workspace = true +tokio = { version = "1.45.1", default-features = false, features = ["net"] } tokio-openssl = { version = "0.6.5", default-features = false } tokio-stream = { version = "0.1", default-features = false, features = ["time"], optional = true } tokio-util = { version = "0.7.0", default-features = false, features = ["time"] } @@ -67,7 +67,7 @@ vrl.workspace = true cfg-if.workspace = true [target.'cfg(target_os = "macos")'.dependencies] -security-framework = "2.10.0" +security-framework = "3.2.0" [target.'cfg(windows)'.dependencies] schannel = "0.1.27" @@ -78,11 +78,11 @@ prost-build.workspace = true [dev-dependencies] base64 = "0.22.1" chrono-tz.workspace = true -criterion = { version = "0.5.1", features = ["html_reports"] } +criterion = { version = "0.7.0", features = ["html_reports"] } env-test-util = "1.0.1" quickcheck = "1" quickcheck_macros = "1" -proptest = "1.6" +proptest = "1.7" similar-asserts = "1.7.0" tokio-test = "0.4.4" toml.workspace = true diff --git a/lib/vector-core/src/config/global_options.rs b/lib/vector-core/src/config/global_options.rs index 80e7316f023bc..df12981f6f626 100644 --- a/lib/vector-core/src/config/global_options.rs +++ b/lib/vector-core/src/config/global_options.rs @@ -31,6 +31,19 @@ pub(crate) enum DataDirError { }, } +/// Specifies the wildcard matching mode, relaxed allows configurations where wildcard doesn not match any existing inputs +#[configurable_component] +#[derive(Clone, Debug, Copy, PartialEq, Eq, Default)] +#[serde(rename_all = "lowercase")] +pub enum WildcardMatching { + /// Strict matching (must match at least one existing input) + #[default] + Strict, + + /// Relaxed matching (must match 0 or more inputs) + Relaxed, +} + /// Global configuration options. // // If this is modified, make sure those changes are reflected in the `ConfigBuilder::append` @@ -48,6 +61,14 @@ pub struct GlobalOptions { #[configurable(metadata(docs::common = false))] pub data_dir: Option, + /// Set wildcard matching mode for inputs + /// + /// Setting this to "relaxed" allows configurations with wildcards that do not match any inputs + /// to be accepted without causing an error. + #[serde(skip_serializing_if = "crate::serde::is_default")] + #[configurable(metadata(docs::common = false, docs::required = false))] + pub wildcard_matching: Option, + /// Default log schema for all events. /// /// This is used if a component does not have its own specific log schema. All events use a log @@ -86,7 +107,7 @@ pub struct GlobalOptions { /// See [End-to-end Acknowledgements][e2e_acks] for more information on how Vector handles event /// acknowledgement. /// - /// [e2e_acks]: https://vector.dev/docs/about/under-the-hood/architecture/end-to-end-acknowledgements/ + /// [e2e_acks]: https://vector.dev/docs/architecture/end-to-end-acknowledgements/ #[serde( default, deserialize_with = "bool_or_struct", @@ -182,6 +203,13 @@ impl GlobalOptions { pub fn merge(&self, with: Self) -> Result> { let mut errors = Vec::new(); + if conflicts( + self.wildcard_matching.as_ref(), + with.wildcard_matching.as_ref(), + ) { + errors.push("conflicting values for 'wildcard_matching' found".to_owned()); + } + if conflicts(self.proxy.http.as_ref(), with.proxy.http.as_ref()) { errors.push("conflicting values for 'proxy.http' found".to_owned()); } @@ -250,6 +278,7 @@ impl GlobalOptions { if errors.is_empty() { Ok(Self { data_dir, + wildcard_matching: self.wildcard_matching.or(with.wildcard_matching), log_schema, telemetry, acknowledgements: self.acknowledgements.merge_default(&with.acknowledgements), diff --git a/lib/vector-core/src/config/log_schema.rs b/lib/vector-core/src/config/log_schema.rs index 2ab6c423a66d0..bbbcfaeba499e 100644 --- a/lib/vector-core/src/config/log_schema.rs +++ b/lib/vector-core/src/config/log_schema.rs @@ -45,6 +45,7 @@ pub fn log_schema() -> &'static LogSchema { #[configurable_component] #[derive(Clone, Debug, Eq, PartialEq)] #[serde(default)] +#[allow(clippy::struct_field_names)] pub struct LogSchema { /// The name of the event field to treat as the event message. /// diff --git a/lib/vector-core/src/config/mod.rs b/lib/vector-core/src/config/mod.rs index 074b8c5fa3cab..874014a96a859 100644 --- a/lib/vector-core/src/config/mod.rs +++ b/lib/vector-core/src/config/mod.rs @@ -13,7 +13,7 @@ pub mod proxy; mod telemetry; use crate::event::LogEvent; -pub use global_options::GlobalOptions; +pub use global_options::{GlobalOptions, WildcardMatching}; pub use log_schema::{init_log_schema, log_schema, LogSchema}; use lookup::{lookup_v2::ValuePath, path, PathPrefix}; pub use output_id::OutputId; @@ -304,7 +304,7 @@ Enabling or disabling acknowledgements at the source level has **no effect** on See [End-to-end Acknowledgements][e2e_acks] for more information on how event acknowledgement is handled. [global_acks]: https://vector.dev/docs/reference/configuration/global-options/#acknowledgements -[e2e_acks]: https://vector.dev/docs/about/under-the-hood/architecture/end-to-end-acknowledgements/" +[e2e_acks]: https://vector.dev/docs/architecture/end-to-end-acknowledgements/" )] #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub struct SourceAcknowledgementsConfig { @@ -352,7 +352,7 @@ impl From for AcknowledgementsConfig { #[configurable( description = "See [End-to-end Acknowledgements][e2e_acks] for more information on how event acknowledgement is handled. -[e2e_acks]: https://vector.dev/docs/about/under-the-hood/architecture/end-to-end-acknowledgements/" +[e2e_acks]: https://vector.dev/docs/architecture/end-to-end-acknowledgements/" )] #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub struct AcknowledgementsConfig { diff --git a/lib/vector-core/src/event/discriminant.rs b/lib/vector-core/src/event/discriminant.rs index 20770506562db..08f7f913222ab 100644 --- a/lib/vector-core/src/event/discriminant.rs +++ b/lib/vector-core/src/event/discriminant.rs @@ -77,7 +77,7 @@ fn f64_eq(this: f64, other: f64) -> bool { } if this != other { return false; - }; + } if (this.is_sign_positive() && other.is_sign_negative()) || (this.is_sign_negative() && other.is_sign_positive()) { diff --git a/lib/vector-core/src/event/metadata.rs b/lib/vector-core/src/event/metadata.rs index c1dd5790236bb..9b784fb9b22f1 100644 --- a/lib/vector-core/src/event/metadata.rs +++ b/lib/vector-core/src/event/metadata.rs @@ -84,11 +84,11 @@ pub(super) struct Inner { #[derive(Clone, Default, Debug, Deserialize, PartialEq, Serialize)] pub struct DatadogMetricOriginMetadata { /// `OriginProduct` - origin_product: Option, + product: Option, /// `OriginCategory` - origin_category: Option, + category: Option, /// `OriginService` - origin_service: Option, + service: Option, } impl DatadogMetricOriginMetadata { @@ -100,25 +100,25 @@ impl DatadogMetricOriginMetadata { #[must_use] pub fn new(product: Option, category: Option, service: Option) -> Self { Self { - origin_product: product, - origin_category: category, - origin_service: service, + product, + category, + service, } } /// Returns a reference to the `OriginProduct`. pub fn product(&self) -> Option { - self.origin_product + self.product } /// Returns a reference to the `OriginCategory`. pub fn category(&self) -> Option { - self.origin_category + self.category } /// Returns a reference to the `OriginService`. pub fn service(&self) -> Option { - self.origin_service + self.service } } @@ -356,7 +356,7 @@ impl EventMetadata { inner.source_event_id = Some(uuid2); } _ => {} // Keep the existing value. - }; + } } /// Update the finalizer(s) status. diff --git a/lib/vector-core/src/event/util/log/all_fields.rs b/lib/vector-core/src/event/util/log/all_fields.rs index 10afadfede4cc..5194f05cb707d 100644 --- a/lib/vector-core/src/event/util/log/all_fields.rs +++ b/lib/vector-core/src/event/util/log/all_fields.rs @@ -144,7 +144,7 @@ impl<'a> FieldsIter<'a> { None => break res.into(), Some(PathComponent::Key(key)) => { if self.quote_invalid_fields && !IS_VALID_PATH_SEGMENT.is_match(key) { - res.push_str(&format!("\"{key}\"")); + write!(res, "\"{key}\"").expect("write to String never fails"); } else { res.push_str(key); } @@ -191,7 +191,7 @@ impl<'a> Iterator for FieldsIter<'a> { *visited = true; break result; } - }; + } } } } diff --git a/lib/vector-core/src/event/vrl_target.rs b/lib/vector-core/src/event/vrl_target.rs index 7ef2aec3b87a6..c79e2df360301 100644 --- a/lib/vector-core/src/event/vrl_target.rs +++ b/lib/vector-core/src/event/vrl_target.rs @@ -1,4 +1,5 @@ use std::borrow::Cow; +use std::num::{NonZero, TryFromIntError}; use std::{collections::BTreeMap, convert::TryFrom, marker::PhantomData}; use lookup::lookup_v2::OwnedSegment; @@ -13,10 +14,11 @@ use super::{metric::TagValue, Event, EventMetadata, LogEvent, Metric, MetricKind use crate::config::{log_schema, LogNamespace}; use crate::schema::Definition; -const VALID_METRIC_PATHS_SET: &str = ".name, .namespace, .timestamp, .kind, .tags"; +const VALID_METRIC_PATHS_SET: &str = ".name, .namespace, .interval_ms, .timestamp, .kind, .tags"; /// We can get the `type` of the metric in Remap, but can't set it. -const VALID_METRIC_PATHS_GET: &str = ".name, .namespace, .timestamp, .kind, .tags, .type"; +const VALID_METRIC_PATHS_GET: &str = + ".name, .namespace, .interval_ms, .timestamp, .kind, .tags, .type"; /// Metrics aren't interested in paths that have a length longer than 3. /// @@ -102,7 +104,7 @@ impl VrlTarget { // We pre-generate [`Value`] types for the metric fields accessed in // the event. This allows us to then return references to those // values, even if the field is accessed more than once. - let value = precompute_metric_value(&metric, info); + let value = precompute_metric_value(&metric, info, multi_value_metric_tags); VrlTarget::Metric { metric, @@ -240,9 +242,13 @@ fn merge_array_definitions(mut definition: Definition) -> Definition { fn set_metric_tag_values(name: String, value: &Value, metric: &mut Metric, multi_value_tags: bool) { if multi_value_tags { - let tag_values = value - .as_array() - .unwrap_or(&[]) + let values = if let Value::Array(values) = value { + values.as_slice() + } else { + std::slice::from_ref(value) + }; + + let tag_values = values .iter() .filter_map(|value| match value { Value::Bytes(bytes) => { @@ -282,10 +288,7 @@ impl Target for VrlTarget { return Err(MetricPathError::SetPathError.to_string()); } - if let Some(paths) = path - .to_alternative_components(MAX_METRIC_PATH_DEPTH) - .first() - { + if let Some(paths) = path.to_alternative_components(MAX_METRIC_PATH_DEPTH) { match paths.as_slice() { ["tags"] => { let value = @@ -319,6 +322,15 @@ impl Target for VrlTarget { metric.series.name.namespace = Some(String::from_utf8_lossy(&value).into_owned()); } + ["interval_ms"] => { + let value: i64 = + value.clone().try_into_i64().map_err(|e| e.to_string())?; + let value: u32 = value + .try_into() + .map_err(|e: TryFromIntError| e.to_string())?; + let value = NonZero::try_from(value).map_err(|e| e.to_string())?; + metric.data.time.interval_ms = Some(value); + } ["timestamp"] => { let value = value.clone().try_timestamp().map_err(|e| e.to_string())?; @@ -407,11 +419,17 @@ impl Target for VrlTarget { if let Some(paths) = target_path .path .to_alternative_components(MAX_METRIC_PATH_DEPTH) - .first() { let removed_value = match paths.as_slice() { ["namespace"] => metric.series.name.namespace.take().map(Into::into), ["timestamp"] => metric.data.time.timestamp.take().map(Into::into), + ["interval_ms"] => metric + .data + .time + .interval_ms + .take() + .map(u32::from) + .map(Into::into), ["tags"] => metric.series.tags.take().map(|map| { map.into_iter_single() .map(|(k, v)| (k, v.into())) @@ -429,10 +447,10 @@ impl Target for VrlTarget { value.remove(&target_path.path, false); - return Ok(removed_value); + Ok(removed_value) + } else { + Ok(None) } - - Ok(None) } }, PathPrefix::Metadata => Ok(self @@ -459,13 +477,14 @@ impl SecretTarget for VrlTarget { /// Retrieves a value from a the provided metric using the path. /// Currently the root path and the following paths are supported: -/// - name -/// - namespace -/// - timestamp -/// - kind -/// - tags -/// - tags. -/// - type +/// - `name` +/// - `namespace` +/// - `interval_ms` +/// - `timestamp` +/// - `kind` +/// - `tags` +/// - `tags.` +/// - `type` /// /// Any other paths result in a `MetricPathError::InvalidPath` being returned. fn target_get_metric<'a>( @@ -478,27 +497,25 @@ fn target_get_metric<'a>( let value = value.get(path); - for paths in path.to_alternative_components(MAX_METRIC_PATH_DEPTH) { - match paths.as_slice() { - ["name"] | ["kind"] | ["type"] | ["tags", _] => return Ok(value), - ["namespace"] | ["timestamp"] | ["tags"] => { - if let Some(value) = value { - return Ok(Some(value)); - } - } - _ => { - return Err(MetricPathError::InvalidPath { - path: &path.to_string(), - expected: VALID_METRIC_PATHS_GET, - } - .to_string()) - } + let Some(paths) = path.to_alternative_components(MAX_METRIC_PATH_DEPTH) else { + return Ok(None); + }; + + match paths.as_slice() { + ["name"] + | ["kind"] + | ["type"] + | ["tags", _] + | ["namespace"] + | ["timestamp"] + | ["interval_ms"] + | ["tags"] => Ok(value), + _ => Err(MetricPathError::InvalidPath { + path: &path.to_string(), + expected: VALID_METRIC_PATHS_GET, } + .to_string()), } - - // We only reach this point if we have requested a tag that doesn't exist or an empty - // field. - Ok(None) } fn target_get_mut_metric<'a>( @@ -511,127 +528,138 @@ fn target_get_mut_metric<'a>( let value = value.get_mut(path); - for paths in path.to_alternative_components(MAX_METRIC_PATH_DEPTH) { - match paths.as_slice() { - ["name"] | ["kind"] | ["tags", _] => return Ok(value), - ["namespace"] | ["timestamp"] | ["tags"] => { - if let Some(value) = value { - return Ok(Some(value)); - } - } - _ => { - return Err(MetricPathError::InvalidPath { - path: &path.to_string(), - expected: VALID_METRIC_PATHS_SET, - } - .to_string()) - } + let Some(paths) = path.to_alternative_components(MAX_METRIC_PATH_DEPTH) else { + return Ok(None); + }; + + match paths.as_slice() { + ["name"] + | ["kind"] + | ["tags", _] + | ["namespace"] + | ["timestamp"] + | ["interval_ms"] + | ["tags"] => Ok(value), + _ => Err(MetricPathError::InvalidPath { + path: &path.to_string(), + expected: VALID_METRIC_PATHS_SET, } + .to_string()), } - - // We only reach this point if we have requested a tag that doesn't exist or an empty - // field. - Ok(None) } /// pre-compute the `Value` structure of the metric. /// /// This structure is partially populated based on the fields accessed by /// the VRL program as informed by `ProgramInfo`. -fn precompute_metric_value(metric: &Metric, info: &ProgramInfo) -> Value { - let mut map = ObjectMap::default(); - - let mut set_name = false; - let mut set_kind = false; - let mut set_type = false; - let mut set_namespace = false; - let mut set_timestamp = false; - let mut set_tags = false; +fn precompute_metric_value(metric: &Metric, info: &ProgramInfo, multi_value_tags: bool) -> Value { + struct MetricProperty { + property: &'static str, + getter: fn(&Metric) -> Option, + set: bool, + } - for target_path in &info.target_queries { - // Accessing a root path requires us to pre-populate all fields. - if target_path == &OwnedTargetPath::event_root() { - if !set_name { - map.insert("name".into(), metric.name().to_owned().into()); + impl MetricProperty { + fn new(property: &'static str, getter: fn(&Metric) -> Option) -> Self { + Self { + property, + getter, + set: false, } + } - if !set_kind { - map.insert("kind".into(), metric.kind().into()); + fn insert(&mut self, metric: &Metric, map: &mut ObjectMap) { + if self.set { + return; } - - if !set_type { - map.insert("type".into(), metric.value().clone().into()); + if let Some(value) = (self.getter)(metric) { + map.insert(self.property.into(), value); + self.set = true; } + } + } - if !set_namespace { - if let Some(namespace) = metric.namespace() { - map.insert("namespace".into(), namespace.to_owned().into()); - } - } + fn get_single_value_tags(metric: &Metric) -> Option { + metric.tags().cloned().map(|tags| { + tags.into_iter_single() + .map(|(tag, value)| (tag.into(), value.into())) + .collect::() + .into() + }) + } - if !set_timestamp { - if let Some(timestamp) = metric.timestamp() { - map.insert("timestamp".into(), timestamp.into()); - } - } + fn get_multi_value_tags(metric: &Metric) -> Option { + metric.tags().cloned().map(|tags| { + tags.iter_sets() + .map(|(tag, tag_set)| { + let array_values: Vec = tag_set + .iter() + .map(|v| match v { + Some(s) => Value::Bytes(s.as_bytes().to_vec().into()), + None => Value::Null, + }) + .collect(); + (tag.into(), Value::Array(array_values)) + }) + .collect::() + .into() + }) + } - if !set_tags { - if let Some(tags) = metric.tags().cloned() { - map.insert( - "tags".into(), - tags.into_iter_single() - .map(|(tag, value)| (tag.into(), value.into())) - .collect::() - .into(), - ); - } - } + let mut name = MetricProperty::new("name", |metric| Some(metric.name().to_owned().into())); + let mut kind = MetricProperty::new("kind", |metric| Some(metric.kind().into())); + let mut type_ = MetricProperty::new("type", |metric| Some(metric.value().clone().into())); + let mut namespace = MetricProperty::new("namespace", |metric| { + metric.namespace().map(String::from).map(Into::into) + }); + let mut interval_ms = + MetricProperty::new("interval_ms", |metric| metric.interval_ms().map(Into::into)); + let mut timestamp = + MetricProperty::new("timestamp", |metric| metric.timestamp().map(Into::into)); + let mut tags = MetricProperty::new( + "tags", + if multi_value_tags { + get_multi_value_tags + } else { + get_single_value_tags + }, + ); + + let mut map = ObjectMap::default(); + for target_path in &info.target_queries { + // Accessing a root path requires us to pre-populate all fields. + if target_path == &OwnedTargetPath::event_root() { + let mut properties = [ + &mut name, + &mut kind, + &mut type_, + &mut namespace, + &mut interval_ms, + &mut timestamp, + &mut tags, + ]; + properties + .iter_mut() + .for_each(|property| property.insert(metric, &mut map)); break; } // For non-root paths, we continuously populate the value with the // relevant data. if let Some(OwnedSegment::Field(field)) = target_path.path.segments.first() { - match field.as_ref() { - "name" if !set_name => { - set_name = true; - map.insert("name".into(), metric.name().to_owned().into()); - } - "kind" if !set_kind => { - set_kind = true; - map.insert("kind".into(), metric.kind().into()); - } - "type" if !set_type => { - set_type = true; - map.insert("type".into(), metric.value().clone().into()); - } - "namespace" if !set_namespace && metric.namespace().is_some() => { - set_namespace = true; - map.insert( - "namespace".into(), - metric.namespace().unwrap().to_owned().into(), - ); - } - "timestamp" if !set_timestamp && metric.timestamp().is_some() => { - set_timestamp = true; - map.insert("timestamp".into(), metric.timestamp().unwrap().into()); - } - "tags" if !set_tags && metric.tags().is_some() => { - set_tags = true; - map.insert( - "tags".into(), - metric - .tags() - .cloned() - .unwrap() - .into_iter_single() - .map(|(tag, value)| (tag.into(), value.into())) - .collect::() - .into(), - ); - } - _ => {} + let property = match field.as_ref() { + "name" => Some(&mut name), + "kind" => Some(&mut kind), + "type" => Some(&mut type_), + "namespace" => Some(&mut namespace), + "timestamp" => Some(&mut timestamp), + "interval_ms" => Some(&mut interval_ms), + "tags" => Some(&mut tags), + _ => None, + }; + if let Some(property) = property { + property.insert(metric, &mut map); } } } @@ -1088,7 +1116,8 @@ mod test { Utc.with_ymd_and_hms(2020, 12, 10, 12, 0, 0) .single() .expect("invalid timestamp"), - )); + )) + .with_interval_ms(Some(NonZero::::new(507).unwrap())); let info = ProgramInfo { fallible: false, @@ -1096,6 +1125,7 @@ mod test { target_queries: vec![ OwnedTargetPath::event(owned_value_path!("name")), OwnedTargetPath::event(owned_value_path!("namespace")), + OwnedTargetPath::event(owned_value_path!("interval_ms")), OwnedTargetPath::event(owned_value_path!("timestamp")), OwnedTargetPath::event(owned_value_path!("kind")), OwnedTargetPath::event(owned_value_path!("type")), @@ -1110,6 +1140,7 @@ mod test { btreemap! { "name" => "zub", "namespace" => "zoob", + "interval_ms" => 507, "timestamp" => Utc.with_ymd_and_hms(2020, 12, 10, 12, 0, 0).single().expect("invalid timestamp"), "tags" => btreemap! { "tig" => "tog" }, "kind" => "absolute", @@ -1125,6 +1156,13 @@ mod test { #[test] fn metric_fields() { + struct Case { + path: OwnedValuePath, + current: Option, + new: Value, + delete: bool, + } + let metric = Metric::new( "name", MetricKind::Absolute, @@ -1133,39 +1171,46 @@ mod test { .with_tags(Some(metric_tags!("tig" => "tog"))); let cases = vec![ - ( - owned_value_path!("name"), // Path - Some(Value::from("name")), // Current value - Value::from("namefoo"), // New value - false, // Test deletion - ), - ( - owned_value_path!("namespace"), - None, - "namespacefoo".into(), - true, - ), - ( - owned_value_path!("timestamp"), - None, - Utc.with_ymd_and_hms(2020, 12, 8, 12, 0, 0) + Case { + path: owned_value_path!("name"), + current: Some(Value::from("name")), + new: Value::from("namefoo"), + delete: false, + }, + Case { + path: owned_value_path!("namespace"), + current: None, + new: "namespacefoo".into(), + delete: true, + }, + Case { + path: owned_value_path!("timestamp"), + current: None, + new: Utc + .with_ymd_and_hms(2020, 12, 8, 12, 0, 0) .single() .expect("invalid timestamp") .into(), - true, - ), - ( - owned_value_path!("kind"), - Some(Value::from("absolute")), - "incremental".into(), - false, - ), - ( - owned_value_path!("tags", "thing"), - None, - "footag".into(), - true, - ), + delete: true, + }, + Case { + path: owned_value_path!("interval_ms"), + current: None, + new: 123_456.into(), + delete: true, + }, + Case { + path: owned_value_path!("kind"), + current: Some(Value::from("absolute")), + new: "incremental".into(), + delete: false, + }, + Case { + path: owned_value_path!("tags", "thing"), + current: None, + new: "footag".into(), + delete: true, + }, ]; let info = ProgramInfo { @@ -1175,13 +1220,20 @@ mod test { OwnedTargetPath::event(owned_value_path!("name")), OwnedTargetPath::event(owned_value_path!("namespace")), OwnedTargetPath::event(owned_value_path!("timestamp")), + OwnedTargetPath::event(owned_value_path!("interval_ms")), OwnedTargetPath::event(owned_value_path!("kind")), ], target_assignments: vec![], }; let mut target = VrlTarget::new(Event::Metric(metric), &info, false); - for (path, current, new, delete) in cases { + for Case { + path, + current, + new, + delete, + } in cases + { let path = OwnedTargetPath::event(path); assert_eq!( @@ -1249,13 +1301,21 @@ mod test { let validpaths_get = [ ".name", ".namespace", + ".interval_ms", ".timestamp", ".kind", ".tags", ".type", ]; - let validpaths_set = [".name", ".namespace", ".timestamp", ".kind", ".tags"]; + let validpaths_set = [ + ".name", + ".namespace", + ".interval_ms", + ".timestamp", + ".kind", + ".tags", + ]; let info = ProgramInfo { fallible: false, diff --git a/lib/vector-core/src/tls/incoming.rs b/lib/vector-core/src/tls/incoming.rs index d13c10717d8bc..290ad3e6f0ca4 100644 --- a/lib/vector-core/src/tls/incoming.rs +++ b/lib/vector-core/src/tls/incoming.rs @@ -320,13 +320,13 @@ impl MaybeTlsIncomingStream { continue; } Err(error) => { - let error = io::Error::new(io::ErrorKind::Other, error); + let error = io::Error::other(error); this.state = StreamState::AcceptError(error.to_string()); Poll::Ready(Err(error)) } }, StreamState::AcceptError(error) => { - Poll::Ready(Err(io::Error::new(io::ErrorKind::Other, error.clone()))) + Poll::Ready(Err(io::Error::other(error.clone()))) } StreamState::Closed => Poll::Ready(Err(io::ErrorKind::BrokenPipe.into())), }; @@ -369,14 +369,12 @@ impl AsyncWrite for MaybeTlsIncomingStream { Poll::Pending } Err(error) => { - let error = io::Error::new(io::ErrorKind::Other, error); + let error = io::Error::other(error); this.state = StreamState::AcceptError(error.to_string()); Poll::Ready(Err(error)) } }, - StreamState::AcceptError(error) => { - Poll::Ready(Err(io::Error::new(io::ErrorKind::Other, error.clone()))) - } + StreamState::AcceptError(error) => Poll::Ready(Err(io::Error::other(error.clone()))), StreamState::Closed => Poll::Ready(Ok(())), } } diff --git a/lib/vector-core/src/tls/settings.rs b/lib/vector-core/src/tls/settings.rs index e0c84dc51308c..9fcfe432c282f 100644 --- a/lib/vector-core/src/tls/settings.rs +++ b/lib/vector-core/src/tls/settings.rs @@ -41,8 +41,9 @@ pub const TEST_PEM_CLIENT_KEY_PATH: &str = #[configurable_component] #[configurable(metadata(docs::advanced))] #[derive(Clone, Debug, Default)] +#[serde(deny_unknown_fields)] pub struct TlsEnableableConfig { - /// Whether or not to require TLS for incoming or outgoing connections. + /// Whether to require TLS for incoming or outgoing connections. /// /// When enabled and used for incoming connections, an identity certificate is also required. See `tls.crt_file` for /// more information. @@ -370,7 +371,7 @@ impl TlsConfig { |der| X509::from_der(&der).map(|x509| vec![x509]), |pem| { pem.match_indices(PEM_START_MARKER) - .map(|(start, _)| X509::from_pem(pem[start..].as_bytes())) + .map(|(start, _)| X509::from_pem(&pem.as_bytes()[start..])) .collect() }, ) diff --git a/lib/vector-core/src/transform/mod.rs b/lib/vector-core/src/transform/mod.rs index 37d694e6c1b9b..87fac605f6b12 100644 --- a/lib/vector-core/src/transform/mod.rs +++ b/lib/vector-core/src/transform/mod.rs @@ -376,7 +376,6 @@ impl TransformOutputsBuf { /// # Panics /// /// Panics if there is no default output. - #[cfg(any(feature = "test", test))] pub fn drain(&mut self) -> impl Iterator + '_ { self.primary_buffer .as_mut() @@ -389,7 +388,6 @@ impl TransformOutputsBuf { /// # Panics /// /// Panics if there is no output with the given name. - #[cfg(any(feature = "test", test))] pub fn drain_named(&mut self, name: &str) -> impl Iterator + '_ { self.named_buffers .get_mut(name) @@ -402,12 +400,10 @@ impl TransformOutputsBuf { /// # Panics /// /// Panics if there is no default output. - #[cfg(any(feature = "test", test))] pub fn take_primary(&mut self) -> OutputBuffer { std::mem::take(self.primary_buffer.as_mut().expect("no default output")) } - #[cfg(any(feature = "test", test))] pub fn take_all_named(&mut self) -> HashMap { std::mem::take(&mut self.named_buffers) } @@ -482,7 +478,6 @@ impl OutputBuffer { }) } - #[cfg(any(feature = "test", test))] pub fn drain(&mut self) -> impl Iterator + '_ { self.0.drain(..).flat_map(EventArray::into_events) } diff --git a/lib/vector-lib/src/lib.rs b/lib/vector-lib/src/lib.rs index f7bda363cc6b7..3d2628c90e80d 100644 --- a/lib/vector-lib/src/lib.rs +++ b/lib/vector-lib/src/lib.rs @@ -34,7 +34,7 @@ pub mod config { clone_input_definitions, init_log_schema, init_telemetry, log_schema, proxy, telemetry, AcknowledgementsConfig, DataType, GlobalOptions, Input, LegacyKey, LogNamespace, LogSchema, OutputId, SourceAcknowledgementsConfig, SourceOutput, Tags, Telemetry, TransformOutput, - MEMORY_BUFFER_DEFAULT_MAX_EVENTS, + WildcardMatching, MEMORY_BUFFER_DEFAULT_MAX_EVENTS, }; } diff --git a/lib/vector-stream/Cargo.toml b/lib/vector-stream/Cargo.toml index 63493bea41cd3..dc08739e1c4b6 100644 --- a/lib/vector-stream/Cargo.toml +++ b/lib/vector-stream/Cargo.toml @@ -10,15 +10,16 @@ async-stream = { version = "0.3.6", default-features = false } futures.workspace = true futures-util = { version = "0.3.29", default-features = false, features = ["std"] } pin-project.workspace = true -tokio = { version = "1.44.2", default-features = false, features = ["net"] } +tokio.workspace = true tokio-util = { version = "0.7.0", default-features = false, features = ["time"] } -tower = { version = "0.4", default-features = false, features = ["util"] } +tower = { version = "0.5.2", default-features = false, features = ["util"] } tracing = { version = "0.1.34", default-features = false } -twox-hash = { version = "2.1.0", default-features = false, features = ["xxhash64"] } +twox-hash = { version = "2.1.1", default-features = false, features = ["xxhash64"] } vector-common = { path = "../vector-common" } vector-core = { path = "../vector-core" } [dev-dependencies] -proptest = "1.6" +proptest = "1.7" rand.workspace = true rand_distr.workspace = true +tokio = {workspace = true, features = ["test-util"] } diff --git a/lib/vector-stream/src/concurrent_map.rs b/lib/vector-stream/src/concurrent_map.rs index 57ce9b8bc47d7..d637537b65dc8 100644 --- a/lib/vector-stream/src/concurrent_map.rs +++ b/lib/vector-stream/src/concurrent_map.rs @@ -83,7 +83,11 @@ where } match ready!(this.in_flight.poll_next_unpin(cx)) { - // Either nothing is in-flight, or nothing is ready. + // If the stream is done and there is no futures managed by FuturesOrdered, + // we must end the stream by returning Poll::Ready(None). + None if this.stream.is_done() => Poll::Ready(None), + // If there are no in-flight futures managed by FuturesOrdered but the underlying + // stream is not done, then we must keep polling that stream. None => Poll::Pending, Some(result) => match result { Ok(item) => Poll::Ready(Some(item)), @@ -101,3 +105,21 @@ where } } } + +#[cfg(test)] +mod tests { + use super::*; + use futures_util::stream::StreamExt; + + #[tokio::test] + async fn test_concurrent_map_on_empty_stream() { + let stream = futures_util::stream::empty::<()>(); + let limit = Some(NonZeroUsize::new(2).unwrap()); + // The `as _` is required to construct a `dyn Future` + let f = |()| Box::pin(async move {}) as _; + let mut concurrent_map = ConcurrentMap::new(stream, limit, f); + + // Assert that the stream does not hang + assert_eq!(concurrent_map.next().await, None); + } +} diff --git a/lib/vector-stream/src/driver.rs b/lib/vector-stream/src/driver.rs index b142deba814f2..45ec1af74741a 100644 --- a/lib/vector-stream/src/driver.rs +++ b/lib/vector-stream/src/driver.rs @@ -226,7 +226,7 @@ where finalizers.update_status(EventStatus::Rejected); } } - }; + } drop(finalizers); // suppress "argument not consumed" warning } diff --git a/lib/vector-tap/Cargo.toml b/lib/vector-tap/Cargo.toml index 83a89dd9c8fcf..7b1640748f49e 100644 --- a/lib/vector-tap/Cargo.toml +++ b/lib/vector-tap/Cargo.toml @@ -10,12 +10,12 @@ license = "MPL-2.0" api = ["dep:async-graphql"] [dependencies] -async-graphql = { version = "7.0.16", default-features = false, features = ["playground"], optional = true} +async-graphql = { version = "7.0.17", default-features = false, features = ["playground"], optional = true} colored = { version = "3.0.0", default-features = false } futures.workspace = true glob.workspace = true serde_yaml = { version = "0.9.34", default-features = false } -tokio = { version = "1.44.2", default-features = false, features = ["time"] } +tokio = { version = "1.45.1", default-features = false, features = ["time"] } tokio-stream = { version = "0.1.17", default-features = false, features = ["sync"] } tokio-tungstenite = { version = "0.20.1", default-features = false } tracing = { version = "0.1.34", default-features = false } @@ -31,4 +31,4 @@ futures-util = "0.3.30" chrono = { workspace = true } portpicker = { path = "../portpicker" } serde_json = { workspace = true } -tokio = { version = "1.44.2", default-features = false, features = ["test-util"] } +tokio = { version = "1.45.1", default-features = false, features = ["test-util"] } diff --git a/lib/vector-tap/src/controller.rs b/lib/vector-tap/src/controller.rs index 656f77c108bc2..cd10074d69f33 100644 --- a/lib/vector-tap/src/controller.rs +++ b/lib/vector-tap/src/controller.rs @@ -26,7 +26,7 @@ type TapSender = tokio_mpsc::Sender; type ShutdownTx = oneshot::Sender<()>; type ShutdownRx = oneshot::Receiver<()>; -const TAP_BUFFER_SIZE: NonZeroUsize = unsafe { NonZeroUsize::new_unchecked(100) }; +const TAP_BUFFER_SIZE: NonZeroUsize = NonZeroUsize::new(100).unwrap(); /// Clients can supply glob patterns to find matched topology components. trait GlobMatcher { @@ -123,7 +123,7 @@ impl TapPayload { invalid_matches: Vec, ) -> Self { let pattern = pattern.into(); - let message = format!("[tap] Warning: source inputs cannot be tapped. Input pattern '{}' matches sources {:?}", pattern, invalid_matches); + let message = format!("[tap] Warning: source inputs cannot be tapped. Input pattern '{pattern}' matches sources {invalid_matches:?}"); Self::Notification(Notification::InvalidMatch(InvalidMatch::new( message, pattern, @@ -138,8 +138,7 @@ impl TapPayload { ) -> Self { let pattern = pattern.into(); let message = format!( - "[tap] Warning: sink outputs cannot be tapped. Output pattern '{}' matches sinks {:?}", - pattern, invalid_matches + "[tap] Warning: sink outputs cannot be tapped. Output pattern '{pattern}' matches sinks {invalid_matches:?}" ); Self::Notification(Notification::InvalidMatch(InvalidMatch::new( message, diff --git a/lib/vector-tap/src/notification.rs b/lib/vector-tap/src/notification.rs index 58fd81766a764..04ba715718993 100644 --- a/lib/vector-tap/src/notification.rs +++ b/lib/vector-tap/src/notification.rs @@ -14,7 +14,7 @@ pub struct Matched { impl Matched { pub fn new(pattern: String) -> Self { Self { - message: format!("[tap] Pattern '{}' successfully matched.", pattern), + message: format!("[tap] Pattern '{pattern}' successfully matched."), pattern, } } @@ -34,8 +34,7 @@ impl NotMatched { pub fn new(pattern: String) -> Self { Self { message: format!( - "[tap] Pattern '{}' failed to match: will retry on configuration reload.", - pattern + "[tap] Pattern '{pattern}' failed to match: will retry on configuration reload." ), pattern, } diff --git a/lib/vector-vrl/tests/resources/json-schema_definition.json b/lib/vector-vrl/tests/resources/json-schema_definition.json new file mode 100644 index 0000000000000..9c4c8a00d0b28 --- /dev/null +++ b/lib/vector-vrl/tests/resources/json-schema_definition.json @@ -0,0 +1 @@ +{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://example.com/product.schema.json","title":"Product","description":"A product from Acme's catalog","type":"object","properties":{"productUser":{"description":"The unique identifier for a product user","type":"string","format":"email"}}} diff --git a/lib/vector-vrl/tests/src/docs.rs b/lib/vector-vrl/tests/src/docs.rs index ec694ce316f6e..0dacafb10ed5a 100644 --- a/lib/vector-vrl/tests/src/docs.rs +++ b/lib/vector-vrl/tests/src/docs.rs @@ -183,7 +183,7 @@ fn test_from_cue_example(category: &'static str, name: String, example: Example) Test { name: title, - category: format!("docs/{}/{}", category, name), + category: format!("docs/{category}/{name}"), error: None, source, object, diff --git a/lib/vector-vrl/tests/src/main.rs b/lib/vector-vrl/tests/src/main.rs index 4fd751796935f..6f0025d3cb054 100644 --- a/lib/vector-vrl/tests/src/main.rs +++ b/lib/vector-vrl/tests/src/main.rs @@ -56,7 +56,7 @@ pub struct Cmd { impl Cmd { fn timezone(&self) -> TimeZone { if let Some(ref tz) = self.timezone { - TimeZone::parse(tz).unwrap_or_else(|| panic!("couldn't parse timezone: {}", tz)) + TimeZone::parse(tz).unwrap_or_else(|| panic!("couldn't parse timezone: {tz}")) } else { TimeZone::Named(Tz::UTC) } diff --git a/lib/vector-vrl/tests/src/test_enrichment.rs b/lib/vector-vrl/tests/src/test_enrichment.rs index 725c4cd4a2ab6..ff803ea090b71 100644 --- a/lib/vector-vrl/tests/src/test_enrichment.rs +++ b/lib/vector-vrl/tests/src/test_enrichment.rs @@ -10,6 +10,7 @@ impl enrichment::Table for TestEnrichmentTable { _case: enrichment::Case, _condition: &'a [enrichment::Condition<'a>], _select: Option<&[String]>, + _wildcard: Option<&Value>, _index: Option, ) -> Result { let mut result = ObjectMap::new(); @@ -25,6 +26,7 @@ impl enrichment::Table for TestEnrichmentTable { _case: enrichment::Case, _condition: &'a [enrichment::Condition<'a>], _select: Option<&[String]>, + _wildcard: Option<&Value>, _index: Option, ) -> Result, String> { let mut result1 = ObjectMap::new(); diff --git a/lib/vector-vrl/web-playground/Cargo.toml b/lib/vector-vrl/web-playground/Cargo.toml index 8f2c875b6e718..30a2c44a7716c 100644 --- a/lib/vector-vrl/web-playground/Cargo.toml +++ b/lib/vector-vrl/web-playground/Cargo.toml @@ -2,6 +2,9 @@ name = "vector-vrl-web-playground" version = "0.1.0" edition = "2021" +repository = "https://github.com/vectordotdev/vector" +description = "A playground for experimenting with VRL" +license = "MPL-2.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -12,6 +15,7 @@ crate-type = ["cdylib"] wasm-bindgen = "0.2" vrl.workspace = true serde.workspace = true +web-sys = { version = "0.3", features = ["Window", "Performance"] } gloo-utils = { version = "0.2", features = ["serde"] } vector-vrl-functions = { path = "../functions" } enrichment = { path = "../../enrichment" } diff --git a/lib/vector-vrl/web-playground/build.rs b/lib/vector-vrl/web-playground/build.rs index b6bc46a33cc80..9f6bc26293068 100644 --- a/lib/vector-vrl/web-playground/build.rs +++ b/lib/vector-vrl/web-playground/build.rs @@ -55,40 +55,39 @@ fn write_vrl_constants(lockfile: &Lockfile, output_file: &mut File) { .find(|&package| package.name.as_str() == "vrl") .expect("missing VRL dependency"); - let vrl_source = vrl_dep.source.clone().expect("missing VRL source id"); - - let (version, link) = match vrl_source.kind() { - SourceKind::Git(_) => { - let precise = vrl_source - .precise() - .expect("git reference should have precise") - .to_string(); - ( - precise.clone(), - format!("{}/tree/{precise}", vrl_source.url()), - ) - } - SourceKind::Registry if vrl_source.is_default_registry() => { - let version = vrl_dep.version.to_string(); - ( - version.to_string(), - format!("https://crates.io/crates/vrl/{version}"), - ) + let (version, link) = if let Some(vrl_source) = vrl_dep.source.clone() { + match vrl_source.kind() { + SourceKind::Git(_) => { + let precise = vrl_source + .precise() + .expect("git reference should have precise") + .to_string(); + ( + precise.clone(), + Some(format!("{}/tree/{precise}", vrl_source.url())), + ) + } + SourceKind::Registry if vrl_source.is_default_registry() => { + let version = vrl_dep.version.to_string(); + ( + version.clone(), + Some(format!("https://crates.io/crates/vrl/{version}")), + ) + } + SourceKind::Path => (vrl_dep.version.to_string(), None), + kind => unimplemented!("unhandled source kind: {:?}", kind), } - SourceKind::Path - | SourceKind::Registry - | SourceKind::SparseRegistry - | SourceKind::LocalRegistry - | SourceKind::Directory => unimplemented!("unhandled source kind: {:?}", vrl_source.kind()), - _ => unimplemented!("unknown source kind: {:?}", vrl_source.kind()), + } else { + (vrl_dep.version.to_string(), None) }; output_file .write_all(create_const_statement("VRL_VERSION", &version).as_bytes()) .expect("Failed to write VRL version constant"); + let link_str = link.as_deref().unwrap_or(""); output_file - .write_all(create_const_statement("VRL_LINK", &link).as_bytes()) + .write_all(create_const_statement("VRL_LINK", link_str).as_bytes()) .expect("Failed to write VRL_LINK constant"); } diff --git a/lib/vector-vrl/web-playground/public/index.css b/lib/vector-vrl/web-playground/public/index.css index caaacc2cd0d3a..364ac714fe491 100644 --- a/lib/vector-vrl/web-playground/public/index.css +++ b/lib/vector-vrl/web-playground/public/index.css @@ -176,7 +176,7 @@ div#output-section { border-bottom: 1px solid var(--datadog-gray-light); } -/* BUTTONS */ +/* BUTTONS */ .btn { display: inline-block; outline: 0; @@ -373,3 +373,50 @@ div#output-section { font-size: 11px; } } + +#toolbar-section { + display: flex; + align-items: center; + gap: 8px; +} + +.timezone-container { + margin-left: auto; + display: flex; + align-items: center; +} + +#timezone-label { + margin-right: 8px; + font-weight: bold; + font-family: "Open Sans", -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen, Ubuntu, + Cantarell, "Helvetica Neue", sans-serif; + font-size: 14px; +} + +#timezone-input { + padding: 0px 10px; + border: 1px solid #ccc; + border-radius: 4px; + background-color: #f8f9fa; + border-width: 1px; + border-style: solid; + font-size: 14px; + font-family: "Open Sans", -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen, Ubuntu, + Cantarell, "Helvetica Neue", sans-serif; + line-height: 1.5; + color: #212529; + height: 36px; + box-sizing: border-box; +} + +#output-cell-title .cell-title { + display: flex; + justify-content: space-between; + align-items: center; +} + +#elapsed-time { + font-weight: normal; + font-size: 12px; +} diff --git a/lib/vector-vrl/web-playground/public/index.html b/lib/vector-vrl/web-playground/public/index.html index 9799dcd4a4e76..7064dc6f4cd2e 100644 --- a/lib/vector-vrl/web-playground/public/index.html +++ b/lib/vector-vrl/web-playground/public/index.html @@ -2,7 +2,6 @@ - VRL playground @@ -53,8 +52,30 @@

VRL Playground

-
- +
+ + + + + + + + + + + + + + + +
+
@@ -74,7 +95,10 @@

VRL Playground

-

Output

+

+ Output + +

diff --git a/lib/vector-vrl/web-playground/public/index.js b/lib/vector-vrl/web-playground/public/index.js index aaf0c87007253..1f08115e8bb6a 100644 --- a/lib/vector-vrl/web-playground/public/index.js +++ b/lib/vector-vrl/web-playground/public/index.js @@ -39,7 +39,6 @@ export class VrlWebPlayground { constructor() { let temp = init().then(() => { this.run_vrl = run_vrl; - this.vector_version = vector_version(); this.vector_link = vector_link(); @@ -210,10 +209,18 @@ export class VrlWebPlayground { return input; } - let res = this.run_vrl(input); + const tz_input = document.getElementById('timezone-input'); + // set default tz if nothing is set. this is going to use default timezone + let timezone = tz_input.value ? tz_input.value : (tz_input.value = "Default"); + let res = this.run_vrl(input, timezone); console.log("[DEBUG::handleRunCode()] Printing out res: ", res); if (res.result) { this.outputEditor.setValue(JSON.stringify(res.result, null, "\t")); + if (res.elapsed_time !== null) { + document.getElementById("elapsed-time").textContent = `Duration: ${res.elapsed_time} milliseconds`; + }else { + document.getElementById("elapsed-time").textContent = ""; + } } else if (res.msg) { // disable json linting for error msgs // since error msgs are not valid json diff --git a/lib/vector-vrl/web-playground/src/lib.rs b/lib/vector-vrl/web-playground/src/lib.rs index a434050aefa3c..14c983762e917 100644 --- a/lib/vector-vrl/web-playground/src/lib.rs +++ b/lib/vector-vrl/web-playground/src/lib.rs @@ -29,17 +29,22 @@ impl Input { } } -// The module returns the result of the last expression and the event that results from the -// applied program +// The module returns the result of the last expression, the resulting event, +// and the execution time. #[derive(Deserialize, Serialize)] pub struct VrlCompileResult { pub output: Value, pub result: Value, + pub elapsed_time: Option, } impl VrlCompileResult { - fn new(output: Value, result: Value) -> Self { - Self { output, result } + fn new(output: Value, result: Value, elapsed_time: Option) -> Self { + Self { + output, + result, + elapsed_time, + } } } @@ -76,7 +81,10 @@ impl VrlDiagnosticResult { } } -fn compile(mut input: Input) -> Result { +fn compile( + mut input: Input, + tz_str: Option, +) -> Result { let mut functions = vrl::stdlib::all(); functions.extend(vector_vrl_functions::all()); functions.extend(enrichment::vrl_functions()); @@ -85,7 +93,24 @@ fn compile(mut input: Input) -> Result { let state = TypeState::default(); let mut runtime = Runtime::default(); let config = CompileConfig::default(); - let timezone = TimeZone::default(); + + let timezone = match tz_str.as_deref() { + // Empty or "Default" tz string will default to tz default + None | Some("") | Some("Default") => TimeZone::default(), + Some(other) => match other.parse() { + Ok(tz) => TimeZone::Named(tz), + Err(_) => { + // Returns error message if tz parsing has failed. + // This avoids head scratching, instead of it silently using the default timezone. + let error_message = format!("Invalid timezone identifier: '{other}'"); + return Err(VrlDiagnosticResult { + list: vec![error_message.clone()], + msg: error_message.clone(), + msg_colorized: error_message, + }); + } + }, + }; let mut target_value = TargetValue { value: event.clone(), @@ -98,18 +123,32 @@ fn compile(mut input: Input) -> Result { Err(diagnostics) => return Err(VrlDiagnosticResult::new(&input.program, diagnostics)), }; - match runtime.resolve(&mut target_value, &program.program, &timezone) { - Ok(result) => Ok(VrlCompileResult::new(result, target_value.value)), + let (result, elapsed_time) = + if let Some(performance) = web_sys::window().and_then(|w| w.performance()) { + let start_time = performance.now(); + let result = runtime.resolve(&mut target_value, &program.program, &timezone); + let end_time = performance.now(); + (result, Some(end_time - start_time)) + } else { + // If performance API is not available, run the program without timing. + let result = runtime.resolve(&mut target_value, &program.program, &timezone); + (result, None) + }; + + match result { + // The final event is in `target_value.value`. + // The value of the last expression is in `res`. + Ok(res) => Ok(VrlCompileResult::new(res, target_value.value, elapsed_time)), Err(err) => Err(VrlDiagnosticResult::new_runtime_error(&input.program, err)), } } // The user-facing function #[wasm_bindgen] -pub fn run_vrl(incoming: &JsValue) -> JsValue { +pub fn run_vrl(incoming: &JsValue, tz_str: &str) -> JsValue { let input: Input = incoming.into_serde().unwrap(); - match compile(input) { + match compile(input, Some(tz_str.to_string())) { Ok(res) => JsValue::from_serde(&res).unwrap(), Err(err) => JsValue::from_serde(&err).unwrap(), } @@ -129,6 +168,7 @@ pub fn vector_link() -> String { pub fn vrl_version() -> String { built_info::VRL_VERSION.to_string() } + #[wasm_bindgen] pub fn vrl_link() -> String { built_info::VRL_LINK.to_string() diff --git a/rfcs/2020-03-06-1999-api-extensions-for-lua-transform.md b/rfcs/2020-03-06-1999-api-extensions-for-lua-transform.md index d2bcb020041b3..4b8b75c032d66 100644 --- a/rfcs/2020-03-06-1999-api-extensions-for-lua-transform.md +++ b/rfcs/2020-03-06-1999-api-extensions-for-lua-transform.md @@ -488,7 +488,7 @@ Events produced by the transforms through calling an emitting function can have Both log and metrics events are encoded using [external tagging](https://serde.rs/enum-representations.html#externally-tagged). -* [Log events](https://vector.dev/docs/about/data-model/log/) could be seen as tables created using +* [Log events](https://vector.dev/docs/architecture/data-model/log/) could be seen as tables created using ```lua { @@ -498,7 +498,7 @@ Both log and metrics events are encoded using [external tagging](https://serde.r } ``` - The content of the `log` field corresponds to the usual [log event](https://vector.dev/docs/about/data-model/log/#examples) structure, with possible nesting of the fields. + The content of the `log` field corresponds to the usual [log event](https://vector.dev/docs/architecture/data-model/log/#examples) structure, with possible nesting of the fields. If a log event is created by the user inside the transform is a table, then, if default fields named according to the [global schema](https://vector.dev/docs/reference/global-options/#log_schema) are not present in such a table, then they are automatically added to the event. This rule does not apply to events having `userdata` type. @@ -532,7 +532,7 @@ Both log and metrics events are encoded using [external tagging](https://serde.r > > And then emits the event. In that case Vector would not automatically insert the `timestamp` field. -* [Metric events](https://vector.dev/docs/about/data-model/metric/) could be seen as tables created using +* [Metric events](https://vector.dev/docs/architecture/data-model/metric/) could be seen as tables created using ```lua { @@ -542,7 +542,7 @@ Both log and metrics events are encoded using [external tagging](https://serde.r } ``` - The content of the `metric` field matches the [metric data model](https://vector.dev/docs/about/data-model/metric). The values use [external tagging](https://serde.rs/enum-representations.html#externally-tagged) with respect to the metric type, see the examples. + The content of the `metric` field matches the [metric data model](https://vector.dev/docs/architecture/data-model/metric). The values use [external tagging](https://serde.rs/enum-representations.html#externally-tagged) with respect to the metric type, see the examples. In case when the metric events are created as tables in user-defined code, the following default values are assumed if they are not provided: @@ -552,7 +552,7 @@ Both log and metrics events are encoded using [external tagging](https://serde.r | `kind` | `absolute` | | `tags` | empty map | - Furthermore, for [`aggregated_histogram`](https://vector.dev/docs/about/data-model/metric/#aggregated_histogram) the `count` field inside the `value` map can be omitted. + Furthermore, for [`aggregated_histogram`](https://vector.dev/docs/architecture/data-model/metric/#aggregated_histogram) the `count` field inside the `value` map can be omitted. **Example: `counter`** @@ -621,7 +621,7 @@ Both log and metrics events are encoded using [external tagging](https://serde.r > } > } > } - > Note that the field [`count`](https://vector.dev/docs/about/data-model/metric/#count) is not required because it can be inferred by Vector automatically by summing up the values from `counts`. + > Note that the field [`count`](https://vector.dev/docs/architecture/data-model/metric/#count) is not required because it can be inferred by Vector automatically by summing up the values from `counts`. **Example: `aggregated_summary`** > The minimal Lua code required to create an aggregated summary metric is the following: @@ -645,14 +645,14 @@ The mapping between Vector data types and Lua data types is the following: | Vector Type | Lua Type | Comment | | :----------- | :-------- | :------- | -| [`String`](https://vector.dev/docs/about/data-model/log/#strings) | [`string`](https://www.lua.org/pil/2.4.html) || -| [`Integer`](https://vector.dev/docs/about/data-model/log/#ints) | [`integer`](https://docs.rs/mlua/0.6.0/mlua/type.Integer.html) || -| [`Float`](https://vector.dev/docs/about/data-model/log/#floats) | [`number`](https://docs.rs/mlua/0.6.0/mlua/type.Number.html) || -| [`Boolean`](https://vector.dev/docs/about/data-model/log/#booleans) | [`boolean`](https://www.lua.org/pil/2.2.html) || -| [`Timestamp`](https://vector.dev/docs/about/data-model/log/#timestamps) | [`userdata`](https://www.lua.org/pil/28.1.html) | There is no dedicated timestamp type in Lua. However, there is a standard library function [`os.date`](https://www.lua.org/manual/5.1/manual.html#pdf-os.date) which returns a table with fields `year`, `month`, `day`, `hour`, `min`, `sec`, and some others. Other standard library functions, such as [`os.time`](https://www.lua.org/manual/5.1/manual.html#pdf-os.time), support tables with these fields as arguments. Because of that, Vector timestamps passed to the transform are represented as `userdata` with the same set of accessible fields. In order to have one-to-one correspondence between Vector timestamps and Lua timestamps, `os.date` function from the standard library is patched to return not a table, but `userdata` with the same set of fields as it usually would return instead. This approach makes it possible to have both compatibility with the standard library functions and a dedicated data type for timestamps. | -| [`Null`](https://vector.dev/docs/about/data-model/log/#null-values) | empty string | In Lua setting a table field to `nil` means deletion of this field. Furthermore, setting an array element to `nil` leads to deletion of this element. In order to avoid inconsistencies, already present `Null` values are visible represented as empty strings from Lua code, and it is impossible to create a new `Null` value in the user-defined code. | -| [`Map`](https://vector.dev/docs/about/data-model/log/#maps) | [`userdata`](https://www.lua.org/pil/28.1.html) or [`table`](https://www.lua.org/pil/2.5.html) | Maps which are parts of events passed to the transform from Vector have `userdata` type. User-created maps have `table` type. Both types are converted to Vector's `Map` type when they are emitted from the transform. | -| [`Array`](https://vector.dev/docs/about/data-model/log/#arrays) | [`sequence`](https://www.lua.org/pil/11.1.html) | Sequences in Lua are a special case of tables. Because of that fact, the indexes can in principle start from any number. However, the convention in Lua is to start indexes from 1 instead of 0, so Vector should adhere it. | +| [`String`](https://vector.dev/docs/architecture/data-model/log/#strings) | [`string`](https://www.lua.org/pil/2.4.html) || +| [`Integer`](https://vector.dev/docs/architecture/data-model/log/#ints) | [`integer`](https://docs.rs/mlua/0.6.0/mlua/type.Integer.html) || +| [`Float`](https://vector.dev/docs/architecture/data-model/log/#floats) | [`number`](https://docs.rs/mlua/0.6.0/mlua/type.Number.html) || +| [`Boolean`](https://vector.dev/docs/architecture/data-model/log/#booleans) | [`boolean`](https://www.lua.org/pil/2.2.html) || +| [`Timestamp`](https://vector.dev/docs/architecture/data-model/log/#timestamps) | [`userdata`](https://www.lua.org/pil/28.1.html) | There is no dedicated timestamp type in Lua. However, there is a standard library function [`os.date`](https://www.lua.org/manual/5.1/manual.html#pdf-os.date) which returns a table with fields `year`, `month`, `day`, `hour`, `min`, `sec`, and some others. Other standard library functions, such as [`os.time`](https://www.lua.org/manual/5.1/manual.html#pdf-os.time), support tables with these fields as arguments. Because of that, Vector timestamps passed to the transform are represented as `userdata` with the same set of accessible fields. In order to have one-to-one correspondence between Vector timestamps and Lua timestamps, `os.date` function from the standard library is patched to return not a table, but `userdata` with the same set of fields as it usually would return instead. This approach makes it possible to have both compatibility with the standard library functions and a dedicated data type for timestamps. | +| [`Null`](https://vector.dev/docs/architecture/data-model/log/#null-values) | empty string | In Lua setting a table field to `nil` means deletion of this field. Furthermore, setting an array element to `nil` leads to deletion of this element. In order to avoid inconsistencies, already present `Null` values are visible represented as empty strings from Lua code, and it is impossible to create a new `Null` value in the user-defined code. | +| [`Map`](https://vector.dev/docs/architecture/data-model/log/#maps) | [`userdata`](https://www.lua.org/pil/28.1.html) or [`table`](https://www.lua.org/pil/2.5.html) | Maps which are parts of events passed to the transform from Vector have `userdata` type. User-created maps have `table` type. Both types are converted to Vector's `Map` type when they are emitted from the transform. | +| [`Array`](https://vector.dev/docs/architecture/data-model/log/#arrays) | [`sequence`](https://www.lua.org/pil/11.1.html) | Sequences in Lua are a special case of tables. Because of that fact, the indexes can in principle start from any number. However, the convention in Lua is to start indexes from 1 instead of 0, so Vector should adhere it. | ### Configuration @@ -679,7 +679,7 @@ The implementation of `lua` transform supports only log events. Processing of lo Events have type [`userdata`](https://www.lua.org/pil/28.1.html) with custom [metamethods](https://www.lua.org/pil/13.html), so they are views to Vector's events. Thus passing an event to Lua has zero cost, so only when fields are actually accessed the data is copied to Lua. -The fields are accessed through string indexes using [Vector's field path notation](https://vector.dev/docs/about/data-model/log/). +The fields are accessed through string indexes using [Vector's field path notation](https://vector.dev/docs/architecture/data-model/log/). ## Sales Pitch diff --git a/rfcs/2020-05-25-2692-more-usable-logevents.md b/rfcs/2020-05-25-2692-more-usable-logevents.md index 73e50f5231dd6..09ab89b35d7e5 100644 --- a/rfcs/2020-05-25-2692-more-usable-logevents.md +++ b/rfcs/2020-05-25-2692-more-usable-logevents.md @@ -162,7 +162,7 @@ There is no guide accompanying this RFC, it only minimally touches user facing s ## Doc Level Proposal -> **Placement:** Insert into [Log Event](https://vector.dev/docs/about/data-model/log/#types)'s [Types](https://vector.dev/docs/about/data-model/log/#types) section +> **Placement:** Insert into [Log Event](https://vector.dev/docs/architecture/data-model/log/#types)'s [Types](https://vector.dev/docs/architecture/data-model/log/#types) section ### Bytes diff --git a/rfcs/2020-07-28-3642-jmx_rfc.md b/rfcs/2020-07-28-3642-jmx_rfc.md index fb3a98d32cd56..b872b276574f5 100644 --- a/rfcs/2020-07-28-3642-jmx_rfc.md +++ b/rfcs/2020-07-28-3642-jmx_rfc.md @@ -223,7 +223,7 @@ principles of Vector: > One Tool. All Data. - One simple tool gets your logs, metrics, and traces > (coming soon) from A to B. -[Vector principles](https://vector.dev/docs/about/what-is-vector/#who-should-use-vector) +[Vector principles](https://vector.dev/docs/) If users are already running Prometheus though, they could opt for the Prometheus path. diff --git a/rfcs/2020-08-21-3092-apache-metrics-source.md b/rfcs/2020-08-21-3092-apache-metrics-source.md index c7ab4a9162ad4..4b285632f121b 100644 --- a/rfcs/2020-08-21-3092-apache-metrics-source.md +++ b/rfcs/2020-08-21-3092-apache-metrics-source.md @@ -166,8 +166,7 @@ principles of Vector: > One Tool. All Data. - One simple tool gets your logs, metrics, and traces > (coming soon) from A to B. -[Vector -principles](https://vector.dev/docs/about/what-is-vector/#who-should-use-vector) +[Vector principles](https://vector.dev/docs/) On the same page, it is mentioned that Vector should be a replacement for Telegraf. diff --git a/rfcs/2020-08-26-3191-host-metrics.md b/rfcs/2020-08-26-3191-host-metrics.md index f494030000fad..58243ddf1eeda 100644 --- a/rfcs/2020-08-26-3191-host-metrics.md +++ b/rfcs/2020-08-26-3191-host-metrics.md @@ -180,7 +180,7 @@ principles of Vector: > (coming soon) from A to B. [Vector -principles](https://vector.dev/docs/about/what-is-vector/#who-should-use-vector) +principles](https://vector.dev/docs/) On the same page, it is mentioned that Vector should be a replacement for Telegraf. diff --git a/rfcs/2020-08-27-3603-postgres-metrics.md b/rfcs/2020-08-27-3603-postgres-metrics.md index cbc7facbdca6c..2b47b4e07e12f 100644 --- a/rfcs/2020-08-27-3603-postgres-metrics.md +++ b/rfcs/2020-08-27-3603-postgres-metrics.md @@ -136,7 +136,7 @@ principles of Vector: > (coming soon) from A to B. [Vector -principles](https://vector.dev/docs/about/what-is-vector/#who-should-use-vector) +principles](https://vector.dev/docs/) On the same page, it is mentioned that Vector should be a replacement for Telegraf. diff --git a/rfcs/2020-08-31-3640-nginx-metrics-source.md b/rfcs/2020-08-31-3640-nginx-metrics-source.md index 5351aa1351660..93551a51492c9 100644 --- a/rfcs/2020-08-31-3640-nginx-metrics-source.md +++ b/rfcs/2020-08-31-3640-nginx-metrics-source.md @@ -131,7 +131,7 @@ principles of Vector: > (coming soon) from A to B. [Vector -principles](https://vector.dev/docs/about/what-is-vector/#who-should-use-vector) +principles](https://vector.dev/docs/) On the same page, it is mentioned that Vector should be a replacement for Telegraf. diff --git a/rfcs/2020-08-31-3641-mongo-metrics.md b/rfcs/2020-08-31-3641-mongo-metrics.md index ca100cf457302..32949f59b1273 100644 --- a/rfcs/2020-08-31-3641-mongo-metrics.md +++ b/rfcs/2020-08-31-3641-mongo-metrics.md @@ -1107,7 +1107,7 @@ principles of Vector: > (coming soon) from A to B. [Vector -principles](https://vector.dev/docs/about/what-is-vector/#who-should-use-vector) +principles](https://vector.dev/docs/) On the same page, it is mentioned that Vector should be a replacement for Telegraf. diff --git a/rfcs/2021-09-01-8547-accept-metrics-in-datadog-agent-source.md b/rfcs/2021-09-01-8547-accept-metrics-in-datadog-agent-source.md index aec7cd9db8e3a..ac95ef626762e 100644 --- a/rfcs/2021-09-01-8547-accept-metrics-in-datadog-agent-source.md +++ b/rfcs/2021-09-01-8547-accept-metrics-in-datadog-agent-source.md @@ -116,7 +116,7 @@ A few details about the Datadog Agents & [Datadog metrics](https://docs.datadogh [here](https://github.com/DataDog/agent-payload/blob/master/proto/metrics/agent_payload.proto#L47-L81). Vector has a nice description of its [metrics data -model](https://vector.dev/docs/about/under-the-hood/architecture/data-model/metric/) and a [concise enum for +model](https://vector.dev/docs/architecture/data-model/metric/) and a [concise enum for representing it](https://github.com/vectordotdev/vector/blob/master/lib/vector-core/src/event/metric.rs#L135-L169). diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 5bedd373fc370..c2ee2a8766ad1 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "1.85" +channel = "1.88" profile = "default" diff --git a/scripts/check_changelog_fragments.sh b/scripts/check_changelog_fragments.sh index 731d4ea34fb3a..916ebe88dd53b 100755 --- a/scripts/check_changelog_fragments.sh +++ b/scripts/check_changelog_fragments.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # This script is intended to run during CI, however it can be run locally by # committing changelog fragments before executing the script. If the script @@ -17,16 +17,18 @@ if [ ! -d "${CHANGELOG_DIR}" ]; then fi # diff-filter=A lists only added files -FRAGMENTS=$(git diff --name-only --diff-filter=A --merge-base origin/master ${CHANGELOG_DIR}) +FRAGMENTS=$(git diff --name-only --diff-filter=A --merge-base "${MERGE_BASE:-origin/master}" ${CHANGELOG_DIR}) if [ -z "$FRAGMENTS" ]; then echo "No changelog fragments detected" - echo "If no changes necessitate user-facing explanations, add the GH label 'no-changelog'" + echo "If no changes necessitate user-facing explanations, add the GH label 'no-changelog'" echo "Otherwise, add changelog fragments to changelog.d/" echo "For details, see 'changelog.d/README.md'" exit 1 fi +[[ "$(wc -l <<< "$FRAGMENTS")" -gt "${MAX_FRAGMENTS:-1000}" ]] && exit 1 + # extract the basename from the file path FRAGMENTS=$(xargs -n1 basename <<< "${FRAGMENTS}") diff --git a/scripts/ci-int-e2e-test.sh b/scripts/ci-int-e2e-test.sh deleted file mode 100755 index 3b569db530f5f..0000000000000 --- a/scripts/ci-int-e2e-test.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# Used in CI to run and stop an integration test and upload the results of it. -# This is useful to allow retrying the integration test at a higher level than -# the nextest and reduce code duplication in the workflow file. - -set -u - -if [[ -z "${CI:-}" ]]; then - echo "Aborted: this script is for use in CI." >&2 - exit 1 -fi - -if [ $# -ne 2 ] -then - echo "usage: $0 [int|e2e] TEST_NAME" - exit 1 -fi - -set -x - -TEST_TYPE=$1 # either "int" or "e2e" -TEST_NAME=$2 - -cargo vdev -v "${TEST_TYPE}" start -a "${TEST_NAME}" -sleep 30 -cargo vdev -v "${TEST_TYPE}" test --retries 2 -a "${TEST_NAME}" -RET=$? -cargo vdev -v "${TEST_TYPE}" stop -a "${TEST_NAME}" -./scripts/upload-test-results.sh -exit $RET diff --git a/scripts/e2e/Dockerfile b/scripts/e2e/Dockerfile index 495c69275cad4..3af5d83489c27 100644 --- a/scripts/e2e/Dockerfile +++ b/scripts/e2e/Dockerfile @@ -1,8 +1,7 @@ -ARG RUST_VERSION +ARG RUST_VERSION=1.85 ARG FEATURES -ARG DEBIAN_RELEASE=slim-bookworm -FROM docker.io/rust:${RUST_VERSION}-${DEBIAN_RELEASE} +FROM docker.io/rust:${RUST_VERSION}-slim-bookworm RUN apt-get update && apt-get -y --no-install-recommends install \ build-essential \ @@ -17,18 +16,10 @@ RUN apt-get update && apt-get -y --no-install-recommends install \ libxxhash-dev \ unzip \ zlib1g-dev \ - zlib1g + zlib1g \ + mold -RUN git clone https://github.com/rui314/mold.git \ - && mkdir mold/build \ - && cd mold/build \ - && git checkout v2.0.0 \ - && ../install-build-deps.sh \ - && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=c++ .. \ - && cmake --build . -j $(nproc) \ - && cmake --install . - -RUN rustup run "${RUST_VERSION}" cargo install cargo-nextest --version 0.9.72 --locked +RUN cargo install cargo-nextest --version 0.9.95 --locked COPY scripts/environment/install-protoc.sh / COPY tests/data/ca/certs /certs @@ -41,6 +32,6 @@ ARG FEATURES RUN --mount=type=cache,target=/vector/target \ --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/usr/local/cargo/git \ - /usr/local/bin/mold -run cargo build --tests --lib --bin vector \ + /usr/bin/mold -run cargo build --tests --lib --bin vector \ --no-default-features --features $FEATURES && \ cp target/debug/vector /usr/bin/vector diff --git a/scripts/e2e/datadog-logs/compose.yaml b/scripts/e2e/datadog-logs/compose.yaml index 56fb68350c004..ae004588f46f3 100644 --- a/scripts/e2e/datadog-logs/compose.yaml +++ b/scripts/e2e/datadog-logs/compose.yaml @@ -1,4 +1,4 @@ -version: '3' +version: '3.8' services: # Generates random log data for consumption by the custom Agent check @@ -17,7 +17,9 @@ services: - "-o" - "/var/log/a_custom.log" volumes: - - log_path:/var/log/ + - type: volume + source: log_path + target: /var/log/ # Tails a custom log created by `log_generator` and sends log data to # the `fakeintake-agent` service @@ -32,12 +34,22 @@ services: - DD_ENABLE_PAYLOADS_SERVICE_CHECKS=false - DD_CONTAINER_EXCLUDE="name:.*" volumes: - # The Agent config file - - ${PWD}/tests/data/e2e/datadog/logs/agent_only.yaml:/etc/datadog-agent/datadog.yaml - # The custom logs check - - ${PWD}/tests/data/e2e/datadog/logs/logs.conf.d:/conf.d:ro - # The custom log to tail, created by the `log_generator` service - - log_path:/var/log/ + # The Agent config file + - type: bind + source: ../../../tests/data/e2e/datadog/logs/agent_only.yaml + target: /etc/datadog-agent/datadog.yaml + read_only: true + + # The custom logs check + - type: bind + source: ../../../tests/data/e2e/datadog/logs/logs.conf.d + target: /conf.d + read_only: true + + # The custom log to tail, created by the `log_generator` service + - type: volume + source: log_path + target: /var/log/ # Tails a custom log created by `log_generator` and sends log data to # the `vector` service @@ -52,12 +64,22 @@ services: - DD_ENABLE_PAYLOADS_SERVICE_CHECKS=false - DD_CONTAINER_EXCLUDE="name:.*" volumes: - # The Agent config file - - ${PWD}/tests/data/e2e/datadog/logs/agent_vector.yaml:/etc/datadog-agent/datadog.yaml - # The custom logs check - - ${PWD}/tests/data/e2e/datadog/logs/logs.conf.d:/conf.d:ro - # The custom log to tail, created by the `log_generator` service - - log_path:/var/log/ + # The Agent config file + - type: bind + source: ../../../tests/data/e2e/datadog/logs/agent_vector.yaml + target: /etc/datadog-agent/datadog.yaml + read_only: true + + # The custom logs check + - type: bind + source: ../../../tests/data/e2e/datadog/logs/logs.conf.d + target: /conf.d + read_only: true + + # The custom log to tail, created by the `log_generator` service + - type: volume + source: log_path + target: /var/log/ # Receives log data from the `datadog-agent-vector` service and sends # to the `fakeintake-vector` service. @@ -65,7 +87,7 @@ services: depends_on: - fakeintake-vector build: - context: ${PWD} + context: ../../.. # re-using the integration test runner image since it already has # compiled vector on it. image: ${CONFIG_VECTOR_IMAGE} @@ -79,19 +101,21 @@ services: - "-c" - "/home/vector/tests/data/e2e/datadog/logs/vector.toml" volumes: - - ${PWD}:/home/vector + - type: bind + source: ../../.. + target: /home/vector # Receives log data from the `datadog-agent` service. Is queried by the test runner # which does the validation of consistency with the other fakeintake service. fakeintake-agent: # TODO: temporarily pegging the image as latest results in failures - image: docker.io/datadog/fakeintake:v77a06f2b + image: docker.io/datadog/fakeintake:ved764626 # Receives log data from the `datadog-agent-vector` service. Is queried by the test runner # which does the validation of consistency with the other fakeintake service. fakeintake-vector: # TODO: temporarily pegging the image as latest results in failures - image: docker.io/datadog/fakeintake:v77a06f2b + image: docker.io/datadog/fakeintake:ved764626 networks: default: diff --git a/scripts/e2e/datadog-metrics/compose.yaml b/scripts/e2e/datadog-metrics/compose.yaml index 5942fda1011e9..80523ac8144f4 100644 --- a/scripts/e2e/datadog-metrics/compose.yaml +++ b/scripts/e2e/datadog-metrics/compose.yaml @@ -27,8 +27,8 @@ services: - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - DD_HOSTNAME=datadog-agent volumes: - # The Agent config file - - ${PWD}/tests/data/e2e/datadog/metrics/agent_only.yaml:/etc/datadog-agent/datadog.yaml + # The Agent config file + - ../../../tests/data/e2e/datadog/metrics/agent_only.yaml:/etc/datadog-agent/datadog.yaml # Sends metric data received from the Emitter to the `vector` service datadog-agent-vector: @@ -39,8 +39,8 @@ services: - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - DD_HOSTNAME=datadog-agent-vector volumes: - # The Agent config file - - ${PWD}/tests/data/e2e/datadog/metrics/agent_vector.yaml:/etc/datadog-agent/datadog.yaml + # The Agent config file + - ../../../tests/data/e2e/datadog/metrics/agent_vector.yaml:/etc/datadog-agent/datadog.yaml # Receives metric data from the `datadog-agent-vector` service and sends # to the `fakeintake-vector` service. @@ -48,7 +48,7 @@ services: depends_on: - fakeintake-vector build: - context: ${PWD} + context: ../../.. # re-using the integration test runner image since it already has # compiled vector on it. image: ${CONFIG_VECTOR_IMAGE} @@ -62,7 +62,7 @@ services: - "-c" - "/home/vector/tests/data/e2e/datadog/metrics/vector.toml" volumes: - - ${PWD}:/home/vector + - ../../..:/home/vector # Receives metric data from the `datadog-agent` service. Is queried by the test runner # which does the validation of consistency with the other fakeintake service. diff --git a/scripts/e2e/opentelemetry-logs/README.md b/scripts/e2e/opentelemetry-logs/README.md new file mode 100644 index 0000000000000..af55f71d0f3d7 --- /dev/null +++ b/scripts/e2e/opentelemetry-logs/README.md @@ -0,0 +1,25 @@ +# OpenTelemetry Vector E2E Log Pipeline Test + +This end-to-end (E2E) test validates that log events generated in a container are correctly ingested by Vector, processed, and forwarded to an OpenTelemetry Collector sink, where they are exported to a file for verification. + +## How this test works + +- **Orchestrates all required services:** + - **Log generator**: Emits fake OTLP logs. + - **Vector**: Receives, transforms, and forwards logs to the OTEL sink and a file. + - **OTEL Collector Source**: Forwards or processes logs upstream. + - **OTEL Collector Sink**: Receives logs from Vector and writes them to a file. +- **Mounts volumes** to share configuration and output files between containers and the host. +- **Exposes ports** for OTLP HTTP ingestion and for accessing Vector/collector APIs if needed. + +## How to Run + +```shell +# from the repo root directory +./scripts/int-e2e-test.sh e2e opentelemetry-logs +``` + +## Notes + +- The test ensures true end-to-end delivery and format compliance for OTLP logs through Vector and the OpenTelemetry Collector stack. +- Adjust the log generator, remap logic, or assertions as needed for your use case. diff --git a/scripts/e2e/opentelemetry-logs/compose.yaml b/scripts/e2e/opentelemetry-logs/compose.yaml new file mode 100644 index 0000000000000..c82b2e8a569d8 --- /dev/null +++ b/scripts/e2e/opentelemetry-logs/compose.yaml @@ -0,0 +1,70 @@ +name: opentelemetry-vector-e2e +services: + otel-collector-source: + container_name: otel-collector-source + image: otel/opentelemetry-collector-contrib:${CONFIG_COLLECTOR_VERSION:-latest} + init: true + volumes: + - type: bind + source: ../../../tests/data/e2e/opentelemetry/logs/collector-source.yaml + target: /etc/otelcol-contrib/config.yaml + read_only: true + ports: + - "${OTEL_COLLECTOR_SOURCE_GRPC_PORT:-4317}:4317" + - "${OTEL_COLLECTOR_SOURCE_HTTP_PORT:-4318}:4318" + command: [ "--config=/etc/otelcol-contrib/config.yaml" ] + + logs-generator: + container_name: logs-generator + build: + context: ./generator + init: true + depends_on: + - otel-collector-source + - vector + - otel-collector-sink + volumes: + - type: bind + source: ./generator + target: /generator + environment: + - PYTHONUNBUFFERED=1 + command: [ "python", "/generator/logs_generator.py", "-n", "100" ] + + otel-collector-sink: + container_name: otel-collector-sink + image: otel/opentelemetry-collector-contrib:${CONFIG_COLLECTOR_VERSION:-latest} + init: true + volumes: + - type: bind + source: ../../../tests/data/e2e/opentelemetry/logs/collector-sink.yaml + target: /etc/otelcol-contrib/config.yaml + read_only: true + - type: bind + source: ../../../tests/data/e2e/opentelemetry/logs/output + target: /output + ports: + - "${OTEL_COLLECTOR_SINK_HTTP_PORT:-5318}:5318" + + vector: + container_name: vector-otel-logs-e2e + build: + context: ../../../ + dockerfile: ./scripts/e2e/Dockerfile + args: + FEATURES: e2e-tests-opentelemetry + RUST_VERSION: ${RUST_VERSION:-1.88} + init: true + volumes: + - type: bind + source: ../../../tests/data/e2e/opentelemetry/logs/vector.yaml + target: /etc/vector/vector.yaml + read_only: true + - type: bind + source: ../../../tests/data/e2e/opentelemetry/logs/output + target: /output + environment: + - VECTOR_LOG=${VECTOR_LOG:-info} + - FEATURES=e2e-tests-opentelemetry + - OTEL_E2E_OUTPUT_PATH + command: [ "vector", "-c", "/etc/vector/vector.yaml" ] diff --git a/scripts/e2e/opentelemetry-logs/generator/Dockerfile b/scripts/e2e/opentelemetry-logs/generator/Dockerfile new file mode 100644 index 0000000000000..349b4a8a669aa --- /dev/null +++ b/scripts/e2e/opentelemetry-logs/generator/Dockerfile @@ -0,0 +1,4 @@ +FROM python:3.11-alpine +WORKDIR /generator +COPY requirements.txt logs_generator.py ./ +RUN pip install --no-cache-dir -r requirements.txt diff --git a/scripts/e2e/opentelemetry-logs/generator/logs_generator.py b/scripts/e2e/opentelemetry-logs/generator/logs_generator.py new file mode 100755 index 0000000000000..a5266485dc64a --- /dev/null +++ b/scripts/e2e/opentelemetry-logs/generator/logs_generator.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 + +import argparse +import json +import random +import time +import uuid + +import requests + +SEVERITIES = ["DEBUG", "INFO", "WARN", "ERROR"] +PATHS = ["/", "/login", "/api/data", "/metrics", "/health"] + +def generate_log(endpoint: str, count: int) -> dict: + now_nanos = time.time_ns() + timestamp = time.strftime("%Y-%m-%dT%H:%M:%S%z") + severity = random.choice(SEVERITIES) + log_id = str(uuid.uuid4())[:8] + + log_data = { + "resourceLogs": [ + { + "resource": { + "attributes": [ + {"key": "service.name", "value": {"stringValue": "opentelemetry-logs"}} + ] + }, + "scopeLogs": [ + { + "scope": {"name": "log-generator"}, + "logRecords": [ + { + "timeUnixNano": now_nanos, + "severityText": severity, + "body": {"stringValue": f"[{log_id}] {severity} log {count} at {timestamp}"}, + "attributes": [ + {"key": "count", "value": {"intValue": count}} + ] + } + ] + } + ] + } + ] + } + + try: + response = requests.post( + endpoint, + data=json.dumps(log_data), + headers={"Content-Type": "application/json"}, + timeout=2 + ) + if response.status_code == 200: + return { + "success": True, + "message": f"Log {count} sent successfully", + "log_id": log_id, + "status_code": response.status_code + } + else: + return { + "success": False, + "message": f"HTTP {response.status_code}: {response.text.strip() or '[empty]'}", + "log_id": log_id, + "status code": response.status_code, + } + + except requests.exceptions.RequestException as e: + return { + "success": False, + "message": f"RequestException: {str(e)}", + "log_id": log_id, + } + + +def non_negative_float(value): + f = float(value) + if f < 0: + raise argparse.ArgumentTypeError(f"Interval must be non-negative, got {value}") + return f + + +def main(): + parser = argparse.ArgumentParser(description="Generate OTLP logs periodically.") + parser.add_argument( + "--interval", + type=non_negative_float, + help="Seconds between log sends (non-negative, optional)" + ) + parser.add_argument("-n", type=int, default=0, help="Total logs to send (0 or negative = infinite)") + parser.add_argument("--host", type=str, default="otel-collector-source", help="Host for the OTLP collector") + parser.add_argument("--port", type=int, default=4318, help="Port for OTLP HTTP logs") + parser.add_argument("--path", type=str, default="/v1/logs", help="OTLP HTTP logs path") + + args = parser.parse_args() + endpoint = f"http://{args.host}:{args.port}{args.path}" + + print(f"Starting log generator → {endpoint}") + + count = 0 + sent = 0 + failed = 0 + + while True: + result = generate_log(endpoint, count) + count += 1 + if result["success"]: + print(f"✅ Sent log {count} (ID: {result['log_id']})") + sent += 1 + else: + print(f"❌ Failed log {count} (ID: {result['log_id']}): {result['message']}") + failed += 1 + + if 0 < args.n <= count: + break + + if args.interval is not None: + time.sleep(args.interval) + + print(f"\n📊 Finished: Sent={sent}, Failed={failed}") + + +if __name__ == "__main__": + main() diff --git a/scripts/e2e/opentelemetry-logs/generator/requirements.txt b/scripts/e2e/opentelemetry-logs/generator/requirements.txt new file mode 100644 index 0000000000000..f2293605cf1b0 --- /dev/null +++ b/scripts/e2e/opentelemetry-logs/generator/requirements.txt @@ -0,0 +1 @@ +requests diff --git a/scripts/e2e/opentelemetry-logs/test.yaml b/scripts/e2e/opentelemetry-logs/test.yaml new file mode 100644 index 0000000000000..70e3862ae3347 --- /dev/null +++ b/scripts/e2e/opentelemetry-logs/test.yaml @@ -0,0 +1,24 @@ +features: +- e2e-tests-opentelemetry + +test: "e2e" + +test_filter: "opentelemetry::logs::" + +runner: + env: + OTEL_COLLECTOR_SOURCE_GRPC_PORT: '4317' + OTEL_COLLECTOR_SOURCE_HTTP_PORT: '4318' + OTEL_COLLECTOR_SINK_HTTP_PORT: '5318' + +matrix: + # Determines which `otel/opentelemetry-collector-contrib` version to use + collector_version: [ 'latest' ] + +# Only trigger this integration test if relevant OTEL source/sink files change +paths: + - "src/sources/opentelemetry/**" + - "src/sinks/opentelemetry/**" + - "src/internal_events/opentelemetry_*" + - "tests/e2e/opentelemetry/logs/**" + - "scripts/e2e/opentelemetry-logs/**" diff --git a/scripts/ensure-wasm-target-installed.sh b/scripts/ensure-wasm-target-installed.sh index e80b06c4d3f1c..da89901163941 100644 --- a/scripts/ensure-wasm-target-installed.sh +++ b/scripts/ensure-wasm-target-installed.sh @@ -1,4 +1,4 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash if [[ "$(rustup target list --installed | grep wasm32-unknown-unknown)" != "wasm32-unknown-unknown" ]] ; then echo "wasm32-unknown-unknown target is not installed" diff --git a/scripts/environment/binstall.sh b/scripts/environment/binstall.sh new file mode 100755 index 0000000000000..def6e9eb7b3cc --- /dev/null +++ b/scripts/environment/binstall.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +set -eux +set -o pipefail + +BINSTALL_VERSION="v1.14.1" +BINSTALL_SHA256SUM_X86_64_LINUX="e1d1231720e6ed497a4b0f8881b08f5df9ce1a938fb3ae6f2444e95eb601fe99" +BINSTALL_SHA256SUM_AARCH64_LINUX="17d69bcc07a0e38c912e7f596ed71b1f5f59dc8980da59890c5bc86c07e8506a" +BINSTALL_SHA256SUM_ARMV7_LINUX="e4ba720023e02b071aa805ae62412e94741c1bb0e0a2bb2b35896fec3d140128" +BINSTALL_SHA256SUM_AARCH64_DARWIN="07d46d31fb68ac10b906c5d39d611ded7787966f4ed15c598cb6175b45a2b069" +BINSTALL_SHA256SUM_X86_64_DARWIN="3de381bdcca08c418dc790d2a283711894a0577c6e55bba0d4e6cb8b0378b36" + +pushd "$(mktemp -d)" + +base_url="https://github.com/cargo-bins/cargo-binstall/releases/download/${BINSTALL_VERSION}/cargo-binstall" + +download() { + curl --retry 3 --proto '=https' --tlsv1.2 -fsSL "$@" +} + +os="$(uname -s)" +machine="$(uname -m)" + +if [ "$os" = "Darwin" ]; then + if [ "$machine" = "arm64" ]; then + url="${base_url}-aarch64-apple-darwin.zip" + download_sha256sum="${BINSTALL_SHA256SUM_AARCH64_DARWIN}" + elif [ "$machine" = "x86_64" ]; then + url="${base_url}-x86_64-apple-darwin.zip" + download_sha256sum="${BINSTALL_SHA256SUM_X86_64_DARWIN}" + else + echo "Unsupported OS ${os} machine ${machine}" + popd + exit 1 + fi + + download -o output.zip "$url" +elif [ "$os" = "Linux" ]; then + if [ "$machine" = "armv7l" ]; then + target="armv7-unknown-linux-musleabihf" + download_sha256sum="${BINSTALL_SHA256SUM_ARMV7_LINUX}" + elif [ "$machine" = "aarch64" ]; then + target="${machine}-unknown-linux-musl" + download_sha256sum="${BINSTALL_SHA256SUM_AARCH64_LINUX}" + elif [ "$machine" = "x86_64" ]; then + target="${machine}-unknown-linux-musl" + download_sha256sum="${BINSTALL_SHA256SUM_X86_64_LINUX}" + else + echo "Unsupported OS ${os} machine ${machine}" + popd + exit 1 + fi + + url="${base_url}-${target}.tgz" + + download -o output.tgz "$url" +# elif [ "${OS-}" = "Windows_NT" ]; then +# target="${machine}-pc-windows-msvc" +# url="${base_url}-${target}.zip" +# download -o output.zip "$url" +else + echo "Unsupported OS ${os}" + popd + exit 1 +fi + +echo "${download_sha256sum} $(echo output.*)" | sha256sum --check + +case "$(echo output.*)" in + *.zip) unzip output.* ;; + *.tgz) tar -xvzf output.* ;; + *) >&2 echo "output.* not found"; exit 1 ;; +esac + +./cargo-binstall --self-install || ./cargo-binstall -y --force cargo-binstall diff --git a/scripts/environment/bootstrap-macos.sh b/scripts/environment/bootstrap-macos.sh index 32a8fd528e1f0..b1ddf5c58c489 100755 --- a/scripts/environment/bootstrap-macos.sh +++ b/scripts/environment/bootstrap-macos.sh @@ -1,4 +1,4 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash set -e -o verbose brew update diff --git a/scripts/environment/bootstrap-ubuntu-24.04.sh b/scripts/environment/bootstrap-ubuntu-24.04.sh index ca481e6219228..664efef7b68ab 100755 --- a/scripts/environment/bootstrap-ubuntu-24.04.sh +++ b/scripts/environment/bootstrap-ubuntu-24.04.sh @@ -1,4 +1,4 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash # Refer to https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2404-Readme.md # for all runner information such as OS version and installed software. @@ -60,18 +60,6 @@ tar \ cp "${TEMP}/cue" /usr/bin/cue rm -rf "$TEMP" -# Grease -# Grease is used for the `make release-github` task. -TEMP=$(mktemp -d) -curl \ - -L https://github.com/vectordotdev/grease/releases/download/v1.0.1/grease-1.0.1-linux-amd64.tar.gz \ - -o "${TEMP}/grease-1.0.1-linux-amd64.tar.gz" -tar \ - -xvf "${TEMP}/grease-1.0.1-linux-amd64.tar.gz" \ - -C "${TEMP}" -cp "${TEMP}/grease/bin/grease" /usr/bin/grease -rm -rf "$TEMP" - # Locales locale-gen en_US.UTF-8 dpkg-reconfigure locales diff --git a/scripts/environment/bootstrap-windows-2022.ps1 b/scripts/environment/bootstrap-windows-2022.ps1 index 27b35d9ff55b9..7af1123b2df35 100644 --- a/scripts/environment/bootstrap-windows-2022.ps1 +++ b/scripts/environment/bootstrap-windows-2022.ps1 @@ -10,7 +10,7 @@ echo "CARGO_BUILD_JOBS=$N_JOBS" | Out-File -FilePath $env:GITHUB_ENV -Encoding u if ($env:RELEASE_BUILDER -ne "true") { # Ensure we have cargo-next test installed. - rustup run stable cargo install cargo-nextest --version 0.9.72 --locked + rustup run stable cargo install cargo-nextest --version 0.9.95 --locked } # Enable retries to avoid transient network issues. diff --git a/scripts/environment/entrypoint.sh b/scripts/environment/entrypoint.sh index ae6b55eb1ce55..b97484e991fc6 100755 --- a/scripts/environment/entrypoint.sh +++ b/scripts/environment/entrypoint.sh @@ -1,7 +1,16 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash # set HOSTNAME to container id for `cross` -HOSTNAME="$(head -1 /proc/self/cgroup|cut -d/ -f3)" -export HOSTNAME +if [ -f /.docker-container-id ]; then + HOSTNAME="$(cat /.docker-container-id)" + export HOSTNAME +fi + +if [ -z "$HOSTNAME" ]; then + echo "Failed to properly set HOSTNAME, cross may not work" + # Fallback if everything else fails + HOSTNAME="vector-environment" + export HOSTNAME +fi exec "$@" diff --git a/scripts/environment/install-protoc.sh b/scripts/environment/install-protoc.sh index 082c3112e3aa1..dc0eaa09037a2 100755 --- a/scripts/environment/install-protoc.sh +++ b/scripts/environment/install-protoc.sh @@ -1,4 +1,4 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash set -o errexit -o verbose # A parameter can be optionally passed to this script to specify an alternative diff --git a/scripts/environment/prepare.sh b/scripts/environment/prepare.sh index f7b0f2508516b..bf39d7205d690 100755 --- a/scripts/environment/prepare.sh +++ b/scripts/environment/prepare.sh @@ -1,40 +1,171 @@ -#! /usr/bin/env bash -set -e -o verbose +#!/usr/bin/env bash +set -euo pipefail + +ALL_MODULES=( + rustup + cargo-deb + cross + cargo-nextest + cargo-deny + cargo-msrv + dd-rust-license-tool + wasm-pack + markdownlint + datadog-ci + release-flags +) + +# By default, install everything +MODULES=( "${ALL_MODULES[@]}" ) + +# Helper to join an array by comma +join_by() { local IFS="$1"; shift; echo "$*"; } + +# If the INSTALL_MODULES env var is set, override MODULES +if [[ -n "${INSTALL_MODULES:-}" ]]; then + IFS=',' read -r -a MODULES <<< "$INSTALL_MODULES" +fi + +# Parse CLI args for --modules or -m +for arg in "$@"; do + case $arg in + --modules=*|-m=*) + val="${arg#*=}" + IFS=',' read -r -a MODULES <<< "$val" + shift + ;; + --help|-h) + cat </dev/null || ./scripts/environment/binstall.sh; then + install=(binstall -y) + else + echo "Failed to install cargo binstall, defaulting to cargo install" + fi + fi +fi +set -e -o verbose +if contains_module cargo-deb; then + if [[ "$(cargo-deb --version 2>/dev/null)" != "2.0.2" ]]; then + rustup run stable cargo "${install[@]}" cargo-deb --version 2.0.2 --force --locked + fi +fi + +if contains_module cross; then + if ! cross --version 2>/dev/null | grep -q '^cross 0.2.5'; then + rustup run stable cargo "${install[@]}" cross --version 0.2.5 --force --locked + fi fi -if [[ "$(cargo-nextest --version)" != "cargo-nextest 0.9.72" ]] ; then - rustup run stable cargo install cargo-nextest --version 0.9.72 --force --locked + +if contains_module cargo-nextest; then + if ! cargo-nextest --version 2>/dev/null | grep -q '^cargo-nextest 0.9.95'; then + rustup run stable cargo "${install[@]}" cargo-nextest --version 0.9.95 --force --locked + fi fi -if [[ "$(cargo-deny --version)" != "cargo-deny 0.16.1" ]] ; then - rustup run stable cargo install cargo-deny --version 0.16.1 --force --locked + +if contains_module cargo-deny; then + if ! cargo-deny --version 2>/dev/null | grep -q '^cargo-deny 0.16.2'; then + rustup run stable cargo "${install[@]}" cargo-deny --version 0.16.2 --force --locked + fi fi -if ! dd-rust-license-tool --help >& /dev/null ; then - rustup run stable cargo install dd-rust-license-tool --version 1.0.2 --force --locked + +if contains_module cargo-msrv; then + if ! cargo-msrv --version 2>/dev/null | grep -q '^cargo-msrv 0.18.4'; then + rustup run stable cargo "${install[@]}" cargo-msrv --version 0.18.4 --force --locked + fi fi -if [[ "$(wasm-pack --version)" != "wasm-pack 0.13.1" ]] ; then - echo "wasm-pack version 0.13.1 is not installed" - cargo install --force --locked --version 0.13.1 wasm-pack -else - echo "wasm-pack version 0.13.1 is installed already" +if contains_module dd-rust-license-tool; then + if ! dd-rust-license-tool --help &>/dev/null; then + rustup run stable cargo install dd-rust-license-tool --version 1.0.2 --force --locked + fi fi -# Currently fixing this to version 0.30 since version 0.31 has introduced -# a change that means it only works with versions of node > 10. -# https://github.com/igorshubovych/markdownlint-cli/issues/258 -# ubuntu 20.04 gives us version 10.19. We can revert once we update the -# ci image to install a newer version of node. -sudo npm -g install markdownlint-cli@0.30 -sudo npm -g install @datadog/datadog-ci +if contains_module wasm-pack; then + if ! wasm-pack --version | grep -q '^wasm-pack 0.13.1'; then + rustup run stable cargo "${install[@]}" --locked --version 0.13.1 wasm-pack + fi +fi -# Make sure our release build settings are present. -. scripts/environment/release-flags.sh +if contains_module markdownlint; then + if [[ "$(markdownlint --version 2>/dev/null)" != "0.45.0" ]]; then + sudo npm install -g markdownlint-cli@0.45.0 + fi +fi + +if contains_module datadog-ci; then + if [[ "$(datadog-ci version 2>/dev/null)" != "v3.16.0" ]]; then + sudo npm install -g @datadog/datadog-ci@3.16.0 + fi +fi diff --git a/scripts/environment/release-flags.sh b/scripts/environment/release-flags.sh index 5ccbc70f60866..4115517c437e1 100755 --- a/scripts/environment/release-flags.sh +++ b/scripts/environment/release-flags.sh @@ -1,4 +1,4 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash set -e -o verbose # We want to ensure we're building using "full" release capabilities when possible, which diff --git a/scripts/environment/setup-helm.sh b/scripts/environment/setup-helm.sh index 66c919f895c83..a16b731957b3c 100755 --- a/scripts/environment/setup-helm.sh +++ b/scripts/environment/setup-helm.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail KUBERNETES_VERSION="v1.18.6" diff --git a/scripts/generate-component-docs.rb b/scripts/generate-component-docs.rb index e023b75d968af..940abc111bf73 100755 --- a/scripts/generate-component-docs.rb +++ b/scripts/generate-component-docs.rb @@ -1693,14 +1693,14 @@ def render_and_import_schema(unwrapped_resolved_schema, friendly_name, config_ma @logger.info "[✓] Imported #{friendly_name} schema to '#{cue_output_file}'." end -def render_and_import_base_component_schema(root_schema, schema_name, component_type) - friendly_name = "base #{component_type} configuration" +def render_and_import_generated_component_schema(root_schema, schema_name, component_type) + friendly_name = "generated #{component_type} configuration" unwrapped_resolved_schema = unwrap_resolved_schema(root_schema, schema_name, friendly_name) render_and_import_schema( unwrapped_resolved_schema, friendly_name, - ["base", "components", "#{component_type}s"], - "components/base/#{component_type}s.cue" + ["generated", "components", "#{component_type}s"], + "components/generated/#{component_type}s.cue" ) end @@ -1710,12 +1710,12 @@ def render_and_import_component_schema(root_schema, schema_name, component_type, render_and_import_schema( unwrapped_resolved_schema, friendly_name, - ["base", "components", "#{component_type}s", component_name], - "components/#{component_type}s/base/#{component_name}.cue" + ["generated", "components", "#{component_type}s", component_name], + "components/#{component_type}s/generated/#{component_name}.cue" ) end -def render_and_import_base_api_schema(root_schema, apis) +def render_and_import_generated_api_schema(root_schema, apis) api_schema = {} apis.each do |component_name, schema_name| friendly_name = "'#{component_name}' #{schema_name} configuration" @@ -1726,12 +1726,12 @@ def render_and_import_base_api_schema(root_schema, apis) render_and_import_schema( api_schema, "configuration", - ["base", "api"], - "base/api.cue" + ["generated", "api"], + "generated/api.cue" ) end -def render_and_import_base_global_option_schema(root_schema, global_options) +def render_and_import_generated_global_option_schema(root_schema, global_options) global_option_schema = {} global_options.each do |component_name, schema_name| @@ -1750,8 +1750,8 @@ def render_and_import_base_global_option_schema(root_schema, global_options) render_and_import_schema( global_option_schema, "configuration", - ["base", "configuration"], - "base/configuration.cue" + ["generated", "configuration"], + "generated/configuration.cue" ) end @@ -1774,7 +1774,7 @@ def render_and_import_base_global_option_schema(root_schema, global_options) # First off, we generate the component type configuration bases. These are the high-level # configuration settings that are universal on a per-component type basis. # -# For example, the "base" configuration for a sink would be the inputs, buffer settings, healthcheck +# For example, the "generated" configuration for a sink would be the inputs, buffer settings, healthcheck # settings, and proxy settings... and then the configuration for a sink would be those, plus # whatever the sink itself defines. component_bases = root_schema['definitions'].filter_map do |key, definition| @@ -1784,7 +1784,7 @@ def render_and_import_base_global_option_schema(root_schema, global_options) .reduce { |acc, item| nested_merge(acc, item) } component_bases.each do |component_type, schema_name| - render_and_import_base_component_schema(root_schema, schema_name, component_type) + render_and_import_generated_component_schema(root_schema, schema_name, component_type) end # Now we'll generate the base configuration for each component. @@ -1808,7 +1808,7 @@ def render_and_import_base_global_option_schema(root_schema, global_options) end .reduce { |acc, item| nested_merge(acc, item) } -render_and_import_base_api_schema(root_schema, apis) +render_and_import_generated_api_schema(root_schema, apis) # At last, we generate the global options configuration. @@ -1819,4 +1819,4 @@ def render_and_import_base_global_option_schema(root_schema, global_options) end .reduce { |acc, item| nested_merge(acc, item) } -render_and_import_base_global_option_schema(root_schema, global_options) +render_and_import_generated_global_option_schema(root_schema, global_options) diff --git a/scripts/int-e2e-test.sh b/scripts/int-e2e-test.sh new file mode 100755 index 0000000000000..c9981e248da88 --- /dev/null +++ b/scripts/int-e2e-test.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# Used in CI to run and stop an integration test and upload the results of it. +# This is useful to allow retrying the integration test at a higher level than +# the nextest and reduce code duplication in the workflow file. + +set -u + +if [ $# -ne 2 ] +then + echo "usage: $0 [int|e2e] TEST_NAME" + exit 1 +fi + +set -x + +SCRIPT_DIR=$(realpath "$(dirname "${BASH_SOURCE[0]}")") +TEST_TYPE=$1 # either "int" or "e2e" +TEST_NAME=$2 + +print_compose_logs_on_failure() { + local LAST_RETURN_CODE=$1 + + if [[ $LAST_RETURN_CODE -ne 0 || "${ACTIONS_RUNNER_DEBUG:-}" == "true" ]]; then + (docker compose --project-name "${TEST_NAME}" logs) || echo "Failed to collect logs" + fi +} + +if [[ "$TEST_NAME" == "opentelemetry-logs" ]]; then + find "${SCRIPT_DIR}/../tests/data/e2e/opentelemetry/logs/output" -type f -name '*.log' -delete + chmod -R 777 "${SCRIPT_DIR}/../tests/data/e2e/opentelemetry/logs/output" +fi + +cargo vdev -v "${TEST_TYPE}" start -a "${TEST_NAME}" +START_RET=$? +print_compose_logs_on_failure $START_RET + +if [[ $START_RET -eq 0 ]]; then + cargo vdev -v "${TEST_TYPE}" test --retries 2 -a "${TEST_NAME}" + RET=$? + print_compose_logs_on_failure $RET +else + echo "Skipping test phase because 'vdev start' failed" + RET=$START_RET +fi + +cargo vdev -v "${TEST_TYPE}" stop -a "${TEST_NAME}" + +# Only upload test results if CI is defined +if [[ -n "${CI:-}" ]]; then + ./scripts/upload-test-results.sh +fi + +exit $RET diff --git a/scripts/integration/Dockerfile b/scripts/integration/Dockerfile index 07ffd19738823..3e6b68dbfb669 100644 --- a/scripts/integration/Dockerfile +++ b/scripts/integration/Dockerfile @@ -1,4 +1,4 @@ -ARG RUST_VERSION +ARG RUST_VERSION=1.85 FROM docker.io/rust:${RUST_VERSION}-slim-bookworm RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -16,7 +16,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ git \ && rm -rf /var/lib/apt/lists/* -RUN rustup run "${RUST_VERSION}" cargo install cargo-nextest --version 0.9.72 --locked +RUN cargo install cargo-nextest --version 0.9.95 --locked COPY scripts/environment/install-protoc.sh / COPY tests/data/ca/certs /certs diff --git a/scripts/integration/amqp/compose.yaml b/scripts/integration/amqp/compose.yaml index 865c35266f15f..a60a27155df0b 100644 --- a/scripts/integration/amqp/compose.yaml +++ b/scripts/integration/amqp/compose.yaml @@ -12,5 +12,4 @@ services: - RABBITMQ_SSL_CACERTFILE=/code/tests/data/ca/intermediate_server/certs/ca-chain.cert.pem - RABBITMQ_SSL_FAIL_IF_NO_PEER_CERT=false volumes: - - ${PWD}:/code - + - ../../..:/code diff --git a/scripts/integration/mqtt/test.yaml b/scripts/integration/mqtt/test.yaml index 607da45eeb5e1..ec0c637e094d1 100644 --- a/scripts/integration/mqtt/test.yaml +++ b/scripts/integration/mqtt/test.yaml @@ -10,3 +10,4 @@ paths: - "src/internal_events/mqtt.rs" - "src/sinks/mqtt/**" - "src/sinks/util/**" +- "src/sources/mqtt/**" diff --git a/scripts/integration/nats/compose.yaml b/scripts/integration/nats/compose.yaml index fb36e78d819d4..9be71dc529380 100644 --- a/scripts/integration/nats/compose.yaml +++ b/scripts/integration/nats/compose.yaml @@ -43,3 +43,10 @@ services: - /usr/share/nats/config/nats-jwt.conf volumes: - ../../../tests/data/nats:/usr/share/nats/config + nats-jetstream-test: + image: docker.io/library/nats:${CONFIG_VERSION} + command: + - --config + - /usr/share/nats/config/nats-jetstream.conf + volumes: + - ../../../tests/data/nats:/usr/share/nats/config diff --git a/scripts/integration/nats/test.yaml b/scripts/integration/nats/test.yaml index 1615b5f244ff6..171a1c4ae6a3e 100644 --- a/scripts/integration/nats/test.yaml +++ b/scripts/integration/nats/test.yaml @@ -1,7 +1,7 @@ features: -- nats-integration-tests + - nats-integration-tests -test_filter: '::nats::' +test_filter: "::nats::" env: NATS_ADDRESS: nats://nats:4222 @@ -11,6 +11,7 @@ env: NATS_TLS_CLIENT_CERT_ADDRESS: nats://nats-tls-client-cert:4222 NATS_TOKEN_ADDRESS: nats://nats-token:4222 NATS_USERPASS_ADDRESS: nats://nats-userpass:4222 + NATS_JETSTREAM_ADDRESS: nats://nats-jetstream-test:4222 matrix: version: [latest] @@ -18,10 +19,10 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/nats.rs" -- "src/sources/nats.rs" -- "src/sources/util/**" -- "src/sinks/nats.rs" -- "src/sinks/util/**" -- "src/nats.rs" -- "scripts/integration/nats/**" + - "src/internal_events/nats.rs" + - "src/sources/nats.rs" + - "src/sources/util/**" + - "src/sinks/nats/**" + - "src/sinks/util/**" + - "src/nats.rs" + - "scripts/integration/nats/**" diff --git a/scripts/integration/redis/compose.yaml b/scripts/integration/redis/compose.yaml index a5dd865e43579..1399a12b2a655 100644 --- a/scripts/integration/redis/compose.yaml +++ b/scripts/integration/redis/compose.yaml @@ -1,5 +1,26 @@ version: '3' services: - redis: + redis-primary: image: docker.io/redis:${CONFIG_VERSION} + container_name: redis-primary + hostname: redis-primary + ports: + - "6379:6379" + + redis-sentinel: + image: docker.io/redis:${CONFIG_VERSION} + container_name: redis-sentinel + hostname: redis-sentinel + depends_on: + - redis-primary + ports: + - "26379:26379" + command: > + sh -c 'echo "bind 0.0.0.0" > /etc/sentinel.conf && + echo "sentinel monitor vector redis-primary 6379 1" >> /etc/sentinel.conf && + echo "sentinel resolve-hostnames yes" >> /etc/sentinel.conf && + echo "sentinel down-after-milliseconds vector 5000" >> /etc/sentinel.conf && + echo "sentinel failover-timeout vector 5000" >> /etc/sentinel.conf && + echo "sentinel parallel-syncs vector 1" >> /etc/sentinel.conf && + redis-sentinel /etc/sentinel.conf' diff --git a/scripts/integration/redis/test.yaml b/scripts/integration/redis/test.yaml index d2d0577e844ca..6b2d9bfa3745a 100644 --- a/scripts/integration/redis/test.yaml +++ b/scripts/integration/redis/test.yaml @@ -4,7 +4,8 @@ features: test_filter: "::redis::" env: - REDIS_URL: redis://redis:6379/0 + REDIS_URL: redis://redis-primary:6379/0 + SENTINEL_URL: redis://redis-sentinel:26379/ matrix: version: [6-alpine] diff --git a/src/api/schema/components/mod.rs b/src/api/schema/components/mod.rs index 8beffd82ce354..b65b2a4cec230 100644 --- a/src/api/schema/components/mod.rs +++ b/src/api/schema/components/mod.rs @@ -235,7 +235,7 @@ pub struct ComponentsSubscription; #[Subscription] impl ComponentsSubscription { /// Subscribes to all newly added components - async fn component_added(&self) -> impl Stream { + async fn component_added(&self) -> impl Stream + use<> { BroadcastStream::new(COMPONENT_CHANGED.subscribe()).filter_map(|c| match c { Ok(ComponentChanged::Added(c)) => Some(c), _ => None, @@ -243,7 +243,7 @@ impl ComponentsSubscription { } /// Subscribes to all removed components - async fn component_removed(&self) -> impl Stream { + async fn component_removed(&self) -> impl Stream + use<> { BroadcastStream::new(COMPONENT_CHANGED.subscribe()).filter_map(|c| match c { Ok(ComponentChanged::Removed(c)) => Some(c), _ => None, diff --git a/src/api/schema/filter.rs b/src/api/schema/filter.rs index c73b88a6c01d6..438a4d7587207 100644 --- a/src/api/schema/filter.rs +++ b/src/api/schema/filter.rs @@ -7,7 +7,7 @@ use super::components::{source, ComponentKind}; /// Takes an `&Option` and returns early if false #[macro_export] macro_rules! filter_check { - ($($match:expr),+) => { + ($($match:expr_2021),+) => { $( if matches!($match, Some(t) if !t) { return false; diff --git a/src/api/schema/gen.rs b/src/api/schema/gen.rs index 33e27f658416d..c359f0db3b640 100644 --- a/src/api/schema/gen.rs +++ b/src/api/schema/gen.rs @@ -110,7 +110,7 @@ async fn main() { fs::write( "lib/vector-api-client/graphql/schema.json", - format!("{}\n", json), + format!("{json}\n"), ) .expect("Couldn't save schema file"); } diff --git a/src/api/schema/health.rs b/src/api/schema/health.rs index ebd24184caf17..36500f21f7afa 100644 --- a/src/api/schema/health.rs +++ b/src/api/schema/health.rs @@ -34,7 +34,7 @@ impl HealthSubscription { async fn heartbeat( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream { + ) -> impl Stream + use<> { IntervalStream::new(tokio::time::interval(Duration::from_millis( interval as u64, ))) diff --git a/src/api/schema/metrics/filter.rs b/src/api/schema/metrics/filter.rs index 52446cdc73ed2..1236119c87af3 100644 --- a/src/api/schema/metrics/filter.rs +++ b/src/api/schema/metrics/filter.rs @@ -303,7 +303,7 @@ pub fn component_sent_events_totals_metrics_with_outputs( match m.value() { MetricValue::Counter { value } if cache - .insert(format!("{}.{}", id, output), *value) + .insert(format!("{id}.{output}"), *value) .unwrap_or(0.00) < *value => { @@ -349,8 +349,7 @@ pub fn component_sent_events_total_throughputs_with_outputs( .iter() .filter_map(|output| { let m = filter_output_metric(metrics.as_ref(), output.as_ref())?; - let throughput = - throughput(&m, format!("{}.{}", id, output), &mut cache)?; + let throughput = throughput(&m, format!("{id}.{output}"), &mut cache)?; Some(OutputThroughput::new(output.clone(), throughput as i64)) }) .collect::>(); diff --git a/src/api/schema/metrics/mod.rs b/src/api/schema/metrics/mod.rs index 08baa7cc12fd7..116f1ca3777c2 100644 --- a/src/api/schema/metrics/mod.rs +++ b/src/api/schema/metrics/mod.rs @@ -63,7 +63,7 @@ impl MetricsSubscription { async fn uptime( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream { + ) -> impl Stream + use<> { get_metrics(interval).filter_map(|m| match m.name() { "uptime_seconds" => Some(Uptime::new(m)), _ => None, @@ -75,7 +75,7 @@ impl MetricsSubscription { async fn received_events_total( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream { + ) -> impl Stream + use<> { get_metrics(interval).filter_map(|m| match m.name() { "component_received_events_total" => Some(ReceivedEventsTotal::new(m)), _ => None, @@ -87,7 +87,7 @@ impl MetricsSubscription { async fn received_events_throughput( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream { + ) -> impl Stream + use<> { counter_throughput(interval, &|m| m.name() == "component_received_events_total") .map(|(_, throughput)| throughput as i64) } @@ -96,7 +96,7 @@ impl MetricsSubscription { async fn component_received_events_throughputs( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> { + ) -> impl Stream> + use<> { component_counter_throughputs(interval, &|m| m.name() == "component_received_events_total") .map(|m| { m.into_iter() @@ -114,7 +114,7 @@ impl MetricsSubscription { async fn component_received_events_totals( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> { + ) -> impl Stream> + use<> { component_counter_metrics(interval, &|m| m.name() == "component_received_events_total").map( |m| { m.into_iter() @@ -129,7 +129,7 @@ impl MetricsSubscription { async fn sent_events_total( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream { + ) -> impl Stream + use<> { get_metrics(interval).filter_map(|m| match m.name() { "component_sent_events_total" => Some(SentEventsTotal::new(m)), _ => None, @@ -141,7 +141,7 @@ impl MetricsSubscription { async fn sent_events_throughput( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream { + ) -> impl Stream + use<> { counter_throughput(interval, &|m| m.name() == "component_sent_events_total") .map(|(_, throughput)| throughput as i64) } @@ -150,7 +150,7 @@ impl MetricsSubscription { async fn component_sent_events_throughputs( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> { + ) -> impl Stream> + use<> { component_sent_events_total_throughputs_with_outputs(interval).map(|m| { m.into_iter() .map(|(key, total_throughput, outputs)| { @@ -164,7 +164,7 @@ impl MetricsSubscription { async fn component_sent_events_totals( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> { + ) -> impl Stream> + use<> { component_sent_events_totals_metrics_with_outputs(interval).map(|ms| { ms.into_iter() .map(|(m, m_by_outputs)| ComponentSentEventsTotal::new(m, m_by_outputs)) @@ -176,7 +176,7 @@ impl MetricsSubscription { async fn component_received_bytes_totals( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> { + ) -> impl Stream> + use<> { component_counter_metrics(interval, &|m| m.name() == "component_received_bytes_total").map( |m| { m.into_iter() @@ -190,7 +190,7 @@ impl MetricsSubscription { async fn component_received_bytes_throughputs( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> { + ) -> impl Stream> + use<> { component_counter_throughputs(interval, &|m| m.name() == "component_received_bytes_total") .map(|m| { m.into_iter() @@ -208,7 +208,7 @@ impl MetricsSubscription { async fn component_sent_bytes_totals( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> { + ) -> impl Stream> + use<> { component_counter_metrics(interval, &|m| m.name() == "component_sent_bytes_total") .map(|m| m.into_iter().map(ComponentSentBytesTotal::new).collect()) } @@ -217,7 +217,7 @@ impl MetricsSubscription { async fn component_sent_bytes_throughputs( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> { + ) -> impl Stream> + use<> { component_counter_throughputs(interval, &|m| m.name() == "component_sent_bytes_total").map( |m| { m.into_iter() @@ -236,7 +236,7 @@ impl MetricsSubscription { async fn errors_total( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream { + ) -> impl Stream + use<> { get_metrics(interval) .filter(|m| m.name().ends_with("_errors_total")) .map(ErrorsTotal::new) @@ -246,7 +246,7 @@ impl MetricsSubscription { async fn allocated_bytes( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream { + ) -> impl Stream + use<> { get_metrics(interval) .filter(|m| m.name() == "component_allocated_bytes") .map(AllocatedBytes::new) @@ -256,7 +256,7 @@ impl MetricsSubscription { async fn component_allocated_bytes( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> { + ) -> impl Stream> + use<> { component_gauge_metrics(interval, &|m| m.name() == "component_allocated_bytes") .map(|m| m.into_iter().map(ComponentAllocatedBytes::new).collect()) } @@ -265,7 +265,7 @@ impl MetricsSubscription { async fn component_errors_totals( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> { + ) -> impl Stream> + use<> { component_counter_metrics(interval, &|m| m.name().ends_with("_errors_total")) .map(|m| m.into_iter().map(ComponentErrorsTotal::new).collect()) } @@ -274,7 +274,7 @@ impl MetricsSubscription { async fn metrics( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream { + ) -> impl Stream + use<> { get_metrics(interval).filter_map(|m| match m.name() { "uptime_seconds" => Some(MetricType::Uptime(m.into())), _ => None, diff --git a/src/api/schema/metrics/source/file.rs b/src/api/schema/metrics/source/file.rs index fa73300ee5cce..93a3141860d5f 100644 --- a/src/api/schema/metrics/source/file.rs +++ b/src/api/schema/metrics/source/file.rs @@ -249,7 +249,7 @@ mod tests { sort::by_fields(&mut files, &fields); for (i, f) in ["1", "2", "3"].iter().enumerate() { - assert_eq!(files[i].name.as_str(), format!("/path/to/file/{}", f)); + assert_eq!(files[i].name.as_str(), format!("/path/to/file/{f}")); } } @@ -268,7 +268,7 @@ mod tests { sort::by_fields(&mut files, &fields); for (i, f) in ["3", "2", "1"].iter().enumerate() { - assert_eq!(files[i].name.as_str(), format!("/path/to/file/{}", f)); + assert_eq!(files[i].name.as_str(), format!("/path/to/file/{f}")); } } diff --git a/src/api/schema/relay.rs b/src/api/schema/relay.rs index 366a71723aafe..83f397a9d8f76 100644 --- a/src/api/schema/relay.rs +++ b/src/api/schema/relay.rs @@ -49,7 +49,7 @@ impl Base64Cursor { let cursor = String::from_utf8(bytes).map_err(|_| Base64CursorError::Invalid)?; let index = cursor .split(':') - .last() + .next_back() .map(|s| s.parse::()) .ok_or(Base64CursorError::Invalid)? .map_err(|_| Base64CursorError::Invalid)?; diff --git a/src/api/tests.rs b/src/api/tests.rs index b9f06ac3ed20e..6d7cd0a20596f 100644 --- a/src/api/tests.rs +++ b/src/api/tests.rs @@ -198,13 +198,13 @@ async fn integration_test_source_metric() { "to_metric", &["in"], LogToMetricConfig { - metrics: vec![MetricConfig { + metrics: Some(vec![MetricConfig { field: "message".try_into().expect("Fixed template string"), name: None, namespace: None, tags: None, metric: MetricTypeConfig::Gauge, - }], + }]), all_metrics: None, }, ); diff --git a/src/app.rs b/src/app.rs index 3707071041510..c5c69a8209a9c 100644 --- a/src/app.rs +++ b/src/app.rs @@ -13,7 +13,6 @@ use tokio::runtime::{self, Runtime}; use tokio::sync::{broadcast::error::RecvError, MutexGuard}; use tokio_stream::wrappers::UnboundedReceiverStream; -use crate::extra_context::ExtraContext; #[cfg(feature = "api")] use crate::{api, internal_events::ApiStarted}; use crate::{ @@ -28,6 +27,7 @@ use crate::{ }, trace, }; +use crate::{config::ComponentType, extra_context::ExtraContext}; #[cfg(unix)] use std::os::unix::process::ExitStatusExt; @@ -379,6 +379,15 @@ async fn handle_signal( reload_config_from_result(topology_controller, new_config).await } + Ok(SignalTo::ReloadEnrichmentTables) => { + let topology_controller = topology_controller.lock().await; + + topology_controller + .topology + .reload_enrichment_tables() + .await; + None + } Err(RecvError::Lagged(amt)) => { warn!("Overflow, dropped {} signals.", amt); None @@ -533,10 +542,33 @@ pub async fn load_configs( let mut watched_component_paths = Vec::new(); if let Some(watcher_conf) = watcher_conf { + for (name, transform) in config.transforms() { + let files = transform.inner.files_to_watch(); + let component_config = ComponentConfig::new( + files.into_iter().cloned().collect(), + name.clone(), + ComponentType::Transform, + ); + watched_component_paths.push(component_config); + } + for (name, sink) in config.sinks() { let files = sink.inner.files_to_watch(); - let component_config = - ComponentConfig::new(files.into_iter().cloned().collect(), name.clone()); + let component_config = ComponentConfig::new( + files.into_iter().cloned().collect(), + name.clone(), + ComponentType::Sink, + ); + watched_component_paths.push(component_config); + } + + for (name, table) in config.enrichment_tables() { + let files = table.inner.files_to_watch(); + let component_config = ComponentConfig::new( + files.into_iter().cloned().collect(), + name.clone(), + ComponentType::EnrichmentTable, + ); watched_component_paths.push(component_config); } @@ -544,6 +576,10 @@ pub async fn load_configs( message = "Starting watcher.", paths = ?watched_paths ); + info!( + message = "Components to watch.", + paths = ?watched_component_paths + ); // Start listening for config changes. config::watcher::spawn_thread( diff --git a/src/async_read.rs b/src/async_read.rs index 00a3c53024684..6dc0f0f7d83dd 100644 --- a/src/async_read.rs +++ b/src/async_read.rs @@ -39,7 +39,7 @@ impl AllowReadUntil { &self.reader } - pub fn get_mut(&mut self) -> &mut S { + pub const fn get_mut(&mut self) -> &mut S { &mut self.reader } } diff --git a/src/aws/auth.rs b/src/aws/auth.rs index 1b8aa111eec33..1ed174096dfa1 100644 --- a/src/aws/auth.rs +++ b/src/aws/auth.rs @@ -71,6 +71,11 @@ pub enum AwsAuthentication { #[configurable(metadata(docs::examples = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"))] secret_access_key: SensitiveString, + /// The AWS session token. + /// See [AWS temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) + #[configurable(metadata(docs::examples = "AQoDYXdz...AQoDYXdz..."))] + session_token: Option, + /// The ARN of an [IAM role][iam_role] to assume. /// /// [iam_role]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html @@ -95,7 +100,7 @@ pub enum AwsAuthentication { /// The optional [RoleSessionName][role_session_name] is a unique session identifier for your assumed role. /// /// Should be unique per principal or reason. - /// If not set, session name will be autogenerated like assume-role-provider-1736428351340 + /// If not set, the session name is autogenerated like assume-role-provider-1736428351340 /// /// [role_session_name]: https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html #[configurable(metadata(docs::examples = "vector-indexer-role"))] @@ -167,7 +172,7 @@ pub enum AwsAuthentication { /// The optional [RoleSessionName][role_session_name] is a unique session identifier for your assumed role. /// /// Should be unique per principal or reason. - /// If not set, session name will be autogenerated like assume-role-provider-1736428351340 + /// If not set, the session name is autogenerated like assume-role-provider-1736428351340 /// /// [role_session_name]: https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html #[configurable(metadata(docs::examples = "vector-indexer-role"))] @@ -275,11 +280,12 @@ impl AwsAuthentication { external_id, region, session_name, + session_token, } => { let provider = SharedCredentialsProvider::new(Credentials::from_keys( access_key_id.inner(), secret_access_key.inner(), - None, + session_token.clone().map(|v| v.inner().into()), )); if let Some(assume_role) = assume_role { let auth_region = region.clone().map(Region::new).unwrap_or(service_region); @@ -372,6 +378,7 @@ impl AwsAuthentication { external_id: None, region: None, session_name: None, + session_token: None, } } } diff --git a/src/aws/mod.rs b/src/aws/mod.rs index 0348f783b0837..dbfc742878ae8 100644 --- a/src/aws/mod.rs +++ b/src/aws/mod.rs @@ -130,7 +130,7 @@ pub trait ClientBuilder { pub fn region_provider( proxy: &ProxyConfig, tls_options: Option<&TlsConfig>, -) -> crate::Result { +) -> crate::Result> { let config = aws_config::provider_config::ProviderConfig::default() .with_http_client(connector(proxy, tls_options)?); diff --git a/src/codecs/encoding/config.rs b/src/codecs/encoding/config.rs index b11fe751a7462..c3d77a5abde2c 100644 --- a/src/codecs/encoding/config.rs +++ b/src/codecs/encoding/config.rs @@ -8,7 +8,8 @@ use vector_lib::configurable::configurable_component; /// Encoding configuration. #[configurable_component] #[derive(Clone, Debug)] -#[configurable(description = "Configures how events are encoded into raw bytes.")] +/// Configures how events are encoded into raw bytes. +/// The selected encoding also determines which input types (logs, metrics, traces) are supported. pub struct EncodingConfig { #[serde(flatten)] encoding: SerializerConfig, diff --git a/src/codecs/encoding/encoder.rs b/src/codecs/encoding/encoder.rs index e461e6834f53b..94ef9fe0408f9 100644 --- a/src/codecs/encoding/encoder.rs +++ b/src/codecs/encoding/encoder.rs @@ -93,12 +93,14 @@ impl Encoder { } /// Get the suffix that encloses a batch of events. - pub const fn batch_suffix(&self) -> &[u8] { - match (&self.framer, &self.serializer) { + pub const fn batch_suffix(&self, empty: bool) -> &[u8] { + match (&self.framer, &self.serializer, empty) { ( Framer::CharacterDelimited(CharacterDelimitedEncoder { delimiter: b',' }), Serializer::Json(_) | Serializer::NativeJson(_), + _, ) => b"]", + (Framer::NewlineDelimited(_), _, false) => b"\n", _ => &[], } } @@ -237,7 +239,7 @@ mod tests { fn encode(&mut self, _: (), dst: &mut BytesMut) -> Result<(), Self::Error> { self.0.encode((), dst)?; let result = if self.1 == self.2 { - Err(Box::new(std::io::Error::new(std::io::ErrorKind::Other, "error")) as _) + Err(Box::new(std::io::Error::other("error")) as _) } else { Ok(()) }; @@ -323,4 +325,23 @@ mod tests { let sink = framed.into_inner(); assert_eq!(sink, b"(foo)(bar)"); } + + #[tokio::test] + async fn test_encode_batch_newline() { + let encoder = Encoder::::new( + Framer::NewlineDelimited(NewlineDelimitedEncoder::default()), + TextSerializerConfig::default().build().into(), + ); + let source = futures::stream::iter(vec![ + Event::Log(LogEvent::from("bar")), + Event::Log(LogEvent::from("baz")), + Event::Log(LogEvent::from("bat")), + ]) + .map(Ok); + let sink: Vec = Vec::new(); + let mut framed = FramedWrite::new(sink, encoder); + source.forward(&mut framed).await.unwrap(); + let sink = framed.into_inner(); + assert_eq!(sink, b"bar\nbaz\nbat\n"); + } } diff --git a/src/codecs/encoding/transformer.rs b/src/codecs/encoding/transformer.rs index 5539df5dd332c..4b9583e1f53e6 100644 --- a/src/codecs/encoding/transformer.rs +++ b/src/codecs/encoding/transformer.rs @@ -396,7 +396,7 @@ mod tests { ), ]; for (fmt, expected) in cases { - let config: String = format!(r#"timestamp_format = "{}""#, fmt); + let config: String = format!(r#"timestamp_format = "{fmt}""#); let transformer: Transformer = toml::from_str(&config).unwrap(); let mut event = base.clone(); transformer.transform(&mut event); diff --git a/src/codecs/ready_frames.rs b/src/codecs/ready_frames.rs index 4aed021b57e83..a707140b23001 100644 --- a/src/codecs/ready_frames.rs +++ b/src/codecs/ready_frames.rs @@ -52,7 +52,7 @@ where } /// Returns a mutable reference to the underlying stream. - pub fn get_mut(&mut self) -> &mut T { + pub const fn get_mut(&mut self) -> &mut T { &mut self.inner } diff --git a/src/common/backoff.rs b/src/common/backoff.rs new file mode 100644 index 0000000000000..9c18593678f6c --- /dev/null +++ b/src/common/backoff.rs @@ -0,0 +1,81 @@ +use std::time::Duration; + +// `tokio-retry` crate +// MIT License +// Copyright (c) 2017 Sam Rijs +// +/// A retry strategy driven by exponential back-off. +/// +/// The power corresponds to the number of past attempts. +#[derive(Debug, Clone)] +pub(crate) struct ExponentialBackoff { + current: u64, + base: u64, + factor: u64, + max_delay: Option, +} + +impl ExponentialBackoff { + /// Constructs a new exponential back-off strategy, + /// given a base duration in milliseconds. + /// + /// The resulting duration is calculated by taking the base to the `n`-th power, + /// where `n` denotes the number of past attempts. + pub(crate) const fn from_millis(base: u64) -> ExponentialBackoff { + ExponentialBackoff { + current: base, + base, + factor: 1u64, + max_delay: None, + } + } + + /// A multiplicative factor that will be applied to the retry delay. + /// + /// For example, using a factor of `1000` will make each delay in units of seconds. + /// + /// Default factor is `1`. + pub(crate) const fn factor(mut self, factor: u64) -> ExponentialBackoff { + self.factor = factor; + self + } + + /// Apply a maximum delay. No retry delay will be longer than this `Duration`. + pub(crate) const fn max_delay(mut self, duration: Duration) -> ExponentialBackoff { + self.max_delay = Some(duration); + self + } + + /// Resents the exponential back-off strategy to its initial state. + pub(crate) const fn reset(&mut self) { + self.current = self.base; + } +} + +impl Iterator for ExponentialBackoff { + type Item = Duration; + + fn next(&mut self) -> Option { + // set delay duration by applying factor + let duration = if let Some(duration) = self.current.checked_mul(self.factor) { + Duration::from_millis(duration) + } else { + Duration::from_millis(u64::MAX) + }; + + // check if we reached max delay + if let Some(ref max_delay) = self.max_delay { + if duration > *max_delay { + return Some(*max_delay); + } + } + + if let Some(next) = self.current.checked_mul(self.base) { + self.current = next; + } else { + self.current = u64::MAX; + } + + Some(duration) + } +} diff --git a/src/common/datadog.rs b/src/common/datadog.rs index 7e58a010494aa..9b84a2f8351dc 100644 --- a/src/common/datadog.rs +++ b/src/common/datadog.rs @@ -93,7 +93,7 @@ pub struct DatadogPoint(pub i64, pub T); /// /// If `endpoint` is not specified, we fallback to `site`. pub(crate) fn get_api_base_endpoint(endpoint: Option<&str>, site: &str) -> String { - endpoint.map_or_else(|| format!("https://api.{}", site), compute_api_endpoint) + endpoint.map_or_else(|| format!("https://api.{site}"), compute_api_endpoint) } /// Computes the Datadog API endpoint from a given endpoint string. diff --git a/src/common/expansion.rs b/src/common/expansion.rs index 11d6940183de7..04c737498f259 100644 --- a/src/common/expansion.rs +++ b/src/common/expansion.rs @@ -25,7 +25,7 @@ pub(crate) fn pair_expansion( // key_* -> key_one, key_two, key_three // * -> one, two, three for (k, v) in output { - let key = slugify_text(&format!("{}{}", opening_prefix, k)); + let key = slugify_text(&format!("{opening_prefix}{k}")); let val = Value::from(v).to_string_lossy().into_owned(); if val == "" { warn!("Encountered \"null\" value for dynamic pair. key: {}", key); diff --git a/src/common/http/server_auth.rs b/src/common/http/server_auth.rs index fda8ccdbf82eb..0b3eda4c43184 100644 --- a/src/common/http/server_auth.rs +++ b/src/common/http/server_auth.rs @@ -181,6 +181,7 @@ impl HttpServerAuthConfig { /// Built auth matcher with validated configuration /// Can be used directly in a component to validate authentication in HTTP requests +#[allow(clippy::large_enum_variant)] #[derive(Clone, Debug)] pub enum HttpServerAuthMatcher { /// Matcher for comparing exact value of Authorization header @@ -198,6 +199,7 @@ impl HttpServerAuthMatcher { &self, address: Option<&SocketAddr>, headers: &HeaderMap, + path: &str, ) -> Result<(), ErrorMessage> { match self { HttpServerAuthMatcher::AuthHeader(expected, err_message) => { @@ -218,7 +220,7 @@ impl HttpServerAuthMatcher { } } HttpServerAuthMatcher::Vrl { program } => { - self.handle_vrl_auth(address, headers, program) + self.handle_vrl_auth(address, headers, path, program) } } } @@ -227,6 +229,7 @@ impl HttpServerAuthMatcher { &self, address: Option<&SocketAddr>, headers: &HeaderMap, + path: &str, program: &Program, ) -> Result<(), ErrorMessage> { let mut target = VrlTarget::new( @@ -250,6 +253,7 @@ impl HttpServerAuthMatcher { "address".into(), address.map_or(Value::Null, |a| Value::from(a.ip().to_string())), ), + ("path".into(), Value::from(path.to_owned())), ]), Default::default(), )), @@ -439,7 +443,7 @@ mod tests { let matcher = basic_auth.build(&Default::default()).unwrap(); - let result = matcher.handle_auth(Some(&next_addr()), &HeaderMap::new()); + let result = matcher.handle_auth(Some(&next_addr()), &HeaderMap::new(), "/"); assert!(result.is_err()); let error = result.unwrap_err(); @@ -458,7 +462,7 @@ mod tests { let mut headers = HeaderMap::new(); headers.insert(AUTHORIZATION, HeaderValue::from_static("Basic wrong")); - let result = matcher.handle_auth(Some(&next_addr()), &headers); + let result = matcher.handle_auth(Some(&next_addr()), &headers, "/"); assert!(result.is_err()); let error = result.unwrap_err(); @@ -482,7 +486,7 @@ mod tests { AUTHORIZATION, Authorization::basic(&username, &password).0.encode(), ); - let result = matcher.handle_auth(Some(&next_addr()), &headers); + let result = matcher.handle_auth(Some(&next_addr()), &headers, "/"); assert!(result.is_ok()); } @@ -497,7 +501,7 @@ mod tests { let mut headers = HeaderMap::new(); headers.insert(AUTHORIZATION, HeaderValue::from_static("test")); - let result = matcher.handle_auth(Some(&next_addr()), &headers); + let result = matcher.handle_auth(Some(&next_addr()), &headers, "/"); assert!(result.is_ok()); } @@ -513,7 +517,7 @@ mod tests { let matcher = custom_auth.build(&Default::default()).unwrap(); let headers = HeaderMap::new(); - let result = matcher.handle_auth(Some(&next_addr()), &headers); + let result = matcher.handle_auth(Some(&next_addr()), &headers, "/"); assert!(result.is_ok()); } @@ -529,7 +533,35 @@ mod tests { let matcher = custom_auth.build(&Default::default()).unwrap(); let headers = HeaderMap::new(); - let result = matcher.handle_auth(None, &headers); + let result = matcher.handle_auth(None, &headers, "/"); + + assert!(result.is_err()); + } + + #[test] + fn custom_auth_matcher_should_be_able_to_check_path() { + let custom_auth = HttpServerAuthConfig::Custom { + source: r#".path == "/ok""#.to_string(), + }; + + let matcher = custom_auth.build(&Default::default()).unwrap(); + + let headers = HeaderMap::new(); + let result = matcher.handle_auth(Some(&next_addr()), &headers, "/ok"); + + assert!(result.is_ok()); + } + + #[test] + fn custom_auth_matcher_should_return_401_with_wrong_path() { + let custom_auth = HttpServerAuthConfig::Custom { + source: r#".path == "/ok""#.to_string(), + }; + + let matcher = custom_auth.build(&Default::default()).unwrap(); + + let headers = HeaderMap::new(); + let result = matcher.handle_auth(Some(&next_addr()), &headers, "/bad"); assert!(result.is_err()); } @@ -544,7 +576,7 @@ mod tests { let mut headers = HeaderMap::new(); headers.insert(AUTHORIZATION, HeaderValue::from_static("wrong value")); - let result = matcher.handle_auth(Some(&next_addr()), &headers); + let result = matcher.handle_auth(Some(&next_addr()), &headers, "/"); assert!(result.is_err()); let error = result.unwrap_err(); @@ -562,7 +594,7 @@ mod tests { let mut headers = HeaderMap::new(); headers.insert(AUTHORIZATION, HeaderValue::from_static("test")); - let result = matcher.handle_auth(Some(&next_addr()), &headers); + let result = matcher.handle_auth(Some(&next_addr()), &headers, "/"); assert!(result.is_err()); let error = result.unwrap_err(); diff --git a/src/common/mod.rs b/src/common/mod.rs index cfaf80c4f9354..1c9a6eb45e6d6 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -18,6 +18,14 @@ pub(crate) mod sqs; #[cfg(any(feature = "sources-aws_s3", feature = "sinks-aws_s3"))] pub(crate) mod s3; +#[cfg(any(feature = "sources-websocket", feature = "sinks-websocket"))] +pub(crate) mod websocket; + +pub(crate) mod backoff; +#[cfg(any(feature = "sources-mqtt", feature = "sinks-mqtt",))] +/// Common MQTT configuration shared by MQTT components. +pub mod mqtt; + #[cfg(any(feature = "transforms-log_to_metric", feature = "sinks-loki"))] pub(crate) mod expansion; diff --git a/src/common/mqtt.rs b/src/common/mqtt.rs new file mode 100644 index 0000000000000..c5e1190b0b067 --- /dev/null +++ b/src/common/mqtt.rs @@ -0,0 +1,132 @@ +use rumqttc::{AsyncClient, EventLoop, MqttOptions}; +use snafu::Snafu; +use vector_config_macros::configurable_component; +use vector_lib::tls::{TlsEnableableConfig, TlsError}; + +use crate::template::TemplateParseError; + +/// Shared MQTT configuration for sources and sinks. +#[configurable_component] +#[derive(Clone, Debug, Derivative)] +#[derivative(Default)] +#[serde(deny_unknown_fields)] +pub struct MqttCommonConfig { + /// MQTT server address (The broker’s domain name or IP address). + #[configurable(metadata(docs::examples = "mqtt.example.com", docs::examples = "127.0.0.1"))] + pub host: String, + + /// TCP port of the MQTT server to connect to. + #[configurable(derived)] + #[serde(default = "default_port")] + #[derivative(Default(value = "default_port()"))] + pub port: u16, + + /// MQTT username. + #[serde(default)] + #[configurable(derived)] + pub user: Option, + + /// MQTT password. + #[serde(default)] + #[configurable(derived)] + pub password: Option, + + /// MQTT client ID. + #[serde(default)] + #[configurable(derived)] + pub client_id: Option, + + /// Connection keep-alive interval. + #[serde(default = "default_keep_alive")] + #[derivative(Default(value = "default_keep_alive()"))] + pub keep_alive: u16, + + /// Maximum packet size + #[serde(default = "default_max_packet_size")] + #[derivative(Default(value = "default_max_packet_size()"))] + pub max_packet_size: usize, + + /// TLS configuration. + #[configurable(derived)] + pub tls: Option, +} + +const fn default_port() -> u16 { + 1883 +} + +const fn default_keep_alive() -> u16 { + 60 +} + +const fn default_max_packet_size() -> usize { + 10 * 1024 +} + +/// MQTT Error Types +#[derive(Debug, Snafu)] +#[snafu(visibility(pub))] +pub enum MqttError { + /// Topic template parsing failed + #[snafu(display("invalid topic template: {source}"))] + TopicTemplate { + /// Source of error + source: TemplateParseError, + }, + /// TLS error + #[snafu(display("TLS error: {source}"))] + Tls { + /// Source of error + source: TlsError, + }, + /// Configuration error + #[snafu(display("MQTT configuration error: {source}"))] + Configuration { + /// Source of error + source: ConfigurationError, + }, +} + +/// MQTT Configuration error types +#[derive(Clone, Debug, Eq, PartialEq, Snafu)] +pub enum ConfigurationError { + /// Empty client ID error + #[snafu(display("Client ID is not allowed to be empty."))] + EmptyClientId, + /// Invalid credentials provided error + #[snafu(display("Username and password must be either both provided or both missing."))] + InvalidCredentials, + /// Invalid client ID provied error + #[snafu(display( + "Client ID must be 1-23 characters long and must consist of only alphanumeric characters." + ))] + InvalidClientId, + /// Credentials provided were incomplete + #[snafu(display("Username and password must be either both or neither provided."))] + IncompleteCredentials, +} + +#[derive(Clone)] +/// Mqtt connector wrapper +pub struct MqttConnector { + /// Mqtt connection options + pub options: MqttOptions, +} + +impl MqttConnector { + /// Creates a new MqttConnector + pub const fn new(options: MqttOptions) -> Self { + Self { options } + } + + /// Connects the connector and generates a client and eventloop + pub fn connect(&self) -> (AsyncClient, EventLoop) { + let (client, eventloop) = AsyncClient::new(self.options.clone(), 1024); + (client, eventloop) + } + + /// TODO: Right now there is no way to implement the healthcheck properly: + pub async fn healthcheck(&self) -> crate::Result<()> { + Ok(()) + } +} diff --git a/src/common/websocket.rs b/src/common/websocket.rs new file mode 100644 index 0000000000000..f545bc92adb1b --- /dev/null +++ b/src/common/websocket.rs @@ -0,0 +1,241 @@ +use std::{ + fmt::Debug, + net::SocketAddr, + num::NonZeroU64, + task::{Context, Poll}, + time::Duration, +}; +use vector_config_macros::configurable_component; + +use snafu::{ResultExt, Snafu}; +use tokio::{net::TcpStream, time}; +use tokio_tungstenite::{ + client_async_with_config, + tungstenite::{ + client::{uri_mode, IntoClientRequest}, + error::{Error as TungsteniteError, ProtocolError, UrlError}, + handshake::client::Request, + protocol::WebSocketConfig, + stream::Mode as UriMode, + }, + WebSocketStream, +}; + +use crate::{ + common::backoff::ExponentialBackoff, + dns, + http::Auth, + internal_events::{WebSocketConnectionEstablished, WebSocketConnectionFailedError}, + tls::{MaybeTlsSettings, MaybeTlsStream, TlsEnableableConfig, TlsError}, +}; + +#[allow(unreachable_pub)] +#[derive(Debug, Snafu)] +#[snafu(visibility(pub))] +pub enum WebSocketError { + #[snafu(display("Creating WebSocket client failed: {}", source))] + CreateFailed { source: TungsteniteError }, + #[snafu(display("Connect error: {}", source))] + ConnectError { source: TlsError }, + #[snafu(display("Unable to resolve DNS: {}", source))] + DnsError { source: dns::DnsError }, + #[snafu(display("No addresses returned."))] + NoAddresses, +} + +#[derive(Clone)] +pub(crate) struct WebSocketConnector { + uri: String, + host: String, + port: u16, + tls: MaybeTlsSettings, + auth: Option, +} + +impl WebSocketConnector { + pub(crate) fn new( + uri: String, + tls: MaybeTlsSettings, + auth: Option, + ) -> Result { + let request = (&uri).into_client_request().context(CreateFailedSnafu)?; + let (host, port) = Self::extract_host_and_port(&request).context(CreateFailedSnafu)?; + + Ok(Self { + uri, + host, + port, + tls, + auth, + }) + } + + fn extract_host_and_port(request: &Request) -> Result<(String, u16), TungsteniteError> { + let host = request + .uri() + .host() + .ok_or(TungsteniteError::Url(UrlError::NoHostName))? + .to_string(); + let mode = uri_mode(request.uri())?; + let port = request.uri().port_u16().unwrap_or(match mode { + UriMode::Tls => 443, + UriMode::Plain => 80, + }); + + Ok((host, port)) + } + + const fn fresh_backoff() -> ExponentialBackoff { + ExponentialBackoff::from_millis(2) + .factor(250) + .max_delay(Duration::from_secs(60)) + } + + async fn tls_connect(&self) -> Result, WebSocketError> { + let ip = dns::Resolver + .lookup_ip(self.host.clone()) + .await + .context(DnsSnafu)? + .next() + .ok_or(WebSocketError::NoAddresses)?; + + let addr = SocketAddr::new(ip, self.port); + self.tls + .connect(&self.host, &addr) + .await + .context(ConnectSnafu) + } + + async fn connect(&self) -> Result>, WebSocketError> { + let mut request = (&self.uri) + .into_client_request() + .context(CreateFailedSnafu)?; + + if let Some(auth) = &self.auth { + auth.apply(&mut request); + } + + let maybe_tls = self.tls_connect().await?; + + let ws_config = WebSocketConfig::default(); + + let (ws_stream, _response) = client_async_with_config(request, maybe_tls, Some(ws_config)) + .await + .context(CreateFailedSnafu)?; + + Ok(ws_stream) + } + + pub(crate) async fn connect_backoff(&self) -> WebSocketStream> { + let mut backoff = Self::fresh_backoff(); + loop { + match self.connect().await { + Ok(ws_stream) => { + emit!(WebSocketConnectionEstablished {}); + return ws_stream; + } + Err(error) => { + emit!(WebSocketConnectionFailedError { + error: Box::new(error) + }); + time::sleep(backoff.next().unwrap()).await; + } + } + } + } + + #[cfg(feature = "sinks-websocket")] + pub(crate) async fn healthcheck(&self) -> crate::Result<()> { + self.connect().await.map(|_| ()).map_err(Into::into) + } +} + +pub(crate) const fn is_closed(error: &TungsteniteError) -> bool { + matches!( + error, + TungsteniteError::ConnectionClosed + | TungsteniteError::AlreadyClosed + | TungsteniteError::Protocol(ProtocolError::ResetWithoutClosingHandshake) + ) +} + +pub(crate) struct PingInterval { + interval: Option, +} + +impl PingInterval { + pub(crate) fn new(period: Option) -> Self { + Self { + interval: period.map(|period| time::interval(Duration::from_secs(period))), + } + } + + pub(crate) fn poll_tick(&mut self, cx: &mut Context<'_>) -> Poll { + match self.interval.as_mut() { + Some(interval) => interval.poll_tick(cx), + None => Poll::Pending, + } + } + + pub(crate) async fn tick(&mut self) -> time::Instant { + std::future::poll_fn(|cx| self.poll_tick(cx)).await + } +} + +/// Shared websocket configuration for sources and sinks. +#[configurable_component] +#[derive(Clone, Debug)] +#[serde(deny_unknown_fields)] +pub struct WebSocketCommonConfig { + /// The WebSocket URI to connect to. + /// + /// This should include the protocol and host, but can also include the port, path, and any other valid part of a URI. + /// **Note**: Using the `wss://` protocol requires enabling `tls`. + #[configurable(metadata(docs::examples = "ws://localhost:8080"))] + #[configurable(metadata(docs::examples = "wss://example.com/socket"))] + pub uri: String, + + /// The interval, in seconds, between sending [Ping][ping]s to the remote peer. + /// + /// If this option is not configured, pings are not sent on an interval. + /// + /// If the `ping_timeout` is not set, pings are still sent but there is no expectation of pong + /// response times. + /// + /// [ping]: https://www.rfc-editor.org/rfc/rfc6455#section-5.5.2 + #[configurable(metadata(docs::type_unit = "seconds"))] + #[configurable(metadata(docs::advanced))] + #[configurable(metadata(docs::examples = 30))] + pub ping_interval: Option, + + /// The number of seconds to wait for a [Pong][pong] response from the remote peer. + /// + /// If a response is not received within this time, the connection is re-established. + /// + /// [pong]: https://www.rfc-editor.org/rfc/rfc6455#section-5.5.3 + // NOTE: this option is not relevant if the `ping_interval` is not configured. + #[configurable(metadata(docs::type_unit = "seconds"))] + #[configurable(metadata(docs::advanced))] + #[configurable(metadata(docs::examples = 5))] + pub ping_timeout: Option, + + /// TLS configuration. + #[configurable(derived)] + pub tls: Option, + + /// HTTP Authentication. + #[configurable(derived)] + pub auth: Option, +} + +impl Default for WebSocketCommonConfig { + fn default() -> Self { + Self { + uri: "ws://127.0.0.1:8080".to_owned(), + ping_interval: None, + ping_timeout: None, + tls: None, + auth: None, + } + } +} diff --git a/src/components/validation/mod.rs b/src/components/validation/mod.rs index 514153ad2b16b..a786a56571120 100644 --- a/src/components/validation/mod.rs +++ b/src/components/validation/mod.rs @@ -319,7 +319,7 @@ fn run_validation(configuration: ValidationConfiguration, test_case_data_path: s } else { let formatted = success .iter() - .map(|s| format!(" - {}\n", s)) + .map(|s| format!(" - {s}\n")) .collect::>(); details.push(format!( @@ -340,7 +340,7 @@ fn run_validation(configuration: ValidationConfiguration, test_case_data_path: s } else { let formatted = failure .iter() - .map(|s| format!(" - {}\n", s)) + .map(|s| format!(" - {s}\n")) .collect::>(); details.push(format!( @@ -368,10 +368,9 @@ fn run_validation(configuration: ValidationConfiguration, test_case_data_path: s ); } } - Err(e) => panic!( - "Failed to complete validation run for component '{}': {}", - component_name, e - ), + Err(e) => { + panic!("Failed to complete validation run for component '{component_name}': {e}") + } } }); } diff --git a/src/components/validation/resources/event.rs b/src/components/validation/resources/event.rs index b7a6e102ea799..46ae48378ae1f 100644 --- a/src/components/validation/resources/event.rs +++ b/src/components/validation/resources/event.rs @@ -101,7 +101,7 @@ impl TestEvent { } } - pub fn get_event(&mut self) -> &mut Event { + pub const fn get_event(&mut self) -> &mut Event { match self { Self::Passthrough(event) => event, Self::FailWithAlternateEncoder(event) => event, diff --git a/src/components/validation/resources/http.rs b/src/components/validation/resources/http.rs index 2bfc04deba62f..0be7d4de4bd3c 100644 --- a/src/components/validation/resources/http.rs +++ b/src/components/validation/resources/http.rs @@ -113,16 +113,19 @@ fn spawn_input_http_server( async move { let mut sendable_events = sendable_events.lock().await; - if let Some(event) = sendable_events.pop_front() { - let mut buffer = BytesMut::new(); - encode_test_event(&mut encoder, &mut buffer, event); - - buffer.into_response() - } else { - // We'll send an empty 200 in the response since some - // sources throw errors for anything other than a valid - // response. - StatusCode::OK.into_response() + match sendable_events.pop_front() { + Some(event) => { + let mut buffer = BytesMut::new(); + encode_test_event(&mut encoder, &mut buffer, event); + + buffer.into_response() + } + _ => { + // We'll send an empty 200 in the response since some + // sources throw errors for anything other than a valid + // response. + StatusCode::OK.into_response() + } } } }, @@ -325,16 +328,19 @@ impl HttpResourceOutputContext<'_> { match hyper::body::to_bytes(request.into_body()).await { Err(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(), Ok(body) => { + let byte_size = body.len(); let mut body = BytesMut::from(&body[..]); loop { match decoder.decode_eof(&mut body) { - Ok(Some((events, byte_size))) => { + // `decoded_byte_size` is the decoded size of an individual frame. `byte_size` represents the size of the + // entire payload which may contain multiple frames and their delimiters. + Ok(Some((events, decoded_byte_size))) => { if should_reject { - info!("HTTP server external output resource decoded {byte_size} bytes but test case configured to reject."); + info!(internal_log_rate_limit = true, "HTTP server external output resource decoded {decoded_byte_size:?} bytes but test case configured to reject.", ); } else { let mut output_runner_metrics = output_runner_metrics.lock().await; - info!("HTTP server external output resource decoded {byte_size} bytes."); + info!(internal_log_rate_limit = true, "HTTP server external output resource decoded {decoded_byte_size:?} bytes."); // Update the runner metrics for the received events. This will later // be used in the Validators, as the "expected" case. diff --git a/src/components/validation/runner/config.rs b/src/components/validation/runner/config.rs index 5f70f56670e99..1afc21e5d5fdb 100644 --- a/src/components/validation/runner/config.rs +++ b/src/components/validation/runner/config.rs @@ -33,8 +33,7 @@ impl TopologyBuilder { let component_configuration = configuration .component_configuration_for_test_case(config_name) .ok_or(format!( - "No test case name defined for configuration {:?}.", - config_name + "No test case name defined for configuration {config_name:?}." ))?; Ok(match component_configuration { diff --git a/src/components/validation/runner/mod.rs b/src/components/validation/runner/mod.rs index 8c5937bbf86cd..1316499bb13ac 100644 --- a/src/components/validation/runner/mod.rs +++ b/src/components/validation/runner/mod.rs @@ -188,13 +188,11 @@ impl Runner { .insert(validator_name.to_string(), validator) .is_some() { - panic!( - "attempted to add duplicate validator '{}' to runner", - validator_name - ); + panic!("attempted to add duplicate validator '{validator_name}' to runner"); } } + #[allow(clippy::print_stdout)] pub async fn run_validation(self) -> Result, vector_lib::Error> { // Initialize our test environment. initialize_test_environment(); @@ -205,8 +203,8 @@ impl Runner { let test_cases = load_component_test_cases(&self.test_case_data_path)?; for test_case in test_cases { - println!(""); - println!(""); + println!(); + println!(); info!( "Running test '{}' case for component '{}' (type: {:?})...", test_case.name, @@ -432,18 +430,10 @@ impl Runner { /// returned explaining the cause. fn load_component_test_cases(test_case_data_path: &PathBuf) -> Result, String> { std::fs::File::open(test_case_data_path) - .map_err(|e| { - format!( - "I/O error during open of component validation test cases file: {}", - e - ) - }) + .map_err(|e| format!("I/O error during open of component validation test cases file: {e}")) .and_then(|file| { serde_yaml::from_reader(file).map_err(|e| { - format!( - "Deserialization error for component validation test cases file: {}", - e - ) + format!("Deserialization error for component validation test cases file: {e}") }) }) } @@ -627,7 +617,9 @@ fn spawn_input_driver( if let Some(ts) = log.remove_timestamp() { let ts = match ts.as_integer() { Some(ts) => chrono::DateTime::from_timestamp_millis(ts) - .expect(&format!("invalid timestamp in input test event {ts}")) + .unwrap_or_else(|| { + panic!("invalid timestamp in input test event {ts}") + }) .into(), None => ts, }; diff --git a/src/components/validation/validators/component_spec/mod.rs b/src/components/validation/validators/component_spec/mod.rs index 98db84a1ce6dd..26ffd292c5cf7 100644 --- a/src/components/validation/validators/component_spec/mod.rs +++ b/src/components/validation/validators/component_spec/mod.rs @@ -256,7 +256,7 @@ fn sum_counters( sum += *value; } } - _ => errs.push(format!("{}: metric value is not a counter", metric_name,)), + _ => errs.push(format!("{metric_name}: metric value is not a counter",)), } } diff --git a/src/conditions/datadog_search.rs b/src/conditions/datadog_search.rs index 83ec3e00923a7..a7ecea5f4f590 100644 --- a/src/conditions/datadog_search.rs +++ b/src/conditions/datadog_search.rs @@ -84,7 +84,7 @@ impl Filter for EventFilter { fn exists(&self, field: Field) -> Result>, PathParseError> { Ok(match field { Field::Tag(tag) => { - let starts_with = format!("{}:", tag); + let starts_with = format!("{tag}:"); any_string_match_multiple(vec!["ddtags", "tags"], move |value| { value == tag || value.starts_with(&starts_with) @@ -125,7 +125,7 @@ impl Filter for EventFilter { } // Individual tags are compared by element key:value. Field::Tag(tag) => { - let value_bytes = Value::Bytes(format!("{}:{}", tag, to_match).into()); + let value_bytes = Value::Bytes(format!("{tag}:{to_match}").into()); array_match_multiple(vec!["ddtags", "tags"], move |values| { values.contains(&value_bytes) @@ -160,13 +160,13 @@ impl Filter for EventFilter { Ok(match field { // Default fields are matched by word boundary. Field::Default(field) => { - let re = word_regex(&format!("{}*", prefix)); + let re = word_regex(&format!("{prefix}*")); string_match(field, move |value| re.is_match(&value)) } // Tags are recursed until a match is found. Field::Tag(tag) => { - let starts_with = format!("{}:{}", tag, prefix); + let starts_with = format!("{tag}:{prefix}"); any_string_match_multiple(vec!["ddtags", "tags"], move |value| { value.starts_with(&starts_with) @@ -202,7 +202,7 @@ impl Filter for EventFilter { string_match(field, move |value| re.is_match(&value)) } Field::Tag(tag) => { - let re = wildcard_regex(&format!("{}:{}", tag, wildcard)); + let re = wildcard_regex(&format!("{tag}:{wildcard}")); any_string_match_multiple(vec!["ddtags", "tags"], move |value| re.is_match(&value)) } @@ -1611,7 +1611,7 @@ mod test { // Every query should build successfully. let cond = config .build(&Default::default()) - .unwrap_or_else(|_| panic!("build failed: {}", source)); + .unwrap_or_else(|_| panic!("build failed: {source}")); assert!( cond.check_with_context(pass.clone()).0.is_ok(), diff --git a/src/conditions/vrl.rs b/src/conditions/vrl.rs index e8170e3ec51bd..267a7e01f19a4 100644 --- a/src/conditions/vrl.rs +++ b/src/conditions/vrl.rs @@ -142,7 +142,7 @@ impl Conditional for Vrl { ) .colored() .to_string(); - format!("source execution aborted: {}", err) + format!("source execution aborted: {err}") } Terminate::Error(err) => { let err = Formatter::new( @@ -153,7 +153,7 @@ impl Conditional for Vrl { ) .colored() .to_string(); - format!("source execution failed: {}", err) + format!("source execution failed: {err}") } }); diff --git a/src/config/api.rs b/src/config/api.rs index 2f7ca7dd074f9..fca088f751b39 100644 --- a/src/config/api.rs +++ b/src/config/api.rs @@ -71,7 +71,7 @@ pub fn default_address() -> Option { /// Default GraphQL API address pub fn default_graphql_url() -> Url { let addr = default_address().unwrap(); - Url::parse(&format!("http://{}/graphql", addr)) + Url::parse(&format!("http://{addr}/graphql")) .expect("Couldn't parse default API URL. Please report this.") } @@ -95,9 +95,7 @@ impl Options { // Prefer non default address (Some(a), Some(b)) => { match (Some(a) == default_address(), Some(b) == default_address()) { - (false, false) => { - return Err(format!("Conflicting `api` address: {}, {} .", a, b)) - } + (false, false) => return Err(format!("Conflicting `api` address: {a}, {b} .")), (false, true) => Some(a), (true, _) => Some(b), } diff --git a/src/config/builder.rs b/src/config/builder.rs index 351122aa5ca7d..33889a67833c1 100644 --- a/src/config/builder.rs +++ b/src/config/builder.rs @@ -231,22 +231,22 @@ impl ConfigBuilder { with.enrichment_tables.keys().for_each(|k| { if self.enrichment_tables.contains_key(k) { - errors.push(format!("duplicate enrichment_table name found: {}", k)); + errors.push(format!("duplicate enrichment_table name found: {k}")); } }); with.sources.keys().for_each(|k| { if self.sources.contains_key(k) { - errors.push(format!("duplicate source id found: {}", k)); + errors.push(format!("duplicate source id found: {k}")); } }); with.sinks.keys().for_each(|k| { if self.sinks.contains_key(k) { - errors.push(format!("duplicate sink id found: {}", k)); + errors.push(format!("duplicate sink id found: {k}")); } }); with.transforms.keys().for_each(|k| { if self.transforms.contains_key(k) { - errors.push(format!("duplicate transform id found: {}", k)); + errors.push(format!("duplicate transform id found: {k}")); } }); with.tests.iter().for_each(|wt| { @@ -256,7 +256,7 @@ impl ConfigBuilder { }); with.secret.keys().for_each(|k| { if self.secret.contains_key(k) { - errors.push(format!("duplicate secret id found: {}", k)); + errors.push(format!("duplicate secret id found: {k}")); } }); if !errors.is_empty() { diff --git a/src/config/cmd.rs b/src/config/cmd.rs index a76d7a4a0843a..e2372931087fa 100644 --- a/src/config/cmd.rs +++ b/src/config/cmd.rs @@ -78,7 +78,7 @@ impl Opts { /// Helper to merge JSON. Handles objects and array concatenation. fn merge_json(a: &mut Value, b: Value) { match (a, b) { - (Value::Object(ref mut a), Value::Object(b)) => { + (Value::Object(a), Value::Object(b)) => { for (k, v) in b { merge_json(a.entry(k).or_insert(Value::Null), v); } @@ -92,7 +92,7 @@ fn merge_json(a: &mut Value, b: Value) { /// Helper to sort array values. fn sort_json_array_values(json: &mut Value) { match json { - Value::Array(ref mut arr) => { + Value::Array(arr) => { for v in arr.iter_mut() { sort_json_array_values(v); } @@ -113,7 +113,7 @@ fn sort_json_array_values(json: &mut Value) { .map(|v| serde_json::from_str(v.as_str()).unwrap()) .collect::>(); } - Value::Object(ref mut json) => { + Value::Object(json) => { for (_, v) in json { sort_json_array_values(v); } @@ -242,13 +242,12 @@ mod tests { r#" [sources.in] type = "demo_logs" - format = "${{{}}}" + format = "${{{env_var}}}" [sinks.out] type = "blackhole" - inputs = ["${{{}}}"] - "#, - env_var, env_var_in_arr + inputs = ["${{{env_var_in_arr}}}"] + "# ); let interpolated_config_source = vars::interpolate( config_source.as_ref(), @@ -319,18 +318,18 @@ mod tests { "{}/{}/{}", sources .iter() - .map(|source| format!("{}:{}", source, source)) + .map(|source| format!("{source}:{source}")) .collect::>() .join(","), transforms .iter() - .map(|transform| format!("{}:{}", transform, transform)) + .map(|transform| format!("{transform}:{transform}")) .chain(vec!["manually-added-remap:remap".to_string()]) .collect::>() .join(","), sinks .iter() - .map(|sink| format!("{}:{}", sink, sink)) + .map(|sink| format!("{sink}:{sink}")) .collect::>() .join(","), ); diff --git a/src/config/compiler.rs b/src/config/compiler.rs index 686572e6e56e3..bffb51bcaa93a 100644 --- a/src/config/compiler.rs +++ b/src/config/compiler.rs @@ -71,7 +71,13 @@ pub fn compile(mut builder: ConfigBuilder) -> Result<(Config, Vec), Vec< ) .collect::>(); - let graph = match Graph::new(&sources_and_table_sources, &transforms, &all_sinks, schema) { + let graph = match Graph::new( + &sources_and_table_sources, + &transforms, + &all_sinks, + schema, + global.wildcard_matching.unwrap_or_default(), + ) { Ok(graph) => graph, Err(graph_errors) => { errors.extend(graph_errors); diff --git a/src/config/diff.rs b/src/config/diff.rs index cce523cfb85ba..fa5f8cf09b8ff 100644 --- a/src/config/diff.rs +++ b/src/config/diff.rs @@ -35,7 +35,7 @@ impl ConfigDiff { } /// Swaps removed with added in Differences. - pub fn flip(mut self) -> Self { + pub const fn flip(mut self) -> Self { self.sources.flip(); self.transforms.flip(); self.sinks.flip(); @@ -189,7 +189,7 @@ impl Difference { self.to_remove.contains(key) } - fn flip(&mut self) { + const fn flip(&mut self) { std::mem::swap(&mut self.to_remove, &mut self.to_add); } diff --git a/src/config/format.rs b/src/config/format.rs index 85a11c25697e5..32c2bd7b0f12e 100644 --- a/src/config/format.rs +++ b/src/config/format.rs @@ -46,7 +46,7 @@ impl FromStr for Format { "toml" => Ok(Format::Toml), "yaml" => Ok(Format::Yaml), "json" => Ok(Format::Json), - _ => Err(format!("Invalid format: {}", s)), + _ => Err(format!("Invalid format: {s}")), } } } @@ -58,7 +58,7 @@ impl fmt::Display for Format { Format::Json => "json", Format::Yaml => "yaml", }; - write!(f, "{}", format) + write!(f, "{format}") } } @@ -163,7 +163,7 @@ mod tests { for (input, expected) in cases { let output = Format::from_path(std::path::PathBuf::from(input)); - assert_eq!(expected, output.ok(), "{}", input) + assert_eq!(expected, output.ok(), "{input}") } } @@ -179,7 +179,7 @@ mod tests { use crate::config::ConfigBuilder; macro_rules! concat_with_newlines { - ($($e:expr,)*) => { concat!( $($e, "\n"),+ ) }; + ($($e:expr_2021,)*) => { concat!( $($e, "\n"),+ ) }; } const SAMPLE_TOML: &str = r#" @@ -316,23 +316,20 @@ mod tests { Ok(expected) => { #[allow(clippy::expect_fun_call)] // false positive let output: ConfigBuilder = output.expect(&format!( - "expected Ok, got Err with format {:?} and input {:?}", - format, input + "expected Ok, got Err with format {format:?} and input {input:?}" )); let output_json = serde_json::to_value(output).unwrap(); let expected_output: ConfigBuilder = deserialize(expected, Format::Toml) .expect("Invalid TOML passed as an expectation"); let expected_json = serde_json::to_value(expected_output).unwrap(); - assert_eq!(expected_json, output_json, "{}", input) + assert_eq!(expected_json, output_json, "{input}") } Err(expected) => assert_eq!( expected, output.expect_err(&format!( - "expected Err, got Ok with format {:?} and input {:?}", - format, input + "expected Err, got Ok with format {format:?} and input {input:?}" )), - "{}", - input + "{input}" ), } } diff --git a/src/config/graph.rs b/src/config/graph.rs index 9ed1af6ff17fe..989f0a9eaf282 100644 --- a/src/config/graph.rs +++ b/src/config/graph.rs @@ -1,6 +1,6 @@ use super::{ schema, ComponentKey, DataType, OutputId, SinkOuter, SourceOuter, SourceOutput, TransformOuter, - TransformOutput, + TransformOutput, WildcardMatching, }; use indexmap::{set::IndexSet, IndexMap}; use std::collections::{HashMap, HashSet, VecDeque}; @@ -26,7 +26,7 @@ impl fmt::Display for Node { Node::Source { outputs } => { write!(f, "component_kind: source\n outputs:")?; for output in outputs { - write!(f, "\n {}", output)?; + write!(f, "\n {output}")?; } Ok(()) } @@ -36,7 +36,7 @@ impl fmt::Display for Node { "component_kind: source\n input_types: {in_ty}\n outputs:" )?; for output in outputs { - write!(f, "\n {}", output)?; + write!(f, "\n {output}")?; } Ok(()) } @@ -65,8 +65,9 @@ impl Graph { transforms: &IndexMap>, sinks: &IndexMap>, schema: schema::Options, + wildcard_matching: WildcardMatching, ) -> Result> { - Self::new_inner(sources, transforms, sinks, false, schema) + Self::new_inner(sources, transforms, sinks, false, schema, wildcard_matching) } pub fn new_unchecked( @@ -74,8 +75,10 @@ impl Graph { transforms: &IndexMap>, sinks: &IndexMap>, schema: schema::Options, + wildcard_matching: WildcardMatching, ) -> Self { - Self::new_inner(sources, transforms, sinks, true, schema).expect("errors ignored") + Self::new_inner(sources, transforms, sinks, true, schema, wildcard_matching) + .expect("errors ignored") } fn new_inner( @@ -84,6 +87,7 @@ impl Graph { sinks: &IndexMap>, ignore_errors: bool, schema: schema::Options, + wildcard_matching: WildcardMatching, ) -> Result> { let mut graph = Graph::default(); let mut errors = Vec::new(); @@ -127,7 +131,7 @@ impl Graph { for (id, config) in transforms.iter() { for input in config.inputs.iter() { - if let Err(e) = graph.add_input(input, id, &available_inputs) { + if let Err(e) = graph.add_input(input, id, &available_inputs, wildcard_matching) { errors.push(e); } } @@ -135,7 +139,7 @@ impl Graph { for (id, config) in sinks { for input in config.inputs.iter() { - if let Err(e) = graph.add_input(input, id, &available_inputs) { + if let Err(e) = graph.add_input(input, id, &available_inputs, wildcard_matching) { errors.push(e); } } @@ -153,6 +157,7 @@ impl Graph { from: &str, to: &ComponentKey, available_inputs: &HashMap, + wildcard_matching: WildcardMatching, ) -> Result<(), String> { if let Some(output_id) = available_inputs.get(from) { self.edges.push(Edge { @@ -166,11 +171,23 @@ impl Graph { Some(Node::Sink { .. }) => "sink", _ => panic!("only transforms and sinks have inputs"), }; + // allow empty result if relaxed wildcard matching is enabled + match wildcard_matching { + WildcardMatching::Relaxed => { + // using value != glob::Pattern::escape(value) to check if value is a glob + // TODO: replace with proper check when https://github.com/rust-lang/glob/issues/72 is resolved + if from != glob::Pattern::escape(from) { + info!("Input \"{from}\" for {output_type} \"{to}\" didn’t match any components, but this was ignored because `relaxed_wildcard_matching` is enabled."); + return Ok(()); + } + } + WildcardMatching::Strict => {} + } info!( "Available components:\n{}", self.nodes .iter() - .map(|(key, node)| format!("\"{}\":\n {}", key, node)) + .map(|(key, node)| format!("\"{key}\":\n {node}")) .collect::>() .join("\n") ); @@ -336,7 +353,7 @@ impl Graph { for id in self.valid_inputs() { if let Some(_other) = mapped.insert(id.to_string(), id.clone()) { - errors.insert(format!("Input specifier {} is ambiguous", id)); + errors.insert(format!("Input specifier {id} is ambiguous")); } } @@ -472,9 +489,14 @@ mod test { } } - fn test_add_input(&mut self, node: &str, input: &str) -> Result<(), String> { + fn test_add_input( + &mut self, + node: &str, + input: &str, + wildcard_matching: WildcardMatching, + ) -> Result<(), String> { let available_inputs = self.input_map().unwrap(); - self.add_input(input, &node.into(), &available_inputs) + self.add_input(input, &node.into(), &available_inputs, wildcard_matching) } } @@ -655,14 +677,22 @@ mod test { // make sure we're good with dotted paths assert_eq!( Ok(()), - graph.test_add_input("errored_log_sink", "log_to_log.errors") + graph.test_add_input( + "errored_log_sink", + "log_to_log.errors", + WildcardMatching::Strict + ) ); // make sure that we're not cool with an unknown dotted path let expected = "Input \"log_to_log.not_errors\" for sink \"bad_log_sink\" doesn't match any components.".to_string(); assert_eq!( Err(expected), - graph.test_add_input("bad_log_sink", "log_to_log.not_errors") + graph.test_add_input( + "bad_log_sink", + "log_to_log.not_errors", + WildcardMatching::Strict + ) ); } @@ -745,6 +775,40 @@ mod test { ); } + #[test] + fn wildcard_matching() { + let mut graph = Graph::default(); + graph.add_source("log_source", DataType::Log); + + // don't add inputs to these yet since they're not validated via these helpers + graph.add_sink("sink", DataType::Log, vec![]); + + // make sure we're not good with non existing inputs with relaxed wildcard matching disabled + let wildcard_matching = WildcardMatching::Strict; + let expected = + "Input \"bad_source-*\" for sink \"sink\" doesn't match any components.".to_string(); + assert_eq!( + Err(expected), + graph.test_add_input("sink", "bad_source-*", wildcard_matching) + ); + + // make sure we're good with non existing inputs with relaxed wildcard matching enabled + let wildcard_matching = WildcardMatching::Relaxed; + assert_eq!( + Ok(()), + graph.test_add_input("sink", "bad_source-*", wildcard_matching) + ); + + // make sure we're not good with non existing inputs that are not wildcards even when relaxed wildcard matching is enabled + let wildcard_matching = WildcardMatching::Relaxed; + let expected = + "Input \"bad_source-1\" for sink \"sink\" doesn't match any components.".to_string(); + assert_eq!( + Err(expected), + graph.test_add_input("sink", "bad_source-1", wildcard_matching) + ); + } + #[test] fn paths_to_sink_simple() { let mut graph = Graph::default(); diff --git a/src/config/loading/loader.rs b/src/config/loading/loader.rs index 7bc9262d600c0..0fcfeb2545c48 100644 --- a/src/config/loading/loader.rs +++ b/src/config/loading/loader.rs @@ -100,8 +100,7 @@ pub(super) mod process { } Err(err) => { errors.push(format!( - "Could not read entry in config dir: {:?}, {}.", - path, err + "Could not read entry in config dir: {path:?}, {err}." )); } }; @@ -164,10 +163,9 @@ pub(super) mod process { path: &Path, format: Format, ) -> Result, Vec> { - if let (Ok(name), Some(file)) = (component_name(path), open_file(path)) { - self.load(file, format).map(|value| Some((name, value))) - } else { - Ok(None) + match (component_name(path), open_file(path)) { + (Ok(name), Some(file)) => self.load(file, format).map(|value| Some((name, value))), + _ => Ok(None), } } diff --git a/src/config/loading/mod.rs b/src/config/loading/mod.rs index a4def7e19715d..9fbccfd9f9332 100644 --- a/src/config/loading/mod.rs +++ b/src/config/loading/mod.rs @@ -276,7 +276,13 @@ pub fn prepare_input(mut input: R) -> Result>(); + let mut vars: HashMap = std::env::vars_os() + .filter_map(|(k, v)| match (k.into_string(), v.into_string()) { + (Ok(k), Ok(v)) => Some((k, v)), + _ => None, + }) + .collect(); + if !vars.contains_key("HOSTNAME") { if let Ok(hostname) = crate::get_hostname() { vars.insert("HOSTNAME".into(), hostname); diff --git a/src/config/loading/secret.rs b/src/config/loading/secret.rs index 2107ec6757b44..c3a9cf8f86bb8 100644 --- a/src/config/loading/secret.rs +++ b/src/config/loading/secret.rs @@ -28,7 +28,7 @@ use crate::{ // - "SECRET[secret_name]" will not match // - "SECRET[.secret.name]" will not match pub static COLLECTOR: LazyLock = - LazyLock::new(|| Regex::new(r"SECRET\[([[:word:]]+)\.([[:word:].]+)\]").unwrap()); + LazyLock::new(|| Regex::new(r"SECRET\[([[:word:]]+)\.([[:word:].-]+)\]").unwrap()); /// Helper type for specifically deserializing secrets backends. #[derive(Debug, Default, Deserialize, Serialize)] @@ -201,6 +201,7 @@ mod tests { collect_secret_keys( indoc! {r" SECRET[first_backend.secret_key] + SECRET[first_backend.secret-key] SECRET[first_backend.another_secret_key] SECRET[second_backend.secret_key] SECRET[second_backend.secret.key] @@ -216,8 +217,9 @@ mod tests { assert!(keys.contains_key("second_backend")); let first_backend_keys = keys.get("first_backend").unwrap(); - assert_eq!(first_backend_keys.len(), 4); + assert_eq!(first_backend_keys.len(), 5); assert!(first_backend_keys.contains("secret_key")); + assert!(first_backend_keys.contains("secret-key")); assert!(first_backend_keys.contains("another_secret_key")); assert!(first_backend_keys.contains("a_third.secret_key")); assert!(first_backend_keys.contains("..an_extra_secret_key")); diff --git a/src/config/loading/secret_backend_example.rs b/src/config/loading/secret_backend_example.rs index 2466d7da387f7..4573df934fb3e 100644 --- a/src/config/loading/secret_backend_example.rs +++ b/src/config/loading/secret_backend_example.rs @@ -33,7 +33,7 @@ async fn main() { ( secret.clone(), ExecResponse { - value: format!("{}.retrieved", secret), + value: format!("{secret}.retrieved"), error: None, }, ) diff --git a/src/config/mod.rs b/src/config/mod.rs index 6478203229475..a718df2c4300c 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -2,6 +2,7 @@ use std::{ collections::{HashMap, HashSet}, fmt::{self, Display, Formatter}, + fs, hash::Hash, net::SocketAddr, path::PathBuf, @@ -21,7 +22,7 @@ use serde::Serialize; use vector_config::configurable_component; pub use vector_lib::config::{ AcknowledgementsConfig, DataType, GlobalOptions, Input, LogNamespace, - SourceAcknowledgementsConfig, SourceOutput, TransformOutput, + SourceAcknowledgementsConfig, SourceOutput, TransformOutput, WildcardMatching, }; pub use vector_lib::configurable::component::{ GenerateConfig, SinkDescription, TransformDescription, @@ -76,23 +77,42 @@ pub use vector_lib::{ id::Inputs, }; +#[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq)] +// // This is not a comprehensive set; variants are added as needed. +pub enum ComponentType { + Transform, + Sink, + EnrichmentTable, +} + #[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq)] pub struct ComponentConfig { pub config_paths: Vec, pub component_key: ComponentKey, + pub component_type: ComponentType, } impl ComponentConfig { - pub const fn new(config_paths: Vec, component_key: ComponentKey) -> Self { + pub fn new( + config_paths: Vec, + component_key: ComponentKey, + component_type: ComponentType, + ) -> Self { + let canonicalized_paths = config_paths + .into_iter() + .filter_map(|p| fs::canonicalize(p).ok()) + .collect(); + Self { - config_paths, + config_paths: canonicalized_paths, component_key, + component_type, } } - pub fn contains(&self, config_paths: &[PathBuf]) -> Option { + pub fn contains(&self, config_paths: &[PathBuf]) -> Option<(ComponentKey, ComponentType)> { if config_paths.iter().any(|p| self.config_paths.contains(p)) { - return Some(self.component_key.clone()); + return Some((self.component_key.clone(), self.component_type.clone())); } None } @@ -259,7 +279,7 @@ impl HealthcheckOptions { } } - fn merge(&mut self, other: Self) { + const fn merge(&mut self, other: Self) { self.enabled &= other.enabled; self.require_healthy |= other.require_healthy; } @@ -355,10 +375,10 @@ impl Display for Protocol { impl Display for Resource { fn fmt(&self, fmt: &mut Formatter<'_>) -> Result<(), fmt::Error> { match self { - Resource::Port(address, protocol) => write!(fmt, "{} {}", protocol, address), + Resource::Port(address, protocol) => write!(fmt, "{protocol} {address}"), Resource::SystemFdOffset(offset) => write!(fmt, "systemd {}th socket", offset + 1), - Resource::Fd(fd) => write!(fmt, "file descriptor: {}", fd), - Resource::DiskBuffer(name) => write!(fmt, "disk buffer {:?}", name), + Resource::Fd(fd) => write!(fmt, "file descriptor: {fd}"), + Resource::DiskBuffer(name) => write!(fmt, "disk buffer {name:?}"), } } } @@ -420,8 +440,7 @@ impl TestDefinition { outputs.push(output_id.clone()); } else { errors.push(format!( - r#"Invalid extract_from target in test '{}': '{}' does not exist"#, - name, from + r#"Invalid extract_from target in test '{name}': '{from}' does not exist"# )); } } @@ -443,8 +462,7 @@ impl TestDefinition { Some(output_id.clone()) } else { errors.push(format!( - r#"Invalid no_outputs_from target in test '{}': '{}' does not exist"#, - name, o + r#"Invalid no_outputs_from target in test '{name}': '{o}' does not exist"# )); None } @@ -1370,9 +1388,9 @@ mod resource_config_tests { let json = serde_json::to_string_pretty(&schema) .expect("rendering root schema to JSON should not fail"); - println!("{}", json); + println!("{json}"); } - Err(e) => eprintln!("error while generating schema: {:?}", e), + Err(e) => eprintln!("error while generating schema: {e:?}"), } } } diff --git a/src/config/schema.rs b/src/config/schema.rs index 2245e84e758f3..bed2c0f532b64 100644 --- a/src/config/schema.rs +++ b/src/config/schema.rs @@ -147,13 +147,12 @@ mod test { let mut errors = vec![]; a.append(b, &mut errors); if errors.is_empty() { - assert_eq!(Some(a), expected, "result mismatch: {}", test); + assert_eq!(Some(a), expected, "result mismatch: {test}"); } else { assert_eq!( errors.is_empty(), expected.is_some(), - "error mismatch: {}", - test + "error mismatch: {test}" ); } } diff --git a/src/config/sink.rs b/src/config/sink.rs index 4ee7f64c13709..b4b6ac23c085c 100644 --- a/src/config/sink.rs +++ b/src/config/sink.rs @@ -37,8 +37,10 @@ impl Configurable for BoxedSink { metadata } - fn generate_schema(gen: &RefCell) -> Result { - vector_lib::configurable::component::SinkDescription::generate_schemas(gen) + fn generate_schema( + generator: &RefCell, + ) -> Result { + vector_lib::configurable::component::SinkDescription::generate_schemas(generator) } } @@ -266,7 +268,7 @@ pub trait SinkConfig: DynClone + NamedComponent + core::fmt::Debug + Send + Sync dyn_clone::clone_trait_object!(SinkConfig); -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct SinkContext { pub healthcheck: SinkHealthcheckOptions, pub globals: GlobalOptions, diff --git a/src/config/source.rs b/src/config/source.rs index 8b91413115b7a..2c540eeea8a18 100644 --- a/src/config/source.rs +++ b/src/config/source.rs @@ -33,8 +33,10 @@ impl Configurable for BoxedSource { metadata } - fn generate_schema(gen: &RefCell) -> Result { - vector_lib::configurable::component::SourceDescription::generate_schemas(gen) + fn generate_schema( + generator: &RefCell, + ) -> Result { + vector_lib::configurable::component::SourceDescription::generate_schemas(generator) } } diff --git a/src/config/transform.rs b/src/config/transform.rs index 4ab7e21f99573..5471c41b1429e 100644 --- a/src/config/transform.rs +++ b/src/config/transform.rs @@ -1,5 +1,6 @@ use std::cell::RefCell; use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; use async_trait::async_trait; use dyn_clone::DynClone; @@ -38,8 +39,10 @@ impl Configurable for BoxedTransform { metadata } - fn generate_schema(gen: &RefCell) -> Result { - vector_lib::configurable::component::TransformDescription::generate_schemas(gen) + fn generate_schema( + generator: &RefCell, + ) -> Result { + vector_lib::configurable::component::TransformDescription::generate_schemas(generator) } } @@ -253,6 +256,11 @@ pub trait TransformConfig: DynClone + NamedComponent + core::fmt::Debug + Send + fn nestable(&self, _parents: &HashSet<&'static str>) -> bool { true } + + /// Gets the files to watch to trigger reload + fn files_to_watch(&self) -> Vec<&PathBuf> { + Vec::new() + } } dyn_clone::clone_trait_object!(TransformConfig); diff --git a/src/config/unit_test/mod.rs b/src/config/unit_test/mod.rs index deafeff02cfcb..cb967a4de5460 100644 --- a/src/config/unit_test/mod.rs +++ b/src/config/unit_test/mod.rs @@ -139,7 +139,7 @@ pub async fn build_unit_tests( let mut test_error = errors.join("\n"); // Indent all line breaks test_error = test_error.replace('\n', "\n "); - test_error.insert_str(0, &format!("Failed to build test '{}':\n ", test_name)); + test_error.insert_str(0, &format!("Failed to build test '{test_name}':\n ")); build_errors.push(test_error); } } @@ -385,6 +385,10 @@ async fn build_unit_test( &transform_only_config.transforms, &transform_only_config.sinks, transform_only_config.schema, + transform_only_config + .global + .wildcard_matching + .unwrap_or_default(), ); let test = test.resolve_outputs(&transform_only_graph)?; @@ -401,6 +405,7 @@ async fn build_unit_test( &config_builder.transforms, &config_builder.sinks, config_builder.schema, + config_builder.global.wildcard_matching.unwrap_or_default(), ); let mut valid_components = get_relevant_test_components( @@ -432,6 +437,7 @@ async fn build_unit_test( &config_builder.transforms, &config_builder.sinks, config_builder.schema, + config_builder.global.wildcard_matching.unwrap_or_default(), ); let valid_inputs = graph.input_map()?; for (_, transform) in config_builder.transforms.iter_mut() { @@ -562,8 +568,7 @@ fn build_outputs( match condition.build(&Default::default()) { Ok(condition) => conditions.push(condition), Err(error) => errors.push(format!( - "failed to create test condition '{}': {}", - index, error + "failed to create test condition '{index}': {error}" )), } } diff --git a/src/config/unit_test/unit_test_components.rs b/src/config/unit_test/unit_test_components.rs index d478c80c87a9a..8daf82b5029d3 100644 --- a/src/config/unit_test/unit_test_components.rs +++ b/src/config/unit_test/unit_test_components.rs @@ -220,8 +220,7 @@ impl StreamSink for UnitTestSink { break; } Err(error) => { - condition_errors - .push(format!(" condition[{}]: {}", j, error)); + condition_errors.push(format!(" condition[{j}]: {error}")); } } } diff --git a/src/config/validation.rs b/src/config/validation.rs index 847bc094d737e..6cae1ecfbf19a 100644 --- a/src/config/validation.rs +++ b/src/config/validation.rs @@ -140,10 +140,7 @@ pub fn check_resources(config: &ConfigBuilder) -> Result<(), Vec> { Err(conflicting_components .into_iter() .map(|(resource, components)| { - format!( - "Resource `{}` is claimed by multiple components: {:?}", - resource, components - ) + format!("Resource `{resource}` is claimed by multiple components: {components:?}") }) .collect()) } diff --git a/src/config/watcher.rs b/src/config/watcher.rs index 245a8a12c7448..641c29343121f 100644 --- a/src/config/watcher.rs +++ b/src/config/watcher.rs @@ -1,5 +1,5 @@ -use crate::config::ComponentConfig; -use std::collections::HashSet; +use crate::config::{ComponentConfig, ComponentType}; +use std::collections::HashMap; use std::{ path::{Path, PathBuf}, time::Duration, @@ -61,7 +61,7 @@ impl Watcher { } } -/// Sends a ReloadFromDisk on config_path changes. +/// Sends a ReloadFromDisk or ReloadEnrichmentTables on config_path changes. /// Accumulates file changes until no change for given duration has occurred. /// Has best effort guarantee of detecting all file changes from the end of /// this function until the main thread stops. @@ -103,7 +103,7 @@ pub fn spawn_thread<'a>( debug!(message = "Consumed file change events for delay.", delay = ?delay); - let component_keys: HashSet<_> = component_configs + let changed_components: HashMap<_, _> = component_configs .clone() .into_iter() .flat_map(|p| p.contains(&event.paths)) @@ -119,15 +119,32 @@ pub fn spawn_thread<'a>( debug!(message = "Reloaded paths."); info!("Configuration file changed."); - if !component_keys.is_empty() { - info!("Component {:?} configuration changed.", component_keys); - _ = signal_tx.send(crate::signal::SignalTo::ReloadComponents(component_keys)).map_err(|error| { - error!(message = "Unable to reload component configuration. Restart Vector to reload it.", cause = %error) - }); + if !changed_components.is_empty() { + info!( + internal_log_rate_limit = true, + "Component {:?} configuration changed.", + changed_components.keys() + ); + if changed_components + .iter() + .all(|(_, t)| *t == ComponentType::EnrichmentTable) + { + info!( + internal_log_rate_limit = true, + "Only enrichment tables have changed." + ); + _ = signal_tx.send(crate::signal::SignalTo::ReloadEnrichmentTables).map_err(|error| { + error!(message = "Unable to reload enrichment tables.", cause = %error, internal_log_rate_limit = true) + }); + } else { + _ = signal_tx.send(crate::signal::SignalTo::ReloadComponents(changed_components.into_keys().collect())).map_err(|error| { + error!(message = "Unable to reload component configuration. Restart Vector to reload it.", cause = %error, internal_log_rate_limit = true) + }); + } } else { _ = signal_tx.send(crate::signal::SignalTo::ReloadFromDisk) .map_err(|error| { - error!(message = "Unable to reload configuration file. Restart Vector to reload it.", cause = %error) + error!(message = "Unable to reload configuration file. Restart Vector to reload it.", cause = %error, internal_log_rate_limit = true) }); } } else { @@ -187,7 +204,7 @@ mod tests { signal::SignalRx, test_util::{temp_dir, temp_file, trace_init}, }; - use std::{fs::File, io::Write, time::Duration}; + use std::{collections::HashSet, fs::File, io::Write, time::Duration}; use tokio::sync::broadcast; async fn test_signal( @@ -221,8 +238,11 @@ mod tests { .iter() .map(|file| File::create(file).unwrap()) .collect(); - let component_config = - ComponentConfig::new(component_file_path.clone(), http_component.clone()); + let component_config = ComponentConfig::new( + component_file_path.clone(), + http_component.clone(), + ComponentType::Sink, + ); let (signal_tx, signal_rx) = broadcast::channel(128); spawn_thread( diff --git a/src/convert_config.rs b/src/convert_config.rs index ec80424c12fd1..526838b06cd43 100644 --- a/src/convert_config.rs +++ b/src/convert_config.rs @@ -271,7 +271,7 @@ mod tests { let input_path = test_data_dir(); let output_dir = tempdir() .expect("Unable to create tempdir for config") - .into_path(); + .keep(); walk_dir_and_convert(&input_path, &output_dir, Format::Yaml).unwrap(); let mut count: usize = 0; diff --git a/src/docker.rs b/src/docker.rs index 58e9c1819f7a8..24a1fc429748e 100644 --- a/src/docker.rs +++ b/src/docker.rs @@ -2,10 +2,13 @@ use std::{collections::HashMap, env, path::PathBuf}; use bollard::{ - container::{Config, CreateContainerOptions}, errors::Error as DockerError, - image::{CreateImageOptions, ListImagesOptions}, models::HostConfig, + query_parameters::{ + CreateContainerOptionsBuilder, CreateImageOptionsBuilder, ListImagesOptionsBuilder, + RemoveContainerOptions, StartContainerOptions, StopContainerOptions, + }, + secret::ContainerCreateBody, Docker, API_DEFAULT_VERSION, }; use futures::StreamExt; @@ -45,7 +48,7 @@ pub fn docker(host: Option, tls: Option) -> crate::Resu let host = host.or_else(|| env::var("DOCKER_HOST").ok()); match host { - None => Docker::connect_with_local_defaults().map_err(Into::into), + None => Docker::connect_with_defaults().map_err(Into::into), Some(host) => { let scheme = host .parse::() @@ -74,19 +77,9 @@ pub fn docker(host: Option, tls: Option) -> crate::Resu .map_err(Into::into) } Some("unix") | Some("npipe") | None => { - // TODO: Use `connect_with_local` on all platforms. - // - // Named pipes are currently disabled in Tokio. Tracking issue: - // https://github.com/fussybeaver/bollard/pull/138 - if cfg!(windows) { - warn!("Named pipes are currently not available on Windows, trying to connecting to Docker with default HTTP settings instead."); - Docker::connect_with_http_defaults().map_err(Into::into) - } else { - Docker::connect_with_local(&host, DEFAULT_TIMEOUT, API_DEFAULT_VERSION) - .map_err(Into::into) - } + Docker::connect_with_defaults().map_err(Into::into) } - Some(scheme) => Err(format!("Unknown scheme: {}", scheme).into()), + Some(scheme) => Err(format!("Unknown scheme: {scheme}").into()), } } } @@ -120,26 +113,24 @@ async fn pull_image(docker: &Docker, image: &str, tag: &str) { vec![format!("{}:{}", image, tag)], ); - let options = Some(ListImagesOptions { - filters, - ..Default::default() - }); + let options = Some(ListImagesOptionsBuilder::new().filters(&filters).build()); let images = docker.list_images(options).await.unwrap(); if images.is_empty() { // If not found, pull it - let options = Some(CreateImageOptions { - from_image: image, - tag, - ..Default::default() - }); + let options = Some( + CreateImageOptionsBuilder::new() + .from_image(image) + .tag(tag) + .build(), + ); docker .create_image(options, None, None) .for_each(|item| async move { let info = item.unwrap(); if let Some(error) = info.error { - panic!("{:?}", error); + panic!("{error:?}"); } }) .await @@ -150,7 +141,7 @@ async fn remove_container(docker: &Docker, id: &str) { trace!("Stopping container."); _ = docker - .stop_container(id, None) + .stop_container(id, None::) .await .map_err(|e| error!(%e)); @@ -158,7 +149,7 @@ async fn remove_container(docker: &Docker, id: &str) { // Don't panic, as this is unrelated to the test _ = docker - .remove_container(id, None) + .remove_container(id, None::) .await .map_err(|e| error!(%e)); } @@ -181,7 +172,7 @@ impl Container { } pub fn bind(mut self, src: impl std::fmt::Display, dst: &str) -> Self { - let bind = format!("{}:{}", src, dst); + let bind = format!("{src}:{dst}"); self.binds.get_or_insert_with(Vec::new).push(bind); self } @@ -196,12 +187,11 @@ impl Container { pull_image(&docker, self.image, self.tag).await; - let options = Some(CreateContainerOptions { - name: format!("vector_test_{}", uuid::Uuid::new_v4()), - platform: None, - }); + let options = CreateContainerOptionsBuilder::new() + .name(&format!("vector_test_{}", uuid::Uuid::new_v4())) + .build(); - let config = Config { + let config = ContainerCreateBody { image: Some(format!("{}:{}", &self.image, &self.tag)), cmd: self.cmd, host_config: Some(HostConfig { @@ -213,10 +203,13 @@ impl Container { ..Default::default() }; - let container = docker.create_container(options, config).await.unwrap(); + let container = docker + .create_container(Some(options), config) + .await + .unwrap(); docker - .start_container::(&container.id, None) + .start_container(&container.id, None::) .await .unwrap(); diff --git a/src/encoding_transcode.rs b/src/encoding_transcode.rs index e093fa04b92f7..92a9c88b6e58b 100644 --- a/src/encoding_transcode.rs +++ b/src/encoding_transcode.rs @@ -290,10 +290,7 @@ mod tests { assert_eq!( d.decode_to_utf8(Bytes::from(problematic_input)), - Bytes::from(format!( - "{}{}123", - REPLACEMENT_CHARACTER, REPLACEMENT_CHARACTER - )) + Bytes::from(format!("{REPLACEMENT_CHARACTER}{REPLACEMENT_CHARACTER}123")) ); } diff --git a/src/enrichment_tables/file.rs b/src/enrichment_tables/file.rs index 32ec9318b88e0..ec234ea10e16c 100644 --- a/src/enrichment_tables/file.rs +++ b/src/enrichment_tables/file.rs @@ -143,10 +143,7 @@ impl FileConfig { .from_utc_datetime( &chrono::NaiveDate::parse_from_str(value, "%Y-%m-%d") .map_err(|_| { - format!( - "unable to parse date {} found in row {}", - value, row - ) + format!("unable to parse date {value} found in row {row}") })? .and_hms_opt(0, 0, 0) .expect("invalid timestamp"), @@ -159,10 +156,7 @@ impl FileConfig { .from_utc_datetime( &chrono::NaiveDate::parse_from_str(value, format) .map_err(|_| { - format!( - "unable to parse date {} found in row {}", - value, row - ) + format!("unable to parse date {value} found in row {row}") })? .and_hms_opt(0, 0, 0) .expect("invalid timestamp"), @@ -174,9 +168,7 @@ impl FileConfig { Conversion::parse(format, timezone).map_err(|err| err.to_string())?; conversion .convert(Bytes::copy_from_slice(value.as_bytes())) - .map_err(|_| { - format!("unable to parse {} found in row {}", value, row) - })? + .map_err(|_| format!("unable to parse {value} found in row {row}"))? } } } @@ -295,20 +287,60 @@ impl File { } /// Does the given row match all the conditions specified? - fn row_equals(&self, case: Case, condition: &[Condition], row: &[Value]) -> bool { + fn row_equals( + &self, + case: Case, + condition: &[Condition], + row: &[Value], + wildcard: Option<&Value>, + ) -> bool { condition.iter().all(|condition| match condition { Condition::Equals { field, value } => match self.column_index(field) { None => false, - Some(idx) => match (case, &row[idx], value) { - (Case::Insensitive, Value::Bytes(bytes1), Value::Bytes(bytes2)) => { - match (std::str::from_utf8(bytes1), std::str::from_utf8(bytes2)) { - (Ok(s1), Ok(s2)) => s1.to_lowercase() == s2.to_lowercase(), - (Err(_), Err(_)) => bytes1 == bytes2, - _ => false, + Some(idx) => { + let current_row_value = &row[idx]; + + // Helper closure for comparing current_row_value with another value, + // respecting the specified case for Value::Bytes. + let compare_values = |val_to_compare: &Value| -> bool { + match (case, current_row_value, val_to_compare) { + ( + Case::Insensitive, + Value::Bytes(bytes_row), + Value::Bytes(bytes_cmp), + ) => { + // Perform case-insensitive comparison for byte strings. + // If both are valid UTF-8, compare their lowercase versions. + // If both are non-UTF-8 bytes, compare them directly. + // If one is UTF-8 and the other is not, they are considered not equal. + match ( + std::str::from_utf8(bytes_row), + std::str::from_utf8(bytes_cmp), + ) { + (Ok(s_row), Ok(s_cmp)) => { + s_row.to_lowercase() == s_cmp.to_lowercase() + } + (Err(_), Err(_)) => bytes_row == bytes_cmp, + _ => false, + } + } + // For Case::Sensitive, or for Case::Insensitive with non-Bytes types, + // perform a direct equality check. + _ => current_row_value == val_to_compare, } + }; + + // First, check if the row value matches the condition's value. + if compare_values(value) { + true + } else if let Some(wc_val) = wildcard { + // If not, and a wildcard is provided, check if the row value matches the wildcard. + compare_values(wc_val) + } else { + // Otherwise, no match. + false } - (_, value1, value2) => value1 == value2, - }, + } }, Condition::BetweenDates { field, from, to } => match self.column_index(field) { None => false, @@ -317,6 +349,20 @@ impl File { _ => false, }, }, + Condition::FromDate { field, from } => match self.column_index(field) { + None => false, + Some(idx) => match row[idx] { + Value::Timestamp(date) => from <= &date, + _ => false, + }, + }, + Condition::ToDate { field, to } => match self.column_index(field) { + None => false, + Some(idx) => match row[idx] { + Value::Timestamp(date) => &date <= to, + _ => false, + }, + }, }) } @@ -362,7 +408,7 @@ impl File { }) .collect::>() .join(", "); - Err(format!("field(s) '{}' missing from dataset", missing)) + Err(format!("field(s) '{missing}' missing from dataset")) } else { Ok(normalized) } @@ -408,12 +454,13 @@ impl File { case: Case, condition: &'a [Condition<'a>], select: Option<&'a [String]>, + wildcard: Option<&'a Value>, ) -> impl Iterator + 'a where I: Iterator> + 'a, { data.filter_map(move |row| { - if self.row_equals(case, condition, row) { + if self.row_equals(case, condition, row, wildcard) { Some(self.add_columns(select, row)) } else { None @@ -445,6 +492,32 @@ impl File { let IndexHandle(handle) = handle; Ok(self.indexes[handle].2.get(&key)) } + + fn indexed_with_wildcard<'a>( + &'a self, + case: Case, + wildcard: &'a Value, + condition: &'a [Condition<'a>], + handle: IndexHandle, + ) -> Result>, String> { + if let Some(result) = self.indexed(case, condition, handle)? { + return Ok(Some(result)); + } + + // If lookup fails and a wildcard is provided, compute hash for the wildcard + let mut wildcard_hash = seahash::SeaHasher::default(); + for header in self.headers.iter() { + if condition.iter().any( + |condition| matches!(condition, Condition::Equals { field, .. } if field == header), + ) { + hash_value(&mut wildcard_hash, case, wildcard)?; + } + } + + let wildcard_key = wildcard_hash.finish(); + let IndexHandle(handle) = handle; + Ok(self.indexes[handle].2.get(&wildcard_key)) + } } /// Adds the bytes from the given value to the hash. @@ -492,22 +565,26 @@ impl Table for File { case: Case, condition: &'a [Condition<'a>], select: Option<&'a [String]>, + wildcard: Option<&Value>, index: Option, ) -> Result { match index { None => { // No index has been passed so we need to do a Sequential Scan. - single_or_err(self.sequential(self.data.iter(), case, condition, select)) + single_or_err(self.sequential(self.data.iter(), case, condition, select, wildcard)) } Some(handle) => { - let result = self - .indexed(case, condition, handle)? - .ok_or_else(|| "no rows found in index".to_string())? - .iter() - .map(|idx| &self.data[*idx]); + let result = if let Some(wildcard) = wildcard { + self.indexed_with_wildcard(case, wildcard, condition, handle)? + } else { + self.indexed(case, condition, handle)? + } + .ok_or_else(|| "no rows found in index".to_string())? + .iter() + .map(|idx| &self.data[*idx]); // Perform a sequential scan over the indexed result. - single_or_err(self.sequential(result, case, condition, select)) + single_or_err(self.sequential(result, case, condition, select, wildcard)) } } } @@ -517,25 +594,33 @@ impl Table for File { case: Case, condition: &'a [Condition<'a>], select: Option<&'a [String]>, + wildcard: Option<&Value>, index: Option, ) -> Result, String> { match index { None => { // No index has been passed so we need to do a Sequential Scan. Ok(self - .sequential(self.data.iter(), case, condition, select) + .sequential(self.data.iter(), case, condition, select, wildcard) .collect()) } Some(handle) => { // Perform a sequential scan over the indexed result. + let indexed_result = if let Some(wildcard) = wildcard { + self.indexed_with_wildcard(case, wildcard, condition, handle)? + } else { + self.indexed(case, condition, handle)? + }; + Ok(self .sequential( - self.indexed(case, condition, handle)? + indexed_result .iter() .flat_map(|results| results.iter().map(|idx| &self.data[*idx])), case, condition, select, + wildcard, ) .collect()) } @@ -790,7 +875,37 @@ mod tests { ("field1".into(), Value::from("zirp")), ("field2".into(), Value::from("zurp")), ])), - file.find_table_row(Case::Sensitive, &[condition], None, None) + file.find_table_row(Case::Sensitive, &[condition], None, None, None) + ); + } + + #[test] + fn finds_row_with_wildcard() { + let file = File::new( + Default::default(), + FileData { + modified: SystemTime::now(), + data: vec![ + vec!["zip".into(), "zup".into()], + vec!["zirp".into(), "zurp".into()], + ], + headers: vec!["field1".to_string(), "field2".to_string()], + }, + ); + + let wildcard = Value::from("zirp"); + + let condition = Condition::Equals { + field: "field1", + value: Value::from("nonexistent"), + }; + + assert_eq!( + Ok(ObjectMap::from([ + ("field1".into(), Value::from("zirp")), + ("field2".into(), Value::from("zurp")), + ])), + file.find_table_row(Case::Sensitive, &[condition], None, Some(&wildcard), None) ); } @@ -864,7 +979,44 @@ mod tests { ("field1".into(), Value::from("zirp")), ("field2".into(), Value::from("zurp")), ])), - file.find_table_row(Case::Sensitive, &[condition], None, Some(handle)) + file.find_table_row(Case::Sensitive, &[condition], None, None, Some(handle)) + ); + } + + #[test] + fn finds_row_with_index_case_sensitive_and_wildcard() { + let mut file = File::new( + Default::default(), + FileData { + modified: SystemTime::now(), + data: vec![ + vec!["zip".into(), "zup".into()], + vec!["zirp".into(), "zurp".into()], + ], + headers: vec!["field1".to_string(), "field2".to_string()], + }, + ); + + let handle = file.add_index(Case::Sensitive, &["field1"]).unwrap(); + let wildcard = Value::from("zirp"); + + let condition = Condition::Equals { + field: "field1", + value: Value::from("nonexistent"), + }; + + assert_eq!( + Ok(ObjectMap::from([ + ("field1".into(), Value::from("zirp")), + ("field2".into(), Value::from("zurp")), + ])), + file.find_table_row( + Case::Sensitive, + &[condition], + None, + Some(&wildcard), + Some(handle) + ) ); } @@ -903,6 +1055,7 @@ mod tests { value: Value::from("zip"), }], None, + None, Some(handle) ) ); @@ -916,6 +1069,7 @@ mod tests { value: Value::from("ZiP"), }], None, + None, Some(handle) ) ); @@ -962,6 +1116,7 @@ mod tests { Case::Sensitive, &[condition], Some(&["field1".to_string(), "field3".to_string()]), + None, Some(handle) ) ); @@ -1002,6 +1157,7 @@ mod tests { value: Value::from("zip"), }], None, + None, Some(handle) ) ); @@ -1024,13 +1180,78 @@ mod tests { value: Value::from("ZiP"), }], None, + None, Some(handle) ) ); } #[test] - fn finds_row_with_dates() { + fn finds_rows_with_index_case_insensitive_and_wildcard() { + let mut file = File::new( + Default::default(), + FileData { + modified: SystemTime::now(), + data: vec![ + vec!["zip".into(), "zup".into()], + vec!["zirp".into(), "zurp".into()], + vec!["zip".into(), "zoop".into()], + ], + headers: vec!["field1".to_string(), "field2".to_string()], + }, + ); + + let handle = file.add_index(Case::Insensitive, &["field1"]).unwrap(); + + assert_eq!( + Ok(vec![ + ObjectMap::from([ + ("field1".into(), Value::from("zip")), + ("field2".into(), Value::from("zup")), + ]), + ObjectMap::from([ + ("field1".into(), Value::from("zip")), + ("field2".into(), Value::from("zoop")), + ]), + ]), + file.find_table_rows( + Case::Insensitive, + &[Condition::Equals { + field: "field1", + value: Value::from("nonexistent"), + }], + None, + Some(&Value::from("zip")), + Some(handle) + ) + ); + + assert_eq!( + Ok(vec![ + ObjectMap::from([ + ("field1".into(), Value::from("zip")), + ("field2".into(), Value::from("zup")), + ]), + ObjectMap::from([ + ("field1".into(), Value::from("zip")), + ("field2".into(), Value::from("zoop")), + ]), + ]), + file.find_table_rows( + Case::Insensitive, + &[Condition::Equals { + field: "field1", + value: Value::from("ZiP"), + }], + None, + Some(&Value::from("ZiP")), + Some(handle) + ) + ); + } + + #[test] + fn finds_row_between_dates() { let mut file = File::new( Default::default(), FileData { @@ -1092,7 +1313,133 @@ mod tests { ) ) ])), - file.find_table_row(Case::Sensitive, &conditions, None, Some(handle)) + file.find_table_row(Case::Sensitive, &conditions, None, None, Some(handle)) + ); + } + + #[test] + fn finds_row_from_date() { + let mut file = File::new( + Default::default(), + FileData { + modified: SystemTime::now(), + data: vec![ + vec![ + "zip".into(), + Value::Timestamp( + chrono::Utc + .with_ymd_and_hms(2015, 12, 7, 0, 0, 0) + .single() + .expect("invalid timestamp"), + ), + ], + vec![ + "zip".into(), + Value::Timestamp( + chrono::Utc + .with_ymd_and_hms(2016, 12, 7, 0, 0, 0) + .single() + .expect("invalid timestamp"), + ), + ], + ], + headers: vec!["field1".to_string(), "field2".to_string()], + }, + ); + + let handle = file.add_index(Case::Sensitive, &["field1"]).unwrap(); + + let conditions = [ + Condition::Equals { + field: "field1", + value: "zip".into(), + }, + Condition::FromDate { + field: "field2", + from: chrono::Utc + .with_ymd_and_hms(2016, 1, 1, 0, 0, 0) + .single() + .expect("invalid timestamp"), + }, + ]; + + assert_eq!( + Ok(ObjectMap::from([ + ("field1".into(), Value::from("zip")), + ( + "field2".into(), + Value::Timestamp( + chrono::Utc + .with_ymd_and_hms(2016, 12, 7, 0, 0, 0) + .single() + .expect("invalid timestamp") + ) + ) + ])), + file.find_table_row(Case::Sensitive, &conditions, None, None, Some(handle)) + ); + } + + #[test] + fn finds_row_to_date() { + let mut file = File::new( + Default::default(), + FileData { + modified: SystemTime::now(), + data: vec![ + vec![ + "zip".into(), + Value::Timestamp( + chrono::Utc + .with_ymd_and_hms(2015, 12, 7, 0, 0, 0) + .single() + .expect("invalid timestamp"), + ), + ], + vec![ + "zip".into(), + Value::Timestamp( + chrono::Utc + .with_ymd_and_hms(2016, 12, 7, 0, 0, 0) + .single() + .expect("invalid timestamp"), + ), + ], + ], + headers: vec!["field1".to_string(), "field2".to_string()], + }, + ); + + let handle = file.add_index(Case::Sensitive, &["field1"]).unwrap(); + + let conditions = [ + Condition::Equals { + field: "field1", + value: "zip".into(), + }, + Condition::ToDate { + field: "field2", + to: chrono::Utc + .with_ymd_and_hms(2016, 1, 1, 0, 0, 0) + .single() + .expect("invalid timestamp"), + }, + ]; + + assert_eq!( + Ok(ObjectMap::from([ + ("field1".into(), Value::from("zip")), + ( + "field2".into(), + Value::Timestamp( + chrono::Utc + .with_ymd_and_hms(2015, 12, 7, 0, 0, 0) + .single() + .expect("invalid timestamp") + ) + ) + ])), + file.find_table_row(Case::Sensitive, &conditions, None, None, Some(handle)) ); } @@ -1117,7 +1464,7 @@ mod tests { assert_eq!( Err("no rows found".to_string()), - file.find_table_row(Case::Sensitive, &[condition], None, None) + file.find_table_row(Case::Sensitive, &[condition], None, None, None) ); } @@ -1144,7 +1491,41 @@ mod tests { assert_eq!( Err("no rows found in index".to_string()), - file.find_table_row(Case::Sensitive, &[condition], None, Some(handle)) + file.find_table_row(Case::Sensitive, &[condition], None, None, Some(handle)) + ); + } + + #[test] + fn doesnt_find_row_with_index_and_wildcard() { + let mut file = File::new( + Default::default(), + FileData { + modified: SystemTime::now(), + data: vec![ + vec!["zip".into(), "zup".into()], + vec!["zirp".into(), "zurp".into()], + ], + headers: vec!["field1".to_string(), "field2".to_string()], + }, + ); + + let handle = file.add_index(Case::Sensitive, &["field1"]).unwrap(); + let wildcard = Value::from("nonexistent"); + + let condition = Condition::Equals { + field: "field1", + value: Value::from("zorp"), + }; + + assert_eq!( + Err("no rows found in index".to_string()), + file.find_table_row( + Case::Sensitive, + &[condition], + None, + Some(&wildcard), + Some(handle) + ) ); } } diff --git a/src/enrichment_tables/geoip.rs b/src/enrichment_tables/geoip.rs index 917fc4885881a..6e09105a8c2c3 100644 --- a/src/enrichment_tables/geoip.rs +++ b/src/enrichment_tables/geoip.rs @@ -4,7 +4,7 @@ //! //! [maxmind]: https://dev.maxmind.com/geoip/geoip2/downloadable //! [geolite]: https://dev.maxmind.com/geoip/geoip2/geolite2/#Download_Access -use std::{collections::BTreeMap, fs, net::IpAddr, sync::Arc, time::SystemTime}; +use std::{collections::BTreeMap, fs, net::IpAddr, path::PathBuf, sync::Arc, time::SystemTime}; use maxminddb::{ geoip2::{AnonymousIp, City, ConnectionType, Isp}, @@ -56,7 +56,7 @@ pub struct GeoipConfig { /// /// [geoip2]: https://dev.maxmind.com/geoip/geoip2/downloadable /// [geolite2]: https://dev.maxmind.com/geoip/geoip2/geolite2/#Download_Access - pub path: String, + pub path: PathBuf, /// The locale to use when querying the database. /// @@ -87,7 +87,7 @@ fn default_locale() -> String { impl GenerateConfig for GeoipConfig { fn generate_config() -> toml::Value { toml::Value::try_from(Self { - path: "/path/to/GeoLite2-City.mmdb".to_string(), + path: "/path/to/GeoLite2-City.mmdb".into(), locale: default_locale(), }) .unwrap() @@ -115,7 +115,7 @@ pub struct Geoip { impl Geoip { /// Creates a new GeoIP struct from the provided config. pub fn new(config: GeoipConfig) -> crate::Result { - let dbreader = Arc::new(Reader::open_readfile(config.path.clone())?); + let dbreader = Arc::new(Reader::open_readfile(&config.path)?); let dbkind = DatabaseKind::try_from(dbreader.metadata.database_type.as_str()).map_err(|_| { format!( @@ -156,7 +156,7 @@ impl Geoip { }; macro_rules! add_field { - ($k:expr, $v:expr) => { + ($k:expr_2021, $v:expr_2021) => { add_field($k, $v.map(Into::into)) }; } @@ -264,9 +264,10 @@ impl Table for Geoip { case: Case, condition: &'a [Condition<'a>], select: Option<&[String]>, + wildcard: Option<&Value>, index: Option, ) -> Result { - let mut rows = self.find_table_rows(case, condition, select, index)?; + let mut rows = self.find_table_rows(case, condition, select, wildcard, index)?; match rows.pop() { Some(row) if rows.is_empty() => Ok(row), @@ -283,6 +284,7 @@ impl Table for Geoip { _: Case, condition: &'a [Condition<'a>], select: Option<&[String]>, + _wildcard: Option<&Value>, _: Option, ) -> Result, String> { match condition.first() { @@ -333,7 +335,8 @@ impl std::fmt::Debug for Geoip { write!( f, "Geoip {} database {})", - self.config.locale, self.config.path + self.config.locale, + self.config.path.display() ) } } @@ -468,7 +471,7 @@ mod tests { #[test] fn custom_mmdb_type_error() { let result = Geoip::new(GeoipConfig { - path: "tests/data/custom-type.mmdb".to_string(), + path: "tests/data/custom-type.mmdb".into(), locale: default_locale(), }); @@ -502,7 +505,7 @@ mod tests { fn find_select(ip: &str, database: &str, select: Option<&[String]>) -> Option { Geoip::new(GeoipConfig { - path: database.to_string(), + path: database.into(), locale: default_locale(), }) .unwrap() @@ -514,6 +517,7 @@ mod tests { }], select, None, + None, ) .unwrap() .pop() diff --git a/src/enrichment_tables/memory/table.rs b/src/enrichment_tables/memory/table.rs index dd8ff7779780a..068f6630bfb75 100644 --- a/src/enrichment_tables/memory/table.rs +++ b/src/enrichment_tables/memory/table.rs @@ -243,9 +243,10 @@ impl Table for Memory { case: Case, condition: &'a [Condition<'a>], select: Option<&'a [String]>, + wildcard: Option<&Value>, index: Option, ) -> Result { - let mut rows = self.find_table_rows(case, condition, select, index)?; + let mut rows = self.find_table_rows(case, condition, select, wildcard, index)?; match rows.pop() { Some(row) if rows.is_empty() => Ok(row), @@ -259,6 +260,7 @@ impl Table for Memory { _case: Case, condition: &'a [Condition<'a>], _select: Option<&'a [String]>, + _wildcard: Option<&Value>, _index: Option, ) -> Result, String> { match condition.first() { @@ -414,7 +416,7 @@ mod tests { ("ttl".into(), Value::from(memory.config.ttl)), ("value".into(), Value::from(5)), ])), - memory.find_table_row(Case::Sensitive, &[condition], None, None) + memory.find_table_row(Case::Sensitive, &[condition], None, None, None) ); } @@ -446,7 +448,7 @@ mod tests { ("ttl".into(), Value::from(ttl - secs_to_subtract)), ("value".into(), Value::from(5)), ])), - memory.find_table_row(Case::Sensitive, &[condition], None, None) + memory.find_table_row(Case::Sensitive, &[condition], None, None, None) ); } @@ -479,7 +481,7 @@ mod tests { ("ttl".into(), Value::from(0)), ("value".into(), Value::from(5)), ])), - memory.find_table_row(Case::Sensitive, &[condition.clone()], None, None) + memory.find_table_row(Case::Sensitive, &[condition.clone()], None, None, None) ); // Force scan @@ -488,7 +490,7 @@ mod tests { // The value is not present anymore assert!(memory - .find_table_rows(Case::Sensitive, &[condition], None, None) + .find_table_rows(Case::Sensitive, &[condition], None, None, None) .unwrap() .pop() .is_none()); @@ -509,7 +511,7 @@ mod tests { }; assert!(memory - .find_table_rows(Case::Sensitive, &[condition], None, None) + .find_table_rows(Case::Sensitive, &[condition], None, None, None) .unwrap() .pop() .is_none()); @@ -541,7 +543,7 @@ mod tests { ("ttl".into(), Value::from(ttl / 2)), ("value".into(), Value::from(5)), ])), - memory.find_table_row(Case::Sensitive, &[condition.clone()], None, None) + memory.find_table_row(Case::Sensitive, &[condition.clone()], None, None, None) ); memory.handle_value(ObjectMap::from([("test_key".into(), Value::from(5))])); @@ -552,7 +554,7 @@ mod tests { ("ttl".into(), Value::from(ttl)), ("value".into(), Value::from(5)), ])), - memory.find_table_row(Case::Sensitive, &[condition], None, None) + memory.find_table_row(Case::Sensitive, &[condition], None, None, None) ); } @@ -569,7 +571,7 @@ mod tests { }; assert!(memory - .find_table_rows(Case::Sensitive, &[condition], None, None) + .find_table_rows(Case::Sensitive, &[condition], None, None, None) .unwrap() .pop() .is_none()); @@ -598,6 +600,7 @@ mod tests { value: Value::from("test_key") }], None, + None, None ) ); @@ -610,6 +613,7 @@ mod tests { value: Value::from("rejected_key") }], None, + None, None ) .unwrap() @@ -627,7 +631,7 @@ mod tests { }; assert!(memory - .find_table_rows(Case::Sensitive, &[condition], None, None) + .find_table_rows(Case::Sensitive, &[condition], None, None, None) .unwrap() .pop() .is_none()); diff --git a/src/enrichment_tables/mmdb.rs b/src/enrichment_tables/mmdb.rs index 7441dd8a3d893..91b8d22a99d8c 100644 --- a/src/enrichment_tables/mmdb.rs +++ b/src/enrichment_tables/mmdb.rs @@ -2,6 +2,7 @@ //! Enrichment data is loaded from any database in [MaxMind][maxmind] format. //! //! [maxmind]: https://maxmind.com +use std::path::PathBuf; use std::{fs, net::IpAddr, sync::Arc, time::SystemTime}; use maxminddb::Reader; @@ -18,13 +19,13 @@ pub struct MmdbConfig { /// Path to the [MaxMind][maxmind] database /// /// [maxmind]: https://maxmind.com - pub path: String, + pub path: PathBuf, } impl GenerateConfig for MmdbConfig { fn generate_config() -> toml::Value { toml::Value::try_from(Self { - path: "/path/to/GeoLite2-City.mmdb".to_string(), + path: "/path/to/GeoLite2-City.mmdb".into(), }) .unwrap() } @@ -50,7 +51,7 @@ pub struct Mmdb { impl Mmdb { /// Creates a new Mmdb struct from the provided config. pub fn new(config: MmdbConfig) -> crate::Result { - let dbreader = Arc::new(Reader::open_readfile(config.path.clone())?); + let dbreader = Arc::new(Reader::open_readfile(&config.path)?); // Check if we can read database with dummy Ip. let ip = IpAddr::V4(std::net::Ipv4Addr::UNSPECIFIED); @@ -98,9 +99,10 @@ impl Table for Mmdb { case: Case, condition: &'a [Condition<'a>], select: Option<&[String]>, + wildcard: Option<&Value>, index: Option, ) -> Result { - let mut rows = self.find_table_rows(case, condition, select, index)?; + let mut rows = self.find_table_rows(case, condition, select, wildcard, index)?; match rows.pop() { Some(row) if rows.is_empty() => Ok(row), @@ -117,6 +119,7 @@ impl Table for Mmdb { _: Case, condition: &'a [Condition<'a>], select: Option<&[String]>, + _wildcard: Option<&Value>, _: Option, ) -> Result, String> { match condition.first() { @@ -164,7 +167,7 @@ impl Table for Mmdb { impl std::fmt::Debug for Mmdb { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "Maxmind database {})", self.config.path) + write!(f, "Maxmind database {})", self.config.path.display()) } } @@ -259,7 +262,7 @@ mod tests { fn find_select(ip: &str, database: &str, select: Option<&[String]>) -> Option { Mmdb::new(MmdbConfig { - path: database.to_string(), + path: database.into(), }) .unwrap() .find_table_rows( @@ -270,6 +273,7 @@ mod tests { }], select, None, + None, ) .unwrap() .pop() diff --git a/src/enrichment_tables/mod.rs b/src/enrichment_tables/mod.rs index d067262a1ea74..f814e175f576e 100644 --- a/src/enrichment_tables/mod.rs +++ b/src/enrichment_tables/mod.rs @@ -1,4 +1,6 @@ //! Functionality to handle enrichment tables. +use std::path::PathBuf; + use enum_dispatch::enum_dispatch; use vector_lib::configurable::configurable_component; pub use vector_lib::enrichment::{Condition, IndexHandle, Table}; @@ -77,3 +79,18 @@ impl GenerateConfig for EnrichmentTables { .unwrap() } } + +impl EnrichmentTables { + /// Gets the files to watch to trigger reload + pub fn files_to_watch(&self) -> Vec<&PathBuf> { + match self { + EnrichmentTables::File(file_config) => vec![&file_config.file.path], + #[cfg(feature = "enrichment-tables-memory")] + EnrichmentTables::Memory(_) => vec![], + #[cfg(feature = "enrichment-tables-geoip")] + EnrichmentTables::Geoip(geoip_config) => vec![&geoip_config.path], + #[cfg(feature = "enrichment-tables-mmdb")] + EnrichmentTables::Mmdb(mmdb_config) => vec![&mmdb_config.path], + } + } +} diff --git a/src/extra_context.rs b/src/extra_context.rs index 025093b2ad22d..b80586bc714fd 100644 --- a/src/extra_context.rs +++ b/src/extra_context.rs @@ -8,7 +8,7 @@ use std::{ /// Structure containing any extra data. /// The data is held in an [`Arc`] so is cheap to clone. -#[derive(Clone, Default)] +#[derive(Clone, Debug, Default)] pub struct ExtraContext(Arc>); type ContextItem = Box; diff --git a/src/gcp.rs b/src/gcp.rs index 120ca0196398d..d484a063ccd35 100644 --- a/src/gcp.rs +++ b/src/gcp.rs @@ -257,7 +257,7 @@ impl InnerCreds { } async fn fetch_token(creds: &Credentials, scope: &Scope) -> crate::Result { - let claims = JwtClaims::new(creds.iss(), scope, creds.token_uri(), None, None); + let claims = JwtClaims::new(creds.iss(), &[scope.clone()], creds.token_uri(), None, None); let rsa_key = creds.rsa_key().context(InvalidRsaKeySnafu)?; let jwt = Jwt::new(claims, rsa_key, None); diff --git a/src/generate.rs b/src/generate.rs index d49dd9c4a8e5c..1da6c414c7640 100644 --- a/src/generate.rs +++ b/src/generate.rs @@ -140,8 +140,7 @@ pub(crate) fn generate_example( let (name, source_type) = if let Some(c_index) = source_expr.find(':') { if c_index == 0 { errs.push(format!( - "failed to generate source '{}': empty name is not allowed", - source_expr + "failed to generate source '{source_expr}': empty name is not allowed" )); continue; } @@ -151,17 +150,14 @@ pub(crate) fn generate_example( chopped_expr.drain(1..).collect(), ) } else { - (format!("source{}", i), source_expr.clone()) + (format!("source{i}"), source_expr.clone()) }; source_names.push(name.clone()); let mut example = match SourceDescription::example(&source_type) { Ok(example) => example, Err(err) => { - errs.push(format!( - "failed to generate source '{}': {}", - source_type, err - )); + errs.push(format!("failed to generate source '{source_type}': {err}")); Value::Table(Map::new()) } }; @@ -186,8 +182,7 @@ pub(crate) fn generate_example( let (name, transform_type) = if let Some(c_index) = transform_expr.find(':') { if c_index == 0 { errs.push(format!( - "failed to generate transform '{}': empty name is not allowed", - transform_expr + "failed to generate transform '{transform_expr}': empty name is not allowed" )); continue; } @@ -197,7 +192,7 @@ pub(crate) fn generate_example( chopped_expr.drain(1..).collect(), ) } else { - (format!("transform{}", i), transform_expr.clone()) + (format!("transform{i}"), transform_expr.clone()) }; transform_names.push(name.clone()); @@ -220,8 +215,7 @@ pub(crate) fn generate_example( Ok(example) => example, Err(err) => { errs.push(format!( - "failed to generate transform '{}': {}", - transform_type, err + "failed to generate transform '{transform_type}': {err}" )); Value::Table(Map::new()) } @@ -252,8 +246,7 @@ pub(crate) fn generate_example( let (name, sink_type) = if let Some(c_index) = sink_expr.find(':') { if c_index == 0 { errs.push(format!( - "failed to generate sink '{}': empty name is not allowed", - sink_expr + "failed to generate sink '{sink_expr}': empty name is not allowed" )); continue; } @@ -263,13 +256,13 @@ pub(crate) fn generate_example( chopped_expr.drain(1..).collect(), ) } else { - (format!("sink{}", i), sink_expr.clone()) + (format!("sink{i}"), sink_expr.clone()) }; let mut example = match SinkDescription::example(&sink_type) { Ok(example) => example, Err(err) => { - errs.push(format!("failed to generate sink '{}': {}", sink_type, err)); + errs.push(format!("failed to generate sink '{sink_type}': {err}")); Value::Table(Map::new()) } }; @@ -354,7 +347,7 @@ pub fn cmd(opts: &Opts) -> exitcode::ExitCode { Ok(s) => { #[allow(clippy::print_stdout)] { - println!("{}", s); + println!("{s}"); } exitcode::OK } @@ -411,15 +404,15 @@ mod tests { #[test] fn generate_all(#[case] format: Format) { for name in SourceDescription::types() { - generate_and_deserialize(format!("{}//", name), format); + generate_and_deserialize(format!("{name}//"), format); } for name in TransformDescription::types() { - generate_and_deserialize(format!("/{}/", name), format); + generate_and_deserialize(format!("/{name}/"), format); } for name in SinkDescription::types() { - generate_and_deserialize(format!("//{}", name), format); + generate_and_deserialize(format!("//{name}"), format); } } diff --git a/src/generate_schema.rs b/src/generate_schema.rs index bddd203cda382..f20a71480c106 100644 --- a/src/generate_schema.rs +++ b/src/generate_schema.rs @@ -46,7 +46,7 @@ pub fn cmd(opts: &Opts) -> exitcode::ExitCode { exitcode::OK } Err(e) => { - eprintln!("error while generating schema: {:?}", e); + eprintln!("error while generating schema: {e:?}"); exitcode::SOFTWARE } } diff --git a/src/graph.rs b/src/graph.rs index 44ae55b19bbff..5d7ad0da03fca 100644 --- a/src/graph.rs +++ b/src/graph.rs @@ -85,10 +85,7 @@ fn node_attributes_to_string(attributes: &HashMap, default_shape if !attrs.contains_key("shape") { attrs.insert("shape".to_string(), default_shape.to_string()); } - attrs - .iter() - .map(|(k, v)| format!("{}=\"{}\"", k, v)) - .join(" ") + attrs.iter().map(|(k, v)| format!("{k}=\"{v}\"")).join(" ") } pub(crate) fn cmd(opts: &Opts) -> exitcode::ExitCode { @@ -103,7 +100,7 @@ pub(crate) fn cmd(opts: &Opts) -> exitcode::ExitCode { Err(errs) => { #[allow(clippy::print_stderr)] for err in errs { - eprintln!("{}", err); + eprintln!("{err}"); } return exitcode::CONFIG; } @@ -147,8 +144,7 @@ fn render_dot(config: config::Config) -> exitcode::ExitCode { ) .expect("write to String never fails"); } else { - writeln!(dot, " \"{}\" -> \"{}\"", input, id) - .expect("write to String never fails"); + writeln!(dot, " \"{input}\" -> \"{id}\"").expect("write to String never fails"); } } } @@ -171,8 +167,7 @@ fn render_dot(config: config::Config) -> exitcode::ExitCode { ) .expect("write to String never fails"); } else { - writeln!(dot, " \"{}\" -> \"{}\"", input, id) - .expect("write to String never fails"); + writeln!(dot, " \"{input}\" -> \"{id}\"").expect("write to String never fails"); } } } @@ -181,7 +176,7 @@ fn render_dot(config: config::Config) -> exitcode::ExitCode { #[allow(clippy::print_stdout)] { - println!("{}", dot); + println!("{dot}"); } exitcode::OK @@ -223,7 +218,7 @@ fn render_mermaid(config: config::Config) -> exitcode::ExitCode { #[allow(clippy::print_stdout)] { - println!("{}", mermaid); + println!("{mermaid}"); } exitcode::OK diff --git a/src/http.rs b/src/http.rs index 76aff937631d3..bbd716a385817 100644 --- a/src/http.rs +++ b/src/http.rs @@ -105,7 +105,7 @@ where let app_name = crate::get_app_name(); let version = crate::get_version(); - let user_agent = HeaderValue::from_str(&format!("{}/{}", app_name, version)) + let user_agent = HeaderValue::from_str(&format!("{app_name}/{version}")) .expect("Invalid header value for user-agent!"); Ok(HttpClient { @@ -277,7 +277,7 @@ impl fmt::Debug for HttpClient { pub enum Auth { /// Basic authentication. /// - /// The username and password are concatenated and encoded via [base64][base64]. + /// The username and password are concatenated and encoded using [base64][base64]. /// /// [base64]: https://en.wikipedia.org/wiki/Base64 Basic { @@ -363,7 +363,7 @@ pub fn get_http_scheme_from_uri(uri: &Uri) -> &'static str { // it also supports arbitrary schemes, which is where we bomb out down here, since we can't generate a static // string for an arbitrary input string... and anything other than "http" and "https" makes no sense for an HTTP // client anyways. - s => panic!("invalid URI scheme for HTTP client: {}", s), + s => panic!("invalid URI scheme for HTTP client: {s}"), }) } @@ -569,6 +569,73 @@ where } } +/// The type of a query parameter's value, determines if it's treated as a plain string or a VRL expression. +#[configurable_component] +#[derive(Clone, Debug, Default, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum ParamType { + /// The parameter value is a plain string. + #[default] + String, + /// The parameter value is a VRL expression that will be evaluated before each request. + Vrl, +} + +impl ParamType { + fn is_default(&self) -> bool { + *self == Self::default() + } +} + +/// Represents a query parameter value, which can be a simple string or a typed object +/// indicating whether the value is a string or a VRL expression. +#[configurable_component] +#[derive(Clone, Debug, Eq, PartialEq)] +#[serde(untagged)] +pub enum ParameterValue { + /// A simple string value. For backwards compatibility. + String(String), + /// A value with an explicit type. + Typed { + /// The raw value of the parameter. + value: String, + /// The type of the parameter, indicating how the `value` should be treated. + #[serde( + default, + skip_serializing_if = "ParamType::is_default", + rename = "type" + )] + r#type: ParamType, + }, +} + +impl ParameterValue { + /// Returns true if the parameter is a VRL expression. + pub const fn is_vrl(&self) -> bool { + match self { + ParameterValue::String(_) => false, + ParameterValue::Typed { r#type, .. } => matches!(r#type, ParamType::Vrl), + } + } + + /// Returns the raw string value of the parameter. + #[allow(clippy::missing_const_for_fn)] + pub fn value(&self) -> &str { + match self { + ParameterValue::String(s) => s, + ParameterValue::Typed { value, .. } => value, + } + } + + /// Consumes the `ParameterValue` and returns the owned raw string value. + pub fn into_value(self) -> String { + match self { + ParameterValue::String(s) => s, + ParameterValue::Typed { value, .. } => value, + } + } +} + /// Configuration of the query parameter value for HTTP requests. #[configurable_component] #[derive(Clone, Debug, Eq, PartialEq)] @@ -576,23 +643,22 @@ where #[configurable(metadata(docs::enum_tag_description = "Query parameter value"))] pub enum QueryParameterValue { /// Query parameter with single value - SingleParam(String), + SingleParam(ParameterValue), /// Query parameter with multiple values - MultiParams(Vec), + MultiParams(Vec), } impl QueryParameterValue { - /// Returns an iterator over string slices of the parameter values - pub fn iter(&self) -> std::iter::Map, fn(&String) -> &str> { + /// Returns an iterator over the contained `ParameterValue`s. + pub fn iter(&self) -> impl Iterator { match self { QueryParameterValue::SingleParam(param) => std::slice::from_ref(param).iter(), QueryParameterValue::MultiParams(params) => params.iter(), } - .map(String::as_str) } - /// Convert to Vec for owned iteration - fn into_vec(self) -> Vec { + /// Convert to `Vec` for owned iteration. + fn into_vec(self) -> Vec { match self { QueryParameterValue::SingleParam(param) => vec![param], QueryParameterValue::MultiParams(params) => params, @@ -600,20 +666,10 @@ impl QueryParameterValue { } } -// Implement IntoIterator for &QueryParameterValue -impl<'a> IntoIterator for &'a QueryParameterValue { - type Item = &'a str; - type IntoIter = std::iter::Map, fn(&String) -> &str>; - - fn into_iter(self) -> Self::IntoIter { - self.iter() - } -} - // Implement IntoIterator for owned QueryParameterValue impl IntoIterator for QueryParameterValue { - type Item = String; - type IntoIter = std::vec::IntoIter; + type Item = ParameterValue; + type IntoIter = std::vec::IntoIter; fn into_iter(self) -> Self::IntoIter { self.into_vec().into_iter() @@ -846,13 +902,13 @@ mod tests { // Responses generated before the client's max connection age has elapsed do not // include a `Connection: close` header in the response. - let req = Request::get(format!("http://{}/", addr)) + let req = Request::get(format!("http://{addr}/")) .body(Body::empty()) .unwrap(); let response = client.send(req).await.unwrap(); assert_eq!(response.headers().get("Connection"), None); - let req = Request::get(format!("http://{}/", addr)) + let req = Request::get(format!("http://{addr}/")) .body(Body::empty()) .unwrap(); let response = client.send(req).await.unwrap(); @@ -861,7 +917,7 @@ mod tests { // The first response generated after the client's max connection age has elapsed should // include the `Connection: close` header. tokio::time::sleep(Duration::from_secs(1)).await; - let req = Request::get(format!("http://{}/", addr)) + let req = Request::get(format!("http://{addr}/")) .body(Body::empty()) .unwrap(); let response = client.send(req).await.unwrap(); @@ -873,7 +929,7 @@ mod tests { // The next request should establish a new connection. // Importantly, this also confirms that each connection has its own independent // connection age timer. - let req = Request::get(format!("http://{}/", addr)) + let req = Request::get(format!("http://{addr}/")) .body(Body::empty()) .unwrap(); let response = client.send(req).await.unwrap(); diff --git a/src/internal_events/adaptive_concurrency.rs b/src/internal_events/adaptive_concurrency.rs index 03836f5d855d1..e5898295ed79d 100644 --- a/src/internal_events/adaptive_concurrency.rs +++ b/src/internal_events/adaptive_concurrency.rs @@ -25,9 +25,9 @@ registered_event! { fn emit(&self, data: AdaptiveConcurrencyLimitData) { self.limit.record(data.concurrency as f64); - let reached_limit = data.reached_limit.then_some(1.0).unwrap_or_default(); + let reached_limit = if data.reached_limit { 1.0 } else { Default::default() }; self.reached_limit.record(reached_limit); - let back_pressure = data.had_back_pressure.then_some(1.0).unwrap_or_default(); + let back_pressure = if data.had_back_pressure { 1.0 } else { Default::default() }; self.back_pressure.record(back_pressure); self.past_rtt_mean.record(data.past_rtt); // past_rtt_deviation is unrecorded diff --git a/src/internal_events/exec.rs b/src/internal_events/exec.rs index 539471719d0e9..fea1f498b725f 100644 --- a/src/internal_events/exec.rs +++ b/src/internal_events/exec.rs @@ -153,7 +153,7 @@ impl ExecFailedToSignalChild { match self { #[cfg(unix)] - SignalError(err) => format!("errno_{}", err), + SignalError(err) => format!("errno_{err}"), #[cfg(unix)] FailedToMarshalPid(_) => String::from("failed_to_marshal_pid"), #[cfg(unix)] @@ -170,9 +170,9 @@ impl std::fmt::Display for ExecFailedToSignalChild { match self { #[cfg(unix)] - SignalError(err) => write!(f, "errno: {}", err), + SignalError(err) => write!(f, "errno: {err}"), #[cfg(unix)] - FailedToMarshalPid(err) => write!(f, "failed to marshal pid to i32: {}", err), + FailedToMarshalPid(err) => write!(f, "failed to marshal pid to i32: {err}"), #[cfg(unix)] NoPid => write!(f, "child had no pid"), #[cfg(windows)] diff --git a/src/internal_events/expansion.rs b/src/internal_events/expansion.rs new file mode 100644 index 0000000000000..62a96afca1068 --- /dev/null +++ b/src/internal_events/expansion.rs @@ -0,0 +1,46 @@ +use metrics::counter; +use vector_lib::internal_event::{error_stage, error_type}; +use vector_lib::internal_event::{ComponentEventsDropped, InternalEvent, UNINTENTIONAL}; + +pub struct PairExpansionError<'a> { + pub key: &'a str, + pub value: &'a str, + pub drop_event: bool, + pub error: serde_json::Error, +} + +impl InternalEvent for PairExpansionError<'_> { + fn emit(self) { + let message = format!("Failed to expand key: `{}`:`{}`", self.key, self.value); + + if self.drop_event { + error!( + message = %message, + error = %self.error, + error_type = error_type::PARSER_FAILED, + stage = error_stage::PROCESSING, + internal_log_rate_limit = true, + ); + + counter!( + "component_errors_total", + "error_type" => error_type::PARSER_FAILED, + "stage" => error_stage::PROCESSING, + ) + .increment(1); + + emit!(ComponentEventsDropped:: { + count: 1, + reason: &message, + }); + } else { + warn!( + message = %message, + error = %self.error, + error_type = error_type::PARSER_FAILED, + stage = error_stage::PROCESSING, + internal_log_rate_limit = true, + ); + } + } +} diff --git a/src/internal_events/file.rs b/src/internal_events/file.rs index fba86d6ad1b72..4697c93c89ca7 100644 --- a/src/internal_events/file.rs +++ b/src/internal_events/file.rs @@ -106,8 +106,10 @@ impl InternalEvent for FileIoError<'_, P> { mod source { use std::{io::Error, path::Path, time::Duration}; + use bytes::BytesMut; use metrics::counter; use vector_lib::file_source::FileSourceInternalEvents; + use vector_lib::internal_event::{ComponentEventsDropped, INTENTIONAL}; use super::{FileOpen, InternalEvent}; use vector_lib::emit; @@ -496,6 +498,38 @@ mod source { } } + #[derive(Debug)] + pub struct FileLineTooBigError<'a> { + pub truncated_bytes: &'a BytesMut, + pub configured_limit: usize, + pub encountered_size_so_far: usize, + } + + impl InternalEvent for FileLineTooBigError<'_> { + fn emit(self) { + error!( + message = "Found line that exceeds max_line_bytes; discarding.", + truncated_bytes = ?self.truncated_bytes, + configured_limit = self.configured_limit, + encountered_size_so_far = self.encountered_size_so_far, + internal_log_rate_limit = true, + error_type = error_type::CONDITION_FAILED, + stage = error_stage::RECEIVING, + ); + counter!( + "component_errors_total", + "error_code" => "reading_line_from_file", + "error_type" => error_type::CONDITION_FAILED, + "stage" => error_stage::RECEIVING, + ) + .increment(1); + emit!(ComponentEventsDropped:: { + count: 1, + reason: "Found line that exceeds max_line_bytes; discarding.", + }); + } + } + #[derive(Clone)] pub struct FileSourceInternalEventsEmitter { pub include_file_metric_tag: bool, @@ -578,5 +612,18 @@ mod source { fn emit_path_globbing_failed(&self, path: &Path, error: &Error) { emit!(PathGlobbingError { path, error }); } + + fn emit_file_line_too_long( + &self, + truncated_bytes: &bytes::BytesMut, + configured_limit: usize, + encountered_size_so_far: usize, + ) { + emit!(FileLineTooBigError { + truncated_bytes, + configured_limit, + encountered_size_so_far + }); + } } } diff --git a/src/internal_events/http_client.rs b/src/internal_events/http_client.rs index a6fde2f1a6dd1..a35d290395be3 100644 --- a/src/internal_events/http_client.rs +++ b/src/internal_events/http_client.rs @@ -103,13 +103,13 @@ impl std::fmt::Display for FormatBody<'_, B> { let size = self.0.size_hint(); match (size.lower(), size.upper()) { (0, None) => write!(fmt, "[unknown]"), - (lower, None) => write!(fmt, "[>={} bytes]", lower), + (lower, None) => write!(fmt, "[>={lower} bytes]"), (0, Some(0)) => write!(fmt, "[empty]"), - (0, Some(upper)) => write!(fmt, "[<={} bytes]", upper), + (0, Some(upper)) => write!(fmt, "[<={upper} bytes]"), - (lower, Some(upper)) if lower == upper => write!(fmt, "[{} bytes]", lower), - (lower, Some(upper)) => write!(fmt, "[{}..={} bytes]", lower, upper), + (lower, Some(upper)) if lower == upper => write!(fmt, "[{lower} bytes]"), + (lower, Some(upper)) => write!(fmt, "[{lower}..={upper} bytes]"), } } } diff --git a/src/internal_events/kubernetes_logs.rs b/src/internal_events/kubernetes_logs.rs index 77dac30b1945b..39d27649cef5e 100644 --- a/src/internal_events/kubernetes_logs.rs +++ b/src/internal_events/kubernetes_logs.rs @@ -1,9 +1,10 @@ use metrics::counter; -use vector_lib::internal_event::InternalEvent; +use vector_lib::internal_event::{InternalEvent, INTENTIONAL}; use vector_lib::{ internal_event::{error_stage, error_type, ComponentEventsDropped, UNINTENTIONAL}, json_size::JsonSize, }; +use vrl::core::Value; use crate::event::Event; @@ -205,3 +206,35 @@ impl InternalEvent for KubernetesLifecycleError { }); } } + +#[derive(Debug)] +pub struct KubernetesMergedLineTooBigError<'a> { + pub event: &'a Value, + pub configured_limit: usize, + pub encountered_size_so_far: usize, +} + +impl InternalEvent for KubernetesMergedLineTooBigError<'_> { + fn emit(self) { + error!( + message = "Found line that exceeds max_merged_line_bytes; discarding.", + event = ?self.event, + configured_limit = self.configured_limit, + encountered_size_so_far = self.encountered_size_so_far, + internal_log_rate_limit = true, + error_type = error_type::CONDITION_FAILED, + stage = error_stage::RECEIVING, + ); + counter!( + "component_errors_total", + "error_code" => "reading_line_from_kubernetes_log", + "error_type" => error_type::CONDITION_FAILED, + "stage" => error_stage::RECEIVING, + ) + .increment(1); + emit!(ComponentEventsDropped:: { + count: 1, + reason: "Found line that exceeds max_merged_line_bytes; discarding.", + }); + } +} diff --git a/src/internal_events/mod.rs b/src/internal_events/mod.rs index 52c5cafb7a6e7..070c6951b2e50 100644 --- a/src/internal_events/mod.rs +++ b/src/internal_events/mod.rs @@ -125,7 +125,7 @@ mod template; mod throttle; mod udp; mod unix; -#[cfg(feature = "sinks-websocket")] +#[cfg(any(feature = "sources-websocket", feature = "sinks-websocket"))] mod websocket; #[cfg(feature = "sinks-websocket-server")] mod websocket_server; @@ -140,6 +140,11 @@ mod window; mod file; mod windows; +#[cfg(any(feature = "transforms-log_to_metric", feature = "sinks-loki"))] +mod expansion; +#[cfg(any(feature = "transforms-log_to_metric", feature = "sinks-loki"))] +pub use self::expansion::*; + #[cfg(feature = "sources-mongodb_metrics")] pub(crate) use mongodb_metrics::*; @@ -263,7 +268,7 @@ pub(crate) use self::tag_cardinality_limit::*; pub(crate) use self::throttle::*; #[cfg(unix)] pub(crate) use self::unix::*; -#[cfg(feature = "sinks-websocket")] +#[cfg(any(feature = "sources-websocket", feature = "sinks-websocket"))] pub(crate) use self::websocket::*; #[cfg(feature = "sinks-websocket-server")] pub(crate) use self::websocket_server::*; diff --git a/src/internal_events/prelude.rs b/src/internal_events/prelude.rs index 8da12ce4b824e..e01bab6933b9f 100644 --- a/src/internal_events/prelude.rs +++ b/src/internal_events/prelude.rs @@ -5,7 +5,7 @@ feature = "sources-utils-http", ))] pub(crate) fn http_error_code(code: u16) -> String { - format!("http_response_{}", code) + format!("http_response_{code}") } pub(crate) fn io_error_code(error: &std::io::Error) -> &'static str { diff --git a/src/internal_events/socket.rs b/src/internal_events/socket.rs index 97a648da2587b..5f2f7aac301e4 100644 --- a/src/internal_events/socket.rs +++ b/src/internal_events/socket.rs @@ -1,3 +1,5 @@ +use std::net::Ipv4Addr; + use metrics::{counter, histogram}; use vector_lib::internal_event::{ComponentEventsDropped, InternalEvent, UNINTENTIONAL}; use vector_lib::{ @@ -120,7 +122,7 @@ impl InternalEvent for SocketBindError { error = %self.error, error_code = "socket_bind", error_type = error_type::IO_FAILED, - stage = error_stage::RECEIVING, + stage = error_stage::INITIALIZING, %mode, internal_log_rate_limit = true, ); @@ -128,8 +130,46 @@ impl InternalEvent for SocketBindError { "component_errors_total", "error_code" => "socket_bind", "error_type" => error_type::IO_FAILED, - "stage" => error_stage::RECEIVING, + "stage" => error_stage::INITIALIZING, + "mode" => mode, + ) + .increment(1); + } +} + +#[derive(Debug)] +pub struct SocketMulticastGroupJoinError { + pub error: E, + pub group_addr: Ipv4Addr, + pub interface: Ipv4Addr, +} + +impl InternalEvent for SocketMulticastGroupJoinError { + fn emit(self) { + // Multicast groups are only used in UDP mode + let mode = SocketMode::Udp.as_str(); + let group_addr = self.group_addr.to_string(); + let interface = self.interface.to_string(); + + error!( + message = "Error joining multicast group.", + error = %self.error, + error_code = "socket_multicast_group_join", + error_type = error_type::IO_FAILED, + stage = error_stage::INITIALIZING, + %mode, + %group_addr, + %interface, + internal_log_rate_limit = true, + ); + counter!( + "component_errors_total", + "error_code" => "socket_multicast_group_join", + "error_type" => error_type::IO_FAILED, + "stage" => error_stage::INITIALIZING, "mode" => mode, + "group_addr" => group_addr, + "interface" => interface, ) .increment(1); } diff --git a/src/internal_events/template.rs b/src/internal_events/template.rs index 82df77cefdb8d..6ecc62b859363 100644 --- a/src/internal_events/template.rs +++ b/src/internal_events/template.rs @@ -13,7 +13,7 @@ impl InternalEvent for TemplateRenderingError<'_> { let mut msg = "Failed to render template".to_owned(); if let Some(field) = self.field { use std::fmt::Write; - _ = write!(msg, " for \"{}\"", field); + _ = write!(msg, " for \"{field}\""); } msg.push('.'); diff --git a/src/internal_events/websocket.rs b/src/internal_events/websocket.rs index 59d64574940f4..3dac157e31472 100644 --- a/src/internal_events/websocket.rs +++ b/src/internal_events/websocket.rs @@ -1,43 +1,49 @@ use std::error::Error; -use std::fmt::Debug; +use std::fmt::{Debug, Display, Formatter, Result}; -use metrics::counter; +use metrics::{counter, histogram}; +use tokio_tungstenite::tungstenite::error::Error as TungsteniteError; use vector_lib::internal_event::InternalEvent; -use vector_lib::internal_event::{error_stage, error_type}; +use vector_common::{ + internal_event::{error_stage, error_type}, + json_size::JsonSize, +}; + +pub const PROTOCOL: &str = "websocket"; #[derive(Debug)] -pub struct WsConnectionEstablished; +pub struct WebSocketConnectionEstablished; -impl InternalEvent for WsConnectionEstablished { +impl InternalEvent for WebSocketConnectionEstablished { fn emit(self) { debug!(message = "Connected."); counter!("connection_established_total").increment(1); } fn name(&self) -> Option<&'static str> { - Some("WsConnectionEstablished") + Some("WebSocketConnectionEstablished") } } #[derive(Debug)] -pub struct WsConnectionFailedError { +pub struct WebSocketConnectionFailedError { pub error: Box, } -impl InternalEvent for WsConnectionFailedError { +impl InternalEvent for WebSocketConnectionFailedError { fn emit(self) { error!( message = "WebSocket connection failed.", error = %self.error, - error_code = "ws_connection_error", + error_code = "websocket_connection_error", error_type = error_type::CONNECTION_FAILED, stage = error_stage::SENDING, internal_log_rate_limit = true, ); counter!( "component_errors_total", - "error_code" => "ws_connection_failed", + "error_code" => "websocket_connection_failed", "error_type" => error_type::CONNECTION_FAILED, "stage" => error_stage::SENDING, ) @@ -45,42 +51,43 @@ impl InternalEvent for WsConnectionFailedError { } fn name(&self) -> Option<&'static str> { - Some("WsConnectionFailed") + Some("WebSocketConnectionFailedError") } } #[derive(Debug)] -pub struct WsConnectionShutdown; +pub struct WebSocketConnectionShutdown; -impl InternalEvent for WsConnectionShutdown { +impl InternalEvent for WebSocketConnectionShutdown { fn emit(self) { warn!(message = "Closed by the server."); counter!("connection_shutdown_total").increment(1); } fn name(&self) -> Option<&'static str> { - Some("WsConnectionShutdown") + Some("WebSocketConnectionShutdown") } } #[derive(Debug)] -pub struct WsConnectionError { +pub struct WebSocketConnectionError { pub error: tokio_tungstenite::tungstenite::Error, } -impl InternalEvent for WsConnectionError { +impl InternalEvent for WebSocketConnectionError { fn emit(self) { error!( message = "WebSocket connection error.", error = %self.error, - error_code = "ws_connection_error", + error_code = "websocket_connection_error", error_type = error_type::WRITER_FAILED, stage = error_stage::SENDING, internal_log_rate_limit = true, ); counter!( "component_errors_total", - "error_code" => "ws_connection_error", + "protocol" => PROTOCOL, + "error_code" => "websocket_connection_error", "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, ) @@ -88,6 +95,152 @@ impl InternalEvent for WsConnectionError { } fn name(&self) -> Option<&'static str> { - Some("WsConnectionError") + Some("WebSocketConnectionError") + } +} + +#[allow(dead_code)] +#[derive(Debug, Copy, Clone)] +pub enum WebSocketKind { + Ping, + Pong, + Text, + Binary, + Close, + Frame, +} + +impl Display for WebSocketKind { + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + write!(f, "{self:?}") + } +} + +#[derive(Debug)] +pub struct WebSocketBytesReceived<'a> { + pub byte_size: usize, + pub url: &'a str, + pub protocol: &'static str, + pub kind: WebSocketKind, +} + +impl InternalEvent for WebSocketBytesReceived<'_> { + fn emit(self) { + trace!( + message = "Bytes received.", + byte_size = %self.byte_size, + url = %self.url, + protocol = %self.protocol, + kind = %self.kind + ); + let counter = counter!( + "component_received_bytes_total", + "url" => self.url.to_string(), + "protocol" => self.protocol, + "kind" => self.kind.to_string() + ); + counter.increment(self.byte_size as u64); + } +} + +#[derive(Debug)] +pub struct WebSocketMessageReceived<'a> { + pub count: usize, + pub byte_size: JsonSize, + pub url: &'a str, + pub protocol: &'static str, + pub kind: WebSocketKind, +} + +impl InternalEvent for WebSocketMessageReceived<'_> { + fn emit(self) { + trace!( + message = "Events received.", + count = %self.count, + byte_size = %self.byte_size, + url = %self.url, + protcol = %self.protocol, + kind = %self.kind + ); + + let histogram = histogram!("component_received_events_count"); + histogram.record(self.count as f64); + let counter = counter!( + "component_received_events_total", + "uri" => self.url.to_string(), + "protocol" => PROTOCOL, + "kind" => self.kind.to_string() + ); + counter.increment(self.count as u64); + let counter = counter!( + "component_received_event_bytes_total", + "url" => self.url.to_string(), + "protocol" => PROTOCOL, + "kind" => self.kind.to_string() + ); + counter.increment(self.byte_size.get() as u64); + } + + fn name(&self) -> Option<&'static str> { + Some("WebSocketMessageReceived") + } +} + +#[derive(Debug)] +pub struct WebSocketReceiveError<'a> { + pub error: &'a TungsteniteError, +} + +impl InternalEvent for WebSocketReceiveError<'_> { + fn emit(self) { + error!( + message = "Error receiving message from websocket.", + error = %self.error, + error_code = "websocket_receive_error", + error_type = error_type::CONNECTION_FAILED, + stage = error_stage::PROCESSING, + internal_log_rate_limit = true, + ); + counter!( + "component_errors_total", + "protocol" => PROTOCOL, + "error_code" => "websocket_receive_error", + "error_type" => error_type::CONNECTION_FAILED, + "stage" => error_stage::PROCESSING, + ) + .increment(1); + } + + fn name(&self) -> Option<&'static str> { + Some("WebSocketReceiveError") + } +} + +#[derive(Debug)] +pub struct WebSocketSendError<'a> { + pub error: &'a TungsteniteError, +} + +impl InternalEvent for WebSocketSendError<'_> { + fn emit(self) { + error!( + message = "Error sending message to websocket.", + error = %self.error, + error_code = "websocket_send_error", + error_type = error_type::CONNECTION_FAILED, + stage = error_stage::PROCESSING, + internal_log_rate_limit = true, + ); + counter!( + "component_errors_total", + "error_code" => "websocket_send_error", + "error_type" => error_type::CONNECTION_FAILED, + "stage" => error_stage::PROCESSING, + ) + .increment(1); + } + + fn name(&self) -> Option<&'static str> { + Some("WebSocketSendError") } } diff --git a/src/internal_events/websocket_server.rs b/src/internal_events/websocket_server.rs index 2fdb79394998e..b928385e669b9 100644 --- a/src/internal_events/websocket_server.rs +++ b/src/internal_events/websocket_server.rs @@ -7,12 +7,12 @@ use vector_lib::internal_event::InternalEvent; use vector_lib::internal_event::{error_stage, error_type}; #[derive(Debug)] -pub struct WsListenerConnectionEstablished { +pub struct WebSocketListenerConnectionEstablished { pub client_count: usize, pub extra_tags: Vec<(String, String)>, } -impl InternalEvent for WsListenerConnectionEstablished { +impl InternalEvent for WebSocketListenerConnectionEstablished { fn emit(self) { debug!( message = format!( @@ -25,17 +25,17 @@ impl InternalEvent for WsListenerConnectionEstablished { } fn name(&self) -> Option<&'static str> { - Some("WsListenerConnectionEstablished") + Some("WebSocketListenerConnectionEstablished") } } #[derive(Debug)] -pub struct WsListenerConnectionFailedError { +pub struct WebSocketListenerConnectionFailedError { pub error: Box, pub extra_tags: Vec<(String, String)>, } -impl InternalEvent for WsListenerConnectionFailedError { +impl InternalEvent for WebSocketListenerConnectionFailedError { fn emit(self) { error!( message = "WebSocket connection failed.", @@ -65,12 +65,12 @@ impl InternalEvent for WsListenerConnectionFailedError { } #[derive(Debug)] -pub struct WsListenerConnectionShutdown { +pub struct WebSocketListenerConnectionShutdown { pub client_count: usize, pub extra_tags: Vec<(String, String)>, } -impl InternalEvent for WsListenerConnectionShutdown { +impl InternalEvent for WebSocketListenerConnectionShutdown { fn emit(self) { info!( message = format!( @@ -83,16 +83,16 @@ impl InternalEvent for WsListenerConnectionShutdown { } fn name(&self) -> Option<&'static str> { - Some("WsListenerConnectionShutdown") + Some("WebSocketListenerConnectionShutdown") } } #[derive(Debug)] -pub struct WsListenerSendError { +pub struct WebSocketListenerSendError { pub error: Box, } -impl InternalEvent for WsListenerSendError { +impl InternalEvent for WebSocketListenerSendError { fn emit(self) { error!( message = "WebSocket message send error.", @@ -117,12 +117,12 @@ impl InternalEvent for WsListenerSendError { } #[derive(Debug)] -pub struct WsListenerMessageSent { +pub struct WebSocketListenerMessageSent { pub message_size: usize, pub extra_tags: Vec<(String, String)>, } -impl InternalEvent for WsListenerMessageSent { +impl InternalEvent for WebSocketListenerMessageSent { fn emit(self) { counter!("websocket_messages_sent_total", &self.extra_tags).increment(1); counter!("websocket_bytes_sent_total", &self.extra_tags) @@ -130,6 +130,6 @@ impl InternalEvent for WsListenerMessageSent { } fn name(&self) -> Option<&'static str> { - Some("WsListenerMessageSent") + Some("WebSocketListenerMessageSent") } } diff --git a/src/internal_telemetry/allocations/allocator/stack.rs b/src/internal_telemetry/allocations/allocator/stack.rs index 5479654501fe7..2ac8bab15410a 100644 --- a/src/internal_telemetry/allocations/allocator/stack.rs +++ b/src/internal_telemetry/allocations/allocator/stack.rs @@ -33,7 +33,7 @@ impl GroupStack { pub fn push(&mut self, group: AllocationGroupId) { self.current_top += 1; if self.current_top >= self.slots.len() { - panic!("tried to push new allocation group to the current stack, but hit the limit of {} entries", N); + panic!("tried to push new allocation group to the current stack, but hit the limit of {N} entries"); } self.slots[self.current_top] = group; } diff --git a/src/internal_telemetry/allocations/allocator/tracing_allocator.rs b/src/internal_telemetry/allocations/allocator/tracing_allocator.rs index 4c48928d079ce..dc189c35c239e 100644 --- a/src/internal_telemetry/allocations/allocator/tracing_allocator.rs +++ b/src/internal_telemetry/allocations/allocator/tracing_allocator.rs @@ -29,55 +29,59 @@ impl GroupedTraceableAllocator { unsafe impl GlobalAlloc for GroupedTraceableAllocator { #[inline] unsafe fn alloc(&self, object_layout: Layout) -> *mut u8 { - if !TRACK_ALLOCATIONS.load(Ordering::Relaxed) { - return self.allocator.alloc(object_layout); + unsafe { + if !TRACK_ALLOCATIONS.load(Ordering::Relaxed) { + return self.allocator.alloc(object_layout); + } + + // Allocate our wrapped layout and make sure the allocation succeeded. + let (actual_layout, offset_to_group_id) = get_wrapped_layout(object_layout); + let actual_ptr = self.allocator.alloc(actual_layout); + if actual_ptr.is_null() { + return actual_ptr; + } + + let group_id_ptr = actual_ptr.add(offset_to_group_id).cast::(); + + let object_size = object_layout.size(); + + try_with_suspended_allocation_group( + #[inline(always)] + |group_id| { + group_id_ptr.write(group_id.as_raw()); + self.tracer.trace_allocation(object_size, group_id); + }, + ); + actual_ptr } - - // Allocate our wrapped layout and make sure the allocation succeeded. - let (actual_layout, offset_to_group_id) = get_wrapped_layout(object_layout); - let actual_ptr = self.allocator.alloc(actual_layout); - if actual_ptr.is_null() { - return actual_ptr; - } - - let group_id_ptr = actual_ptr.add(offset_to_group_id).cast::(); - - let object_size = object_layout.size(); - - try_with_suspended_allocation_group( - #[inline(always)] - |group_id| { - group_id_ptr.write(group_id.as_raw()); - self.tracer.trace_allocation(object_size, group_id); - }, - ); - actual_ptr } #[inline] unsafe fn dealloc(&self, object_ptr: *mut u8, object_layout: Layout) { - if !TRACK_ALLOCATIONS.load(Ordering::Relaxed) { - self.allocator.dealloc(object_ptr, object_layout); - return; + unsafe { + if !TRACK_ALLOCATIONS.load(Ordering::Relaxed) { + self.allocator.dealloc(object_ptr, object_layout); + return; + } + // Regenerate the wrapped layout so we know where we have to look, as the pointer we've given relates to the + // requested layout, not the wrapped layout that was actually allocated. + let (wrapped_layout, offset_to_group_id) = get_wrapped_layout(object_layout); + + let raw_group_id = object_ptr.add(offset_to_group_id).cast::().read(); + + // Deallocate before tracking, just to make sure we're reclaiming memory as soon as possible. + self.allocator.dealloc(object_ptr, wrapped_layout); + + let object_size = object_layout.size(); + let source_group_id = AllocationGroupId::from_raw(raw_group_id); + + try_with_suspended_allocation_group( + #[inline(always)] + |_| { + self.tracer.trace_deallocation(object_size, source_group_id); + }, + ); } - // Regenerate the wrapped layout so we know where we have to look, as the pointer we've given relates to the - // requested layout, not the wrapped layout that was actually allocated. - let (wrapped_layout, offset_to_group_id) = get_wrapped_layout(object_layout); - - let raw_group_id = object_ptr.add(offset_to_group_id).cast::().read(); - - // Deallocate before tracking, just to make sure we're reclaiming memory as soon as possible. - self.allocator.dealloc(object_ptr, wrapped_layout); - - let object_size = object_layout.size(); - let source_group_id = AllocationGroupId::from_raw(raw_group_id); - - try_with_suspended_allocation_group( - #[inline(always)] - |_| { - self.tracer.trace_deallocation(object_size, source_group_id); - }, - ); } } diff --git a/src/lib.rs b/src/lib.rs index 59f91992a1f87..cc2bb8b59ae54 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -187,12 +187,12 @@ pub fn get_version() -> String { // or full debug symbols. See the Cargo Book profiling section for value meaning: // https://doc.rust-lang.org/cargo/reference/profiles.html#debug let build_string = match built_info::DEBUG { - "1" => format!("{} debug=line", build_string), - "2" | "true" => format!("{} debug=full", build_string), + "1" => format!("{build_string} debug=line"), + "2" | "true" => format!("{build_string} debug=full"), _ => build_string, }; - format!("{} ({})", pkg_version, build_string) + format!("{pkg_version} ({build_string})") } /// Includes information about the current build. diff --git a/src/line_agg.rs b/src/line_agg.rs index 35d37a4fb60b1..25693fb21783a 100644 --- a/src/line_agg.rs +++ b/src/line_agg.rs @@ -183,10 +183,13 @@ where // If we're in draining mode, short circuit here. if let Some(to_drain) = &mut this.draining { - if let Some(val) = to_drain.pop() { - return Poll::Ready(Some(val)); - } else { - return Poll::Ready(None); + match to_drain.pop() { + Some(val) => { + return Poll::Ready(Some(val)); + } + _ => { + return Poll::Ready(None); + } } } @@ -749,7 +752,7 @@ mod tests { "START msg 1".to_string(), // will be stashed ]; for i in 0..n { - lines.push(format!("line {}", i)); + lines.push(format!("line {i}")); } let config = Config { start_pattern: Regex::new("").unwrap(), @@ -760,7 +763,7 @@ mod tests { let mut expected = "START msg 1".to_string(); for i in 0..n { - write!(expected, "\nline {}", i).expect("write to String never fails"); + write!(expected, "\nline {i}").expect("write to String never fails"); } let (mut send, recv) = futures::channel::mpsc::unbounded(); diff --git a/src/list.rs b/src/list.rs index 566b07215d486..3cedbc91a9d0f 100644 --- a/src/list.rs +++ b/src/list.rs @@ -40,22 +40,22 @@ pub fn cmd(opts: &Opts) -> exitcode::ExitCode { Format::Text => { println!("Sources:"); for name in sources { - println!("- {}", name); + println!("- {name}"); } println!("\nTransforms:"); for name in transforms { - println!("- {}", name); + println!("- {name}"); } println!("\nSinks:"); for name in sinks { - println!("- {}", name); + println!("- {name}"); } println!("\nEnrichment tables:"); for name in enrichment_tables { - println!("- {}", name); + println!("- {name}"); } } Format::Json => { diff --git a/src/nats.rs b/src/nats.rs index 93a0a765ede55..5048cc3836315 100644 --- a/src/nats.rs +++ b/src/nats.rs @@ -65,7 +65,7 @@ impl std::fmt::Display for NatsAuthConfig { CredentialsFile { .. } => "credentials_file", Nkey { .. } => "nkey", }; - write!(f, "{}", word) + write!(f, "{word}") } } diff --git a/src/secrets/directory.rs b/src/secrets/directory.rs index e9ebaebbdd771..8170c208fd125 100644 --- a/src/secrets/directory.rs +++ b/src/secrets/directory.rs @@ -43,7 +43,7 @@ impl SecretBackend for DirectoryBackend { &contents }; if secret.is_empty() { - return Err(format!("secret in file '{}' was empty", k).into()); + return Err(format!("secret in file '{k}' was empty").into()); } secrets.insert(k, secret.to_string()); } diff --git a/src/secrets/exec.rs b/src/secrets/exec.rs index 485a5038b690d..29fb2c1585075 100644 --- a/src/secrets/exec.rs +++ b/src/secrets/exec.rs @@ -76,18 +76,18 @@ impl SecretBackend for ExecBackend { for k in secret_keys.into_iter() { if let Some(secret) = output.get_mut(&k) { if let Some(e) = &secret.error { - return Err(format!("secret for key '{}' was not retrieved: {}", k, e).into()); + return Err(format!("secret for key '{k}' was not retrieved: {e}").into()); } if let Some(v) = secret.value.take() { if v.is_empty() { - return Err(format!("secret for key '{}' was empty", k).into()); + return Err(format!("secret for key '{k}' was empty").into()); } secrets.insert(k.to_string(), v); } else { - return Err(format!("secret for key '{}' was empty", k).into()); + return Err(format!("secret for key '{k}' was empty").into()); } } else { - return Err(format!("secret for key '{}' was not retrieved", k).into()); + return Err(format!("secret for key '{k}' was not retrieved").into()); } } Ok(secrets) @@ -146,7 +146,7 @@ async fn query_backend( match stdout { None => break, Some(Ok(b)) => output.extend(b), - Some(Err(e)) => return Err(format!("Error while reading from an exec backend stdout: {}.", e).into()), + Some(Err(e)) => return Err(format!("Error while reading from an exec backend stdout: {e}.").into()), } } _ = &mut timeout => { diff --git a/src/secrets/file.rs b/src/secrets/file.rs index 221a293622cb6..c103385493791 100644 --- a/src/secrets/file.rs +++ b/src/secrets/file.rs @@ -34,11 +34,11 @@ impl SecretBackend for FileBackend { for k in secret_keys.into_iter() { if let Some(secret) = output.get(&k) { if secret.is_empty() { - return Err(format!("secret for key '{}' was empty", k).into()); + return Err(format!("secret for key '{k}' was empty").into()); } secrets.insert(k, secret.to_string()); } else { - return Err(format!("secret for key '{}' was not retrieved", k).into()); + return Err(format!("secret for key '{k}' was not retrieved").into()); } } Ok(secrets) diff --git a/src/serde.rs b/src/serde.rs index 6a0fd579cdd15..2bee11583e0ad 100644 --- a/src/serde.rs +++ b/src/serde.rs @@ -90,7 +90,7 @@ impl Fields { FieldsOrValue::Value(v) => Box::new(std::iter::once((k, v))), FieldsOrValue::Fields(f) => Box::new( f.all_fields() - .map(move |(nested_k, v)| (format!("{}.{}", k, nested_k), v)), + .map(move |(nested_k, v)| (format!("{k}.{nested_k}"), v)), ), } }) diff --git a/src/signal.rs b/src/signal.rs index 4075c545a15ce..af0d01d1c3d60 100644 --- a/src/signal.rs +++ b/src/signal.rs @@ -21,6 +21,8 @@ pub enum SignalTo { ReloadFromConfigBuilder(ConfigBuilder), /// Signal to reload config from the filesystem. ReloadFromDisk, + /// Signal to reload all enrichment tables. + ReloadEnrichmentTables, /// Signal to shutdown process. Shutdown(Option), /// Shutdown process immediately. @@ -36,6 +38,7 @@ impl PartialEq for SignalTo { // TODO: This will require a lot of plumbing but ultimately we can derive equality for config builders. (ReloadFromConfigBuilder(_), ReloadFromConfigBuilder(_)) => true, (ReloadFromDisk, ReloadFromDisk) => true, + (ReloadEnrichmentTables, ReloadEnrichmentTables) => true, (Shutdown(a), Shutdown(b)) => a == b, (Quit, Quit) => true, _ => false, @@ -173,7 +176,7 @@ impl SignalHandler { /// Signals from OS/user. #[cfg(unix)] -fn os_signals(runtime: &Runtime) -> impl Stream { +fn os_signals(runtime: &Runtime) -> impl Stream + use<> { use tokio::signal::unix::{signal, SignalKind}; // The `signal` function must be run within the context of a Tokio runtime. diff --git a/src/sinks/amqp/channel.rs b/src/sinks/amqp/channel.rs new file mode 100644 index 0000000000000..6bb8f6f47c475 --- /dev/null +++ b/src/sinks/amqp/channel.rs @@ -0,0 +1,87 @@ +use super::config::AmqpSinkConfig; +use super::service::AmqpError; +use crate::amqp::AmqpConfig; +use deadpool::managed::Pool; +use lapin::options::ConfirmSelectOptions; + +pub type AmqpSinkChannels = Pool; + +pub(super) fn new_channel_pool(config: &AmqpSinkConfig) -> crate::Result { + let max_channels = config.max_channels.try_into().map_err(|_| { + Box::new(AmqpError::PoolError { + error: "max_channels must fit into usize".into(), + }) + })?; + if max_channels == 0 { + return Err(Box::new(AmqpError::PoolError { + error: "max_channels must be positive".into(), + })); + } + let channel_manager = AmqpSinkChannelManager::new(&config.connection); + let channels = Pool::builder(channel_manager) + .max_size(max_channels) + .runtime(deadpool::Runtime::Tokio1) + .build()?; + debug!("AMQP channel pool created with max size: {}", max_channels); + Ok(channels) +} + +/// A channel pool manager for the AMQP sink. +/// This manager is responsible for creating and recycling AMQP channels. +/// It uses the `deadpool` crate to manage the channels. +pub(crate) struct AmqpSinkChannelManager { + config: AmqpConfig, +} + +impl deadpool::managed::Manager for AmqpSinkChannelManager { + type Type = lapin::Channel; + type Error = AmqpError; + + async fn create(&self) -> Result { + let channel = Self::new_channel(&self.config).await?; + info!( + message = "Created a new channel to the AMQP broker.", + id = channel.id(), + internal_log_rate_limit = true, + ); + Ok(channel) + } + + async fn recycle( + &self, + channel: &mut Self::Type, + _: &deadpool::managed::Metrics, + ) -> deadpool::managed::RecycleResult { + let state = channel.status().state(); + if state == lapin::ChannelState::Connected { + Ok(()) + } else { + Err((AmqpError::ChannelClosed { state }).into()) + } + } +} + +impl AmqpSinkChannelManager { + /// Creates a new channel pool manager for the AMQP sink. + pub fn new(config: &AmqpConfig) -> Self { + Self { + config: config.clone(), + } + } + + /// Creates a new AMQP channel using the configuration of this sink. + async fn new_channel(config: &AmqpConfig) -> Result { + let (_, channel) = config + .connect() + .await + .map_err(|e| AmqpError::ConnectFailed { error: e })?; + + // Enable confirmations on the channel. + channel + .confirm_select(ConfirmSelectOptions::default()) + .await + .map_err(|e| AmqpError::ConnectFailed { error: Box::new(e) })?; + + Ok(channel) + } +} diff --git a/src/sinks/amqp/config.rs b/src/sinks/amqp/config.rs index 50ea63104f836..afca8b47ccae5 100644 --- a/src/sinks/amqp/config.rs +++ b/src/sinks/amqp/config.rs @@ -1,7 +1,7 @@ //! Configuration functionality for the `AMQP` sink. +use super::channel::AmqpSinkChannels; use crate::{amqp::AmqpConfig, sinks::prelude::*}; use lapin::{types::ShortString, BasicProperties}; -use std::sync::Arc; use vector_lib::{ codecs::TextSerializerConfig, internal_event::{error_stage, error_type}, @@ -90,6 +90,14 @@ pub struct AmqpSinkConfig { skip_serializing_if = "crate::serde::is_default" )] pub(crate) acknowledgements: AcknowledgementsConfig, + + /// Maximum number of AMQP channels to keep active (channels are created as needed). + #[serde(default = "default_max_channels")] + pub(crate) max_channels: u32, +} + +const fn default_max_channels() -> u32 { + 4 } impl Default for AmqpSinkConfig { @@ -101,6 +109,7 @@ impl Default for AmqpSinkConfig { encoding: TextSerializerConfig::default().into(), connection: AmqpConfig::default(), acknowledgements: AcknowledgementsConfig::default(), + max_channels: default_max_channels(), } } } @@ -111,7 +120,8 @@ impl GenerateConfig for AmqpSinkConfig { r#"connection_string = "amqp://localhost:5672/%2f" routing_key = "user_id" exchange = "test" - encoding.codec = "json""#, + encoding.codec = "json" + max_channels = 4"#, ) .unwrap() } @@ -122,7 +132,7 @@ impl GenerateConfig for AmqpSinkConfig { impl SinkConfig for AmqpSinkConfig { async fn build(&self, _cx: SinkContext) -> crate::Result<(VectorSink, Healthcheck)> { let sink = AmqpSink::new(self.clone()).await?; - let hc = healthcheck(Arc::clone(&sink.channel)).boxed(); + let hc = healthcheck(sink.channels.clone()).boxed(); Ok((VectorSink::from_event_streamsink(sink), hc)) } @@ -135,9 +145,11 @@ impl SinkConfig for AmqpSinkConfig { } } -pub(super) async fn healthcheck(channel: Arc) -> crate::Result<()> { +pub(super) async fn healthcheck(channels: AmqpSinkChannels) -> crate::Result<()> { trace!("Healthcheck started."); + let channel = channels.get().await?; + if !channel.status().connected() { return Err(Box::new(std::io::Error::new( std::io::ErrorKind::BrokenPipe, diff --git a/src/sinks/amqp/encoder.rs b/src/sinks/amqp/encoder.rs index 7529fa12b6ffa..3655ab3d933f1 100644 --- a/src/sinks/amqp/encoder.rs +++ b/src/sinks/amqp/encoder.rs @@ -26,7 +26,7 @@ impl encoding::Encoder for AmqpEncoder { let mut encoder = self.encoder.clone(); encoder .encode(input, &mut body) - .map_err(|_| io::Error::new(io::ErrorKind::Other, "unable to encode"))?; + .map_err(|_| io::Error::other("unable to encode"))?; let body = body.freeze(); write_all(writer, 1, body.as_ref())?; diff --git a/src/sinks/amqp/integration_tests.rs b/src/sinks/amqp/integration_tests.rs index e919c5b88e92e..be30e93a6a37b 100644 --- a/src/sinks/amqp/integration_tests.rs +++ b/src/sinks/amqp/integration_tests.rs @@ -3,6 +3,7 @@ use crate::{ amqp::await_connection, config::{SinkConfig, SinkContext}, shutdown::ShutdownSignal, + sinks::amqp::channel::new_channel_pool, template::{Template, UnsignedIntTemplate}, test_util::{ components::{run_and_assert_sink_compliance, SINK_TAGS}, @@ -12,7 +13,7 @@ use crate::{ }; use config::AmqpPropertiesConfig; use futures::StreamExt; -use std::{collections::HashSet, sync::Arc, time::Duration}; +use std::{collections::HashSet, time::Duration}; use vector_lib::{config::LogNamespace, event::LogEvent}; pub fn make_config() -> AmqpSinkConfig { @@ -24,8 +25,7 @@ pub fn make_config() -> AmqpSinkConfig { let pass = std::env::var("AMQP_PASSWORD").unwrap_or_else(|_| "guest".to_string()); let host = std::env::var("AMQP_HOST").unwrap_or_else(|_| "rabbitmq".to_string()); let vhost = std::env::var("AMQP_VHOST").unwrap_or_else(|_| "%2f".to_string()); - config.connection.connection_string = - format!("amqp://{}:{}@{}:5672/{}", user, pass, host, vhost); + config.connection.connection_string = format!("amqp://{user}:{pass}@{host}:5672/{vhost}"); config } @@ -37,8 +37,8 @@ async fn healthcheck() { let mut config = make_config(); config.exchange = Template::try_from(exchange.as_str()).unwrap(); await_connection(&config.connection).await; - let (_conn, channel) = config.connection.connect().await.unwrap(); - super::config::healthcheck(Arc::new(channel)).await.unwrap(); + let channels = new_channel_pool(&config).unwrap(); + super::config::healthcheck(channels).await.unwrap(); } #[tokio::test] diff --git a/src/sinks/amqp/mod.rs b/src/sinks/amqp/mod.rs index 749f892f1ccd2..6b478c4adc10d 100644 --- a/src/sinks/amqp/mod.rs +++ b/src/sinks/amqp/mod.rs @@ -1,5 +1,6 @@ //! `AMQP` sink. //! Handles version AMQP 0.9.1 which is used by RabbitMQ. +mod channel; mod config; mod encoder; mod request_builder; @@ -15,7 +16,5 @@ use snafu::Snafu; #[derive(Debug, Snafu)] enum BuildError { #[snafu(display("creating amqp producer failed: {}", source))] - AmqpCreateFailed { - source: Box, - }, + AmqpCreateFailed { source: vector_common::Error }, } diff --git a/src/sinks/amqp/service.rs b/src/sinks/amqp/service.rs index 44c475f208cae..cee7d20344bd7 100644 --- a/src/sinks/amqp/service.rs +++ b/src/sinks/amqp/service.rs @@ -5,10 +5,9 @@ use bytes::Bytes; use futures::future::BoxFuture; use lapin::{options::BasicPublishOptions, BasicProperties}; use snafu::Snafu; -use std::{ - sync::Arc, - task::{Context, Poll}, -}; +use std::task::{Context, Poll}; + +use super::channel::AmqpSinkChannels; /// The request contains the data to send to `AMQP` together /// with the information need to route the message. @@ -79,11 +78,11 @@ impl DriverResponse for AmqpResponse { /// The tower service that handles the actual sending of data to `AMQP`. pub(super) struct AmqpService { - pub(super) channel: Arc, + pub(super) channels: AmqpSinkChannels, } #[derive(Debug, Snafu)] -pub(super) enum AmqpError { +pub enum AmqpError { #[snafu(display("Failed retrieving Acknowledgement: {}", error))] AcknowledgementFailed { error: lapin::Error }, @@ -92,6 +91,15 @@ pub(super) enum AmqpError { #[snafu(display("Received Negative Acknowledgement from AMQP broker."))] Nack, + + #[snafu(display("Failed to open AMQP channel: {}", error))] + ConnectFailed { error: vector_common::Error }, + + #[snafu(display("Channel is not writeable: {:?}", state))] + ChannelClosed { state: lapin::ChannelState }, + + #[snafu(display("Channel pool error: {}", error))] + PoolError { error: vector_common::Error }, } impl Service for AmqpService { @@ -106,9 +114,13 @@ impl Service for AmqpService { } fn call(&mut self, req: AmqpRequest) -> Self::Future { - let channel = Arc::clone(&self.channel); + let channel = self.channels.clone(); Box::pin(async move { + let channel = channel.get().await.map_err(|error| AmqpError::PoolError { + error: Box::new(error), + })?; + let byte_size = req.body.len(); let fut = channel .basic_publish( diff --git a/src/sinks/amqp/sink.rs b/src/sinks/amqp/sink.rs index 1537713be206c..ca40fe46b21e5 100644 --- a/src/sinks/amqp/sink.rs +++ b/src/sinks/amqp/sink.rs @@ -1,10 +1,10 @@ //! The sink for the `AMQP` sink that wires together the main stream that takes the //! event and sends it to `AMQP`. use crate::sinks::prelude::*; -use lapin::{options::ConfirmSelectOptions, BasicProperties}; +use lapin::BasicProperties; use serde::Serialize; -use std::sync::Arc; +use super::channel::AmqpSinkChannels; use super::{ config::{AmqpPropertiesConfig, AmqpSinkConfig}, encoder::AmqpEncoder, @@ -27,7 +27,7 @@ pub(super) struct AmqpEvent { } pub(super) struct AmqpSink { - pub(super) channel: Arc, + pub(super) channels: AmqpSinkChannels, exchange: Template, routing_key: Option