diff --git a/.gitignore b/.gitignore index 6372e0a5..32751a99 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ debug/ target/ data/ +.env.lambda .env metastore.yaml diff --git a/config/.env.exmaple b/config/.env.example similarity index 62% rename from config/.env.exmaple rename to config/.env.example index 9b66cd05..6a2c7ba1 100644 --- a/config/.env.exmaple +++ b/config/.env.example @@ -1,3 +1,4 @@ METASTORE_CONFIG=config/metastore.yaml JWT_SECRET=secret -TRACING_LEVEL=info \ No newline at end of file +TRACING_LEVEL=debug +RUST_LOG=info diff --git a/config/diesel.toml b/config/diesel.toml deleted file mode 100644 index 768ceacc..00000000 --- a/config/diesel.toml +++ /dev/null @@ -1,5 +0,0 @@ -[migrations_directory] -dir = "../crates/queries/migrations" - -[print_schema] -file = "../crates/queries/src/models/diesel_schema.rs" diff --git a/config/otel-example.yaml b/config/otel-example.yaml new file mode 100644 index 00000000..e45c3b6c --- /dev/null +++ b/config/otel-example.yaml @@ -0,0 +1,25 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: localhost:4317 + http: + endpoint: localhost:4318 + +processors: + batch: + +exporters: + otlp: + endpoint: "${env:OTEL_EXPORTER_ENDPOINT}" + headers: + # explicit header example, feel free to set own set of headers in the same way + # everything can be hardcoded as well, without parametrization via env vars + x-honeycomb-team: "${env:OTEL_EXPORTER_API_KEY}" + +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [otlp] \ No newline at end of file diff --git a/crates/embucket-lambda/Cargo.toml b/crates/embucket-lambda/Cargo.toml index 861856c2..e16d6369 100644 --- a/crates/embucket-lambda/Cargo.toml +++ b/crates/embucket-lambda/Cargo.toml @@ -59,4 +59,10 @@ timeout = 30 tracing = "Active" # Note: include path is relative to workspace root # Must run deploy from workspace root: cargo lambda deploy --binary-name bootstrap -include = ["config"] +include = ["config/metastore.yaml"] + +[package.metadata.lambda.deploy.env] +LOG_FORMAT = "json" +METASTORE_CONFIG = "config/metastore.yaml" +TRACING_LEVEL = "debug" +RUST_LOG = "info" \ No newline at end of file diff --git a/crates/embucket-lambda/Makefile b/crates/embucket-lambda/Makefile index 4740bc94..dcbd151a 100644 --- a/crates/embucket-lambda/Makefile +++ b/crates/embucket-lambda/Makefile @@ -1,15 +1,23 @@ .PHONY: build deploy test logs clean -# Function name (override with: make deploy FUNCTION_NAME=your-function) +# Following vars to be set externally: +# FUNCTION_NAME - override name of the lambda function +# ENV_FILE - override env file, relatively to project root +# FEATURES - cargo features to enable, comma separated +# LAYERS - additional layer ARNs to include, comma separated +# AWS_LAMBDA_ROLE_ARN - IAM role ARN for the lambda function +# WITH_OTEL_CONFIG - otel config path, file from /config/ dir + FUNCTION_NAME ?= embucket-lambda -ENV_FILE ?= config/.env -AWS_LAMBDA_ROLE_ARN_PARAM := $(if $(AWS_LAMBDA_ROLE_ARN),--iam-role $(AWS_LAMBDA_ROLE_ARN)) -OTEL_COLLECTOR_LAYERS_PARAM := $(if $(OTEL_COLLECTOR_LAYERS),--layer-arn $(OTEL_COLLECTOR_LAYERS)) -# supported features: "streaming" -FEATURES_PARAM := $(if $(FEATURES),--features $(FEATURES)) +ENV_FILE ?= config/.env.lambda +OTEL_COLLECTOR_LAYER=arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1 +OTEL_CONFIG_ENV_PARAM=$(if $(WITH_OTEL_CONFIG),--env-var OPENTELEMETRY_COLLECTOR_CONFIG_URI=/var/task/$(WITH_OTEL_CONFIG)) build: - cd ../.. && cargo lambda build --release -p embucket-lambda --arm64 -o zip --manifest-path crates/embucket-lambda/Cargo.toml $(FEATURES_PARAM) + cd ../.. && cargo lambda build --release -p embucket-lambda --arm64 \ + $(if $(FEATURES),--features $(FEATURES)) \ + $(if $(WITH_OTEL_CONFIG),--include $(WITH_OTEL_CONFIG)) \ + -o zip --manifest-path crates/embucket-lambda/Cargo.toml # Deploy to AWS (must run from workspace root for include paths to work) deploy: build deploy-only public-url @@ -17,23 +25,15 @@ deploy: build deploy-only public-url # Quick deploy without rebuild deploy-only: - cd ../.. && cargo lambda deploy $(OTEL_COLLECTOR_LAYERS_PARAM) $(AWS_LAMBDA_ROLE_ARN_PARAM) --env-file $(ENV_FILE) --binary-name bootstrap $(FUNCTION_NAME) + cd ../.. && cargo lambda deploy --admerge \ + --env-file $(ENV_FILE) \ + $(OTEL_CONFIG_ENV_PARAM) \ + $(if $(WITH_OTEL_CONFIG),--layer-arn $(OTEL_COLLECTOR_LAYER)) \ + $(if $(AWS_LAMBDA_ROLE_ARN),--iam-role $(AWS_LAMBDA_ROLE_ARN)) \ + $(if $(LAYERS),--layer-arn $(LAYERS)) \ + --binary-name bootstrap $(FUNCTION_NAME) aws logs create-log-group --log-group-name "/aws/lambda/$(FUNCTION_NAME)" >/dev/null 2>&1 || true -public-url: - @set -e; \ - echo "Ensuring Function URL config exists for $(FUNCTION_NAME) (auth: NONE)..." ; \ - aws lambda create-function-url-config --function-name "$(FUNCTION_NAME)" --auth-type NONE >/dev/null 2>&1 || \ - aws lambda update-function-url-config --function-name "$(FUNCTION_NAME)" --auth-type NONE >/dev/null ; \ - echo "Ensuring public invoke permission exists..." ; \ - aws lambda add-permission --function-name "$(FUNCTION_NAME)" \ - --statement-id AllowPublicURLInvoke \ - --action lambda:InvokeFunctionUrl \ - --principal "*" \ - --function-url-auth-type NONE >/dev/null 2>&1 || true ; \ - URL="$$(aws lambda get-function-url-config --function-name "$(FUNCTION_NAME)" --query 'FunctionUrl' --output text)"; \ - echo "$$URL" - # Watch locally for development watch: cargo lambda watch @@ -53,3 +53,34 @@ verify: # Clean build artifacts clean: cargo clean + +############################### +# Non-standard targets below # +############################### + +public-url: + echo "Ensuring Function URL config exists for $(FUNCTION_NAME) (auth: NONE)..." ; \ + aws lambda create-function-url-config \ + --function-name "$(FUNCTION_NAME)" \ + --auth-type NONE >/dev/null 2>&1 || \ + aws lambda update-function-url-config --function-name "$(FUNCTION_NAME)" --auth-type NONE >/dev/null ; \ + echo "Ensuring public invoke permission exists..." ; \ + aws lambda add-permission --function-name "$(FUNCTION_NAME)" \ + --statement-id AllowPublicURLInvoke \ + --action lambda:InvokeFunctionUrl \ + --principal "*" \ + --function-url-auth-type NONE >/dev/null 2>&1 || true ; \ + URL="$$(aws lambda get-function-url-config --function-name "$(FUNCTION_NAME)" --query 'FunctionUrl' --output text)"; \ + echo "$$URL" + +streaming: INVOKE_MODE=RESPONSE_STREAM +streaming: invoke-mode + +buffered: INVOKE_MODE=BUFFERED +buffered: invoke-mode + +invoke-mode: + echo "Enabling $(INVOKE_MODE) invoke mode for $(FUNCTION_NAME)..." + @aws lambda update-function-url-config \ + --function-name "$(FUNCTION_NAME)" \ + --invoke-mode $(INVOKE_MODE) diff --git a/crates/embucket-lambda/README.md b/crates/embucket-lambda/README.md index 6816919a..08e27d50 100644 --- a/crates/embucket-lambda/README.md +++ b/crates/embucket-lambda/README.md @@ -70,12 +70,13 @@ cargo lambda deploy --binary-name bootstrap - Timeout: `timeout = 30` - Included files: `include = ["config"]` -**Environment Variables** -- Set envs using `ENV_FILE=.env` environment variable: - ``` sh - ENV_FILE=".env.dev" make deploy - ``` -- It will deploy envs from `.env` if `ENV_FILE` not specified +**Environment Variables** (in `Cargo.toml`, `.env` envs will be combined) +- Set envs in `Cargo.toml` +- Provide envs at deploy: `ENV_FILE=config/.env.lambda make deploy` + +**Invoke mode** (Max response size up to 6 MB / 200 MB) +- `RESPONSE_STREAM` - Ensure you build with streaming feature `FEATURES=streaming make build deploy streaming`, otherwise it will not work +- `BUFFERED` - Basic response is 6MB, ensure lambda built without streaming feature ### Observability @@ -83,26 +84,28 @@ cargo lambda deploy --binary-name bootstrap We send events, spans to stdout log in json format, and in case if AWS X-Ray is enabled it enhances traces. - `RUST_LOG` - Controls verbosity log level. Default to "INFO", possible values: "OFF", "ERROR", "WARN", "INFO", "DEBUG", "TRACE". -#### OpenTelemetry configuration +#### OpenTelemetry traces +Send spans to external opentelemetry collector. +- `TRACING_LEVEL` - Controls verbosity level. Default to "INFO", possible values: "OFF", "ERROR", "WARN", "INFO", "DEBUG", "TRACE". + +#### OpenTelemetry configuration via OTLP/gRPC To work with Opentelemtry, you need an Opentelemetry Collector running in your environment with open telemetry config. The easiest way is to add two layers to your lambda deployment. One of which would be your config file with the remote exporter. -1. Create a folder called collector-config and add a file called `config.yml` with the OpenTelemetry Collector [**configuration**](https://opentelemetry.io/docs/languages/sdk-configuration/otlp-exporter/). -2. After which zip the folder with this ocmmand: `zip -r .zip collector-config` -3. Then publish it to AWS (change the file name and layer name if you want): `aws lambda publish-layer-version - --layer-name - --zip-file fileb://.zip - --compatible-runtimes provided.al2 provided.al2023 - --compatible-architectures arm64` -4. After which provide this as an external env variable (the first layer is the collector itself): `OTEL_COLLECTOR_LAYERS=arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1,arn:aws:lambda:::layer::` -5. Now you can deploy the function with the new layer. +* Configure `OTEL_EXPORTER_ENDPOINT`, `OTEL_EXPORTER_API_KEY` env vars and add them to your `.env.lambda` file. +* Deploy with enabled telemetry using `make build deploy WITH_OTEL_CONFIG=config/otel-example.yaml`. + - Specify opentelemetry config using `WITH_OTEL_CONFIG` makefile variable, for example use `config/otel-example.yaml` as is (works with honeycomb.io), or adapt to your needs. + - `WITH_OTEL_CONFIG` - specify path to a mentioned config, file should be in `config` folder + - [opentelemetry-lambda](https://github.com/open-telemetry/opentelemetry-lambda) collector extension layer will be deployed + - Config file also will be deployed as part of lambda deployment; Makefile will set env var `OPENTELEMETRY_COLLECTOR_CONFIG_URI` automatically. -If you later update the configratuin and publish the layer again remember to change the layer `` number, after the first publish it is `1`. +##### Setting these environment variables most likely will break telemetry setup: +* OTEL_EXPORTER_OTLP_ENDPOINT -#### Exporting telemetry spans to [**honeycomb.io**](https://docs.honeycomb.io/send-data/opentelemetry/collector/) +#### Example config for `opentelemetry-lambda` collector exporting spans to [**honeycomb.io**](https://docs.honeycomb.io/send-data/opentelemetry/collector/) -OpenTelemrty Collector config example for Honeycomb: +config/otel-example.yaml: ```yaml receivers: otlp: @@ -117,10 +120,11 @@ processors: exporters: otlp: - # You can name these envs anything you want as long as they are the same as in .env file - endpoint: "${env:HONEYCOMB_ENDPOINT_URL}" + endpoint: "${env:OTEL_EXPORTER_ENDPOINT}" headers: - x-honeycomb-team: "${env:HONEYCOMB_API_KEY}" + # explicit header example, feel free to set own set of headers in the same way + # everything can be hardcoded as well, without parametrization via env vars + x-honeycomb-team: "${env:OTEL_EXPORTER_API_KEY}" service: pipelines: @@ -131,8 +135,8 @@ service: ``` - Environment variables configuration: - * `HONEYCOMB_API_KEY` - this is the full ingestion key (not the key id or management key) - * `HONEYCOMB_ENDPOINT_URL` - check the region it can start be `api.honeycomb.io` or `api.eu1.honeycomb.io` + * `OTEL_EXPORTER_API_KEY` - this is the full ingestion key (not the key id or management key) + * `OTEL_EXPORTER_ENDPOINT` - check the region it can start be `api.honeycomb.io` or `api.eu1.honeycomb.io`, choose **gRPC** * `OTEL_SERVICE_NAME` - is the x-honeycomb-dataset name ### Test locally diff --git a/crates/embucket-lambda/src/config.rs b/crates/embucket-lambda/src/config.rs index bfc058ad..489d6c47 100644 --- a/crates/embucket-lambda/src/config.rs +++ b/crates/embucket-lambda/src/config.rs @@ -61,7 +61,7 @@ impl EnvConfig { object_store_connect_timeout_secs: parse_env("OBJECT_STORE_CONNECT_TIMEOUT_SECS") .unwrap_or(3), otel_exporter_otlp_protocol: parse_env("OTEL_EXPORTER_OTLP_PROTOCOL") - .unwrap_or("grpc".to_string()), + .unwrap_or_else(|| "grpc".to_string()), tracing_level: env_or_default("TRACING_LEVEL", "INFO"), } } diff --git a/crates/embucketd/src/main.rs b/crates/embucketd/src/main.rs index e113e53a..c78cb6a9 100644 --- a/crates/embucketd/src/main.rs +++ b/crates/embucketd/src/main.rs @@ -222,7 +222,7 @@ fn setup_tracing(opts: &cli::CliOpts) -> SdkTracerProvider { .build() .expect("Failed to create OTLP HTTP exporter") } - protocol => panic!("Unsupported OTLP protocol: {}", protocol), + protocol => panic!("Unsupported OTLP protocol: {protocol}"), }; let resource = Resource::builder().with_service_name("Em").build(); diff --git a/crates/queries/README.md b/crates/queries/README.md index f09a8dd6..44fc3279 100644 --- a/crates/queries/README.md +++ b/crates/queries/README.md @@ -40,7 +40,15 @@ apt install -y libpq-dev Refer here how to install diesel cli: https://diesel.rs/guides/getting-started#installing-diesel-cli -Diesel config is in repo root in `config/diesel.toml` file. +Put diesel config to the repo root into `config/diesel.toml`: +``` +[migrations_directory] +dir = "../crates/queries/migrations" + +[print_schema] +file = "../crates/queries/src/models/diesel_schema.rs" +``` + Before running diesel cli set DATABASE_URL env var or create .env file: ```bash