Lambda opentelemetry config support

.gitignore

@@ -3,6 +3,7 @@ debug/
 target/
 data/
 
+.env.lambda
 .env
 metastore.yaml
 

config/.env.exmaple → config/.env.example

@@ -1,3 +1,4 @@
 METASTORE_CONFIG=config/metastore.yaml
 JWT_SECRET=secret
-TRACING_LEVEL=info
+TRACING_LEVEL=debug
+RUST_LOG=info

config/otel-example.yaml

@@ -0,0 +1,25 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: localhost:4317
+      http:
+        endpoint: localhost:4318
+
+processors:
+  batch:
+
+exporters:
+  otlp:
+    endpoint: "${env:OTEL_EXPORTER_ENDPOINT}"
+    headers:
+      # explicit header example, feel free to set own set of headers in the same way
+      # everything can be hardcoded as well, without parametrization via env vars
+      x-honeycomb-team: "${env:OTEL_EXPORTER_API_KEY}"
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [otlp]

crates/embucket-lambda/Cargo.toml

@@ -59,4 +59,10 @@ timeout = 30
 tracing = "Active"
 # Note: include path is relative to workspace root
 # Must run deploy from workspace root: cargo lambda deploy --binary-name bootstrap
-include = ["config"]
+include = ["config/metastore.yaml"]
+
+[package.metadata.lambda.deploy.env]
+LOG_FORMAT = "json"
+METASTORE_CONFIG = "config/metastore.yaml"
+TRACING_LEVEL = "debug"
+RUST_LOG = "info"

crates/embucket-lambda/Makefile

@@ -1,39 +1,39 @@
 .PHONY: build deploy test logs clean
 
-# Function name (override with: make deploy FUNCTION_NAME=your-function)
+# Following vars to be set externally:
+#  FUNCTION_NAME - override name of the lambda function
+#  ENV_FILE - override env file, relatively to project root
+#  FEATURES - cargo features to enable, comma separated
+#  LAYERS - additional layer ARNs to include, comma separated
+#  AWS_LAMBDA_ROLE_ARN - IAM role ARN for the lambda function
+#  WITH_OTEL_CONFIG - otel config path, file from <projectroot>/config/ dir
+
 FUNCTION_NAME ?= embucket-lambda
-ENV_FILE ?= config/.env
-AWS_LAMBDA_ROLE_ARN_PARAM := $(if $(AWS_LAMBDA_ROLE_ARN),--iam-role $(AWS_LAMBDA_ROLE_ARN))
-OTEL_COLLECTOR_LAYERS_PARAM := $(if $(OTEL_COLLECTOR_LAYERS),--layer-arn $(OTEL_COLLECTOR_LAYERS))
-# supported features: "streaming"
-FEATURES_PARAM := $(if $(FEATURES),--features $(FEATURES))
+ENV_FILE ?= config/.env.lambda
+OTEL_COLLECTOR_LAYER=arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1
+OTEL_CONFIG_ENV_PARAM=$(if $(WITH_OTEL_CONFIG),--env-var OPENTELEMETRY_COLLECTOR_CONFIG_URI=/var/task/$(WITH_OTEL_CONFIG))
 
 build:
-	cd ../.. && cargo lambda build --release -p embucket-lambda --arm64 -o zip --manifest-path crates/embucket-lambda/Cargo.toml $(FEATURES_PARAM)
+	cd ../.. && cargo lambda build --release -p embucket-lambda --arm64 \
+		$(if $(FEATURES),--features $(FEATURES)) \
+		$(if $(WITH_OTEL_CONFIG),--include $(WITH_OTEL_CONFIG)) \
+		-o zip --manifest-path crates/embucket-lambda/Cargo.toml
 
 # Deploy to AWS (must run from workspace root for include paths to work)
 deploy: build deploy-only public-url
 	@echo "Deployed $(FUNCTION_NAME) to AWS"
 
 # Quick deploy without rebuild
 deploy-only:
-	cd ../.. && cargo lambda deploy $(OTEL_COLLECTOR_LAYERS_PARAM) $(AWS_LAMBDA_ROLE_ARN_PARAM) --env-file $(ENV_FILE) --binary-name bootstrap $(FUNCTION_NAME)
+	cd ../.. && cargo lambda deploy --admerge \
+		--env-file $(ENV_FILE) \
+		$(OTEL_CONFIG_ENV_PARAM) \
+		$(if $(WITH_OTEL_CONFIG),--layer-arn $(OTEL_COLLECTOR_LAYER)) \
+		$(if $(AWS_LAMBDA_ROLE_ARN),--iam-role $(AWS_LAMBDA_ROLE_ARN)) \
+		$(if $(LAYERS),--layer-arn $(LAYERS)) \
+		--binary-name bootstrap $(FUNCTION_NAME)
 	aws logs create-log-group --log-group-name "/aws/lambda/$(FUNCTION_NAME)" >/dev/null 2>&1 || true
 
-public-url:
-	@set -e; \
-	echo "Ensuring Function URL config exists for $(FUNCTION_NAME) (auth: NONE)..." ; \
-	aws lambda create-function-url-config --function-name "$(FUNCTION_NAME)" --auth-type NONE >/dev/null 2>&1 || \
-	  aws lambda update-function-url-config --function-name "$(FUNCTION_NAME)" --auth-type NONE >/dev/null ; \
-	echo "Ensuring public invoke permission exists..." ; \
-	aws lambda add-permission --function-name "$(FUNCTION_NAME)" \
-	  --statement-id AllowPublicURLInvoke \
-	  --action lambda:InvokeFunctionUrl \
-	  --principal "*" \
-	  --function-url-auth-type NONE >/dev/null 2>&1 || true ; \
-	URL="$$(aws lambda get-function-url-config --function-name "$(FUNCTION_NAME)" --query 'FunctionUrl' --output text)"; \
-	echo "$$URL"
-
 # Watch locally for development
 watch:
 	cargo lambda watch
@@ -53,3 +53,34 @@ verify:
 # Clean build artifacts
 clean:
 	cargo clean
+
+###############################
+# Non-standard targets below #
+###############################
+
+public-url:
+	echo "Ensuring Function URL config exists for $(FUNCTION_NAME) (auth: NONE)..." ; \
+	aws lambda create-function-url-config \
+		--function-name "$(FUNCTION_NAME)" \
+		--auth-type NONE >/dev/null 2>&1 || \
+	  aws lambda update-function-url-config --function-name "$(FUNCTION_NAME)" --auth-type NONE >/dev/null ; \
+	echo "Ensuring public invoke permission exists..." ; \
+	aws lambda add-permission --function-name "$(FUNCTION_NAME)" \
+	  --statement-id AllowPublicURLInvoke \
+	  --action lambda:InvokeFunctionUrl \
+	  --principal "*" \
+	  --function-url-auth-type NONE >/dev/null 2>&1 || true ; \
+	URL="$$(aws lambda get-function-url-config --function-name "$(FUNCTION_NAME)" --query 'FunctionUrl' --output text)"; \
+	echo "$$URL"
+
+streaming: INVOKE_MODE=RESPONSE_STREAM
+streaming: invoke-mode
+
+buffered: INVOKE_MODE=BUFFERED
+buffered: invoke-mode
+
+invoke-mode:
+	echo "Enabling $(INVOKE_MODE) invoke mode for $(FUNCTION_NAME)..."
+	@aws lambda update-function-url-config \
+  		--function-name "$(FUNCTION_NAME)" \
+		--invoke-mode $(INVOKE_MODE)

crates/embucket-lambda/README.md

@@ -70,39 +70,42 @@ cargo lambda deploy --binary-name bootstrap
 - Timeout: `timeout = 30`
 - Included files: `include = ["config"]`
 
-**Environment Variables**
-- Set envs using `ENV_FILE=.env` environment variable: 
-  ``` sh
-  ENV_FILE=".env.dev" make deploy
-  ```
-- It will deploy envs from `.env` if `ENV_FILE` not specified
+**Environment Variables** (in `Cargo.toml`, `.env` envs will be combined)
+- Set envs in `Cargo.toml`
+- Provide envs at deploy: `ENV_FILE=config/.env.lambda make deploy`
+
+**Invoke mode** (Max response size up to 6 MB / 200 MB)
+- `RESPONSE_STREAM` - Ensure you build with streaming feature `FEATURES=streaming make build deploy streaming`, otherwise it will not work
+- `BUFFERED` - Basic response is 6MB, ensure lambda built without streaming feature
 
 ### Observability
 
 #### AWS traces
 We send events, spans to stdout log in json format, and in case if AWS X-Ray is enabled it enhances traces.
 - `RUST_LOG` - Controls verbosity log level. Default to "INFO", possible values: "OFF", "ERROR", "WARN", "INFO", "DEBUG", "TRACE".
 
-#### OpenTelemetry configuration
+#### OpenTelemetry traces
+Send spans to external opentelemetry collector.
+- `TRACING_LEVEL` - Controls verbosity level. Default to "INFO", possible values: "OFF", "ERROR", "WARN", "INFO", "DEBUG", "TRACE".
+
+#### OpenTelemetry configuration via OTLP/gRPC
 
 To work with Opentelemtry, you need an Opentelemetry Collector running in your environment with open telemetry config. 
 The easiest way is to add two layers to your lambda deployment. One of which would be your config file with the remote exporter.
 
-1. Create a folder called collector-config and add a file called `config.yml` with the OpenTelemetry Collector [**configuration**](https://opentelemetry.io/docs/languages/sdk-configuration/otlp-exporter/).
-2. After which zip the folder with this ocmmand: `zip -r <filename>.zip collector-config`
-3. Then publish it to AWS (change the file name and layer name if you want): `aws lambda publish-layer-version 
-  --layer-name <layername> 
-  --zip-file fileb://<filename>.zip 
-  --compatible-runtimes provided.al2 provided.al2023 
-  --compatible-architectures arm64`
-4. After which provide this as an external env variable (the first layer is the collector itself): `OTEL_COLLECTOR_LAYERS=arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1,arn:aws:lambda:<region>:<account_id>:layer:<layername>:<version>`
-5. Now you can deploy the function with the new layer. 
+* Configure `OTEL_EXPORTER_ENDPOINT`, `OTEL_EXPORTER_API_KEY` env vars and add them to your `.env.lambda` file.
+* Deploy with enabled telemetry using `make build deploy WITH_OTEL_CONFIG=config/otel-example.yaml`.
+  - Specify opentelemetry config using `WITH_OTEL_CONFIG` makefile variable, for example use `config/otel-example.yaml` as is (works with honeycomb.io), or adapt to your needs.
+  - `WITH_OTEL_CONFIG` - specify path to a mentioned config, file should be in `config` folder
+  - [opentelemetry-lambda](https://github.com/open-telemetry/opentelemetry-lambda) collector extension layer will be deployed
+  - Config file also will be deployed as part of lambda deployment; Makefile will set env var `OPENTELEMETRY_COLLECTOR_CONFIG_URI` automatically.
 
-If you later update the configratuin and publish the layer again remember to change the layer `<version>` number, after the first publish it is `1`.
+##### Setting these environment variables most likely will break telemetry setup:
+* OTEL_EXPORTER_OTLP_ENDPOINT
 
-#### Exporting telemetry spans to [**honeycomb.io**](https://docs.honeycomb.io/send-data/opentelemetry/collector/)
+#### Example config for `opentelemetry-lambda` collector exporting spans to [**honeycomb.io**](https://docs.honeycomb.io/send-data/opentelemetry/collector/)
 
-OpenTelemrty Collector config example for Honeycomb:
+config/otel-example.yaml:
 ```yaml
 receivers:
   otlp:
@@ -117,10 +120,11 @@ processors:
 
 exporters:
   otlp:
-    # You can name these envs anything you want as long as they are the same as in .env file
-    endpoint: "${env:HONEYCOMB_ENDPOINT_URL}"
+    endpoint: "${env:OTEL_EXPORTER_ENDPOINT}"
     headers:
-      x-honeycomb-team: "${env:HONEYCOMB_API_KEY}"
+      # explicit header example, feel free to set own set of headers in the same way
+      # everything can be hardcoded as well, without parametrization via env vars
+      x-honeycomb-team: "${env:OTEL_EXPORTER_API_KEY}"
 
 service:
   pipelines:
@@ -131,8 +135,8 @@ service:
 ```
 
 - Environment variables configuration:
-  * `HONEYCOMB_API_KEY` - this is the full ingestion key (not the key id or management key)
-  * `HONEYCOMB_ENDPOINT_URL` - check the region it can start be `api.honeycomb.io` or `api.eu1.honeycomb.io`
+  * `OTEL_EXPORTER_API_KEY` - this is the full ingestion key (not the key id or management key)
+  * `OTEL_EXPORTER_ENDPOINT` - check the region it can start be `api.honeycomb.io` or `api.eu1.honeycomb.io`, choose **gRPC**
   * `OTEL_SERVICE_NAME` - is the x-honeycomb-dataset name
 
 ### Test locally

crates/embucket-lambda/src/config.rs

@@ -61,7 +61,7 @@ impl EnvConfig {
             object_store_connect_timeout_secs: parse_env("OBJECT_STORE_CONNECT_TIMEOUT_SECS")
                 .unwrap_or(3),
             otel_exporter_otlp_protocol: parse_env("OTEL_EXPORTER_OTLP_PROTOCOL")
-                .unwrap_or("grpc".to_string()),
+                .unwrap_or_else(|| "grpc".to_string()),
             tracing_level: env_or_default("TRACING_LEVEL", "INFO"),
         }
     }

crates/embucketd/src/main.rs

@@ -222,7 +222,7 @@ fn setup_tracing(opts: &cli::CliOpts) -> SdkTracerProvider {
                 .build()
                 .expect("Failed to create OTLP HTTP exporter")
         }
-        protocol => panic!("Unsupported OTLP protocol: {}", protocol),
+        protocol => panic!("Unsupported OTLP protocol: {protocol}"),
     };
 
     let resource = Resource::builder().with_service_name("Em").build();

crates/queries/README.md

@@ -40,7 +40,15 @@ apt install -y libpq-dev
 Refer here how to install diesel cli:
 https://diesel.rs/guides/getting-started#installing-diesel-cli
 
-Diesel config is in repo root in `config/diesel.toml` file. 
+Put diesel config to the repo root into `config/diesel.toml`:
+```
+[migrations_directory]
+dir = "../crates/queries/migrations" 
+
+[print_schema]
+file = "../crates/queries/src/models/diesel_schema.rs"
+```
+
 
 Before running diesel cli set DATABASE_URL env var or create .env file:
 ```bash