From e71dd3bb1a153cd104908180e2c15ca57a6f7f6a Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 5 Feb 2026 17:46:12 +0300 Subject: [PATCH 1/7] migrate to latest worker-deployment cli --- README.md | 4 ++-- deploy-gcp.yml | 10 +++++++--- deploy.yml | 17 +++++++++++------ docs/auth/gcp.md | 6 +++--- 4 files changed, 23 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 0427be21..b618b9e1 100644 --- a/README.md +++ b/README.md @@ -106,10 +106,10 @@ For easier deployment with automatic credential detection, use the [`@udx/worker npm install -g @udx/worker-deployment # Generate config -worker-config +worker config # Run with automatic GCP authentication -worker-run +worker run ``` Features: diff --git a/deploy-gcp.yml b/deploy-gcp.yml index 074f7c50..e5925a66 100644 --- a/deploy-gcp.yml +++ b/deploy-gcp.yml @@ -1,14 +1,18 @@ # npm install -g @udx/worker-deployment -# worker-run --config=deploy-gcp.yml +# worker run --config=deploy-gcp.yml --- kind: workerDeployConfig version: udx.io/worker-v1/deploy config: - # Docker image + # Docker image to run image: "usabilitydynamics/udx-worker:latest" env: ACTORS_CLEANUP: "false" - command: "gcloud auth list" + # Prefer command + args for special characters + command: "gcloud" + args: + - "auth" + - "list" diff --git a/deploy.yml b/deploy.yml index d54e1991..4375754a 100644 --- a/deploy.yml +++ b/deploy.yml @@ -1,28 +1,33 @@ # npm install -g @udx/worker-deployment -# gcloud auth login -# gcloud auth application-default login -# worker-run +# worker config +# worker run --- kind: workerDeployConfig version: udx.io/worker-v1/deploy config: - # Docker image + # Docker image to run image: "usabilitydynamics/udx-worker:latest" env: TEST_ENV_SECRET: "gcp/rabbit-ci-dev/worker-secret-test" TEST_ENV_JSON_KEY: "gcp/rabbit-ci-dev/worker-secret-json-key" - # Mount volumes + # Volume mounts (optional) + # Format: "host_path:container_path" or "host_path:container_path:ro" # volumes: # - "./worker.yaml:/home/udx/.config/worker/worker.yaml" + # Ports to expose (optional) # ports: # - "80:80" - # Command to run + # Command to run (optional - if not specified, uses container default) + # Tip: keep command to the executable and put complex values in args # command: "/usr/local/bin/init.sh" + # args: + # - "--example-flag" + # Service account impersonation (requires gcloud auth on host) service_account: email: "worker-site@rabbit-ci-dev.iam.gserviceaccount.com" diff --git a/docs/auth/gcp.md b/docs/auth/gcp.md index 1713cbed..36b4ee9f 100644 --- a/docs/auth/gcp.md +++ b/docs/auth/gcp.md @@ -120,7 +120,7 @@ config: ```bash # Run with automatic impersonation -worker-run --config=deploy.yml +worker run --config=deploy.yml ``` **Manual Docker Usage:** @@ -172,12 +172,12 @@ npm install -g @udx/worker-deployment **Quick Start:** ```bash # Generate config template -worker-config +worker config # Edit deploy.yml with your settings # Run with automatic credential detection -worker-run +worker run ``` **Features:** From c51f4cb8d813efc74852eef6b8c32ab8760e90e3 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 5 Feb 2026 23:07:49 +0300 Subject: [PATCH 2/7] docs: reorganize and standardize documentation --- README.md | 16 +- docs/CLI.md | 28 --- docs/config.md | 197 ------------------ docs/container-structure.md | 116 ----------- docs/deploy/README.md | 34 +++ docs/deploy/image-override.md | 58 ++++++ docs/deploy/kubernetes.md | 74 +++++++ docs/deploy/worker-deployment.md | 58 ++++++ docs/development/README.md | 33 +++ docs/development/child-images.md | 76 +++++++ docs/development/core-image.md | 69 ++++++ docs/git-help.md | 159 -------------- docs/index.md | 34 +++ docs/notes.md | 91 -------- docs/reference/cli.md | 38 ++++ docs/reference/container-structure.md | 73 +++++++ docs/runtime/config.md | 81 +++++++ docs/runtime/services.md | 91 ++++++++ docs/services.md | 185 ---------------- src/examples/README.md | 18 ++ src/examples/deploy-image-override/README.md | 29 +++ .../deploy-image-override/deploy.template.yml | 8 + src/examples/simple-service/README.md | 43 ++++ 23 files changed, 826 insertions(+), 783 deletions(-) delete mode 100644 docs/CLI.md delete mode 100644 docs/config.md delete mode 100644 docs/container-structure.md create mode 100644 docs/deploy/README.md create mode 100644 docs/deploy/image-override.md create mode 100644 docs/deploy/kubernetes.md create mode 100644 docs/deploy/worker-deployment.md create mode 100644 docs/development/README.md create mode 100644 docs/development/child-images.md create mode 100644 docs/development/core-image.md delete mode 100644 docs/git-help.md create mode 100644 docs/index.md delete mode 100644 docs/notes.md create mode 100644 docs/reference/cli.md create mode 100644 docs/reference/container-structure.md create mode 100644 docs/runtime/config.md create mode 100644 docs/runtime/services.md delete mode 100644 docs/services.md create mode 100644 src/examples/README.md create mode 100644 src/examples/deploy-image-override/README.md create mode 100644 src/examples/deploy-image-override/deploy.template.yml create mode 100644 src/examples/simple-service/README.md diff --git a/README.md b/README.md index b618b9e1..98c4611c 100644 --- a/README.md +++ b/README.md @@ -136,20 +136,22 @@ make log FOLLOW_LOGS=true make test ``` -More examples available in [src/examples](src/examples). +More examples available in [src/examples/README.md](src/examples/README.md). ## ๐Ÿ“š Documentation ### Core Concepts +- [Docs Index](docs/index.md) - Start here +- [Runtime: Services](docs/runtime/services.md) - `services.yaml` +- [Runtime: Config](docs/runtime/config.md) - `worker.yaml` +- [Deployment](docs/deploy/README.md) - `deploy.yml` and `worker-deployment` - [Authorization](docs/authorization.md) - Credential management -- [Configuration](docs/config.md) - Worker setup -- [Services](docs/services.md) - Service management -- [CLI Reference](docs/cli.md) - Command line usage +- [CLI Reference](docs/reference/cli.md) - Command line usage ### Additional Resources -- [Container Structure](docs/container-structure.md) - Directory layout -- [Development Notes](docs/notes.md) - Best practices -- [Git Tips](docs/git-help.md) - Version control helpers +- [Container Structure](docs/reference/container-structure.md) - Directory layout +- [Development](docs/development/README.md) - Build, run, test, child images +- [Examples](src/examples/README.md) - Runnable samples ## ๐Ÿ› ๏ธ Development diff --git a/docs/CLI.md b/docs/CLI.md deleted file mode 100644 index b2d16ebb..00000000 --- a/docs/CLI.md +++ /dev/null @@ -1,28 +0,0 @@ -# UDX Worker CLI Documentation - -The UDX Worker provides a command-line interface for managing services, environment variables, generating Software Bill of Materials (SBOM), and more. - -## Available Commands - -| Command | Description | -|----------|-------------| -| `auth` | Manage authentication operations | -| `config` | Manage configuration operations | -| `env` | Manage environment variables | -| `health` | Monitor system health status | -| `help` | Show help for any command | -| `sbom` | Manage Software Bill of Materials | -| `service`| Manage service operations | -| `version`| Show version information | - -## Usage - -To see detailed help and usage information for any command, run it without arguments: - -```bash -# Show auth command help -worker auth - -# Show service command help -worker service -``` \ No newline at end of file diff --git a/docs/config.md b/docs/config.md deleted file mode 100644 index 48bfbe30..00000000 --- a/docs/config.md +++ /dev/null @@ -1,197 +0,0 @@ -# Worker Configuration - -## Overview - -The UDX Worker uses `worker.yaml` as its primary configuration file, allowing you to: - -- Define environment variables -- Reference secrets from various providers -- Configure worker behavior - -## File Location - -```bash -/home/udx/.config/worker/worker.yaml -``` - -## Configuration Structure - -| Section | Purpose | Required | -| ---------------- | ----------------------------- | -------- | -| `kind` | Configuration type identifier | Yes | -| `version` | Schema version | Yes | -| `config.env` | Environment variables | No | -| `config.secrets` | Secret references | No | - -## Basic Example - -```yaml -kind: workerConfig -version: udx.io/worker-v1/config -config: - env: - AZURE_CLIENT_ID: "12345678-1234-1234-1234-1234567890ab" - AWS_REGION: "us-west-2" - secrets: - DB_PASSWORD: "aws/prod/db_password" - API_KEY: "azure/kv-prod/api-key" -``` - -## Secret Provider References - -### Azure Key Vault - -```yaml -secrets: - CLIENT_SECRET: "azure/{vault_name}/{secret_name}" - API_KEY: "azure/kv-prod/api-key" -``` - -### AWS Secrets Manager - -```yaml -secrets: - DB_PASSWORD: "aws/{path}/{secret_name}" - ACCESS_KEY: "aws/prod/access-key" -``` - -### Google Cloud Secret Manager - -```yaml -secrets: - SERVICE_KEY: "gcp/{project_id}/{secret_name}" - AUTH_TOKEN: "gcp/my-project/auth-token" -``` - -### Bitwarden Vault - -```yaml -secrets: - MASTER_KEY: "bitwarden/{vault_name}/{secret_name}" - LICENSE_KEY: "bitwarden/prod/license-key" -``` - -## Environment Variables - -Environment variables can be defined in two ways: - -### 1. Direct Values - -```yaml -config: - env: - # Cloud Provider Settings - AZURE_TENANT_ID: "tenant-id" - AWS_REGION: "us-west-2" - GCP_PROJECT: "my-project" - - # Application Settings - LOG_LEVEL: "info" - MAX_WORKERS: "5" - ENABLE_METRICS: "true" -``` - -### 2. Secret References - -Environment variables can also reference secrets using the same provider format as the `secrets` section: - -```yaml -config: - env: - # Reference secrets directly in env variables - DATABASE_URL: "gcp/my-project/db-connection-string" - API_TOKEN: "azure/kv-prod/api-token" - AWS_SECRET_KEY: "aws/prod/secret-access-key" - VAULT_PASSWORD: "bitwarden/prod/vault-pass" - - # Mix with regular values - LOG_LEVEL: "info" -``` - -The worker will automatically detect secret references (format: `provider/vault/secret`) in environment variables and resolve them at runtime. - -## Deployment Environment Variables - -Environment variables can also be set at the deployment level (e.g., Kubernetes, Docker Compose) and will **take priority** over `worker.yaml` configuration. - -### Priority Order - -1. **Deployment environment variables** (highest priority) -2. `worker.yaml` config.secrets -3. `worker.yaml` config.env - -### Example: Environment Override - -```yaml -# worker.yaml (production defaults) -config: - secrets: - ES_PASSWORD: "gcp/prod-project/es-password" -``` - -```yaml -# Kubernetes deployment (staging override) -spec: - containers: - - name: worker - env: - - name: ES_PASSWORD - value: "gcp/staging-project/es-password" -``` - -**Result**: The staging secret reference from Kubernetes will be used, not the production one from `worker.yaml`. - -### Use Cases - -- **Environment-specific overrides**: Different secrets per environment (dev/staging/prod) -- **Sensitive values**: Keep secrets out of config files entirely -- **Dynamic configuration**: Runtime values that change per deployment -- **Testing**: Override config values without modifying files - -### Behavior - -- Deployment env vars with secret references are automatically detected and resolved -- Deployment env vars with static values are used as-is -- If a deployment env var exists, the corresponding `worker.yaml` entry is skipped - -## Best Practices - -1. **Secret Management** - - - Never store sensitive values as plain text - - Use secret references in any of these locations: - - `config.secrets`: Explicit separation of secrets - - `config.env`: Unified configuration with secret references - - Deployment env vars: Runtime overrides (highest priority) - - All methods support the same provider format: `provider/vault/secret` - -2. **Environment-Specific Configuration** - - - Use `worker.yaml` for shared/default configuration - - Use deployment env vars for environment-specific overrides - - Example pattern: - ```yaml - # worker.yaml: production defaults - config: - secrets: - DB_PASSWORD: "gcp/prod/db-pass" - - # K8s staging: override with deployment env - env: - - name: DB_PASSWORD - value: "gcp/staging/db-pass" - ``` - -3. **Environment Variables** - - - Use `config.env` for non-sensitive configuration OR secret references - - Use deployment env vars to override per environment - - Document any required variables - - Remember: deployment env vars always win - -4. **File Handling** - - - Keep `worker.yaml` in version control (without sensitive data) - - Use secret references instead of plain text values - - Validate configuration before deployment - - Use deployment env vars for truly sensitive overrides diff --git a/docs/container-structure.md b/docs/container-structure.md deleted file mode 100644 index 34c0cf79..00000000 --- a/docs/container-structure.md +++ /dev/null @@ -1,116 +0,0 @@ -# Container Directory Structure - -This document outlines the directory structure of the UDX Worker container and provides guidance for creating child images. - -## Base Directory Structure - -The worker container uses the following directory structure: - -``` -/ -โ”œโ”€โ”€ opt/worker/ # Base directory for worker-specific files -โ”‚ โ”œโ”€โ”€ apps/ # Worker applications and plugins -โ”‚ โ””โ”€โ”€ data/ # Worker data storage and processing -โ”œโ”€โ”€ etc/worker/ # Worker configuration files -โ”œโ”€โ”€ usr/local/worker/ -โ”‚ โ”œโ”€โ”€ bin/ # Worker executable files -โ”‚ โ”œโ”€โ”€ lib/ # Worker library files -โ”‚ โ””โ”€โ”€ etc/ # Additional worker configuration -โ””โ”€โ”€ usr/local/configs/ # Cloud provider configurations - โ”œโ”€โ”€ gcloud/ # Google Cloud SDK config - โ”œโ”€โ”€ aws/ # AWS CLI config - โ””โ”€โ”€ azure/ # Azure CLI config -``` - -### Directory Purposes - -#### Worker-Specific Directories - -- `WORKER_BASE_DIR=/opt/worker` - - Main directory for worker-specific files - -- `WORKER_APP_DIR=/opt/worker/apps` - - Contains worker-specific applications and plugins - - Used for extending worker functionality - - **Not intended** for application code in child images - -- `WORKER_DATA_DIR=/opt/worker/data` - - Used for worker data storage and processing - - Temporary and persistent data used by the worker - -- `WORKER_CONFIG_DIR=/etc/worker` - - Contains worker configuration files - -- `WORKER_LIB_DIR=/usr/local/worker/lib` - - Worker library files and shared code - -- `WORKER_BIN_DIR=/usr/local/worker/bin` - - Worker executable files - - Added to system PATH - -- `WORKER_ETC_DIR=/usr/local/worker/etc` - - Additional worker configuration files - -#### Cloud Configuration Directories - -- `CLOUDSDK_CONFIG=/usr/local/configs/gcloud` - - Google Cloud SDK configuration - -- `AWS_CONFIG_FILE=/usr/local/configs/aws` - - AWS CLI configuration - -- `AZURE_CONFIG_DIR=/usr/local/configs/azure` - - Azure CLI configuration - -## Child Image Development - -When developing child images (e.g., PHP, Node.js), follow these guidelines: - -### Directory Structure Guidelines - -1. **Preserve Worker Directories** - - Maintain all worker directories as they are - - Do not modify or remove any worker-specific paths - - These directories are essential for worker functionality - -2. **Application Code Placement** - - Use framework/language-specific conventional directories for your application code - - Do not place application code in worker directories - -### Examples by Language - -#### PHP Applications -``` -/ -โ”œโ”€โ”€ var/www/ # PHP application code (standard location) -โ””โ”€โ”€ opt/worker/ # Worker directories (preserved) -``` - -#### Node.js Applications -``` -/ -โ”œโ”€โ”€ usr/src/app/ # Node.js application code (standard location) -โ””โ”€โ”€ opt/worker/ # Worker directories (preserved) -``` - -#### Python Applications -``` -/ -โ”œโ”€โ”€ usr/src/app/ # Python application code (standard location) -โ””โ”€โ”€ opt/worker/ # Worker directories (preserved) -``` - -### Best Practices - -1. **Separation of Concerns** - - Keep application code separate from worker functionality - - Use standard language/framework conventions for your application - - Don't mix application data with worker data - -2. **Configuration** - - Use appropriate configuration directories for your application - - Don't modify worker configurations unless specifically required - -3. **Data Storage** - - Use appropriate data directories for your application - - Don't use `WORKER_DATA_DIR` for application data storage diff --git a/docs/deploy/README.md b/docs/deploy/README.md new file mode 100644 index 00000000..59e96a94 --- /dev/null +++ b/docs/deploy/README.md @@ -0,0 +1,34 @@ +# Deployment + +## Overview + +Deployment is external to the worker container. This directory covers how to run the worker image consistently across local machines, CI/CD, and Kubernetes. + +## When To Use + +Use these docs when you need to: + +- Run the worker locally with a consistent config. +- Deploy in CI/CD (GitHub Actions, etc.). +- Mount runtime configs into Kubernetes. + +## Key Concepts + +- `deploy.yml` chooses the image, mounts runtime configs, and defines command/args. +- `worker.yaml` and `services.yaml` are runtime configs inside the container. + +## Examples + +- Local/CI usage: `docs/deploy/worker-deployment.md` +- CI image overrides: `docs/deploy/image-override.md` +- Kubernetes ConfigMaps: `docs/deploy/kubernetes.md` + +## Common Pitfalls + +- Editing runtime configs when you meant to change deployment behavior. +- Hardcoding image tags in CI instead of rendering `deploy.yml`. + +## Related Docs + +- `docs/runtime/config.md` +- `docs/runtime/services.md` diff --git a/docs/deploy/image-override.md b/docs/deploy/image-override.md new file mode 100644 index 00000000..fe1c4e8f --- /dev/null +++ b/docs/deploy/image-override.md @@ -0,0 +1,58 @@ +# CI/CD Image Override + +## Overview + +Override the worker image at deploy time by rendering a deploy template with an environment variable. + +## When To Use + +Use this when you: + +- Tag images per build and want to deploy that tag. +- Need to switch images without changing runtime configs. + +## Key Concepts + +- Keep `deploy.template.yml` in repo. +- Render to `deploy.yml` in CI. + +## Examples + +Template (`deploy.template.yml`): + +```yaml +kind: workerDeployConfig +version: udx.io/worker-v1/deploy +config: + image: "${WORKER_IMAGE}" + command: "echo" + args: + - "Hello from CI/CD" +``` + +Render + run: + +```bash +export WORKER_IMAGE="usabilitydynamics/udx-worker:latest" +envsubst < deploy.template.yml > deploy.yml +worker run --config=deploy.yml +``` + +If `envsubst` is unavailable: + +```bash +export WORKER_IMAGE="usabilitydynamics/udx-worker:latest" +sed "s|\${WORKER_IMAGE}|${WORKER_IMAGE}|g" deploy.template.yml > deploy.yml +worker run --config=deploy.yml +``` + +Example directory: `src/examples/deploy-image-override/` + +## Common Pitfalls + +- Committing generated `deploy.yml` with build-specific tags. +- Forgetting to set `WORKER_IMAGE` in CI. + +## Related Docs + +- `docs/deploy/worker-deployment.md` diff --git a/docs/deploy/kubernetes.md b/docs/deploy/kubernetes.md new file mode 100644 index 00000000..011c54f2 --- /dev/null +++ b/docs/deploy/kubernetes.md @@ -0,0 +1,74 @@ +# Kubernetes Deployment + +## Overview + +Kubernetes can mount `worker.yaml` and `services.yaml` into the container using ConfigMaps/Secrets. This keeps runtime configuration separate from the image. + +## When To Use + +Use this when you deploy the worker as a Kubernetes Deployment and want environment-specific configuration without rebuilding images. + +## Key Concepts + +- Store runtime configs as ConfigMaps (or Secrets for sensitive data). +- Mount into `/home/udx/.config/worker/`. + +## Examples + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: udx-worker-config +data: + worker.yaml: | + kind: workerConfig + version: udx.io/worker-v1/config + config: + env: + LOG_LEVEL: "info" + services.yaml: | + kind: workerService + version: udx.io/worker-v1/service + services: + - name: "logger" + command: "bash -c 'echo \"[startup]\"; while true; do echo \"[tick]\"; sleep 5; done'" + autostart: true + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: udx-worker +spec: + replicas: 1 + selector: + matchLabels: + app: udx-worker + template: + metadata: + labels: + app: udx-worker + spec: + containers: + - name: worker + image: usabilitydynamics/udx-worker:latest + volumeMounts: + - name: worker-config + mountPath: /home/udx/.config/worker + readOnly: true + volumes: + - name: worker-config + configMap: + name: udx-worker-config +``` + +## Common Pitfalls + +- Mounting configs to the wrong path. +- Putting secrets in ConfigMaps instead of Kubernetes Secrets. + +## Related Docs + +- `docs/runtime/config.md` +- `docs/runtime/services.md` diff --git a/docs/deploy/worker-deployment.md b/docs/deploy/worker-deployment.md new file mode 100644 index 00000000..47240118 --- /dev/null +++ b/docs/deploy/worker-deployment.md @@ -0,0 +1,58 @@ +# worker-deployment CLI + +## Overview + +`@udx/worker-deployment` standardizes how you run the worker image across different hosts and use cases. It keeps image selection, mounts, and runtime args in a single `deploy.yml` file. + +## When To Use + +Use `worker-deployment` when you want: + +- A consistent local run command (`worker run`). +- The same config to work in CI/CD runners. +- A portable deployment format across laptops and ephemeral hosts. + +## Key Concepts + +- `deploy.yml` is the deployment config. +- `worker.yaml` and `services.yaml` are runtime configs mounted into the container. + +## Examples + +### Quick Start + +```bash +npm install -g @udx/worker-deployment + +# Generate a template +worker config + +# Edit deploy.yml, then run +worker run +``` + +### Minimal Config + +```yaml +kind: workerDeployConfig +version: udx.io/worker-v1/deploy +config: + image: "usabilitydynamics/udx-worker:latest" + volumes: + - "./.config/worker:/home/udx/.config/worker:ro" + command: "echo" + args: + - "Hello from deploy.yml" +``` + +## Common Pitfalls + +- Forgetting to mount `worker.yaml`/`services.yaml` into the container. +- Putting runtime logic in `deploy.yml` instead of `services.yaml`. + +## Related Docs + +- `docs/deploy/README.md` +- `docs/deploy/image-override.md` +- `docs/runtime/config.md` +- `docs/runtime/services.md` diff --git a/docs/development/README.md b/docs/development/README.md new file mode 100644 index 00000000..8b09984a --- /dev/null +++ b/docs/development/README.md @@ -0,0 +1,33 @@ +# Development + +## Overview + +This section covers how to build the core worker image and how to create child images for workload-specific tooling. + +## When To Use + +Use these docs when you need to: + +- Modify the core worker runtime. +- Add dependencies for a specific workload. +- Build and test images locally. + +## Key Concepts + +- **Core image**: modify this repo when changing worker behavior. +- **Child image**: extend the core image for extra dependencies. + +## Examples + +- Core image workflow: `docs/development/core-image.md` +- Child image workflow: `docs/development/child-images.md` + +## Common Pitfalls + +- Using child images to change core behavior. +- Forgetting to rebuild after Dockerfile changes. + +## Related Docs + +- `docs/deploy/README.md` +- `docs/reference/container-structure.md` diff --git a/docs/development/child-images.md b/docs/development/child-images.md new file mode 100644 index 00000000..7890eb23 --- /dev/null +++ b/docs/development/child-images.md @@ -0,0 +1,76 @@ +# Develop Child Images + +## Overview + +Child images solve a common problem: the base worker image is intentionally minimal, but many workloads require extra tools. A child image adds only what you need without changing the core worker. + +## When To Use + +Use a child image when: + +- You need extra dependencies (SDKs, language runtimes, build tools). +- You want faster startup and predictable CI/CD. +- You want to keep the core worker image stable. + +Avoid a child image when: + +- You are changing worker core behavior. +- The dependency is only needed for a one-off task. + +## Key Concepts + +- Child images extend `usabilitydynamics/udx-worker`. +- `worker gen` creates scaffolding to get started quickly. + +## Examples + +### Generate Scaffolding + +```bash +npm install -g @udx/worker-deployment + +# Generate a child image repo skeleton (dry-run + prompt) +worker gen repo + +# Generate a Dockerfile only (dry-run + prompt) +worker gen dockerfile +``` + +### Minimal Workflow + +1. Generate a repo or Dockerfile. +2. Add dependencies. +3. Build and tag the image. +4. Deploy using `deploy.yml`. + +Example Dockerfile: + +```dockerfile +FROM usabilitydynamics/udx-worker:latest +RUN apt-get update && apt-get install -y jq +``` + +Build: + +```bash +docker build -t my-org/udx-worker-custom:latest . +``` + +Deploy (excerpt): + +```yaml +kind: workerDeployConfig +version: udx.io/worker-v1/deploy +config: + image: "my-org/udx-worker-custom:latest" +``` + +## Common Pitfalls + +- Baking secrets into the image. +- Forgetting to update `deploy.yml` with the child image. + +## Related Docs + +- `docs/deploy/worker-deployment.md` +- `docs/reference/container-structure.md` diff --git a/docs/development/core-image.md b/docs/development/core-image.md new file mode 100644 index 00000000..e889ebff --- /dev/null +++ b/docs/development/core-image.md @@ -0,0 +1,69 @@ +# Develop the Core Worker Image + +## Overview + +This repo builds the base `usabilitydynamics/udx-worker` image. Use this when you need to change the worker runtime itself. + +## When To Use + +Use this when you are modifying: + +- Supervisor/service behavior +- Auth/secrets handling +- CLI commands + +## Key Concepts + +- `make build` builds the image. +- `make run` runs it locally with mounted configs. +- `make test` runs container-based tests. + +## Examples + +### Build + +```bash +make build +``` + +Common options (see `Makefile.variables`): + +- `DOCKER_IMAGE` (default image name/tag) +- `MULTIPLATFORM=true` (buildx multi-arch) +- `DEBUG=true` (verbose build output) + +### Run + +```bash +make run +``` + +Interactive shell: + +```bash +make run-it +``` + +Follow logs: + +```bash +make log FOLLOW_LOGS=true +``` + +### Tests + +```bash +make test +``` + +The test target mounts `src/tests` and example configs into the container and runs `/home/udx/tests/main.sh`. + +## Common Pitfalls + +- Forgetting to rebuild after changes. +- Not mounting runtime config before testing. + +## Related Docs + +- `docs/runtime/services.md` +- `docs/runtime/config.md` diff --git a/docs/git-help.md b/docs/git-help.md deleted file mode 100644 index 6f34af09..00000000 --- a/docs/git-help.md +++ /dev/null @@ -1,159 +0,0 @@ -# Git Quick Reference - -## ๐Ÿงน Cleanup Operations - -### Clean Ignored Files -Remove files from Git index while keeping them in working directory: - -```bash -# Remove from index -git rm -r --cached . - -# Commit the cleanup -git add . -git commit -m "chore: clean ignored files" -``` - -> ๐Ÿ’ก Useful when `.gitignore` is updated but files are still tracked - -## ๐Ÿ”„ Commit Management - -### Amend Last Commit - -```bash -# Change commit message only -git commit --amend -m "new message" - -# Add staged changes to last commit -git add . -git commit --amend --no-edit -``` - -โš ๏ธ **Warning**: Don't amend pushed commits unless working alone - -### Force Push Changes - -```bash -# Force push with lease (safer than -f) -git push --force-with-lease - -# Force push (use with extreme caution) -git push -f -``` - -> ๐Ÿ›ก๏ธ Always prefer `--force-with-lease` over `-f` to prevent overwriting others' work - -## ๐ŸŒฟ Branch Operations - -### Move Commits Between Branches - -```bash -# 1. Find unpushed commits -git log branch-name --not --remotes --oneline - -# 2. Save current work -git stash - -# 3. Switch and apply -git checkout target-branch -git cherry-pick - -# 4. Restore work -git stash pop -``` - -## ๐Ÿš€ Common Workflows - -### Feature Branch Workflow - -```bash -# 1. Create feature branch -git checkout -b feature/name - -# 2. Make changes and commit -git add . -git commit -m "feat: add new feature" - -# 3. Update with main -git fetch origin -git rebase origin/main - -# 4. Push changes -git push -u origin feature/name -``` - -### Commit Message Format - -```bash -# Format -(): - -# Types -feat: New feature -fix: Bug fix -docs: Documentation -style: Formatting -refactor: Code restructure -test: Tests -chore: Maintenance -``` - -## ๐Ÿ” Inspection Commands - -### View Changes - -```bash -# Show staged changes -git diff --staged - -# Show changes in last commit -git show HEAD - -# Show file history -git log -p filename -``` - -### Branch Information - -```bash -# List all branches -git branch -vv - -# Show merged branches -git branch --merged - -# Show unmerged branches -git branch --no-merged -``` - -## โšก Tips and Tricks - -1. **Stash Management**: - ```bash - # Named stash - git stash save "feature work in progress" - - # List stashes - git stash list - - # Apply specific stash - git stash apply stash@{n} - ``` - -2. **Quick Fixes**: - ```bash - # Undo last commit but keep changes - git reset --soft HEAD^ - - # Discard all local changes - git reset --hard HEAD - ``` - -3. **Search History**: - ```bash - # Search commit messages - git log --grep="keyword" - - # Search code changes - git log -S"code string" - ``` diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..76132df4 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,34 @@ +# UDX Worker Documentation + +## Overview + +This documentation is organized by concern so it is clear what happens **inside** the worker container (runtime) vs what happens **outside** at deployment time. + +## When To Use + +Start here if you need to: + +- Configure runtime behavior (`services.yaml`, `worker.yaml`). +- Deploy the worker image across environments. +- Build the core image or child images. + +## Key Concepts + +- Runtime config lives inside the container. +- Deployment config selects the image and mounts runtime files. + +## Examples + +- Runtime: `docs/runtime/services.md` +- Deployment: `docs/deploy/README.md` +- Development: `docs/development/README.md` + +## Common Pitfalls + +- Mixing runtime config with deployment config. +- Editing the image when you only need a runtime change. + +## Related Docs + +- `docs/runtime/config.md` +- `docs/deploy/worker-deployment.md` diff --git a/docs/notes.md b/docs/notes.md deleted file mode 100644 index f5c061dc..00000000 --- a/docs/notes.md +++ /dev/null @@ -1,91 +0,0 @@ -# Container Development Best Practices - -## User Management - -### Non-Root User Benefits -- ๐Ÿ›ก๏ธ **Enhanced Security**: Limits potential damage from security breaches -- ๐Ÿ”’ **Reduced Privileges**: Prevents unauthorized system modifications -- ๐Ÿ”„ **Best Practice**: Follows container security principles - -### User Configuration - -```dockerfile -ARG USER=udx -ARG UID=500 -ARG GID=500 - -RUN groupadd -g $GID $USER && \ - useradd -u $UID -g $GID -m $USER -``` - -## Dockerfile Arguments - -### Benefits of Using ARGs -- ๐Ÿ”ง **Parameterization**: Easy to customize builds -- ๐ŸŽฏ **Flexibility**: Adapt to different environments -- ๐Ÿ—๏ธ **Reusability**: Same Dockerfile, different configurations - -### Common ARGs -```dockerfile -ARG USER=udx -ARG UID=500 -ARG GID=500 -ARG HOME=/home/udx -``` - -## Permission Management - -### UID/GID Importance -- ๐Ÿ“ **File Access**: Consistent access across host and containers -- ๐Ÿค **Shared Resources**: Proper permissions for mounted volumes -- ๐Ÿ” **Security**: Controlled access to resources - -### Best Practices -1. Use consistent UID/GID across environments -2. Document required permissions -3. Verify file ownership after operations - -## Container Security - -### Running as Non-Root -- Prevents privileged access -- Limits system modification capabilities -- Follows principle of least privilege - -### Security Checklist -- [ ] Use non-root user -- [ ] Set appropriate file permissions -- [ ] Limit mounted volumes -- [ ] Use read-only filesystems where possible - -## Environment Setup - -### Container User Context -- Application-specific user -- Limited privileges -- No system administration capabilities - -### Directory Permissions -```bash -chown -R $USER:$USER /app -chmod -R 755 /app -``` - -## Tips and Tricks - -1. **Testing User Setup**: - ```bash - docker run --rm -it myimage whoami - docker run --rm -it myimage id - ``` - -2. **Debugging Permissions**: - ```bash - docker run --rm -it myimage ls -la /app - docker run --rm -it myimage stat /app - ``` - -3. **Volume Mounting**: - ```bash - docker run -v $(pwd):/app:ro myimage # Read-only mount - ``` \ No newline at end of file diff --git a/docs/reference/cli.md b/docs/reference/cli.md new file mode 100644 index 00000000..58350d86 --- /dev/null +++ b/docs/reference/cli.md @@ -0,0 +1,38 @@ +# UDX Worker CLI + +## Overview + +The UDX Worker provides a command-line interface for managing services, environment variables, and health checks. + +## When To Use + +Use the CLI when you need to: + +- Inspect or manage running services. +- Check or resolve environment and secrets. +- Review health and SBOM output. + +## Key Concepts + +- Commands are namespaced (`worker service`, `worker env`, `worker auth`). +- Most commands provide help when run without arguments. + +## Examples + +```bash +# Show auth command help +worker auth + +# Show service command help +worker service +``` + +## Common Pitfalls + +- Forgetting to mount runtime config before running commands. +- Using the CLI outside the container when the worker is not running. + +## Related Docs + +- `docs/runtime/services.md` +- `docs/runtime/config.md` diff --git a/docs/reference/container-structure.md b/docs/reference/container-structure.md new file mode 100644 index 00000000..94385ab3 --- /dev/null +++ b/docs/reference/container-structure.md @@ -0,0 +1,73 @@ +# Container Structure + +## Overview + +This document outlines the directory structure of the UDX Worker container and how to place application code in child images. + +## When To Use + +Use this when you need to: + +- Understand where worker internals live. +- Build child images without breaking worker paths. + +## Key Concepts + +- Worker directories must be preserved. +- Application code should live in language-standard paths. + +## Examples + +### Base Directory Structure + +``` +/ +โ”œโ”€โ”€ opt/worker/ # Base directory for worker-specific files +โ”‚ โ”œโ”€โ”€ apps/ # Worker applications and plugins +โ”‚ โ””โ”€โ”€ data/ # Worker data storage and processing +โ”œโ”€โ”€ etc/worker/ # Worker configuration files +โ”œโ”€โ”€ usr/local/worker/ +โ”‚ โ”œโ”€โ”€ bin/ # Worker executable files +โ”‚ โ”œโ”€โ”€ lib/ # Worker library files +โ”‚ โ””โ”€โ”€ etc/ # Additional worker configuration +โ””โ”€โ”€ usr/local/configs/ # Cloud provider configurations + โ”œโ”€โ”€ gcloud/ # Google Cloud SDK config + โ”œโ”€โ”€ aws/ # AWS CLI config + โ””โ”€โ”€ azure/ # Azure CLI config +``` + +### Child Image Placement + +PHP: + +``` +/ +โ”œโ”€โ”€ var/www/ # PHP application code +โ””โ”€โ”€ opt/worker/ # Worker directories (preserved) +``` + +Node.js: + +``` +/ +โ”œโ”€โ”€ usr/src/app/ # Node.js application code +โ””โ”€โ”€ opt/worker/ # Worker directories (preserved) +``` + +Python: + +``` +/ +โ”œโ”€โ”€ usr/src/app/ # Python application code +โ””โ”€โ”€ opt/worker/ # Worker directories (preserved) +``` + +## Common Pitfalls + +- Modifying or removing worker directories. +- Storing application code under `/opt/worker`. + +## Related Docs + +- `docs/development/child-images.md` +- `docs/deploy/README.md` diff --git a/docs/runtime/config.md b/docs/runtime/config.md new file mode 100644 index 00000000..1c302c53 --- /dev/null +++ b/docs/runtime/config.md @@ -0,0 +1,81 @@ +# Worker Configuration (`worker.yaml`) + +## Overview + +`worker.yaml` is the primary runtime configuration file. It defines environment variables and secret references used **inside** the worker container. + +## When To Use + +Use this when you need to: + +- Define runtime environment variables. +- Reference secrets from cloud providers. +- Override defaults at runtime without rebuilding images. + +## Key Concepts + +- Runtime-only config: `/home/udx/.config/worker/worker.yaml`. +- Deployment env vars override `worker.yaml` values. +- Secret references use `provider/vault/secret` format. + +## Examples + +### Basic + +```yaml +kind: workerConfig +version: udx.io/worker-v1/config +config: + env: + AZURE_CLIENT_ID: "12345678-1234-1234-1234-1234567890ab" + AWS_REGION: "us-west-2" + secrets: + DB_PASSWORD: "aws/prod/db_password" + API_KEY: "azure/kv-prod/api-key" +``` + +### Secret References in Env + +```yaml +config: + env: + DATABASE_URL: "gcp/my-project/db-connection-string" + API_TOKEN: "azure/kv-prod/api-token" + LOG_LEVEL: "info" +``` + +### Runtime Environment and Precedence + +1. **Deployment environment variables** (highest priority) +2. `worker.yaml` `config.secrets` +3. `worker.yaml` `config.env` + +Example override: + +```yaml +# worker.yaml (production defaults) +config: + secrets: + ES_PASSWORD: "gcp/prod-project/es-password" +``` + +```yaml +# Kubernetes deployment (staging override) +spec: + containers: + - name: worker + env: + - name: ES_PASSWORD + value: "gcp/staging-project/es-password" +``` + +## Common Pitfalls + +- Storing plaintext secrets in `worker.yaml`. +- Forgetting that deployment env vars override runtime config. + +## Related Docs + +- `docs/runtime/services.md` +- `docs/deploy/README.md` +- `docs/deploy/kubernetes.md` diff --git a/docs/runtime/services.md b/docs/runtime/services.md new file mode 100644 index 00000000..1c69d28c --- /dev/null +++ b/docs/runtime/services.md @@ -0,0 +1,91 @@ +# Service Configuration (`services.yaml`) + +## Overview + +`services.yaml` defines **processes that run inside the worker container**. It does not select the container image or perform deployment. + +## When To Use + +Use this when you need to: + +- Define one or more long-running services inside the container. +- Configure commands, restart policy, and env vars for each service. + +## Key Concepts + +- Runtime-only: it lives inside the container at `/home/udx/.config/worker/`. +- Image selection happens at deployment time (see `docs/deploy/README.md`). + +## Examples + +### Basic + +```yaml +kind: workerService +version: udx.io/worker-v1/service +services: + - name: "web-server" + command: "python app.py" + autostart: true + autorestart: true + envs: + - "PORT=8080" + - "DEBUG=true" +``` + +### Writing to Stdout + +```yaml +kind: workerService +version: udx.io/worker-v1/service +services: + - name: "logger" + command: "bash -c 'echo "[startup] logger"; while true; do echo "[tick] $(date)"; sleep 5; done'" + autostart: true + autorestart: true +``` + +Then view output: + +```bash +worker service logs logger +``` + +Example scripts: `src/examples/simple-service/` + +### Multiple Services + +```yaml +kind: workerService +version: udx.io/worker-v1/service +services: + - name: "api-server" + command: "npm start" + autostart: true + autorestart: true + stopasgroup: true + killasgroup: true + envs: + - "PORT=3000" + - "NODE_ENV=production" + + - name: "worker-queue" + command: "python worker.py" + autostart: true + envs: + - "QUEUE_URL=redis://localhost:6379" + + - name: "monitoring" + command: "./monitor.sh" + ignore: true +``` + +## Common Pitfalls + +- Using `services.yaml` to select the image (use `deploy.yml` instead). +- Forgetting to mount `services.yaml` into the container. + +## Related Docs + +- `docs/runtime/config.md` +- `docs/deploy/README.md` diff --git a/docs/services.md b/docs/services.md deleted file mode 100644 index ac593c67..00000000 --- a/docs/services.md +++ /dev/null @@ -1,185 +0,0 @@ -# Service Configuration - -## Overview - -The UDX Worker uses `services.yaml` to define and manage multiple services. Each service can be configured with its own runtime settings, environment variables, and behavior policies. - -## File Location - -```bash -/home/udx/.config/worker/services.yaml -``` - -## Configuration Structure - -| Field | Type | Required | Default | Description | -| ---------- | ------ | -------- | ------- | ---------------------------------- | -| `kind` | string | Yes | - | Must be `workerService` | -| `version` | string | Yes | - | Must be `udx.io/worker-v1/service` | -| `services` | array | Yes | - | List of service definitions | - -### Service Definition Fields - -| Field | Type | Required | Default | Description | -| ------------- | ------- | -------- | ------- | ------------------------- | -| `name` | string | Yes | - | Unique service identifier | -| `command` | string | Yes | - | Command to execute | -| `ignore` | boolean | No | `false` | Skip service management | -| `autostart` | boolean | No | `true` | Start on worker launch | -| `autorestart` | boolean | No | `false` | Restart on failure | -| `envs` | array | No | `[]` | Environment variables | -| `stopasgroup` | boolean | No | `false` | Stop process group | -| `killasgroup` | boolean | No | `false` | Kill process group | - -## Basic Example - -```yaml -kind: workerService -version: udx.io/worker-v1/service -services: - - name: "web-server" - command: "python app.py" - autostart: true - autorestart: true - envs: - - "PORT=8080" - - "DEBUG=true" -``` - -## Advanced Examples - -### Multiple Services - -```yaml -kind: workerService -version: udx.io/worker-v1/service -services: - - name: "api-server" - command: "npm start" - autostart: true - autorestart: true - stopasgroup: true - killasgroup: true - envs: - - "PORT=3000" - - "NODE_ENV=production" - - - name: "worker-queue" - command: "python worker.py" - autostart: true - envs: - - "QUEUE_URL=redis://localhost:6379" - - - name: "monitoring" - command: "./monitor.sh" - ignore: true # Temporarily disabled -``` - -### Service with Complex Command - -```yaml -services: - - name: "data-processor" - command: "bash -c 'source .env && python -m processor.main --config=prod.json'" - autostart: true - autorestart: true - envs: - - "PYTHONPATH=/app" - - "LOG_LEVEL=info" -``` - -## Best Practices - -1. **Service Naming** - - - Use descriptive, lowercase names - - Separate words with hyphens - - Keep names concise but meaningful - -2. **Command Definition** - - - Use absolute paths when possible - - Quote commands with spaces or special characters - - Consider using shell scripts for complex commands - -3. **Environment Variables** - - - Use uppercase for variable names - - Group related variables together - - Document required variables - -4. **Restart Policies** - - Enable `autorestart` for critical services - - Use `ignore` for maintenance or debugging - - Consider dependencies between services - -## Service Status - -When running `worker service list`, services show the following status indicators: - -| Symbol | Status | Description | -| ------ | -------- | ---------------------------------------------- | -| โœ… | RUNNING | Service is running normally | -| โ›” | STOPPED | Service was stopped with `worker service stop` | -| ๐Ÿ’€ | FATAL | Service exited (any exit code) | -| ๐Ÿ”„ | RETRY | Service is retrying (with autorestart: true) | -| โš ๏ธ | STARTING | Service is starting up | - -### Service Types - -1. **Long-running Services** - - ```yaml - - name: "web-server" - command: "python server.py" - autorestart: true # Restarts on exit - ``` - - - Shows as RUNNING (โœ…) while active - - Shows as RETRY (๐Ÿ”„) then FATAL (๐Ÿ’€) if it keeps failing - -2. **One-shot Tasks** - ```yaml - - name: "setup" - command: "./setup.sh" - autorestart: false # Runs once - ``` - - Use `worker service stop` for clean completion (โ›”) - - Otherwise shows as FATAL (๐Ÿ’€) after exit - -Note: Exit codes (0 or non-zero) don't affect the final status. What matters is: - -- Whether the service keeps running (RUNNING โœ…) -- How it stops (STOPPED โ›” vs FATAL ๐Ÿ’€) - -Example: - -```bash -# Long-running service and one-shot task -$ worker service list -โœ… service1 RUNNING pid 123 0:01:23 -โ›” service2 STOPPED Apr 08 11:30 AM - -# Long-running service and failed task -$ worker service list -โœ… service1 RUNNING pid 123 0:01:23 -๐Ÿ’€ service2 FATAL Apr 08 11:30 AM -``` - -## Monitoring and Management - -Use the following CLI commands to manage services: - -```bash -# List all services -worker service list - -# Check specific service status -worker service status web-server - -# View service logs -worker service logs web-server - -# Restart a service -worker service restart web-server -``` diff --git a/src/examples/README.md b/src/examples/README.md new file mode 100644 index 00000000..dbb30d14 --- /dev/null +++ b/src/examples/README.md @@ -0,0 +1,18 @@ +# Examples + +## simple-service + +A set of small service scripts used to demonstrate supervisor behavior: + +- `10_long_running.sh` +- `20_clean_exit.sh` +- `30_syntax_error.sh` +- `40_connection_error.sh` +- `50_rapid_exit.sh` + +## deploy-image-override + +Shows how to override the worker image in CI/CD using a deploy template: + +- `deploy.template.yml` +- `README.md` diff --git a/src/examples/deploy-image-override/README.md b/src/examples/deploy-image-override/README.md new file mode 100644 index 00000000..3f05a663 --- /dev/null +++ b/src/examples/deploy-image-override/README.md @@ -0,0 +1,29 @@ +# Deploy Image Override (CI/CD) + +This example shows how to override the worker image at deploy time using an environment variable and a deploy template. + +Related docs: `docs/deploy/README.md` + +## Steps + +```bash +cd src/examples/deploy-image-override + +export WORKER_IMAGE="usabilitydynamics/udx-worker:latest" + +# Render deploy.yml from the template +envsubst < deploy.template.yml > deploy.yml + +# Run container using worker-deployment +worker run --config=deploy.yml +``` + +## Notes + +- `services.yaml` controls processes inside the container, not the container image. +- The image is selected in `deploy.yml` (worker-deployment). +- If `envsubst` is unavailable, use: + +```bash +sed "s|\${WORKER_IMAGE}|${WORKER_IMAGE}|g" deploy.template.yml > deploy.yml +``` diff --git a/src/examples/deploy-image-override/deploy.template.yml b/src/examples/deploy-image-override/deploy.template.yml new file mode 100644 index 00000000..87e7108b --- /dev/null +++ b/src/examples/deploy-image-override/deploy.template.yml @@ -0,0 +1,8 @@ +--- +kind: workerDeployConfig +version: udx.io/worker-v1/deploy +config: + image: "${WORKER_IMAGE}" + command: "echo" + args: + - "Hello from CI/CD" diff --git a/src/examples/simple-service/README.md b/src/examples/simple-service/README.md new file mode 100644 index 00000000..44044644 --- /dev/null +++ b/src/examples/simple-service/README.md @@ -0,0 +1,43 @@ +# Simple Service Examples + +These scripts demonstrate common service behaviors for `services.yaml`. + +Related docs: `docs/runtime/services.md` + +## Scripts + +- `10_long_running.sh` - long-running process +- `20_clean_exit.sh` - exits cleanly +- `30_syntax_error.sh` - fails due to syntax error +- `40_connection_error.sh` - simulates a connection failure +- `50_rapid_exit.sh` - exits quickly + +## Usage + +Create a `services.yaml` that points to one or more scripts: + +```yaml +kind: workerService +version: udx.io/worker-v1/service +services: + - name: "long-running" + command: "/home/udx/10_long_running.sh" + autostart: true + autorestart: true +``` + +Mount the directory when running the container: + +```bash +docker run -d \ + --name my-service \ + -v "$(pwd)/src/examples/simple-service:/home/udx" \ + -v "$(pwd)/.config/worker:/home/udx/.config/worker" \ + usabilitydynamics/udx-worker:latest +``` + +Then check logs: + +```bash +worker service logs long-running +``` From 91cd820e8551b0ce0fe596501b18f266db28dd49 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 5 Feb 2026 23:08:00 +0300 Subject: [PATCH 3/7] remove Bitwarden integration --- Dockerfile | 14 +----- docs/auth/README.md | 41 ++++++++-------- docs/auth/bitwarden.md | 14 ------ docs/authorization.md | 86 ++++++++++----------------------- etc/configs/worker/default.yaml | 1 - lib/auth/bitwarden.sh | 60 ----------------------- lib/cleanup.sh | 5 -- lib/cli/auth.sh | 12 +---- lib/secrets/bitwarden.sh | 32 ------------ lib/utils.sh | 4 +- 10 files changed, 49 insertions(+), 220 deletions(-) delete mode 100644 docs/auth/bitwarden.md delete mode 100644 lib/auth/bitwarden.sh delete mode 100644 lib/secrets/bitwarden.sh diff --git a/Dockerfile b/Dockerfile index 14433b21..cba13b6e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -101,18 +101,6 @@ RUN ARCH=$(uname -m) && \ ./aws/install && \ rm -rf awscliv2.zip aws /tmp/* /var/tmp/* -# Install Bitwarden CLI (architecture-aware) -RUN ARCH=$(uname -m) && \ - if [ "$ARCH" = "x86_64" ]; then \ - curl -Lso /usr/local/bin/bw "https://vault.bitwarden.com/download/linux/amd64/bw"; \ - elif [ "$ARCH" = "aarch64" ]; then \ - curl -Lso /usr/local/bin/bw "https://vault.bitwarden.com/download/linux/arm64/bw"; \ - else \ - echo "Unsupported architecture: $ARCH" && exit 1; \ - fi && \ - chmod +x /usr/local/bin/bw && \ - rm -rf /tmp/* /var/tmp/* - # Create a new user and group with specific UID and GID, and set permissions RUN groupadd -g ${GID} ${USER} && \ useradd -l -m -u ${UID} -g ${GID} -s /bin/bash ${USER} @@ -212,4 +200,4 @@ USER ${USER} ENTRYPOINT ["/usr/local/worker/bin/entrypoint.sh"] # Set the default command -CMD ["tail", "-f", "/dev/null"] \ No newline at end of file +CMD ["tail", "-f", "/dev/null"] diff --git a/docs/auth/README.md b/docs/auth/README.md index f29fdf4b..17927107 100644 --- a/docs/auth/README.md +++ b/docs/auth/README.md @@ -1,34 +1,31 @@ # Provider Authentication Guides +## Overview + This directory contains detailed authentication documentation for each supported cloud provider. -## Available Guides +## When To Use + +Use these guides when you need provider-specific setup, credential formats, or CLI integration details. -- **[GCP Authentication](gcp.md)** - Complete guide for Google Cloud Platform - - Service Account Keys - - Workload Identity Tokens - - Service Account Impersonation - - worker-deployment CLI integration +## Key Concepts -- **[Azure Authentication](azure.md)** - Coming soon -- **[AWS Authentication](aws.md)** - Coming soon -- **[Bitwarden Authentication](bitwarden.md)** - Coming soon +- Each provider has its own auth flow and credential structure. +- Worker-deployment CLI can simplify authentication setup. -## Quick Links +## Examples -- [Main Authorization Guide](../authorization.md) - Overview and general credential formats -- [Worker Configuration](../config.md) - Worker configuration reference -- [CLI Documentation](../CLI.md) - Worker CLI commands +- `docs/auth/gcp.md` - GCP authentication guide +- `docs/auth/azure.md` - Azure authentication guide (coming soon) +- `docs/auth/aws.md` - AWS authentication guide (coming soon) -## Contributing +## Common Pitfalls -When adding a new provider authentication guide, please follow this structure: +- Mixing provider-specific formats. +- Passing plain secrets directly in config files. -1. **Authentication Methods** - List all supported authentication methods -2. **JSON Format Examples** - Show credential structure -3. **Usage Examples** - Provide practical examples (env vars, file paths, base64) -4. **Features** - Highlight key features and capabilities -5. **Best Practices** - Security and operational recommendations -6. **CLI Integration** - If applicable, show worker-deployment CLI usage +## Related Docs -See [gcp.md](gcp.md) as a reference template. +- `docs/authorization.md` +- `docs/runtime/config.md` +- `docs/reference/cli.md` diff --git a/docs/auth/bitwarden.md b/docs/auth/bitwarden.md deleted file mode 100644 index f1d2a040..00000000 --- a/docs/auth/bitwarden.md +++ /dev/null @@ -1,14 +0,0 @@ -# Bitwarden Authentication - -> **Coming Soon**: Detailed Bitwarden authentication documentation - -## Quick Reference - -**Environment Variable:** `BITWARDEN_CREDS` - -**Supported Formats:** -- JSON -- Base64-encoded JSON -- File path - -For now, see the [general authorization guide](../authorization.md) for credential format examples. diff --git a/docs/authorization.md b/docs/authorization.md index daf5233e..d9e48a9d 100644 --- a/docs/authorization.md +++ b/docs/authorization.md @@ -4,30 +4,29 @@ The UDX Worker supports multiple cloud providers and services through environment-based credential management. -## Supported Providers +## When To Use -| Provider | Environment Variable | Description | -| --------- | -------------------- | ---------------------------------------- | -| Azure | `AZURE_CREDS` | Azure cloud credentials | -| AWS | `AWS_CREDS` | Amazon Web Services credentials | -| GCP | `GCP_CREDS` | Google Cloud Platform credentials | -| Bitwarden | `BITWARDEN_CREDS` | Bitwarden secrets management credentials | +Use this when you need to: -> **๐Ÿ’ก Tip**: For simplified deployment with automatic credential detection, use the [`@udx/worker-deployment`](https://www.npmjs.com/package/@udx/worker-deployment) CLI tool. +- Provide credentials to the worker container. +- Understand supported providers and formats. -## Credential Formats +## Key Concepts -Credentials can be provided in three formats: +- Credentials can be provided via env vars. +- Secrets can be JSON, Base64, or file paths. -| Format | Description | Use Case | -| --------- | ------------------- | ---------------------------- | -| JSON | Plain JSON string | Direct configuration | -| Base64 | Base64-encoded JSON | Secure environment variables | -| File Path | Path to JSON file | Local development | +## Examples -## Format Examples +### Supported Providers -### 1. JSON Format +| Provider | Environment Variable | Description | +| -------- | -------------------- | --------------------------------- | +| Azure | `AZURE_CREDS` | Azure cloud credentials | +| AWS | `AWS_CREDS` | Amazon Web Services credentials | +| GCP | `GCP_CREDS` | Google Cloud Platform credentials | + +### JSON Format ```json { @@ -38,60 +37,25 @@ Credentials can be provided in three formats: } ``` -### 2. Base64 Encoded Format - -```bash -# Original JSON -{ - "client_id": "CLIENT_ID", - "client_secret": "CLIENT_SECRET", - "tenant_id": "TENANT_ID", - "subscription_id": "SUBSCRIPTION_ID" -} - -# Base64 encoded value -ewogICAgImNsaWVudF9pZCI6ICJDTElFTlRfSUQiLAogICAgImNsaWVudF9zZWNyZXQiOiAiQ0xJRU5UX1NFQ1JFVCIsCiAgICAidGVuYW50X2lkIjogIlRFTkFOVF9JRCIsCiAgICAic3Vic2NyaXB0aW9uX2lkIjogIlNVQlNDUklQVElPTl9JRCIKfQ== -``` - -**Generate Base64 Format:** +### Base64 Format ```bash echo -n '{"client_id":"CLIENT_ID","client_secret":"CLIENT_SECRET","tenant_id":"TENANT_ID","subscription_id":"SUBSCRIPTION_ID"}' | base64 ``` -### 3. File Path Format +### File Path ```bash -# Environment variable value AZURE_CREDS="/path/to/azure_credentials.json" - -# Credential file content (azure_credentials.json) -{ - "client_id": "CLIENT_ID", - "client_secret": "CLIENT_SECRET", - "tenant_id": "TENANT_ID", - "subscription_id": "SUBSCRIPTION_ID" -} ``` -> **Note**: Always use absolute paths in production environments to avoid path resolution issues. - -## Provider-Specific Authentication - -For detailed authentication guides for each provider, see: - -- **[GCP Authentication](auth/gcp.md)** - Service account keys, workload identity, impersonation -- **[Azure Authentication](auth/azure.md)** - Coming soon -- **[AWS Authentication](auth/aws.md)** - Coming soon -- **[Bitwarden Authentication](auth/bitwarden.md)** - Coming soon - -## Credential Management +## Common Pitfalls -| Flag | Default | Description | -| ---------------- | ------- | ------------------------------------------------------------------------- | -| `ACTORS_CLEANUP` | `true` | Controls how cloud provider credentials are handled after authentication: | +- Using relative credential paths in production. +- Storing secrets in version control. -- When `true` (default): All temporary credentials and login sessions are cleaned up after use, improving security by not leaving credentials on disk -- When `false`: Credentials are preserved on disk for reuse (e.g., GCP credentials are stored at `$HOME/creds/gcp_creds.json` and `GOOGLE_APPLICATION_CREDENTIALS` is set) +## Related Docs -This flag is particularly useful for long-running processes or development environments where frequent re-authentication would be inefficient. \ No newline at end of file +- `docs/runtime/config.md` +- `docs/deploy/README.md` +- `docs/auth/README.md` diff --git a/etc/configs/worker/default.yaml b/etc/configs/worker/default.yaml index 86488b1a..c30a3ca3 100644 --- a/etc/configs/worker/default.yaml +++ b/etc/configs/worker/default.yaml @@ -9,5 +9,4 @@ config: creds: "${AZURE_CREDS}" - type: aws creds: "${AWS_CREDS}" - - type: bitwarden creds: "${BITWARDEN_CREDS}" diff --git a/lib/auth/bitwarden.sh b/lib/auth/bitwarden.sh deleted file mode 100644 index cd221b05..00000000 --- a/lib/auth/bitwarden.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source "${WORKER_LIB_DIR}/utils.sh" - -# Function to authenticate Bitwarden using API key or master password -# -# Example usage of the function -# bitwarden_authenticate "/path/to/your/bitwarden_creds.json" - -bitwarden_authenticate() { - local creds_file="$1" - - if [[ ! -f "$creds_file" ]]; then - log_error "Bitwarden Authentication" "Credentials file not found: $creds_file" - return 1 - fi - - # Read the credentials file - local creds_content - creds_content=$(cat "$creds_file") - - if [[ -z "$creds_content" ]]; then - log_error "Bitwarden Authentication" "Credentials file is empty: $creds_file" - return 1 - fi - - # Extract API key or master password - local api_key master_password - api_key=$(echo "$creds_content" | jq -r '.apiKey // empty') - master_password=$(echo "$creds_content" | jq -r '.masterPassword // empty') - - if [[ -z "$api_key" && -z "$master_password" ]]; then - log_error "Bitwarden Authentication" "Either API key or master password must be provided in the credentials file." - return 1 - fi - - # Authenticate with Bitwarden and get the session key - local session_key - if [[ -n "$api_key" ]]; then - session_key=$(bw login --apikey "$api_key" --raw 2>/dev/null) - elif [[ -n "$master_password" ]]; then - local email - email=$(echo "$creds_content" | jq -r '.email // empty') - if [[ -z "$email" ]]; then - log_error "Bitwarden Authentication" "Email must be provided with the master password." - return 1 - fi - session_key=$(bw login "$email" "$master_password" --raw 2>/dev/null) - fi - - if [[ -z "$session_key" ]]; then - log_error "Bitwarden Authentication" "Failed to authenticate with Bitwarden." - return 1 - fi - - log_success "Bitwarden Authentication" "Authenticated with Bitwarden." - export BW_SESSION="$session_key" - return 0 -} \ No newline at end of file diff --git a/lib/cleanup.sh b/lib/cleanup.sh index 274ccf79..87de9fdf 100644 --- a/lib/cleanup.sh +++ b/lib/cleanup.sh @@ -131,11 +131,6 @@ cleanup_actors() { any_cleanup=true fi ;; - bitwarden) - if cleanup_provider "bw" "bw logout --force" "bw status" "Bitwarden"; then - any_cleanup=true - fi - ;; *) log_warn "Unsupported or unavailable actor type for cleanup: $provider" ;; diff --git a/lib/cli/auth.sh b/lib/cli/auth.sh index d1c0f981..23cd77c2 100644 --- a/lib/cli/auth.sh +++ b/lib/cli/auth.sh @@ -105,7 +105,7 @@ show_auth_status() { check_provider_status "aws" check_provider_status "gcp" check_provider_status "azure" - check_provider_status "bitwarden" + fi # Close JSON array if json format @@ -171,16 +171,13 @@ logout_provider() { gcp) cleanup_provider "gcloud" "gcloud auth revoke --all" "gcloud auth list" "GCP" ;; - bitwarden) - cleanup_provider "bw" "bw logout --force" "bw status" "Bitwarden" - ;; esac } if [ -n "$target_provider" ]; then do_provider_logout "$target_provider" else - for provider in aws gcp azure bitwarden; do + for provider in aws gcp azure; do do_provider_logout "$provider" done fi @@ -210,11 +207,6 @@ check_provider_auth() { return 0 fi ;; - bitwarden) - if bw status | grep -q "unlocked"; then - return 0 - fi - ;; esac return 1 diff --git a/lib/secrets/bitwarden.sh b/lib/secrets/bitwarden.sh deleted file mode 100644 index 1e9f376e..00000000 --- a/lib/secrets/bitwarden.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source "${WORKER_LIB_DIR}/utils.sh" - -# Function to resolve Bitwarden secret -resolve_bitwarden_secret() { - local collection_name="$1" - local secret_name="$2" - local secret_value - - if [[ -z "$collection_name" || -z "$secret_name" ]]; then - log_error "Bitwarden" "Invalid Bitwarden collection name or secret name" >&2 - return 1 - fi - - # Retrieve the secret using Bitwarden CLI - log_info "Bitwarden" "Retrieving secret from Bitwarden: collection_name=$collection_name, secret_name=$secret_name" >&2 - if ! secret_value=$(bw get item "$secret_name" --organizationid "$collection_name" --output text 2>&1); then - log_error "Bitwarden" "Failed to retrieve secret from Bitwarden: collection_name=$collection_name, secret_name=$secret_name" >&2 - log_error "Bitwarden" "Bitwarden CLI output: $secret_value" >&2 - return 1 - fi - - if [ -z "$secret_value" ]; then - log_error "Bitwarden" "Secret value is empty for $secret_name in collection $collection_name" >&2 - return 1 - fi - - printf "%s" "$secret_value" - return 0 -} \ No newline at end of file diff --git a/lib/utils.sh b/lib/utils.sh index ceeba1b4..c2a35320 100644 --- a/lib/utils.sh +++ b/lib/utils.sh @@ -3,7 +3,7 @@ # Supported secret providers (used for secret reference detection) # Only declare if not already defined (prevents errors when sourced multiple times) if [[ -z "${SUPPORTED_SECRET_PROVIDERS+x}" ]]; then - readonly SUPPORTED_SECRET_PROVIDERS="gcp|azure|aws|bitwarden" +readonly SUPPORTED_SECRET_PROVIDERS="gcp|azure|aws" fi # Function to resolve placeholders with environment variables @@ -43,4 +43,4 @@ log_warn() { log_error() { # Kept >&2 to send to stderr printf "\033[1;31m[ERROR]\033[0m %s: %s\n" "$1" "$2" >&2 -} \ No newline at end of file +} From df9cfebd6b3099f7f8c5e98637838f64dd224c85 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 5 Feb 2026 23:20:03 +0300 Subject: [PATCH 4/7] tweak --- etc/configs/worker/default.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/etc/configs/worker/default.yaml b/etc/configs/worker/default.yaml index c30a3ca3..5ec380b4 100644 --- a/etc/configs/worker/default.yaml +++ b/etc/configs/worker/default.yaml @@ -9,4 +9,3 @@ config: creds: "${AZURE_CREDS}" - type: aws creds: "${AWS_CREDS}" - creds: "${BITWARDEN_CREDS}" From eaea3c0f90152e6a848538e2820aac3add03faf0 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 5 Feb 2026 23:29:42 +0300 Subject: [PATCH 5/7] upgrade python dependencies --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index cba13b6e..46972180 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,8 +50,8 @@ RUN apt-get update && \ unzip=6.0-28ubuntu7 \ nano=8.4-1 \ vim=2:9.1.0967-1ubuntu6 \ - python3.13=3.13.7-1ubuntu0.1 \ - python3.13-venv=3.13.7-1ubuntu0.1 \ + python3.13=3.13.7-1ubuntu0.3 \ + python3.13-venv=3.13.7-1ubuntu0.3 \ python3-pip=25.1.1+dfsg-1ubuntu2 \ supervisor=4.2.5-3 && \ # Install Azure CLI in venv with optimizations for scanning From 97e3bbb872e3ed3e8ee0b48e0303fa07919d5dbd Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 5 Feb 2026 23:44:55 +0300 Subject: [PATCH 6/7] ci: migrate to docker-ops workflow --- .github/workflows/build-and-test.yml | 106 ------------------ .github/workflows/docker-ops.yml | 36 ++++++ .github/workflows/release.yml | 160 --------------------------- 3 files changed, 36 insertions(+), 266 deletions(-) delete mode 100644 .github/workflows/build-and-test.yml create mode 100644 .github/workflows/docker-ops.yml delete mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml deleted file mode 100644 index b7fbf4a1..00000000 --- a/.github/workflows/build-and-test.yml +++ /dev/null @@ -1,106 +0,0 @@ ---- -name: Build and Test Docker Image - -"on": - push: - branches-ignore: - - latest - paths: - - '.github/workflows/build-and-test.yml' - - 'Dockerfile' - - 'bin/**' - - 'lib/**' - - 'src/**' - - 'etc/**' - - 'tests/**' - - 'Makefile' - - 'Makefile.variables' - -jobs: - build: - runs-on: ubuntu-24.04 - permissions: - contents: write - - steps: - - name: Checkout code - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Dev Pipeline - run: | - echo "AZURE_CREDS=$AZURE_CREDS" > .env - make dev-pipeline - env: - AZURE_CREDS: ${{ secrets.AZURE_CREDS }} - working-directory: . - - - name: Install Trivy - run: | - curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | \ - sudo sh -s -- -b /usr/local/bin - - - name: Trivy Vulnerability Scanning - run: | - export TRIVY_DISABLE_VEX_NOTICE=true - - # Disable exit on error for the retry logic - set +e - - # Retry logic for Trivy - max_retries=5 - attempt=1 - success=false - - while [ $attempt -le $max_retries ]; do - echo "Running Trivy scan, attempt $attempt..." - - # Run the Trivy scan and capture the exit status - trivy image --severity CRITICAL --exit-code 1 --quiet \ - usabilitydynamics/udx-worker:latest | tee trivy.log | grep -v 'INFO' - scan_exit_code=$? - - # Check if CRITICAL vulnerabilities were detected - if grep -E "Total: [1-9]" trivy.log; then - echo "CRITICAL vulnerabilities detected! Exiting." - exit 1 - fi - - # Handle a successful scan (no critical vulnerabilities found) - if [ $scan_exit_code -eq 0 ]; then - echo "No CRITICAL vulnerabilities found." - success=true - break - else - echo "Trivy scan encountered an error, retrying in 2 minutes..." - sleep 120 - attempt=$((attempt+1)) - fi - done - - # Exit if all retries fail without a successful scan - if [ "$success" = false ]; then - echo "Failed to complete Trivy scan after $max_retries attempts." - exit 1 - fi - - - name: Trivy SBOM Generation - run: | - export TRIVY_DISABLE_VEX_NOTICE=true - trivy image --format spdx-json --output sbom.json usabilitydynamics/udx-worker:latest 2>/dev/null - - echo "SBOM Top Packages Summary:" - echo "| Package Name | Version |" - echo "|-------------------|-----------|" - - jq -r '.packages[] | select(.versionInfo != null) | "\(.name) | \(.versionInfo)"' sbom.json | sort | uniq | head -n 20 | column -t -s '|' - - - name: Upload SBOM Artifact - uses: actions/upload-artifact@v6 - with: - name: sbom - path: sbom.json diff --git a/.github/workflows/docker-ops.yml b/.github/workflows/docker-ops.yml new file mode 100644 index 00000000..c099c85f --- /dev/null +++ b/.github/workflows/docker-ops.yml @@ -0,0 +1,36 @@ +--- +name: Docker Ops + +"on": + push: + branches: + - "**" + paths: + - ".github/workflows/docker-ops.yml" + - "Dockerfile" + - "bin/**" + - "lib/**" + - "src/**" + - "etc/**" + - "tests/**" + - "Makefile" + - "Makefile.variables" + - "ci/**" + workflow_dispatch: + +jobs: + docker-ops: + permissions: + contents: write + security-events: write + id-token: write + uses: udx/reusable-workflows/.github/workflows/docker-ops.yml@master + with: + image_name: usabilitydynamics/udx-worker + docker_login: usabilitydynamics + docker_org: usabilitydynamics + docker_repo: udx-worker + release_branch: latest + version_config_path: ci/git-version.yml + secrets: + docker_token: ${{ secrets.DOCKER_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 55fa5366..00000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,160 +0,0 @@ ---- -name: Release - -"on": - push: - branches: - - "latest" - paths: - - '.github/workflows/release.yml' - - 'Dockerfile' - - 'bin/**' - - 'lib/**' - - 'src/**' - - 'etc/**' - - 'tests/**' - - 'Makefile' - - LICENSE - -jobs: - docker-release: - runs-on: ubuntu-24.04 - permissions: - id-token: write - contents: write - - outputs: - semVer: ${{ steps.gitversion.outputs.semVer }} - changelog: ${{ steps.changelog.outputs.changelog }} - - steps: - - name: Checkout code - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - driver: docker-container - - - name: Install GitVersion - uses: gittools/actions/gitversion/setup@v4.1.0 - with: - versionSpec: "6.1.0" - - - name: Clear GitVersion Cache - run: rm -rf .git/gitversion_cache - - - name: Determine Version - id: gitversion - uses: gittools/actions/gitversion/execute@v4.1.0 - with: - useConfigFile: true - configFilePath: ci/git-version.yml - - - name: Generate Changelog - id: changelog - run: | - git log $(git describe --tags --abbrev=0)..HEAD -- . \ - --pretty=format:"- %s" > changelog.txt - CHANGELOG=$(cat changelog.txt | jq -sRr @uri) - echo "changelog<> $GITHUB_ENV - echo "$CHANGELOG" >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - - name: Log in to Docker Hub - uses: docker/login-action@v3 - with: - username: "usabilitydynamics" - password: ${{ secrets.DOCKER_TOKEN }} - - - name: Build and Push Docker Image - id: docker_push - uses: docker/build-push-action@v6 - with: - context: . - platforms: linux/amd64, linux/arm64 - push: true - sbom: true - provenance: true - tags: | - usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} - usabilitydynamics/udx-worker:latest - - - name: Install Trivy - run: | - curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | \ - sudo sh -s -- -b /usr/local/bin - - - name: Generate SBOM with Retry Logic - id: generate-sbom - run: | - export TRIVY_DISABLE_VEX_NOTICE=true - max_retries=10 - attempt=1 - success=false - while [ $attempt -le $max_retries ]; do - echo "Generating SBOM, attempt $attempt..." - output=$(trivy image --format spdx-json --output sbom.json usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} 2>&1) - sbom_exit_code=$? - if [ $sbom_exit_code -eq 0 ]; then - echo "SBOM generation successful." - success=true - break - else - echo "Retrying in 120 seconds..." - sleep 120 - attempt=$((attempt+1)) - fi - done - if [ "$success" = false ]; then - exit 1 - fi - - - name: Upload SBOM Artifact - uses: actions/upload-artifact@v6 - with: - name: sbom - path: sbom.json - - - name: Log out from Docker Hub - run: docker logout - - github-release: - runs-on: ubuntu-24.04 - needs: docker-release - permissions: - contents: write - - steps: - - name: Checkout code - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Configure Git for Pushing - run: | - git config --global user.email "worker@udx.io" - git config --global user.name "UDX Worker" - - - name: Download SBOM Artifact - uses: actions/download-artifact@v7 - with: - name: sbom - - - name: Create GitHub Release - uses: softprops/action-gh-release@v2 - with: - tag_name: ${{ needs.docker-release.outputs.semVer }} - body: | - Release version ${{ needs.docker-release.outputs.semVer }}. - [View on Docker Hub](https://hub.docker.com/r/usabilitydynamics/udx-worker/tags?page=1&ordering=last_updated). - ${{ needs.docker-release.outputs.changelog }} - draft: false - prerelease: false - files: sbom.json - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} From 011e01c6ee00516c38ba3839835a6d1accaf5304 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Fri, 6 Feb 2026 00:25:05 +0300 Subject: [PATCH 7/7] test: prune examples and stabilize service test --- Makefile | 4 ++-- src/examples/README.md | 6 ++++++ .../simple-config/.config/worker/worker.yaml | 4 +--- src/examples/simple-config/README.md | 19 +++++++++++++++++++ src/tests/modules/40_service.sh | 14 ++++++++++---- 5 files changed, 38 insertions(+), 9 deletions(-) create mode 100644 src/examples/simple-config/README.md diff --git a/Makefile b/Makefile index d2c61930..6387c2d2 100644 --- a/Makefile +++ b/Makefile @@ -102,7 +102,7 @@ clean: test: clean @printf "$(COLOR_BLUE)$(SYM_ARROW) Running tests...$(COLOR_RESET)\n" @$(MAKE) run \ - VOLUMES="$(PWD)/src/tests:/home/udx/tests $(PWD)/src/examples/simple-config/.config/worker/worker.yaml:/home/udx/.config/worker/worker.yaml $(PWD)/src/examples/simple-service/.config/worker/services.yaml:/home/udx/.config/worker/services.yaml $(PWD)/src/examples/simple-service/index.sh:/home/udx/index.sh" \ + VOLUMES="$(PWD)/src/tests:/home/udx/tests $(PWD)/src/examples/simple-config/.config/worker/worker.yaml:/home/udx/.config/worker/worker.yaml $(PWD)/src/examples/simple-service/.config/worker/services.yaml:/home/udx/.config/worker/services.yaml" \ COMMAND="/home/udx/tests/main.sh" @printf "$(COLOR_BLUE)$(SYM_ARROW) Following test output...$(COLOR_RESET)\n" @docker logs -f $(CONTAINER_NAME) & LOGS_PID=$$!; \ @@ -113,4 +113,4 @@ test: clean @printf "$(COLOR_GREEN)$(SYM_SUCCESS) Tests completed successfully$(COLOR_RESET)\n" dev-pipeline: build test - @printf "$(COLOR_GREEN)$(SYM_SUCCESS) Development pipeline completed successfully$(COLOR_RESET)\n" \ No newline at end of file + @printf "$(COLOR_GREEN)$(SYM_SUCCESS) Development pipeline completed successfully$(COLOR_RESET)\n" diff --git a/src/examples/README.md b/src/examples/README.md index dbb30d14..96b3b68e 100644 --- a/src/examples/README.md +++ b/src/examples/README.md @@ -10,6 +10,12 @@ A set of small service scripts used to demonstrate supervisor behavior: - `40_connection_error.sh` - `50_rapid_exit.sh` +## simple-config + +Minimal `worker.yaml` used by tests and quick local runs: + +- `.config/worker/worker.yaml` + ## deploy-image-override Shows how to override the worker image in CI/CD using a deploy template: diff --git a/src/examples/simple-config/.config/worker/worker.yaml b/src/examples/simple-config/.config/worker/worker.yaml index 601c9ef7..b8cea08f 100644 --- a/src/examples/simple-config/.config/worker/worker.yaml +++ b/src/examples/simple-config/.config/worker/worker.yaml @@ -4,6 +4,4 @@ version: udx.io/worker-v1/config config: env: CREATED: "2025-02-21T07:57:59Z" - secrets: - TEST_ENV: "azure/kv-udx-worker-tooling/test-secret" - ANOTHER_TEST_ENV: "azure/kv-udx-worker-tooling/test-secret" + EXAMPLE_MODE: "simple-config" diff --git a/src/examples/simple-config/README.md b/src/examples/simple-config/README.md new file mode 100644 index 00000000..feb17e56 --- /dev/null +++ b/src/examples/simple-config/README.md @@ -0,0 +1,19 @@ +# Simple Config + +Minimal config used by tests and quick local runs. This keeps the example +small and avoids external secret providers. + +## Files + +- `.config/worker/worker.yaml` + +## Usage + +Mount it into the container: + +```bash +docker run -d \ + --name worker-simple-config \ + -v "$(pwd)/src/examples/simple-config/.config/worker:/home/udx/.config/worker" \ + usabilitydynamics/udx-worker:latest +``` diff --git a/src/tests/modules/40_service.sh b/src/tests/modules/40_service.sh index e11faf05..63112262 100755 --- a/src/tests/modules/40_service.sh +++ b/src/tests/modules/40_service.sh @@ -34,10 +34,16 @@ if [ -z "$CONFIG_OUTPUT" ]; then print_error "service config produced no output" exit 1 fi -if echo "$STATUS_OUTPUT" | grep -q "Simple service is running"; then - print_error "service should be stopped" - exit 1 -fi + +# Validate expected service names exist in JSON output +EXPECTED_SERVICES="10-long-running 20-clean-exit 30-syntax-error 40-connection-error 50-rapid-exit" +CONFIG_JSON=$(worker service config --format json 2>&1) +for svc in $EXPECTED_SERVICES; do + if ! echo "$CONFIG_JSON" | jq -e --arg svc "$svc" '.content.services[] | select(.name == $svc) | .name' > /dev/null; then + print_error "service config missing expected service: $svc" + exit 1 + fi +done # All tests passed print_success "All service tests passed"