From 82774429895f2e2306a73b921fa635a4e4596108 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:01:55 +0000 Subject: [PATCH 01/37] Initial plan From 048d9f499249df4641a07f603430219776ce366f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:03:56 +0000 Subject: [PATCH 02/37] Initial plan From d334eced43a0867d73cee84c61bb6fe0017440da Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:05:45 +0000 Subject: [PATCH 03/37] Initial plan From a2d66a1c639fa8779a990b9c7bc66a5e78b9e119 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:06:52 +0000 Subject: [PATCH 04/37] Add configurable HTTP and HTTPS ports for Caddy container via environment variables Co-authored-by: jaredlander <628977+jaredlander@users.noreply.github.com> --- .../shinyproxyoperator/impl/docker/CaddyConfig.kt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/CaddyConfig.kt b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/CaddyConfig.kt index fb5a5aa..5baa649 100644 --- a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/CaddyConfig.kt +++ b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/CaddyConfig.kt @@ -59,6 +59,8 @@ class CaddyConfig(private val dockerClient: DockerClient, mainDataDir: Path, con private val fileManager = FileManager() private val caddyImage: String = config.readConfigValue("docker.io/library/caddy:2.8", "SPO_CADDY_IMAGE") { it } private val enableTls = config.readConfigValue(false, "SPO_CADDY_ENABLE_TLS") { it.toBoolean() } + private val caddyPortHttp: Int = config.readConfigValue(80, "SPO_CADDY_PORT_HTTP") { it.toInt() } + private val caddyPortHttps: Int = config.readConfigValue(443, "SPO_CADDY_PORT_HTTPS") { it.toInt() } private val client: OkHttpClient = OkHttpClient.Builder() .connectTimeout(3, TimeUnit.SECONDS) .readTimeout(3, TimeUnit.SECONDS) @@ -101,7 +103,7 @@ class CaddyConfig(private val dockerClient: DockerClient, mainDataDir: Path, con } private fun generateServer(): Map { - val listen = if (enableTls) listOf(":443") else listOf(":80") + val listen = if (enableTls) listOf(":$caddyPortHttps") else listOf(":$caddyPortHttp") return mapOf("listen" to listen, "routes" to generateRoutes(), "tls_connection_policies" to generateTlsConnectionPolicies()) } @@ -277,7 +279,7 @@ class CaddyConfig(private val dockerClient: DockerClient, mainDataDir: Path, con logger.info { "[Caddy] Pulling image" } dockerActions.pullImage(caddyImage) - val ports = if (enableTls) listOf("80", "443") else listOf("80") + val ports = if (enableTls) listOf("$caddyPortHttp", "$caddyPortHttps") else listOf("$caddyPortHttp") val hostConfig = HostConfig.builder() .networkMode(DockerOrchestrator.SHARED_NETWORK_NAME) .binds(HostConfig.Bind.builder() From c9b92d8c73ed0a4dbbae549f2245a3a1b3a15c58 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:10:52 +0000 Subject: [PATCH 05/37] Add environment variables support for ShinyProxy container Co-authored-by: jaredlander <628977+jaredlander@users.noreply.github.com> --- .../impl/docker/DockerOrchestrator.kt | 13 +++++- .../shinyproxyoperator/model/ShinyProxy.kt | 8 ++++ .../impl/docker/MainIntegrationTest.kt | 26 ++++++++++++ .../impl/docker/helpers/DockerAssertions.kt | 7 ++-- .../docker/simple_config_with_env.yaml | 40 +++++++++++++++++++ 5 files changed, 90 insertions(+), 4 deletions(-) create mode 100644 src/test/resources/docker/simple_config_with_env.yaml diff --git a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt index d99ef94..171d189 100644 --- a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt +++ b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt @@ -297,11 +297,22 @@ class DockerOrchestrator(channel: Channel, .build()) } + // Build environment variables list: default ones + user-provided ones + val envVars = mutableListOf( + "PROXY_VERSION=${version}", + "PROXY_REALM_ID=${shinyProxy.realmId}", + "SPRING_CONFIG_IMPORT=/opt/shinyproxy/generated.yml" + ) + // Add user-provided environment variables + shinyProxy.env.forEach { (key, value) -> + envVars.add("${key}=${value}") + } + val containerConfig = ContainerConfig.builder() .image(shinyProxy.image) .hostConfig(hostConfigBuilder.build()) .labels(shinyProxy.labels + LabelFactory.labelsForShinyProxyInstance(shinyProxyInstance, version)) - .env("PROXY_VERSION=${version}", "PROXY_REALM_ID=${shinyProxy.realmId}", "SPRING_CONFIG_IMPORT=/opt/shinyproxy/generated.yml") + .env(envVars) .user(dataDirUid.toString()) .build() diff --git a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/model/ShinyProxy.kt b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/model/ShinyProxy.kt index 98b9102..c1dc38b 100644 --- a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/model/ShinyProxy.kt +++ b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/model/ShinyProxy.kt @@ -136,6 +136,14 @@ class ShinyProxy(private val spec: JsonNode, val name: String, val namespace: St return@lazy listOf() } + @get:JsonIgnore + val env: Map by lazy { + if (getSpec().get("env")?.isObject == true) { + return@lazy jacksonObjectMapper().convertValue(getSpec().get("env")) + } + return@lazy mapOf() + } + fun getSpec(): JsonNode { return spec } diff --git a/src/test/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/MainIntegrationTest.kt b/src/test/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/MainIntegrationTest.kt index fe01049..a487e61 100644 --- a/src/test/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/MainIntegrationTest.kt +++ b/src/test/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/MainIntegrationTest.kt @@ -640,4 +640,30 @@ class MainIntegrationTest : IntegrationTestBase() { )) } + @Test + fun `test with environment variables`() = setup { dataDir, inputDir, operator, eventController, dockerAssertions, _, _ -> + val hash = createInputFile(inputDir, "simple_config_with_env.yaml", "realm1.shinyproxy.yaml") + val shinyProxyInstance = ShinyProxyInstance("realm1", "default", "default-realm1", hash, true, 0) + + scope.launch { + operator.init() + operator.run() + } + + eventController.waitForNextReconcile(hash) + + val shinyProxyContainer = getSingleShinyProxyContainer(shinyProxyInstance) + dockerAssertions.assertRedisContainer() + dockerAssertions.assertCaddyContainer("simple_test_caddy.json", mapOf("#CONTAINER_IP#" to shinyProxyContainer.getSharedNetworkIpAddress()!!)) + dockerAssertions.assertShinyProxyContainer( + shinyProxyContainer, + shinyProxyInstance, + mapOf( + "MY_CUSTOM_VAR" to "custom_value", + "ANOTHER_VAR" to "another_value", + "DEBUG_MODE" to "true" + ) + ) + } + } diff --git a/src/test/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/helpers/DockerAssertions.kt b/src/test/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/helpers/DockerAssertions.kt index 879540f..2b26f0c 100644 --- a/src/test/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/helpers/DockerAssertions.kt +++ b/src/test/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/helpers/DockerAssertions.kt @@ -115,7 +115,7 @@ class DockerAssertions(private val base: IntegrationTestBase, } } - fun assertShinyProxyContainer(shinyProxyContainer: Container, shinyProxyInstance: ShinyProxyInstance) { + fun assertShinyProxyContainer(shinyProxyContainer: Container, shinyProxyInstance: ShinyProxyInstance, additionalEnv: Map = mapOf()) { val containerInfo = base.dockerClient.inspectContainer(shinyProxyContainer.id()) assertEquals(true, containerInfo.state().running()) assertEquals("sp-shared-network", containerInfo.hostConfig().networkMode()) @@ -137,11 +137,12 @@ class DockerAssertions(private val base: IntegrationTestBase, "openanalytics.eu/sp-realm-id" to shinyProxyInstance.realmId, "openanalytics.eu/sp-version" to null ), containerInfo.config().labels()) - assertEnv(mapOf( + val expectedEnv = mapOf( "PROXY_VERSION" to null, "PROXY_REALM_ID" to shinyProxyInstance.realmId, "SPRING_CONFIG_IMPORT" to "/opt/shinyproxy/generated.yml" - ), containerInfo.config().env()) + ) + additionalEnv + assertEnv(expectedEnv, containerInfo.config().env()) } } diff --git a/src/test/resources/docker/simple_config_with_env.yaml b/src/test/resources/docker/simple_config_with_env.yaml new file mode 100644 index 0000000..5ee3f4d --- /dev/null +++ b/src/test/resources/docker/simple_config_with_env.yaml @@ -0,0 +1,40 @@ +spring: + session: + store-type: redis +fqdn: itest.local +image: openanalytics/shinyproxy:3.2.1 +env: + MY_CUSTOM_VAR: "custom_value" + ANOTHER_VAR: "another_value" + DEBUG_MODE: "true" +proxy: + store-mode: Redis + realm-id: realm1 + title: Open Analytics Shiny Proxy + logoUrl: http://www.openanalytics.eu/sites/www.openanalytics.eu/themes/oa/logo.png + landingPage: / + heartbeatRate: 10000 + heartbeatTimeout: -1 + port: 8080 + authentication: simple + containerBackend: docker + stop-proxies-on-shutdown: false + default-stop-proxy-on-logout: false + docker: + internal-networking: true + users: + - name: demo + password: demo + groups: scientists + - name: demo2 + password: demo2 + groups: mathematicians + specs: + - id: 01_hello + displayName: Hello Application + description: Application which demonstrates the basics of a Shiny app + containerCmd: [ "R", "-e", "shinyproxy::run_01_hello()" ] + containerImage: openanalytics/shinyproxy-integration-test-app + - id: 06_tabsets + container-cmd: [ "R", "-e", "shinyproxy::run_06_tabsets()" ] + container-image: openanalytics/shinyproxy-integration-test-app From 00d494760a5f47d7e854388328e06388053c9483 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:11:10 +0000 Subject: [PATCH 06/37] Add comprehensive Copilot instructions for ShinyProxy Operator Co-authored-by: jaredlander <628977+jaredlander@users.noreply.github.com> --- .github/copilot-instructions.md | 191 ++++++++++++++++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100644 .github/copilot-instructions.md diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..d81b944 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,191 @@ +# ShinyProxy Operator - Copilot Instructions + +## Project Overview + +ShinyProxy Operator is a Kubernetes/Docker operator for managing ShinyProxy deployments. It's written in Kotlin and uses coroutines for asynchronous operations. + +## Technology Stack + +- **Language**: Kotlin 2.1 with language version 2.1 +- **JDK**: Java 21 (required) +- **Build Tool**: Maven 3.x +- **Container Orchestration**: Kubernetes (via Fabric8 client) and Docker +- **Logging**: kotlin-logging with Log4j 2 +- **Testing**: JUnit 5 (Jupiter) for unit and integration tests +- **Async**: Kotlin Coroutines + +## Build and Test Commands + +### Building the Project +```bash +mvn -U clean install -DskipTests +``` + +The build produces: `target/shinyproxy-operator-jar-with-dependencies.jar` + +### Running Tests +```bash +mvn test +``` + +Tests require: +- Docker network: `sp-shared-network` +- Docker plugin: `grafana/loki-docker-driver:3.2.1` +- Environment variable: `SPO_DOCKER_GID` (group ID for docker group) + +### License Header Management +```bash +mvn validate license:format +``` + +Automatically updates copyright headers. Year updates are handled automatically - don't change year manually. + +## Code Style and Conventions + +### File Structure +- Source code: `src/main/kotlin/` +- Test code: `src/test/kotlin/` +- Package: `eu.openanalytics.shinyproxyoperator` + +### Formatting Guidelines +- **Indentation**: 4 spaces (not tabs) +- **Max line length**: 120 characters +- **Charset**: UTF-8 +- **Line endings**: LF (Unix-style) +- **Trailing whitespace**: Remove +- **Final newline**: Always insert + +### Kotlin Conventions +- Use `object` for singletons (e.g., `LabelFactory`) +- Use `const val` for compile-time constants +- Prefer `when` expressions over if-else chains +- Use `kotlinx.coroutines` for async operations +- Use `suspend` functions appropriately +- Use kotlin-logging's KotlinLogging for logging: `private val logger = KotlinLogging.logger {}` + +### Copyright Headers +Every Kotlin source file must include the Apache License 2.0 header. Use the template in `LICENSE_HEADER`. The license plugin handles year updates automatically. + +Example header structure: +```kotlin +/* + * ShinyProxy-Operator + * + * Copyright (C) 2021-2025 Open Analytics + * + * =========================================================================== + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Apache License as published by + * The Apache Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Apache License for more details. + * + * You should have received a copy of the Apache License + * along with this program. If not, see + */ +``` + +## Architecture + +### Core Components +- **Operator Interface**: `IOperator` - main entry point +- **Orchestrators**: Platform-specific implementations + - `KubernetesOperator`: For Kubernetes deployments + - `DockerOperator`: For Docker deployments +- **Event System**: `ShinyProxyEvent`, `ShinyProxyEventType`, `IEventController` +- **Model**: `ShinyProxy`, `ShinyProxyInstance`, `ShinyProxyStatus` + +### Configuration +- Configuration via environment variables (see `Config` class) +- Key variable: `SPO_ORCHESTRATOR` - determines platform (kubernetes/docker) + +### Implementations +- `src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/kubernetes/` - Kubernetes-specific code +- `src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/` - Docker-specific code + +## Testing + +### Test Structure +- Integration tests: `MainIntegrationTest.kt` in both impl/kubernetes and impl/docker +- Helper classes in `src/test/kotlin/eu/openanalytics/shinyproxyoperator/helpers/` +- Test base classes provide setup/teardown functionality + +### Testing Patterns +- Use `@Test` annotation from JUnit 5 +- Use Kotlin test assertions: `assertEquals`, `assertTrue`, `assertFalse`, `assertNotNull`, `assertNull` +- Integration tests use `setup()` helper with closures +- Tests use coroutines with `withTimeout` for async operations +- Helper extensions in `helpers/extensions.kt` + +### Test Requirements +- Tests run in a Minikube environment (for Kubernetes tests) +- Docker daemon required (for Docker tests) +- Specific images pre-pulled before tests + +## Dependencies + +### Key Libraries +- **Fabric8 Kubernetes Client** (7.1.0): Kubernetes API interactions +- **Docker Client** (7.0.8-OA-5): Docker API interactions (custom OpenAnalytics version) +- **Jackson** (2.18.3): JSON processing +- **Log4j** (2.24.3): Logging backend +- **Kotlin Coroutines** (1.10.1): Async programming +- **JUnit Jupiter** (5.11.4): Testing framework + +### Repository Configuration +The project uses OpenAnalytics Nexus repositories for the custom Docker client version: +- Releases: https://nexus.openanalytics.eu/repository/releases +- Snapshots: https://nexus.openanalytics.eu/repository/snapshots + +## Common Patterns + +### Logging +```kotlin +private val logger = KotlinLogging.logger {} +logger.info { "Message with lazy evaluation" } +logger.warn { "Warning: ${variable}" } +``` + +### Error Handling +- Use `InternalException` for operator-specific errors +- Catch and log exceptions appropriately +- Use `exitProcess(1)` for fatal errors in main + +### Labels +Use `LabelFactory` for consistent label creation: +- `labelsForShinyProxyInstance()` - for instances +- `labelsForShinyProxy()` - for resources + +### Configuration Reading +```kotlin +val value = config.readConfigValue(default, "ENV_VAR_NAME") { it.lowercase() } +``` + +## Documentation + +Main documentation is hosted at: https://shinyproxy.io/documentation/shinyproxy-operator/ + +Additional docs in `docs/` directory: +- Deployment configurations +- Prometheus integration + +## CI/CD + +GitHub Actions workflow in `.github/workflows/workflows.yaml`: +- Runs on Java 21 +- Tests against multiple Kubernetes versions (1.30.11, 1.31.7, 1.32.3) +- Uses Minikube for integration tests +- Caches Maven dependencies + +## Important Notes + +- This is a critical infrastructure component for ShinyProxy deployments +- Changes should maintain backward compatibility +- Integration tests are comprehensive but can be time-consuming +- The operator manages both Kubernetes and Docker environments +- Resource cleanup is important in tests to prevent test pollution From 4ca39df5deb086f3adf816a73968c85868a84e0e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:12:28 +0000 Subject: [PATCH 07/37] Fix Kotlin version references in Copilot instructions Co-authored-by: jaredlander <628977+jaredlander@users.noreply.github.com> --- .github/copilot-instructions.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index d81b944..79ce75f 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -6,7 +6,7 @@ ShinyProxy Operator is a Kubernetes/Docker operator for managing ShinyProxy depl ## Technology Stack -- **Language**: Kotlin 2.1 with language version 2.1 +- **Language**: Kotlin 2.1.10 with language version 2.1 - **JDK**: Java 21 (required) - **Build Tool**: Maven 3.x - **Container Orchestration**: Kubernetes (via Fabric8 client) and Docker @@ -135,6 +135,7 @@ Example header structure: - **Jackson** (2.18.3): JSON processing - **Log4j** (2.24.3): Logging backend - **Kotlin Coroutines** (1.10.1): Async programming +- **Kotlin** (2.1.10): Kotlin language and standard library - **JUnit Jupiter** (5.11.4): Testing framework ### Repository Configuration From c9d50fd82417721fe6306d158c684b76c9eefe8e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:12:31 +0000 Subject: [PATCH 08/37] Add documentation for environment variables feature Co-authored-by: jaredlander <628977+jaredlander@users.noreply.github.com> --- README.md | 4 ++ docs/environment-variables.md | 72 +++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 docs/environment-variables.md diff --git a/README.md b/README.md index a222754..d654c22 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,10 @@ page in the documentation to understand why this is so great. See the [website](https://shinyproxy.io/documentation/shinyproxy-operator/kubernetes/) for all documentation. +### Additional Documentation + +- [Environment Variables for ShinyProxy Container](docs/environment-variables.md) - How to set environment variables for the ShinyProxy container in Docker mode + ## Support See the [website](https://shinyproxy.io/support/) on how to get support. diff --git a/docs/environment-variables.md b/docs/environment-variables.md new file mode 100644 index 0000000..44d98b0 --- /dev/null +++ b/docs/environment-variables.md @@ -0,0 +1,72 @@ +# Environment Variables for ShinyProxy Container + +Starting with version 2.3.1, the ShinyProxy Operator supports setting environment variables for the ShinyProxy container when using Docker backend. + +## Usage + +You can add environment variables to your ShinyProxy configuration by adding an `env` section at the root level of the spec. The environment variables are specified as a key-value map. + +### Example Configuration + +```yaml +apiVersion: openanalytics.eu/v1 +kind: ShinyProxy +metadata: + name: shinyproxy + namespace: default +spec: + # Environment variables for the ShinyProxy container + env: + MY_CUSTOM_VARIABLE: "custom_value" + DATABASE_URL: "jdbc:postgresql://localhost:5432/mydb" + DEBUG_MODE: "true" + LOG_LEVEL: "INFO" + + fqdn: shinyproxy-demo.local + image: openanalytics/shinyproxy:3.2.1 + + proxy: + title: ShinyProxy + authentication: simple + containerBackend: docker + docker: + internal-networking: true + users: + - name: demo + password: demo + groups: scientists + specs: + - id: 01_hello + display-name: Hello Application + container-cmd: ["R", "-e", "shinyproxy::run_01_hello()"] + container-image: openanalytics/shinyproxy-demo +``` + +## Notes + +- Environment variables are passed directly to the ShinyProxy container +- The operator will automatically add required environment variables such as `PROXY_VERSION`, `PROXY_REALM_ID`, and `SPRING_CONFIG_IMPORT` +- User-provided environment variables can override these defaults if needed (though this is not recommended) +- This feature is currently only available for Docker backend deployments +- For Kubernetes deployments, use `kubernetesPodTemplateSpecPatches` to add environment variables + +## Use Cases + +Common use cases for environment variables include: + +1. **Configuring external services**: Database URLs, API endpoints, etc. +2. **Feature flags**: Enable or disable features based on environment +3. **Logging configuration**: Set log levels or destinations +4. **Secrets management**: Pass sensitive configuration (though consider using secure secret management instead) +5. **Java/Spring Boot properties**: Override default Spring Boot configuration + +## Example with Spring Boot Properties + +You can use environment variables to override Spring Boot properties: + +```yaml +env: + SPRING_PROFILES_ACTIVE: "production" + LOGGING_LEVEL_EU_OPENANALYTICS: "DEBUG" + SERVER_PORT: "8080" +``` From 94316e3d5f72811bde02b81beca428b8e06a3466 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:14:05 +0000 Subject: [PATCH 09/37] Add validation and security checks for environment variables Co-authored-by: jaredlander <628977+jaredlander@users.noreply.github.com> --- docs/environment-variables.md | 5 ++- .../impl/docker/DockerOrchestrator.kt | 33 ++++++++++++++++--- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 44d98b0..77987bf 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -46,7 +46,10 @@ spec: - Environment variables are passed directly to the ShinyProxy container - The operator will automatically add required environment variables such as `PROXY_VERSION`, `PROXY_REALM_ID`, and `SPRING_CONFIG_IMPORT` -- User-provided environment variables can override these defaults if needed (though this is not recommended) +- **Security**: User-provided environment variables cannot override reserved environment variables (`PROXY_VERSION`, `PROXY_REALM_ID`, `SPRING_CONFIG_IMPORT`) +- **Validation**: Environment variable keys must follow standard naming conventions (alphanumeric characters and underscores, starting with a letter or underscore) +- **Validation**: Environment variable values cannot contain newline characters (to prevent injection attacks) +- Invalid environment variables will be skipped with a warning logged - This feature is currently only available for Docker backend deployments - For Kubernetes deployments, use `kubernetesPodTemplateSpecPatches` to add environment variables diff --git a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt index 171d189..d33e113 100644 --- a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt +++ b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt @@ -298,13 +298,36 @@ class DockerOrchestrator(channel: Channel, } // Build environment variables list: default ones + user-provided ones - val envVars = mutableListOf( - "PROXY_VERSION=${version}", - "PROXY_REALM_ID=${shinyProxy.realmId}", - "SPRING_CONFIG_IMPORT=/opt/shinyproxy/generated.yml" + val defaultEnvVars = mapOf( + "PROXY_VERSION" to version.toString(), + "PROXY_REALM_ID" to shinyProxy.realmId, + "SPRING_CONFIG_IMPORT" to "/opt/shinyproxy/generated.yml" ) - // Add user-provided environment variables + + val envVars = mutableListOf() + + // Add user-provided environment variables first, but exclude reserved keys shinyProxy.env.forEach { (key, value) -> + // Prevent overriding default environment variables + if (!defaultEnvVars.containsKey(key)) { + // Validate key: only allow alphanumeric characters and underscores + if (key.matches(Regex("^[A-Za-z_][A-Za-z0-9_]*$"))) { + // Validate value: prevent newline characters that could enable injection + if (!value.contains('\n') && !value.contains('\r')) { + envVars.add("${key}=${value}") + } else { + logger.warn { "${logPrefix(shinyProxyInstance)} [Docker] Skipping environment variable '$key' due to invalid value containing newline characters" } + } + } else { + logger.warn { "${logPrefix(shinyProxyInstance)} [Docker] Skipping environment variable '$key' due to invalid key name" } + } + } else { + logger.warn { "${logPrefix(shinyProxyInstance)} [Docker] Ignoring attempt to override reserved environment variable '$key'" } + } + } + + // Add default environment variables (these take precedence) + defaultEnvVars.forEach { (key, value) -> envVars.add("${key}=${value}") } From 831cc57a528faf4e54a7455b31133b8b9c638210 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:16:57 +0000 Subject: [PATCH 10/37] Optimize validation and improve type safety Co-authored-by: jaredlander <628977+jaredlander@users.noreply.github.com> --- .../impl/docker/DockerOrchestrator.kt | 39 ++++++++++++------- .../shinyproxyoperator/model/ShinyProxy.kt | 4 +- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt index d33e113..418fd0f 100644 --- a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt +++ b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt @@ -69,6 +69,13 @@ class DockerOrchestrator(channel: Channel, private val dataDir: Path, private val inputDir: Path) : IOrchestrator { + companion object { + // Regex for validating environment variable key names + private val ENV_KEY_PATTERN = Regex("^[A-Za-z_][A-Za-z0-9_]*$") + // Reserved environment variable keys that cannot be overridden + private val RESERVED_ENV_KEYS = setOf("PROXY_VERSION", "PROXY_REALM_ID", "SPRING_CONFIG_IMPORT") + } + private val dockerGID: Int = config.readConfigValue(null, "SPO_DOCKER_GID") { it.toInt() } private val dockerSocket: String = config.readConfigValue("/var/run/docker.sock", "SPO_DOCKER_SOCKET") { it } private val disableICC: Boolean = config.readConfigValue(false, "SPO_DISABLE_ICC") { it.toBoolean() } @@ -308,21 +315,19 @@ class DockerOrchestrator(channel: Channel, // Add user-provided environment variables first, but exclude reserved keys shinyProxy.env.forEach { (key, value) -> - // Prevent overriding default environment variables - if (!defaultEnvVars.containsKey(key)) { - // Validate key: only allow alphanumeric characters and underscores - if (key.matches(Regex("^[A-Za-z_][A-Za-z0-9_]*$"))) { - // Validate value: prevent newline characters that could enable injection - if (!value.contains('\n') && !value.contains('\r')) { - envVars.add("${key}=${value}") - } else { - logger.warn { "${logPrefix(shinyProxyInstance)} [Docker] Skipping environment variable '$key' due to invalid value containing newline characters" } - } - } else { + when { + RESERVED_ENV_KEYS.contains(key) -> { + logger.warn { "${logPrefix(shinyProxyInstance)} [Docker] Ignoring attempt to override reserved environment variable '$key'" } + } + !ENV_KEY_PATTERN.matches(key) -> { logger.warn { "${logPrefix(shinyProxyInstance)} [Docker] Skipping environment variable '$key' due to invalid key name" } } - } else { - logger.warn { "${logPrefix(shinyProxyInstance)} [Docker] Ignoring attempt to override reserved environment variable '$key'" } + !isValidEnvValue(value) -> { + logger.warn { "${logPrefix(shinyProxyInstance)} [Docker] Skipping environment variable '$key' due to invalid value containing newline characters" } + } + else -> { + envVars.add("${key}=${value}") + } } } @@ -598,4 +603,12 @@ class DockerOrchestrator(channel: Channel, } } + /** + * Validates that an environment variable value is safe and does not contain + * characters that could enable injection attacks. + */ + private fun isValidEnvValue(value: String): Boolean { + return !value.contains('\n') && !value.contains('\r') + } + } diff --git a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/model/ShinyProxy.kt b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/model/ShinyProxy.kt index c1dc38b..3d2db4c 100644 --- a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/model/ShinyProxy.kt +++ b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/model/ShinyProxy.kt @@ -103,7 +103,7 @@ class ShinyProxy(private val spec: JsonNode, val name: String, val namespace: St @get:JsonIgnore val labels: Map by lazy { if (getSpec().get("labels")?.isObject == true) { - return@lazy jacksonObjectMapper().convertValue(getSpec().get("labels")) + return@lazy jacksonObjectMapper().convertValue>(getSpec().get("labels")) } return@lazy mapOf() } @@ -139,7 +139,7 @@ class ShinyProxy(private val spec: JsonNode, val name: String, val namespace: St @get:JsonIgnore val env: Map by lazy { if (getSpec().get("env")?.isObject == true) { - return@lazy jacksonObjectMapper().convertValue(getSpec().get("env")) + return@lazy jacksonObjectMapper().convertValue>(getSpec().get("env")) } return@lazy mapOf() } From 6d0f9a91a547a0c8f17270b0a7c4459ed123caff Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:18:54 +0000 Subject: [PATCH 11/37] Clarify environment variable ordering comments Co-authored-by: jaredlander <628977+jaredlander@users.noreply.github.com> --- .../shinyproxyoperator/impl/docker/DockerOrchestrator.kt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt index 418fd0f..0ba6b0e 100644 --- a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt +++ b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt @@ -313,7 +313,8 @@ class DockerOrchestrator(channel: Channel, val envVars = mutableListOf() - // Add user-provided environment variables first, but exclude reserved keys + // Add user-provided environment variables, excluding reserved keys + // Reserved keys are filtered here to prevent any accidental override shinyProxy.env.forEach { (key, value) -> when { RESERVED_ENV_KEYS.contains(key) -> { @@ -331,7 +332,9 @@ class DockerOrchestrator(channel: Channel, } } - // Add default environment variables (these take precedence) + // Add required default environment variables + // Note: Docker uses the last occurrence when duplicates exist, but we've already + // filtered reserved keys above to prevent conflicts defaultEnvVars.forEach { (key, value) -> envVars.add("${key}=${value}") } From ffad72960f760188cb4569e925f18675ee24d5d1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:28:38 +0000 Subject: [PATCH 12/37] Pass environment variables from operator to ShinyProxy containers - Reverted previous implementation that read from ShinyProxy spec - Now reads environment variables from operator's environment with SHINYPROXY_ENV_ prefix - Environment variables are passed to ShinyProxy container with prefix stripped - This allows using docker-compose env files for secrets without plain text in configs - Added comprehensive documentation with examples Co-authored-by: jaredlander <628977+jaredlander@users.noreply.github.com> --- README.md | 4 +- docs/environment-variables-docker.md | 189 ++++++++++++++++++ docs/environment-variables.md | 75 ------- .../impl/docker/DockerOrchestrator.kt | 58 ++---- .../shinyproxyoperator/model/ShinyProxy.kt | 10 +- .../impl/docker/MainIntegrationTest.kt | 26 --- .../impl/docker/helpers/DockerAssertions.kt | 7 +- .../docker/simple_config_with_env.yaml | 40 ---- 8 files changed, 208 insertions(+), 201 deletions(-) create mode 100644 docs/environment-variables-docker.md delete mode 100644 docs/environment-variables.md delete mode 100644 src/test/resources/docker/simple_config_with_env.yaml diff --git a/README.md b/README.md index d654c22..e8cd1aa 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,9 @@ page in the documentation to understand why this is so great. See the [website](https://shinyproxy.io/documentation/shinyproxy-operator/kubernetes/) for all documentation. -### Additional Documentation +### Docker Mode: Passing Environment Variables -- [Environment Variables for ShinyProxy Container](docs/environment-variables.md) - How to set environment variables for the ShinyProxy container in Docker mode +When using the operator in Docker mode, you can pass environment variables from the operator's environment to ShinyProxy containers. This is useful for injecting secrets without storing them in plain text. See [Environment Variables for Docker Mode](docs/environment-variables-docker.md) for details. ## Support diff --git a/docs/environment-variables-docker.md b/docs/environment-variables-docker.md new file mode 100644 index 0000000..3cff909 --- /dev/null +++ b/docs/environment-variables-docker.md @@ -0,0 +1,189 @@ +# Environment Variables for ShinyProxy Container (Docker Mode) + +## Overview + +When running the ShinyProxy Operator in Docker mode, you can pass environment variables from the operator's environment to the ShinyProxy container. This is useful for injecting sensitive configuration values (like database passwords, API keys, etc.) without storing them in plain text in configuration files. + +## How It Works + +Any environment variable set in the operator's environment that starts with the prefix `SHINYPROXY_ENV_` will be automatically passed to the ShinyProxy container with the prefix stripped. + +**Example:** +- Operator environment: `SHINYPROXY_ENV_DATABASE_URL=jdbc:postgresql://db:5432/myapp` +- ShinyProxy container receives: `DATABASE_URL=jdbc:postgresql://db:5432/myapp` + +## Usage with Docker Compose + +This feature is designed to work seamlessly with Docker Compose and environment files: + +### Example docker-compose.yml + +```yaml +version: '3.8' + +services: + shinyproxy-operator: + image: openanalytics/shinyproxy-operator:latest + environment: + # Operator configuration + - SPO_DOCKER_GID=999 + + # Environment variables to pass to ShinyProxy container + - SHINYPROXY_ENV_DATABASE_URL=jdbc:postgresql://db:5432/myapp + - SHINYPROXY_ENV_DATABASE_USER=shinyproxy + - SHINYPROXY_ENV_DATABASE_PASSWORD=secret123 + - SHINYPROXY_ENV_SPRING_PROFILES_ACTIVE=production + - SHINYPROXY_ENV_LOG_LEVEL=INFO + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./config:/opt/shinyproxy-operator/config + - ./data:/opt/shinyproxy-operator/data +``` + +### Using Environment Files + +For better security, store sensitive values in an environment file: + +**shinyproxy.env:** +```bash +SHINYPROXY_ENV_DATABASE_PASSWORD=super_secret_password +SHINYPROXY_ENV_API_KEY=my_api_key_12345 +SHINYPROXY_ENV_REDIS_PASSWORD=redis_secret +``` + +**docker-compose.yml:** +```yaml +version: '3.8' + +services: + shinyproxy-operator: + image: openanalytics/shinyproxy-operator:latest + env_file: + - shinyproxy.env + environment: + - SPO_DOCKER_GID=999 + - SHINYPROXY_ENV_SPRING_PROFILES_ACTIVE=production + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./config:/opt/shinyproxy-operator/config + - ./data:/opt/shinyproxy-operator/data +``` + +**Important:** Make sure to add `shinyproxy.env` to your `.gitignore` file to avoid committing secrets to version control. + +## Use Cases + +### 1. Database Credentials + +```yaml +environment: + - SHINYPROXY_ENV_SPRING_DATASOURCE_URL=jdbc:postgresql://db:5432/myapp + - SHINYPROXY_ENV_SPRING_DATASOURCE_USERNAME=dbuser + - SHINYPROXY_ENV_SPRING_DATASOURCE_PASSWORD=dbpass +``` + +### 2. External Service API Keys + +```yaml +environment: + - SHINYPROXY_ENV_EXTERNAL_API_KEY=your_api_key_here + - SHINYPROXY_ENV_EXTERNAL_API_ENDPOINT=https://api.example.com +``` + +### 3. Spring Boot Configuration + +```yaml +environment: + - SHINYPROXY_ENV_SPRING_PROFILES_ACTIVE=production + - SHINYPROXY_ENV_LOGGING_LEVEL_EU_OPENANALYTICS=DEBUG + - SHINYPROXY_ENV_SERVER_PORT=8080 +``` + +### 4. Redis Configuration (when using Redis for session storage) + +```yaml +environment: + - SHINYPROXY_ENV_SPRING_REDIS_PASSWORD=${REDIS_PASSWORD} + - SHINYPROXY_ENV_SPRING_REDIS_HOST=redis + - SHINYPROXY_ENV_SPRING_REDIS_PORT=6379 +``` + +## Important Notes + +- **Prefix Required:** Only environment variables starting with `SHINYPROXY_ENV_` are passed through +- **Prefix Stripped:** The `SHINYPROXY_ENV_` prefix is automatically removed when setting the variable in the ShinyProxy container +- **Logging:** Each passed environment variable is logged (variable name only, not the value) when the container is created +- **Docker Only:** This feature is only available for Docker backend deployments +- **Kubernetes:** For Kubernetes deployments, use `kubernetesPodTemplateSpecPatches` to inject environment variables or reference secrets + +## Security Best Practices + +1. **Use Environment Files:** Store secrets in `.env` files that are excluded from version control +2. **File Permissions:** Restrict permissions on environment files (e.g., `chmod 600 shinyproxy.env`) +3. **Docker Secrets:** Consider using Docker Swarm secrets for production deployments +4. **Rotate Credentials:** Regularly rotate sensitive credentials +5. **Minimal Access:** Only pass environment variables that are actually needed by ShinyProxy + +## Troubleshooting + +### Environment variables not appearing in ShinyProxy + +1. Verify the prefix is correct: `SHINYPROXY_ENV_` (note the underscore at the end) +2. Check the operator logs for messages about passed environment variables +3. Ensure the environment variable is set in the operator's environment, not in the ShinyProxy config + +### Conflicts with existing variables + +If you set an environment variable that conflicts with system variables (like `PROXY_VERSION`, `PROXY_REALM_ID`, or `SPRING_CONFIG_IMPORT`), the system variables will take precedence. The operator will log a warning if this occurs. + +## Example: Full Docker Compose Setup + +```yaml +version: '3.8' + +services: + redis: + image: redis:7-alpine + command: redis-server --requirepass ${REDIS_PASSWORD} + volumes: + - redis-data:/data + + database: + image: postgres:15-alpine + environment: + - POSTGRES_DB=shinyproxy + - POSTGRES_USER=shinyproxy + - POSTGRES_PASSWORD=${DB_PASSWORD} + volumes: + - db-data:/var/lib/postgresql/data + + shinyproxy-operator: + image: openanalytics/shinyproxy-operator:latest + environment: + # Operator configuration + - SPO_DOCKER_GID=999 + + # ShinyProxy environment variables + - SHINYPROXY_ENV_SPRING_REDIS_PASSWORD=${REDIS_PASSWORD} + - SHINYPROXY_ENV_SPRING_DATASOURCE_URL=jdbc:postgresql://database:5432/shinyproxy + - SHINYPROXY_ENV_SPRING_DATASOURCE_USERNAME=shinyproxy + - SHINYPROXY_ENV_SPRING_DATASOURCE_PASSWORD=${DB_PASSWORD} + - SHINYPROXY_ENV_SPRING_PROFILES_ACTIVE=production + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./config:/opt/shinyproxy-operator/config + - ./data:/opt/shinyproxy-operator/data + depends_on: + - redis + - database + +volumes: + redis-data: + db-data: +``` + +**.env file:** +```bash +REDIS_PASSWORD=secure_redis_password +DB_PASSWORD=secure_database_password +``` diff --git a/docs/environment-variables.md b/docs/environment-variables.md deleted file mode 100644 index 77987bf..0000000 --- a/docs/environment-variables.md +++ /dev/null @@ -1,75 +0,0 @@ -# Environment Variables for ShinyProxy Container - -Starting with version 2.3.1, the ShinyProxy Operator supports setting environment variables for the ShinyProxy container when using Docker backend. - -## Usage - -You can add environment variables to your ShinyProxy configuration by adding an `env` section at the root level of the spec. The environment variables are specified as a key-value map. - -### Example Configuration - -```yaml -apiVersion: openanalytics.eu/v1 -kind: ShinyProxy -metadata: - name: shinyproxy - namespace: default -spec: - # Environment variables for the ShinyProxy container - env: - MY_CUSTOM_VARIABLE: "custom_value" - DATABASE_URL: "jdbc:postgresql://localhost:5432/mydb" - DEBUG_MODE: "true" - LOG_LEVEL: "INFO" - - fqdn: shinyproxy-demo.local - image: openanalytics/shinyproxy:3.2.1 - - proxy: - title: ShinyProxy - authentication: simple - containerBackend: docker - docker: - internal-networking: true - users: - - name: demo - password: demo - groups: scientists - specs: - - id: 01_hello - display-name: Hello Application - container-cmd: ["R", "-e", "shinyproxy::run_01_hello()"] - container-image: openanalytics/shinyproxy-demo -``` - -## Notes - -- Environment variables are passed directly to the ShinyProxy container -- The operator will automatically add required environment variables such as `PROXY_VERSION`, `PROXY_REALM_ID`, and `SPRING_CONFIG_IMPORT` -- **Security**: User-provided environment variables cannot override reserved environment variables (`PROXY_VERSION`, `PROXY_REALM_ID`, `SPRING_CONFIG_IMPORT`) -- **Validation**: Environment variable keys must follow standard naming conventions (alphanumeric characters and underscores, starting with a letter or underscore) -- **Validation**: Environment variable values cannot contain newline characters (to prevent injection attacks) -- Invalid environment variables will be skipped with a warning logged -- This feature is currently only available for Docker backend deployments -- For Kubernetes deployments, use `kubernetesPodTemplateSpecPatches` to add environment variables - -## Use Cases - -Common use cases for environment variables include: - -1. **Configuring external services**: Database URLs, API endpoints, etc. -2. **Feature flags**: Enable or disable features based on environment -3. **Logging configuration**: Set log levels or destinations -4. **Secrets management**: Pass sensitive configuration (though consider using secure secret management instead) -5. **Java/Spring Boot properties**: Override default Spring Boot configuration - -## Example with Spring Boot Properties - -You can use environment variables to override Spring Boot properties: - -```yaml -env: - SPRING_PROFILES_ACTIVE: "production" - LOGGING_LEVEL_EU_OPENANALYTICS: "DEBUG" - SERVER_PORT: "8080" -``` diff --git a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt index 0ba6b0e..a51217d 100644 --- a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt +++ b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt @@ -69,13 +69,6 @@ class DockerOrchestrator(channel: Channel, private val dataDir: Path, private val inputDir: Path) : IOrchestrator { - companion object { - // Regex for validating environment variable key names - private val ENV_KEY_PATTERN = Regex("^[A-Za-z_][A-Za-z0-9_]*$") - // Reserved environment variable keys that cannot be overridden - private val RESERVED_ENV_KEYS = setOf("PROXY_VERSION", "PROXY_REALM_ID", "SPRING_CONFIG_IMPORT") - } - private val dockerGID: Int = config.readConfigValue(null, "SPO_DOCKER_GID") { it.toInt() } private val dockerSocket: String = config.readConfigValue("/var/run/docker.sock", "SPO_DOCKER_SOCKET") { it } private val disableICC: Boolean = config.readConfigValue(false, "SPO_DISABLE_ICC") { it.toBoolean() } @@ -304,40 +297,23 @@ class DockerOrchestrator(channel: Channel, .build()) } - // Build environment variables list: default ones + user-provided ones - val defaultEnvVars = mapOf( - "PROXY_VERSION" to version.toString(), - "PROXY_REALM_ID" to shinyProxy.realmId, - "SPRING_CONFIG_IMPORT" to "/opt/shinyproxy/generated.yml" + // Build environment variables list: default ones + operator environment variables + val envVars = mutableListOf( + "PROXY_VERSION=${version}", + "PROXY_REALM_ID=${shinyProxy.realmId}", + "SPRING_CONFIG_IMPORT=/opt/shinyproxy/generated.yml" ) - val envVars = mutableListOf() - - // Add user-provided environment variables, excluding reserved keys - // Reserved keys are filtered here to prevent any accidental override - shinyProxy.env.forEach { (key, value) -> - when { - RESERVED_ENV_KEYS.contains(key) -> { - logger.warn { "${logPrefix(shinyProxyInstance)} [Docker] Ignoring attempt to override reserved environment variable '$key'" } - } - !ENV_KEY_PATTERN.matches(key) -> { - logger.warn { "${logPrefix(shinyProxyInstance)} [Docker] Skipping environment variable '$key' due to invalid key name" } - } - !isValidEnvValue(value) -> { - logger.warn { "${logPrefix(shinyProxyInstance)} [Docker] Skipping environment variable '$key' due to invalid value containing newline characters" } - } - else -> { - envVars.add("${key}=${value}") - } + // Add environment variables from the operator's environment + // Any environment variable with prefix SHINYPROXY_ENV_ will be passed to the container + // with the prefix stripped (e.g., SHINYPROXY_ENV_DATABASE_URL -> DATABASE_URL) + System.getenv().forEach { (key, value) -> + if (key.startsWith("SHINYPROXY_ENV_")) { + val targetKey = key.removePrefix("SHINYPROXY_ENV_") + envVars.add("${targetKey}=${value}") + logger.info { "${logPrefix(shinyProxyInstance)} [Docker] Passing environment variable '$targetKey' to ShinyProxy container" } } } - - // Add required default environment variables - // Note: Docker uses the last occurrence when duplicates exist, but we've already - // filtered reserved keys above to prevent conflicts - defaultEnvVars.forEach { (key, value) -> - envVars.add("${key}=${value}") - } val containerConfig = ContainerConfig.builder() .image(shinyProxy.image) @@ -606,12 +582,4 @@ class DockerOrchestrator(channel: Channel, } } - /** - * Validates that an environment variable value is safe and does not contain - * characters that could enable injection attacks. - */ - private fun isValidEnvValue(value: String): Boolean { - return !value.contains('\n') && !value.contains('\r') - } - } diff --git a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/model/ShinyProxy.kt b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/model/ShinyProxy.kt index 3d2db4c..98b9102 100644 --- a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/model/ShinyProxy.kt +++ b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/model/ShinyProxy.kt @@ -103,7 +103,7 @@ class ShinyProxy(private val spec: JsonNode, val name: String, val namespace: St @get:JsonIgnore val labels: Map by lazy { if (getSpec().get("labels")?.isObject == true) { - return@lazy jacksonObjectMapper().convertValue>(getSpec().get("labels")) + return@lazy jacksonObjectMapper().convertValue(getSpec().get("labels")) } return@lazy mapOf() } @@ -136,14 +136,6 @@ class ShinyProxy(private val spec: JsonNode, val name: String, val namespace: St return@lazy listOf() } - @get:JsonIgnore - val env: Map by lazy { - if (getSpec().get("env")?.isObject == true) { - return@lazy jacksonObjectMapper().convertValue>(getSpec().get("env")) - } - return@lazy mapOf() - } - fun getSpec(): JsonNode { return spec } diff --git a/src/test/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/MainIntegrationTest.kt b/src/test/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/MainIntegrationTest.kt index a487e61..fe01049 100644 --- a/src/test/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/MainIntegrationTest.kt +++ b/src/test/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/MainIntegrationTest.kt @@ -640,30 +640,4 @@ class MainIntegrationTest : IntegrationTestBase() { )) } - @Test - fun `test with environment variables`() = setup { dataDir, inputDir, operator, eventController, dockerAssertions, _, _ -> - val hash = createInputFile(inputDir, "simple_config_with_env.yaml", "realm1.shinyproxy.yaml") - val shinyProxyInstance = ShinyProxyInstance("realm1", "default", "default-realm1", hash, true, 0) - - scope.launch { - operator.init() - operator.run() - } - - eventController.waitForNextReconcile(hash) - - val shinyProxyContainer = getSingleShinyProxyContainer(shinyProxyInstance) - dockerAssertions.assertRedisContainer() - dockerAssertions.assertCaddyContainer("simple_test_caddy.json", mapOf("#CONTAINER_IP#" to shinyProxyContainer.getSharedNetworkIpAddress()!!)) - dockerAssertions.assertShinyProxyContainer( - shinyProxyContainer, - shinyProxyInstance, - mapOf( - "MY_CUSTOM_VAR" to "custom_value", - "ANOTHER_VAR" to "another_value", - "DEBUG_MODE" to "true" - ) - ) - } - } diff --git a/src/test/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/helpers/DockerAssertions.kt b/src/test/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/helpers/DockerAssertions.kt index 2b26f0c..879540f 100644 --- a/src/test/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/helpers/DockerAssertions.kt +++ b/src/test/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/helpers/DockerAssertions.kt @@ -115,7 +115,7 @@ class DockerAssertions(private val base: IntegrationTestBase, } } - fun assertShinyProxyContainer(shinyProxyContainer: Container, shinyProxyInstance: ShinyProxyInstance, additionalEnv: Map = mapOf()) { + fun assertShinyProxyContainer(shinyProxyContainer: Container, shinyProxyInstance: ShinyProxyInstance) { val containerInfo = base.dockerClient.inspectContainer(shinyProxyContainer.id()) assertEquals(true, containerInfo.state().running()) assertEquals("sp-shared-network", containerInfo.hostConfig().networkMode()) @@ -137,12 +137,11 @@ class DockerAssertions(private val base: IntegrationTestBase, "openanalytics.eu/sp-realm-id" to shinyProxyInstance.realmId, "openanalytics.eu/sp-version" to null ), containerInfo.config().labels()) - val expectedEnv = mapOf( + assertEnv(mapOf( "PROXY_VERSION" to null, "PROXY_REALM_ID" to shinyProxyInstance.realmId, "SPRING_CONFIG_IMPORT" to "/opt/shinyproxy/generated.yml" - ) + additionalEnv - assertEnv(expectedEnv, containerInfo.config().env()) + ), containerInfo.config().env()) } } diff --git a/src/test/resources/docker/simple_config_with_env.yaml b/src/test/resources/docker/simple_config_with_env.yaml deleted file mode 100644 index 5ee3f4d..0000000 --- a/src/test/resources/docker/simple_config_with_env.yaml +++ /dev/null @@ -1,40 +0,0 @@ -spring: - session: - store-type: redis -fqdn: itest.local -image: openanalytics/shinyproxy:3.2.1 -env: - MY_CUSTOM_VAR: "custom_value" - ANOTHER_VAR: "another_value" - DEBUG_MODE: "true" -proxy: - store-mode: Redis - realm-id: realm1 - title: Open Analytics Shiny Proxy - logoUrl: http://www.openanalytics.eu/sites/www.openanalytics.eu/themes/oa/logo.png - landingPage: / - heartbeatRate: 10000 - heartbeatTimeout: -1 - port: 8080 - authentication: simple - containerBackend: docker - stop-proxies-on-shutdown: false - default-stop-proxy-on-logout: false - docker: - internal-networking: true - users: - - name: demo - password: demo - groups: scientists - - name: demo2 - password: demo2 - groups: mathematicians - specs: - - id: 01_hello - displayName: Hello Application - description: Application which demonstrates the basics of a Shiny app - containerCmd: [ "R", "-e", "shinyproxy::run_01_hello()" ] - containerImage: openanalytics/shinyproxy-integration-test-app - - id: 06_tabsets - container-cmd: [ "R", "-e", "shinyproxy::run_06_tabsets()" ] - container-image: openanalytics/shinyproxy-integration-test-app From c1f78fa25eab28265f3e3b515233228d8234f6ec Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:30:36 +0000 Subject: [PATCH 13/37] Add validation for environment variable keys and values - Validate that target key is not empty after stripping prefix - Validate that values don't contain newline characters (security) - Log warnings for invalid environment variables - Update documentation with validation rules Co-authored-by: jaredlander <628977+jaredlander@users.noreply.github.com> --- docs/environment-variables-docker.md | 3 +++ .../impl/docker/DockerOrchestrator.kt | 15 +++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/docs/environment-variables-docker.md b/docs/environment-variables-docker.md index 3cff909..35ab9a5 100644 --- a/docs/environment-variables-docker.md +++ b/docs/environment-variables-docker.md @@ -112,6 +112,9 @@ environment: - **Prefix Required:** Only environment variables starting with `SHINYPROXY_ENV_` are passed through - **Prefix Stripped:** The `SHINYPROXY_ENV_` prefix is automatically removed when setting the variable in the ShinyProxy container +- **Variable Name Validation:** The variable name after stripping the prefix must not be empty (i.e., don't use `SHINYPROXY_ENV_` as the exact variable name) +- **Value Validation:** Environment variable values cannot contain newline characters (`\n` or `\r`) for security reasons +- **Invalid Variables:** Environment variables that don't pass validation are logged with a warning and skipped - **Logging:** Each passed environment variable is logged (variable name only, not the value) when the container is created - **Docker Only:** This feature is only available for Docker backend deployments - **Kubernetes:** For Kubernetes deployments, use `kubernetesPodTemplateSpecPatches` to inject environment variables or reference secrets diff --git a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt index a51217d..00d2e67 100644 --- a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt +++ b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt @@ -310,8 +310,19 @@ class DockerOrchestrator(channel: Channel, System.getenv().forEach { (key, value) -> if (key.startsWith("SHINYPROXY_ENV_")) { val targetKey = key.removePrefix("SHINYPROXY_ENV_") - envVars.add("${targetKey}=${value}") - logger.info { "${logPrefix(shinyProxyInstance)} [Docker] Passing environment variable '$targetKey' to ShinyProxy container" } + // Validate that the target key is not empty and value doesn't contain newlines + when { + targetKey.isEmpty() -> { + logger.warn { "${logPrefix(shinyProxyInstance)} [Docker] Ignoring invalid environment variable with empty key: $key" } + } + value.contains('\n') || value.contains('\r') -> { + logger.warn { "${logPrefix(shinyProxyInstance)} [Docker] Skipping environment variable '$targetKey' due to invalid value containing newline characters" } + } + else -> { + envVars.add("${targetKey}=${value}") + logger.info { "${logPrefix(shinyProxyInstance)} [Docker] Passing environment variable '$targetKey' to ShinyProxy container" } + } + } } } From 9c8df936136d5cfec223e1da959b31506f5b7d2a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 16:58:18 +0000 Subject: [PATCH 14/37] Initial plan From bc16cfc1cccadcd40bfb081e2d5bd4977abd5726 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 17:02:36 +0000 Subject: [PATCH 15/37] Add GitHub Actions workflow for building and pushing Docker images Co-authored-by: jaredlander <628977+jaredlander@users.noreply.github.com> --- .github/workflows/build-and-push-docker.yaml | 89 ++++++++++++ docs/CI-CD.md | 143 +++++++++++++++++++ 2 files changed, 232 insertions(+) create mode 100644 .github/workflows/build-and-push-docker.yaml create mode 100644 docs/CI-CD.md diff --git a/.github/workflows/build-and-push-docker.yaml b/.github/workflows/build-and-push-docker.yaml new file mode 100644 index 0000000..f0975f8 --- /dev/null +++ b/.github/workflows/build-and-push-docker.yaml @@ -0,0 +1,89 @@ +name: Build and Push Docker Image + +on: + push: + branches: + - main + - develop + tags: + - 'v*' + pull_request: + branches: + - main + - develop + workflow_dispatch: + +jobs: + build-and-push: + runs-on: ubuntu-latest + + steps: + - name: Checkout shinyproxy-operator + uses: actions/checkout@v4 + with: + path: shinyproxy-operator + + - name: Set up JDK 21 + uses: actions/setup-java@v4 + with: + java-version: 21 + distribution: temurin + + - name: Cache Maven packages + uses: actions/cache@v4 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2 + + - name: Build shinyproxy-operator JAR + run: | + cd shinyproxy-operator + mvn -U clean install -DskipTests + ls -lh target/ + + - name: Checkout shinyproxy-docker + uses: actions/checkout@v4 + with: + repository: openanalytics/shinyproxy-docker + path: shinyproxy-docker + + - name: Copy JAR to Docker build context + run: | + cp shinyproxy-operator/target/shinyproxy-operator-jar-with-dependencies.jar shinyproxy-docker/Operator/ + ls -lh shinyproxy-docker/Operator/ + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Azure Container Registry + uses: docker/login-action@v3 + with: + registry: landeranalytics.azurecr.io + username: ${{ secrets.ACR_USERNAME }} + password: ${{ secrets.ACR_PASSWORD }} + + - name: Extract metadata for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: landeranalytics.azurecr.io/shinyproxy-operator + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=sha,prefix={{branch}}- + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: shinyproxy-docker/Operator + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + JAR_LOCATION=shinyproxy-operator-jar-with-dependencies.jar + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/docs/CI-CD.md b/docs/CI-CD.md new file mode 100644 index 0000000..b5bd892 --- /dev/null +++ b/docs/CI-CD.md @@ -0,0 +1,143 @@ +# CI/CD Documentation + +## Build and Push Docker Image Workflow + +This repository includes a GitHub Actions workflow that automatically builds the shinyproxy-operator JAR and creates a Docker image that is pushed to Azure Container Registry. + +### Workflow File + +The workflow is defined in `.github/workflows/build-and-push-docker.yaml`. + +### Triggers + +The workflow is triggered on: +- Pushes to `main` or `develop` branches +- Pull requests to `main` or `develop` branches +- Tag pushes matching `v*` pattern (e.g., `v1.0.0`) +- Manual workflow dispatch + +### Workflow Steps + +1. **Checkout shinyproxy-operator**: Clones this repository +2. **Set up JDK 21**: Installs Java 21 (Temurin distribution) +3. **Cache Maven packages**: Caches Maven dependencies for faster builds +4. **Build shinyproxy-operator JAR**: Runs `mvn -U clean install -DskipTests` to build the JAR +5. **Checkout shinyproxy-docker**: Clones the [openanalytics/shinyproxy-docker](https://github.com/openanalytics/shinyproxy-docker) repository +6. **Copy JAR to Docker build context**: Copies the built JAR to the Operator directory in shinyproxy-docker +7. **Set up Docker Buildx**: Configures Docker Buildx for advanced build features +8. **Login to Azure Container Registry**: Authenticates with the Azure Container Registry +9. **Extract metadata for Docker**: Generates Docker tags and labels based on Git context +10. **Build and push Docker image**: Builds the Docker image and pushes it to the registry + +### Required Secrets + +The workflow requires the following secrets to be configured in the GitHub repository: + +#### `ACR_USERNAME` +- **Description**: Username for Azure Container Registry authentication +- **Type**: Repository Secret +- **Usage**: Used to authenticate with `landeranalytics.azurecr.io` + +#### `ACR_PASSWORD` +- **Description**: Password for Azure Container Registry authentication +- **Type**: Repository Secret +- **Usage**: Used to authenticate with `landeranalytics.azurecr.io` + +### Setting Up Secrets + +To configure these secrets: + +1. Go to the GitHub repository +2. Navigate to **Settings** > **Secrets and variables** > **Actions** +3. Click **New repository secret** +4. Add each secret with the name and value: + - Name: `ACR_USERNAME`, Value: Your Azure Container Registry username + - Name: `ACR_PASSWORD`, Value: Your Azure Container Registry password + +### Docker Image Tags + +The workflow automatically generates tags based on the Git context: + +- **Branch pushes**: `` (e.g., `main`, `develop`) +- **Pull requests**: `pr-` (e.g., `pr-123`) +- **Version tags**: Multiple formats for semantic versioning + - `` (e.g., `1.2.3`) + - `.` (e.g., `1.2`) + - `` (e.g., `1`) +- **SHA-based**: `-` (e.g., `main-abc1234`) + +### Docker Registry + +Images are pushed to: `landeranalytics.azurecr.io/shinyproxy-operator` + +### Build Arguments + +The Docker build uses the following build argument: +- `JAR_LOCATION=shinyproxy-operator-jar-with-dependencies.jar`: Specifies the local JAR file to use instead of downloading from Nexus + +### Caching + +The workflow uses GitHub Actions cache for: +- Maven dependencies (`.m2` directory) +- Docker layer caching (GitHub Actions cache backend) + +This improves build times for subsequent runs. + +### Manual Workflow Dispatch + +You can manually trigger the workflow: + +1. Go to the **Actions** tab in the GitHub repository +2. Select the "Build and Push Docker Image" workflow +3. Click **Run workflow** +4. Select the branch to run from +5. Click **Run workflow** + +### Troubleshooting + +#### Build Failures + +If the Maven build fails: +- Check that the POM file is valid +- Verify that all dependencies are accessible +- Review the build logs for specific error messages + +#### Docker Push Failures + +If pushing to the registry fails: +- Verify that `ACR_USERNAME` and `ACR_PASSWORD` secrets are correctly set +- Ensure the service principal has push permissions to the registry +- Check that the registry URL is correct: `landeranalytics.azurecr.io` + +#### JAR Not Found + +If the JAR file is not found during the copy step: +- Verify the Maven build completed successfully +- Check that the JAR name matches: `shinyproxy-operator-jar-with-dependencies.jar` +- Review the build output for the correct target directory + +### Local Development + +To replicate the workflow locally: + +```bash +# Step 1: Build the operator +git clone https://github.com/jaredlander/shinyproxy-operator.git +cd shinyproxy-operator +mvn -U clean install -DskipTests + +# Step 2: Clone shinyproxy-docker +cd .. +git clone https://github.com/openanalytics/shinyproxy-docker.git +cd shinyproxy-docker/Operator + +# Step 3: Copy the JAR +cp ../../shinyproxy-operator/target/shinyproxy-operator-jar-with-dependencies.jar . + +# Step 4: Build the Docker image +docker build -t shinyproxy-operator-dev --build-arg JAR_LOCATION=shinyproxy-operator-jar-with-dependencies.jar . + +# Step 5: Tag and push (optional) +docker tag shinyproxy-operator-dev landeranalytics.azurecr.io/shinyproxy-operator:dev +docker push landeranalytics.azurecr.io/shinyproxy-operator:dev +``` From e6107966b85a0df90d6b0e58132ff28762bbf5b0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 17:03:29 +0000 Subject: [PATCH 16/37] Update README with CI/CD documentation reference Co-authored-by: jaredlander <628977+jaredlander@users.noreply.github.com> --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index a222754..cdfed61 100644 --- a/README.md +++ b/README.md @@ -35,3 +35,17 @@ The build will result in a single `.jar` file: ## Java Version This project requires JDK 21. + +## CI/CD + +This repository includes automated GitHub Actions workflows for building and deploying the ShinyProxy Operator. + +### Docker Image Build and Push + +A workflow automatically builds the operator JAR, creates a Docker image, and pushes it to Azure Container Registry. See [docs/CI-CD.md](docs/CI-CD.md) for detailed documentation on: + +- Workflow configuration +- Required secrets setup +- Docker image tags +- Manual workflow dispatch +- Local development replication From a463380a1641eca16ce15675cb169ee2316cb968 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 17:04:07 +0000 Subject: [PATCH 17/37] Add build instructions and containerized build environment - Added Dockerfile.build for building in a container with Maven and JDK 21 - Added docker-compose.build.yml for easy Docker-based builds - Added devbox.json configuration for Devbox development environment - Added BUILD.md with comprehensive build instructions for all options - Users can now build without installing Maven/JDK locally Co-authored-by: jaredlander <628977+jaredlander@users.noreply.github.com> --- BUILD.md | 194 +++++++++++++++++++++++++++++++++++++++ Dockerfile.build | 23 +++++ devbox.json | 31 +++++++ docker-compose.build.yml | 15 +++ 4 files changed, 263 insertions(+) create mode 100644 BUILD.md create mode 100644 Dockerfile.build create mode 100644 devbox.json create mode 100644 docker-compose.build.yml diff --git a/BUILD.md b/BUILD.md new file mode 100644 index 0000000..e6cd600 --- /dev/null +++ b/BUILD.md @@ -0,0 +1,194 @@ +# Building ShinyProxy Operator + +This guide provides multiple options for building the ShinyProxy Operator without needing to install build tools locally. + +## Prerequisites + +Choose one of the following options: +- **Option 1**: Docker (recommended for most users) +- **Option 2**: Devbox (for development environments) +- **Option 3**: Local build (requires JDK 21 and Maven) + +--- + +## Option 1: Build with Docker (Recommended) + +This is the easiest option if you have Docker installed. + +### Quick Build with Docker Compose + +```bash +# Build the project +docker-compose -f docker-compose.build.yml up + +# The JAR will be in ./target/shinyproxy-operator-jar-with-dependencies.jar +``` + +### Alternative: Build with Dockerfile Directly + +```bash +# Build the Docker image +docker build -f Dockerfile.build -t shinyproxy-operator-builder . + +# Run the build +docker run --rm -v "$(pwd)/target:/build/target" shinyproxy-operator-builder + +# The JAR will be in ./target/shinyproxy-operator-jar-with-dependencies.jar +``` + +### Extract JAR from Container (if needed) + +```bash +# Start the container +docker run -d --name builder shinyproxy-operator-builder + +# Copy the JAR out +docker cp builder:/build/target/shinyproxy-operator-jar-with-dependencies.jar . + +# Clean up +docker rm builder +``` + +--- + +## Option 2: Build with Devbox + +[Devbox](https://www.jetify.com/devbox) provides an isolated development environment without Docker. + +### Install Devbox + +```bash +curl -fsSL https://get.jetify.com/devbox | bash +``` + +### Build the Project + +```bash +# Enter the devbox shell (installs JDK 21 and Maven automatically) +devbox shell + +# Build the project +devbox run build + +# Or build without tests (faster) +devbox run build-fast + +# The JAR will be at ./target/shinyproxy-operator-jar-with-dependencies.jar +``` + +--- + +## Option 3: Local Build + +If you have JDK 21 and Maven installed locally: + +```bash +mvn -U clean install +``` + +The JAR will be at `target/shinyproxy-operator-jar-with-dependencies.jar`. + +--- + +## Verifying the Build + +After building, verify the JAR exists: + +```bash +ls -lh target/shinyproxy-operator-jar-with-dependencies.jar +``` + +You should see a file of approximately 50-100 MB. + +--- + +## Running the Built JAR + +```bash +java -jar target/shinyproxy-operator-jar-with-dependencies.jar +``` + +--- + +## Building a Docker Image + +If you want to create a Docker image with the operator: + +```bash +# First build the JAR using one of the methods above + +# Then create a runtime Dockerfile (example): +cat > Dockerfile.runtime <<'EOF' +FROM eclipse-temurin:21-jre-alpine + +WORKDIR /opt/shinyproxy-operator + +COPY target/shinyproxy-operator-jar-with-dependencies.jar shinyproxy-operator.jar + +ENTRYPOINT ["java", "-jar", "shinyproxy-operator.jar"] +EOF + +# Build the runtime image +docker build -f Dockerfile.runtime -t shinyproxy-operator:local . +``` + +--- + +## Troubleshooting + +### Docker build fails with network errors + +If you experience network issues during the Maven build: + +```bash +# Build with host network (Linux only) +docker build --network=host -f Dockerfile.build -t shinyproxy-operator-builder . +``` + +### Permission issues with target directory + +```bash +# On Linux/Mac, you might need to fix permissions +sudo chown -R $USER:$USER target/ +``` + +### Out of memory during build + +```bash +# Increase Docker memory limit or add Maven options +docker run --rm -v "$(pwd)/target:/build/target" \ + -e MAVEN_OPTS="-Xmx2048m" \ + shinyproxy-operator-builder +``` + +--- + +## Development Workflow + +For ongoing development: + +1. **Use Devbox** for the best development experience with automatic dependency management +2. **Use Docker Compose** for one-off builds without installing tools +3. **Use Local Build** if you already have the tools installed + +--- + +## Testing Your Changes + +After building with your environment variable changes: + +1. Create a test docker-compose.yml with the operator +2. Set `SHINYPROXY_ENV_*` environment variables +3. Verify they are passed to the ShinyProxy container + +Example: +```yaml +services: + shinyproxy-operator: + image: shinyproxy-operator:local + environment: + - SPO_DOCKER_GID=999 + - SHINYPROXY_ENV_TEST_VAR=test_value + volumes: + - /var/run/docker.sock:/var/run/docker.sock +``` diff --git a/Dockerfile.build b/Dockerfile.build new file mode 100644 index 0000000..c3bd55f --- /dev/null +++ b/Dockerfile.build @@ -0,0 +1,23 @@ +# Dockerfile for building the ShinyProxy Operator +# This provides a containerized build environment with all necessary tools + +FROM maven:3.9.9-eclipse-temurin-21 + +# Set working directory +WORKDIR /build + +# Copy project files +COPY pom.xml . +COPY src ./src +COPY LICENSE_HEADER . + +# Download dependencies (this layer will be cached if pom.xml doesn't change) +RUN mvn dependency:go-offline -B + +# Build the project +RUN mvn -U clean install -DskipTests + +# The JAR will be at /build/target/shinyproxy-operator-jar-with-dependencies.jar +# You can extract it using: docker cp :/build/target/shinyproxy-operator-jar-with-dependencies.jar . + +CMD ["sh", "-c", "echo 'Build complete! JAR location:' && ls -lh target/*.jar"] diff --git a/devbox.json b/devbox.json new file mode 100644 index 0000000..d3cc97a --- /dev/null +++ b/devbox.json @@ -0,0 +1,31 @@ +{ + "$schema": "https://raw.githubusercontent.com/jetify-com/devbox/0.13.7/.schema/devbox.schema.json", + "packages": [ + "openjdk21@latest", + "maven@latest" + ], + "shell": { + "init_hook": [ + "echo 'Welcome to ShinyProxy Operator development environment!'", + "echo 'Java version:'", + "java -version", + "echo ''", + "echo 'Maven version:'", + "mvn -version", + "echo ''", + "echo 'To build the project, run: mvn -U clean install'", + "echo 'The JAR will be at: target/shinyproxy-operator-jar-with-dependencies.jar'" + ], + "scripts": { + "build": [ + "mvn -U clean install" + ], + "build-fast": [ + "mvn clean install -DskipTests" + ], + "test": [ + "mvn test" + ] + } + } +} diff --git a/docker-compose.build.yml b/docker-compose.build.yml new file mode 100644 index 0000000..0ba91e0 --- /dev/null +++ b/docker-compose.build.yml @@ -0,0 +1,15 @@ +# Docker Compose file for building the ShinyProxy Operator +# Usage: docker-compose -f docker-compose.build.yml up + +version: '3.8' + +services: + builder: + build: + context: . + dockerfile: Dockerfile.build + image: shinyproxy-operator-builder:local + volumes: + # Mount the target directory to get the built JAR + - ./target:/build/target + container_name: shinyproxy-operator-builder From aac7bb0cf536e9c039840c82c1cfe3df68a2e929 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 17:05:37 +0000 Subject: [PATCH 18/37] Remove debug ls commands from workflow for cleaner logs Co-authored-by: jaredlander <628977+jaredlander@users.noreply.github.com> --- .github/workflows/build-and-push-docker.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build-and-push-docker.yaml b/.github/workflows/build-and-push-docker.yaml index f0975f8..15edf88 100644 --- a/.github/workflows/build-and-push-docker.yaml +++ b/.github/workflows/build-and-push-docker.yaml @@ -40,7 +40,6 @@ jobs: run: | cd shinyproxy-operator mvn -U clean install -DskipTests - ls -lh target/ - name: Checkout shinyproxy-docker uses: actions/checkout@v4 @@ -51,7 +50,6 @@ jobs: - name: Copy JAR to Docker build context run: | cp shinyproxy-operator/target/shinyproxy-operator-jar-with-dependencies.jar shinyproxy-docker/Operator/ - ls -lh shinyproxy-docker/Operator/ - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 From 651408529f984be11f4bd3232b119c0f6f91f3d3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 17:06:50 +0000 Subject: [PATCH 19/37] Fix cache key path and Docker tag format for better compatibility Co-authored-by: jaredlander <628977+jaredlander@users.noreply.github.com> --- .github/workflows/build-and-push-docker.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-push-docker.yaml b/.github/workflows/build-and-push-docker.yaml index 15edf88..f008eab 100644 --- a/.github/workflows/build-and-push-docker.yaml +++ b/.github/workflows/build-and-push-docker.yaml @@ -33,7 +33,7 @@ jobs: uses: actions/cache@v4 with: path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + key: ${{ runner.os }}-m2-${{ hashFiles('shinyproxy-operator/pom.xml') }} restore-keys: ${{ runner.os }}-m2 - name: Build shinyproxy-operator JAR @@ -72,7 +72,7 @@ jobs: type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} type=semver,pattern={{major}} - type=sha,prefix={{branch}}- + type=sha,format=short - name: Build and push Docker image uses: docker/build-push-action@v5 From 547c3688fbff42a301d68cbc413e2c9fe30aac67 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 17:08:13 +0000 Subject: [PATCH 20/37] Add explicit GITHUB_TOKEN permissions for security Co-authored-by: jaredlander <628977+jaredlander@users.noreply.github.com> --- .github/workflows/build-and-push-docker.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build-and-push-docker.yaml b/.github/workflows/build-and-push-docker.yaml index f008eab..1627b6d 100644 --- a/.github/workflows/build-and-push-docker.yaml +++ b/.github/workflows/build-and-push-docker.yaml @@ -17,6 +17,10 @@ jobs: build-and-push: runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: - name: Checkout shinyproxy-operator uses: actions/checkout@v4 From 0e800392321724e9f5b15cd11784f471c6b4938c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 17:15:12 +0000 Subject: [PATCH 21/37] Update build commands to use 'mvn clean package -DskipTests' - Changed from 'mvn clean install' to 'mvn clean package -DskipTests' for faster builds - 'package' is sufficient for creating the JAR without installing to local repository - Updated Dockerfile.build, BUILD.md, and devbox.json consistently - Addresses user feedback on build command Co-authored-by: jaredlander <628977+jaredlander@users.noreply.github.com> --- BUILD.md | 6 +++++- Dockerfile.build | 2 +- devbox.json | 6 +++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/BUILD.md b/BUILD.md index e6cd600..58027da 100644 --- a/BUILD.md +++ b/BUILD.md @@ -83,7 +83,11 @@ devbox run build-fast If you have JDK 21 and Maven installed locally: ```bash -mvn -U clean install +# Build without tests (faster) +mvn clean package -DskipTests + +# Or build with tests +mvn clean package ``` The JAR will be at `target/shinyproxy-operator-jar-with-dependencies.jar`. diff --git a/Dockerfile.build b/Dockerfile.build index c3bd55f..d8bb6b4 100644 --- a/Dockerfile.build +++ b/Dockerfile.build @@ -15,7 +15,7 @@ COPY LICENSE_HEADER . RUN mvn dependency:go-offline -B # Build the project -RUN mvn -U clean install -DskipTests +RUN mvn clean package -DskipTests # The JAR will be at /build/target/shinyproxy-operator-jar-with-dependencies.jar # You can extract it using: docker cp :/build/target/shinyproxy-operator-jar-with-dependencies.jar . diff --git a/devbox.json b/devbox.json index d3cc97a..1020130 100644 --- a/devbox.json +++ b/devbox.json @@ -13,15 +13,15 @@ "echo 'Maven version:'", "mvn -version", "echo ''", - "echo 'To build the project, run: mvn -U clean install'", + "echo 'To build the project, run: mvn clean package -DskipTests'", "echo 'The JAR will be at: target/shinyproxy-operator-jar-with-dependencies.jar'" ], "scripts": { "build": [ - "mvn -U clean install" + "mvn clean package" ], "build-fast": [ - "mvn clean install -DskipTests" + "mvn clean package -DskipTests" ], "test": [ "mvn test" From a079841b1d75c690db6e15f362ea6288168866ae Mon Sep 17 00:00:00 2001 From: Jared Lander Date: Thu, 4 Dec 2025 13:05:47 -0500 Subject: [PATCH 22/37] Changed the build process to either use devbox, or to have docker build simply output the artifact. Removed the docker compose option. --- BUILD.md | 91 +++++----------------------------------- Dockerfile.build | 9 ++-- docker-compose.build.yml | 15 ------- 3 files changed, 14 insertions(+), 101 deletions(-) delete mode 100644 docker-compose.build.yml diff --git a/BUILD.md b/BUILD.md index 58027da..32f86cf 100644 --- a/BUILD.md +++ b/BUILD.md @@ -15,42 +15,12 @@ Choose one of the following options: This is the easiest option if you have Docker installed. -### Quick Build with Docker Compose - -```bash -# Build the project -docker-compose -f docker-compose.build.yml up - -# The JAR will be in ./target/shinyproxy-operator-jar-with-dependencies.jar -``` - -### Alternative: Build with Dockerfile Directly - -```bash -# Build the Docker image -docker build -f Dockerfile.build -t shinyproxy-operator-builder . - -# Run the build -docker run --rm -v "$(pwd)/target:/build/target" shinyproxy-operator-builder - -# The JAR will be in ./target/shinyproxy-operator-jar-with-dependencies.jar -``` - -### Extract JAR from Container (if needed) - ```bash -# Start the container -docker run -d --name builder shinyproxy-operator-builder - -# Copy the JAR out -docker cp builder:/build/target/shinyproxy-operator-jar-with-dependencies.jar . +docker build -f Dockerfile.build --target artifacts -t shinyproxy-operator-builder -o ./. . -# Clean up -docker rm builder +# the jar will be at ./shinyproxy-operator-jar-with-dependencies.jar ``` ---- - ## Option 2: Build with Devbox [Devbox](https://www.jetify.com/devbox) provides an isolated development environment without Docker. @@ -73,70 +43,29 @@ devbox run build # Or build without tests (faster) devbox run build-fast -# The JAR will be at ./target/shinyproxy-operator-jar-with-dependencies.jar -``` - ---- - -## Option 3: Local Build - -If you have JDK 21 and Maven installed locally: - -```bash -# Build without tests (faster) -mvn clean package -DskipTests +# Or to use previous build steps (faster still) +devbox run package-fast -# Or build with tests -mvn clean package +# The JAR will be at ./target/shinyproxy-operator-jar-with-dependencies.jar ``` -The JAR will be at `target/shinyproxy-operator-jar-with-dependencies.jar`. - ---- - ## Verifying the Build After building, verify the JAR exists: ```bash +# if using docker +ls -lh shinyproxy-operator-jar-with-dependencies.jar + +# if using devbox ls -lh target/shinyproxy-operator-jar-with-dependencies.jar ``` You should see a file of approximately 50-100 MB. ---- - -## Running the Built JAR - -```bash -java -jar target/shinyproxy-operator-jar-with-dependencies.jar -``` - ---- - ## Building a Docker Image -If you want to create a Docker image with the operator: - -```bash -# First build the JAR using one of the methods above - -# Then create a runtime Dockerfile (example): -cat > Dockerfile.runtime <<'EOF' -FROM eclipse-temurin:21-jre-alpine - -WORKDIR /opt/shinyproxy-operator - -COPY target/shinyproxy-operator-jar-with-dependencies.jar shinyproxy-operator.jar - -ENTRYPOINT ["java", "-jar", "shinyproxy-operator.jar"] -EOF - -# Build the runtime image -docker build -f Dockerfile.runtime -t shinyproxy-operator:local . -``` - ---- +This needs to be copied into the image at https://github.com/openanalytics/shinyproxy-docker/. ## Troubleshooting diff --git a/Dockerfile.build b/Dockerfile.build index d8bb6b4..af8423d 100644 --- a/Dockerfile.build +++ b/Dockerfile.build @@ -1,7 +1,7 @@ # Dockerfile for building the ShinyProxy Operator # This provides a containerized build environment with all necessary tools -FROM maven:3.9.9-eclipse-temurin-21 +FROM maven:3.9.9-eclipse-temurin-21 AS builder # Set working directory WORKDIR /build @@ -17,7 +17,6 @@ RUN mvn dependency:go-offline -B # Build the project RUN mvn clean package -DskipTests -# The JAR will be at /build/target/shinyproxy-operator-jar-with-dependencies.jar -# You can extract it using: docker cp :/build/target/shinyproxy-operator-jar-with-dependencies.jar . - -CMD ["sh", "-c", "echo 'Build complete! JAR location:' && ls -lh target/*.jar"] +# Export stage - only contains the JAR artifact +FROM scratch AS artifacts +COPY --from=builder /build/target/shinyproxy-operator-jar-with-dependencies.jar /shinyproxy-operator-jar-with-dependencies.jar diff --git a/docker-compose.build.yml b/docker-compose.build.yml deleted file mode 100644 index 0ba91e0..0000000 --- a/docker-compose.build.yml +++ /dev/null @@ -1,15 +0,0 @@ -# Docker Compose file for building the ShinyProxy Operator -# Usage: docker-compose -f docker-compose.build.yml up - -version: '3.8' - -services: - builder: - build: - context: . - dockerfile: Dockerfile.build - image: shinyproxy-operator-builder:local - volumes: - # Mount the target directory to get the built JAR - - ./target:/build/target - container_name: shinyproxy-operator-builder From 675883de2307455667c4833f5826b8b2bd5505ff Mon Sep 17 00:00:00 2001 From: Jared Lander Date: Thu, 4 Dec 2025 13:13:33 -0500 Subject: [PATCH 23/37] Using correct name for jdk in devbox. Added debox scripts for setting up mvn dependencies and for packaging without tests. Updated BUILD.md to reflect these changes. --- BUILD.md | 3 + devbox.json | 12 +++- devbox.lock | 177 ++++++++++++++++++++++++++++++++++++++++++++++++++++ pom.xml | 1 + 4 files changed, 192 insertions(+), 1 deletion(-) create mode 100644 devbox.lock diff --git a/BUILD.md b/BUILD.md index 32f86cf..eb9464c 100644 --- a/BUILD.md +++ b/BUILD.md @@ -37,6 +37,9 @@ curl -fsSL https://get.jetify.com/devbox | bash # Enter the devbox shell (installs JDK 21 and Maven automatically) devbox shell +# Install the needed dependencies +devbox run install-deps + # Build the project devbox run build diff --git a/devbox.json b/devbox.json index 1020130..8a9236e 100644 --- a/devbox.json +++ b/devbox.json @@ -1,7 +1,8 @@ { "$schema": "https://raw.githubusercontent.com/jetify-com/devbox/0.13.7/.schema/devbox.schema.json", "packages": [ - "openjdk21@latest", + "javaPackages.compiler.openjdk21@latest", + "binutils@latest", "maven@latest" ], "shell": { @@ -17,12 +18,21 @@ "echo 'The JAR will be at: target/shinyproxy-operator-jar-with-dependencies.jar'" ], "scripts": { + "install-deps": [ + "mvn dependency:go-offline -B" + ], "build": [ "mvn clean package" ], "build-fast": [ "mvn clean package -DskipTests" ], + "package": [ + "mvn package" + ], + "package-fast": [ + "mvn package -DskipTests" + ], "test": [ "mvn test" ] diff --git a/devbox.lock b/devbox.lock new file mode 100644 index 0000000..29c703b --- /dev/null +++ b/devbox.lock @@ -0,0 +1,177 @@ +{ + "lockfile_version": "1", + "packages": { + "binutils@latest": { + "last_modified": "2025-11-23T21:50:36Z", + "resolved": "github:NixOS/nixpkgs/ee09932cedcef15aaf476f9343d1dea2cb77e261#binutils", + "source": "devbox-search", + "version": "2.44", + "systems": { + "aarch64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/0hz32dza8cmfq4i823mcz4kqpk54bils-binutils-wrapper-2.44", + "default": true + }, + { + "name": "man", + "path": "/nix/store/16cprzg9s7bw501rppw2bcmf3wpwsxw8-binutils-wrapper-2.44-man", + "default": true + }, + { + "name": "info", + "path": "/nix/store/y9kanlp4jzfzwjriga6jplw8jk9hx5dk-binutils-wrapper-2.44-info" + } + ], + "store_path": "/nix/store/0hz32dza8cmfq4i823mcz4kqpk54bils-binutils-wrapper-2.44" + }, + "aarch64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/fhmb82a3q7nbsz3w9yj28nyscykrj87p-binutils-wrapper-2.44", + "default": true + }, + { + "name": "man", + "path": "/nix/store/136rg4hbbzzgfk9fzpih43wnsb7k04q5-binutils-wrapper-2.44-man", + "default": true + }, + { + "name": "info", + "path": "/nix/store/glmqwm69w9b7wdil3iqncr1ach6py0zh-binutils-wrapper-2.44-info" + } + ], + "store_path": "/nix/store/fhmb82a3q7nbsz3w9yj28nyscykrj87p-binutils-wrapper-2.44" + }, + "x86_64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/jprgd6iai892g6dhajv9k9izxwin5g74-binutils-wrapper-2.44", + "default": true + }, + { + "name": "man", + "path": "/nix/store/g4xhkh8nh9c61xfd428a96c5j97gzyd1-binutils-wrapper-2.44-man", + "default": true + }, + { + "name": "info", + "path": "/nix/store/75plj78svg9ala98s5cvk8dma4bd7syn-binutils-wrapper-2.44-info" + } + ], + "store_path": "/nix/store/jprgd6iai892g6dhajv9k9izxwin5g74-binutils-wrapper-2.44" + }, + "x86_64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/xwydcyvlsa3cvssk0y5llgdhlhjvmqdm-binutils-wrapper-2.44", + "default": true + }, + { + "name": "man", + "path": "/nix/store/2gjcdfkx83vgaszic2vn2cnsfl5n1mzk-binutils-wrapper-2.44-man", + "default": true + }, + { + "name": "info", + "path": "/nix/store/5c4dylia9lsg6qbp3a9rdvv9nr0xp3jz-binutils-wrapper-2.44-info" + } + ], + "store_path": "/nix/store/xwydcyvlsa3cvssk0y5llgdhlhjvmqdm-binutils-wrapper-2.44" + } + } + }, + "github:NixOS/nixpkgs/nixpkgs-unstable": { + "last_modified": "2025-12-03T20:43:00Z", + "resolved": "github:NixOS/nixpkgs/ebc94f855ef25347c314258c10393a92794e7ab9?lastModified=1764794580&narHash=sha256-UMVihg0OQ980YqmOAPz%2BzkuCEb9hpE5Xj2v%2BZGNjQ%2BM%3D" + }, + "javaPackages.compiler.openjdk21@latest": { + "last_modified": "2025-11-23T21:50:36Z", + "resolved": "github:NixOS/nixpkgs/ee09932cedcef15aaf476f9343d1dea2cb77e261#javaPackages.compiler.openjdk21", + "source": "devbox-search", + "version": "21.0.9+10", + "systems": { + "aarch64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/jlxgl3k14x3zhfa00rsygfw3fgp6xgg8-openjdk-21.0.9+10", + "default": true + }, + { + "name": "debug", + "path": "/nix/store/qmczwsgqyfq49baj86rwza8nc49ag282-openjdk-21.0.9+10-debug" + } + ], + "store_path": "/nix/store/jlxgl3k14x3zhfa00rsygfw3fgp6xgg8-openjdk-21.0.9+10" + }, + "x86_64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/65qpdkc33j5wqzxvz8c23zhgms8hl35y-openjdk-21.0.9+10", + "default": true + }, + { + "name": "debug", + "path": "/nix/store/iqrpym2396ppr5498xci88zf6kc37yxl-openjdk-21.0.9+10-debug" + } + ], + "store_path": "/nix/store/65qpdkc33j5wqzxvz8c23zhgms8hl35y-openjdk-21.0.9+10" + } + } + }, + "maven@latest": { + "last_modified": "2025-11-23T21:50:36Z", + "resolved": "github:NixOS/nixpkgs/ee09932cedcef15aaf476f9343d1dea2cb77e261#maven", + "source": "devbox-search", + "version": "3.9.11", + "systems": { + "aarch64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/xqja439g5mrggz87a4f4hp9rdwjmkwj4-maven-3.9.11", + "default": true + } + ], + "store_path": "/nix/store/xqja439g5mrggz87a4f4hp9rdwjmkwj4-maven-3.9.11" + }, + "aarch64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/sjhc4f9kvcr9vz3skyl3dvhir2dby2f1-maven-3.9.11", + "default": true + } + ], + "store_path": "/nix/store/sjhc4f9kvcr9vz3skyl3dvhir2dby2f1-maven-3.9.11" + }, + "x86_64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/90qdbpmykzmgxm2df56axf8lxh1bdv5k-maven-3.9.11", + "default": true + } + ], + "store_path": "/nix/store/90qdbpmykzmgxm2df56axf8lxh1bdv5k-maven-3.9.11" + }, + "x86_64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/4djac4688b16msvhkmyp217ad9kc1m5s-maven-3.9.11", + "default": true + } + ], + "store_path": "/nix/store/4djac4688b16msvhkmyp217ad9kc1m5s-maven-3.9.11" + } + } + } + } +} diff --git a/pom.xml b/pom.xml index 6c72a54..7fbbb94 100644 --- a/pom.xml +++ b/pom.xml @@ -366,6 +366,7 @@ .gitignore .editorconfig JenkinsfileSCM + .devbox/** From 1ffb71c96cb559255b838afab67ebee490787cbd Mon Sep 17 00:00:00 2001 From: Jared Lander Date: Thu, 4 Dec 2025 13:16:59 -0500 Subject: [PATCH 24/37] Cleaned up docs to match the docker steps better. --- BUILD.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/BUILD.md b/BUILD.md index eb9464c..bec6508 100644 --- a/BUILD.md +++ b/BUILD.md @@ -78,7 +78,7 @@ If you experience network issues during the Maven build: ```bash # Build with host network (Linux only) -docker build --network=host -f Dockerfile.build -t shinyproxy-operator-builder . +docker build --network=host -f Dockerfile.build --target artifacts -t shinyproxy-operator-builder -o ./. . ``` ### Permission issues with target directory @@ -92,9 +92,7 @@ sudo chown -R $USER:$USER target/ ```bash # Increase Docker memory limit or add Maven options -docker run --rm -v "$(pwd)/target:/build/target" \ - -e MAVEN_OPTS="-Xmx2048m" \ - shinyproxy-operator-builder +docker build -f Dockerfile.build --target artifacts --build-arg MAVEN_OPTS="-Xmx2048m" -t shinyproxy-operator-builder -o ./. . ``` --- @@ -104,8 +102,7 @@ docker run --rm -v "$(pwd)/target:/build/target" \ For ongoing development: 1. **Use Devbox** for the best development experience with automatic dependency management -2. **Use Docker Compose** for one-off builds without installing tools -3. **Use Local Build** if you already have the tools installed +1. **Use Docker** for one-off builds without installing tools --- From ebefa80fe65ebc80a0091015a4e399860a330693 Mon Sep 17 00:00:00 2001 From: Jared Lander Date: Thu, 4 Dec 2025 13:48:58 -0500 Subject: [PATCH 25/37] Added instructions for build the docker image for the actual oeprator. --- BUILD.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/BUILD.md b/BUILD.md index bec6508..c78a66c 100644 --- a/BUILD.md +++ b/BUILD.md @@ -70,6 +70,24 @@ You should see a file of approximately 50-100 MB. This needs to be copied into the image at https://github.com/openanalytics/shinyproxy-docker/. +First clonse the repo + +```bash +# clone the repo +git clone git@github.com:openanalytics/shinyproxy-docker.git +cd shinyproxy-docker +``` + +Copy the jar and build the image + +```bash +# copy the jar into the build context +cp ../shinyproxy-operator/shinyproxy-operator-jar-with-dependencies.jar Operator/. + +# build the docker image +docker build -t shinyproxy-operator-dev:v2.3.1 --build-arg JAR_LOCATION=shinyproxy-operator-jar-with-dependencies.jar Operator +``` + ## Troubleshooting ### Docker build fails with network errors @@ -118,7 +136,7 @@ Example: ```yaml services: shinyproxy-operator: - image: shinyproxy-operator:local + image: shinyproxy-operator-dev:v2.3.1 environment: - SPO_DOCKER_GID=999 - SHINYPROXY_ENV_TEST_VAR=test_value From d2a55164ca2d2ba8017abdeb83ebeeebc7664c86 Mon Sep 17 00:00:00 2001 From: Jared Lander Date: Thu, 4 Dec 2025 13:49:27 -0500 Subject: [PATCH 26/37] Running dependency management berore copying the source so it will happen less often. --- Dockerfile.build | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile.build b/Dockerfile.build index af8423d..eadce36 100644 --- a/Dockerfile.build +++ b/Dockerfile.build @@ -8,12 +8,13 @@ WORKDIR /build # Copy project files COPY pom.xml . -COPY src ./src -COPY LICENSE_HEADER . # Download dependencies (this layer will be cached if pom.xml doesn't change) RUN mvn dependency:go-offline -B +COPY src ./src +COPY LICENSE_HEADER . + # Build the project RUN mvn clean package -DskipTests From dc555e96b8a0180f68cf8914c178f8114b609832 Mon Sep 17 00:00:00 2001 From: Jared Lander Date: Thu, 4 Dec 2025 13:50:02 -0500 Subject: [PATCH 27/37] Making it so that port mappings are for external ports, but internally caddy and shinyproxy still use the regular ports they expect. --- .../impl/docker/CaddyConfig.kt | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/CaddyConfig.kt b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/CaddyConfig.kt index 5baa649..d845d09 100644 --- a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/CaddyConfig.kt +++ b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/CaddyConfig.kt @@ -103,7 +103,7 @@ class CaddyConfig(private val dockerClient: DockerClient, mainDataDir: Path, con } private fun generateServer(): Map { - val listen = if (enableTls) listOf(":$caddyPortHttps") else listOf(":$caddyPortHttp") + val listen = if (enableTls) listOf(":443") else listOf(":80") return mapOf("listen" to listen, "routes" to generateRoutes(), "tls_connection_policies" to generateTlsConnectionPolicies()) } @@ -279,7 +279,17 @@ class CaddyConfig(private val dockerClient: DockerClient, mainDataDir: Path, con logger.info { "[Caddy] Pulling image" } dockerActions.pullImage(caddyImage) - val ports = if (enableTls) listOf("$caddyPortHttp", "$caddyPortHttps") else listOf("$caddyPortHttp") + val portBindings = if (enableTls) { + mapOf( + "80" to listOf(PortBinding.of("0.0.0.0", caddyPortHttp.toString())), + "443" to listOf(PortBinding.of("0.0.0.0", caddyPortHttps.toString())) + ) + } else { + mapOf("80" to listOf(PortBinding.of("0.0.0.0", caddyPortHttp.toString()))) + } + + val ports = if (enableTls) listOf("80", "443") else listOf("80") + val hostConfig = HostConfig.builder() .networkMode(DockerOrchestrator.SHARED_NETWORK_NAME) .binds(HostConfig.Bind.builder() @@ -298,7 +308,7 @@ class CaddyConfig(private val dockerClient: DockerClient, mainDataDir: Path, con .from(dataDir.resolve("certs").toString()) .to("/certs") .build() - ).portBindings(ports.associateWith { listOf(PortBinding.of("0.0.0.0", it)) }) + ).portBindings(portBindings) .restartPolicy(HostConfig.RestartPolicy.always()) .build() From 32d79fe714c09ebd25ffa528a63e88cbe6b8c221 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 19:00:57 +0000 Subject: [PATCH 28/37] Change branch from develop to dev and make registry URL configurable via secret Co-authored-by: jaredlander <628977+jaredlander@users.noreply.github.com> --- .github/workflows/build-and-push-docker.yaml | 8 ++++---- docs/CI-CD.md | 21 +++++++++++++------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-and-push-docker.yaml b/.github/workflows/build-and-push-docker.yaml index 1627b6d..da25e56 100644 --- a/.github/workflows/build-and-push-docker.yaml +++ b/.github/workflows/build-and-push-docker.yaml @@ -4,13 +4,13 @@ on: push: branches: - main - - develop + - dev tags: - 'v*' pull_request: branches: - main - - develop + - dev workflow_dispatch: jobs: @@ -61,7 +61,7 @@ jobs: - name: Login to Azure Container Registry uses: docker/login-action@v3 with: - registry: landeranalytics.azurecr.io + registry: ${{ secrets.ACR_REGISTRY }} username: ${{ secrets.ACR_USERNAME }} password: ${{ secrets.ACR_PASSWORD }} @@ -69,7 +69,7 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: landeranalytics.azurecr.io/shinyproxy-operator + images: ${{ secrets.ACR_REGISTRY }}/shinyproxy-operator tags: | type=ref,event=branch type=ref,event=pr diff --git a/docs/CI-CD.md b/docs/CI-CD.md index b5bd892..06276f0 100644 --- a/docs/CI-CD.md +++ b/docs/CI-CD.md @@ -33,15 +33,21 @@ The workflow is triggered on: The workflow requires the following secrets to be configured in the GitHub repository: +#### `ACR_REGISTRY` +- **Description**: Azure Container Registry URL +- **Type**: Repository Secret +- **Usage**: The registry URL (e.g., `landeranalytics.azurecr.io`) +- **Example**: `landeranalytics.azurecr.io` + #### `ACR_USERNAME` - **Description**: Username for Azure Container Registry authentication - **Type**: Repository Secret -- **Usage**: Used to authenticate with `landeranalytics.azurecr.io` +- **Usage**: Used to authenticate with the Azure Container Registry #### `ACR_PASSWORD` - **Description**: Password for Azure Container Registry authentication - **Type**: Repository Secret -- **Usage**: Used to authenticate with `landeranalytics.azurecr.io` +- **Usage**: Used to authenticate with the Azure Container Registry ### Setting Up Secrets @@ -51,6 +57,7 @@ To configure these secrets: 2. Navigate to **Settings** > **Secrets and variables** > **Actions** 3. Click **New repository secret** 4. Add each secret with the name and value: + - Name: `ACR_REGISTRY`, Value: Your Azure Container Registry URL (e.g., `landeranalytics.azurecr.io`) - Name: `ACR_USERNAME`, Value: Your Azure Container Registry username - Name: `ACR_PASSWORD`, Value: Your Azure Container Registry password @@ -68,7 +75,7 @@ The workflow automatically generates tags based on the Git context: ### Docker Registry -Images are pushed to: `landeranalytics.azurecr.io/shinyproxy-operator` +Images are pushed to the Azure Container Registry specified by the `ACR_REGISTRY` secret, under the repository name `shinyproxy-operator`. ### Build Arguments @@ -105,9 +112,9 @@ If the Maven build fails: #### Docker Push Failures If pushing to the registry fails: -- Verify that `ACR_USERNAME` and `ACR_PASSWORD` secrets are correctly set +- Verify that `ACR_REGISTRY`, `ACR_USERNAME`, and `ACR_PASSWORD` secrets are correctly set - Ensure the service principal has push permissions to the registry -- Check that the registry URL is correct: `landeranalytics.azurecr.io` +- Check that the registry URL in `ACR_REGISTRY` is correct and accessible #### JAR Not Found @@ -138,6 +145,6 @@ cp ../../shinyproxy-operator/target/shinyproxy-operator-jar-with-dependencies.ja docker build -t shinyproxy-operator-dev --build-arg JAR_LOCATION=shinyproxy-operator-jar-with-dependencies.jar . # Step 5: Tag and push (optional) -docker tag shinyproxy-operator-dev landeranalytics.azurecr.io/shinyproxy-operator:dev -docker push landeranalytics.azurecr.io/shinyproxy-operator:dev +docker tag shinyproxy-operator-dev /shinyproxy-operator:dev +docker push /shinyproxy-operator:dev ``` From ec0e7952ec992b1a4d8915711d084454e9ef4cc1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 19:02:55 +0000 Subject: [PATCH 29/37] Update documentation to consistently use 'dev' instead of 'develop' Co-authored-by: jaredlander <628977+jaredlander@users.noreply.github.com> --- docs/CI-CD.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/CI-CD.md b/docs/CI-CD.md index 06276f0..2206d19 100644 --- a/docs/CI-CD.md +++ b/docs/CI-CD.md @@ -11,8 +11,8 @@ The workflow is defined in `.github/workflows/build-and-push-docker.yaml`. ### Triggers The workflow is triggered on: -- Pushes to `main` or `develop` branches -- Pull requests to `main` or `develop` branches +- Pushes to `main` or `dev` branches +- Pull requests to `main` or `dev` branches - Tag pushes matching `v*` pattern (e.g., `v1.0.0`) - Manual workflow dispatch @@ -65,7 +65,7 @@ To configure these secrets: The workflow automatically generates tags based on the Git context: -- **Branch pushes**: `` (e.g., `main`, `develop`) +- **Branch pushes**: `` (e.g., `main`, `dev`) - **Pull requests**: `pr-` (e.g., `pr-123`) - **Version tags**: Multiple formats for semantic versioning - `` (e.g., `1.2.3`) From 5e642b502bd7bce19a9ff1d6086207fb7f20c2ee Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 19:08:42 +0000 Subject: [PATCH 30/37] Add dev branch specific tags and rename ACR to CR for generic registry support Co-authored-by: jaredlander <628977+jaredlander@users.noreply.github.com> --- .github/workflows/build-and-push-docker.yaml | 13 +++--- README.md | 2 +- docs/CI-CD.md | 42 +++++++++++--------- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/.github/workflows/build-and-push-docker.yaml b/.github/workflows/build-and-push-docker.yaml index da25e56..2700624 100644 --- a/.github/workflows/build-and-push-docker.yaml +++ b/.github/workflows/build-and-push-docker.yaml @@ -58,24 +58,27 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Login to Azure Container Registry + - name: Login to Container Registry uses: docker/login-action@v3 with: - registry: ${{ secrets.ACR_REGISTRY }} - username: ${{ secrets.ACR_USERNAME }} - password: ${{ secrets.ACR_PASSWORD }} + registry: ${{ secrets.CR_REGISTRY }} + username: ${{ secrets.CR_USERNAME }} + password: ${{ secrets.CR_PASSWORD }} - name: Extract metadata for Docker id: meta uses: docker/metadata-action@v5 with: - images: ${{ secrets.ACR_REGISTRY }}/shinyproxy-operator + images: ${{ secrets.CR_REGISTRY }}/shinyproxy-operator tags: | type=ref,event=branch type=ref,event=pr type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} type=semver,pattern={{major}} + type=semver,pattern=dev-{{version}},enable=${{ github.ref == 'refs/heads/dev' }} + type=semver,pattern=dev-{{major}}.{{minor}},enable=${{ github.ref == 'refs/heads/dev' }} + type=semver,pattern=dev-{{major}},enable=${{ github.ref == 'refs/heads/dev' }} type=sha,format=short - name: Build and push Docker image diff --git a/README.md b/README.md index cdfed61..4440a01 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ This repository includes automated GitHub Actions workflows for building and dep ### Docker Image Build and Push -A workflow automatically builds the operator JAR, creates a Docker image, and pushes it to Azure Container Registry. See [docs/CI-CD.md](docs/CI-CD.md) for detailed documentation on: +A workflow automatically builds the operator JAR, creates a Docker image, and pushes it to a Container Registry. See [docs/CI-CD.md](docs/CI-CD.md) for detailed documentation on: - Workflow configuration - Required secrets setup diff --git a/docs/CI-CD.md b/docs/CI-CD.md index 2206d19..10fb511 100644 --- a/docs/CI-CD.md +++ b/docs/CI-CD.md @@ -2,7 +2,7 @@ ## Build and Push Docker Image Workflow -This repository includes a GitHub Actions workflow that automatically builds the shinyproxy-operator JAR and creates a Docker image that is pushed to Azure Container Registry. +This repository includes a GitHub Actions workflow that automatically builds the shinyproxy-operator JAR and creates a Docker image that is pushed to a Container Registry. ### Workflow File @@ -25,7 +25,7 @@ The workflow is triggered on: 5. **Checkout shinyproxy-docker**: Clones the [openanalytics/shinyproxy-docker](https://github.com/openanalytics/shinyproxy-docker) repository 6. **Copy JAR to Docker build context**: Copies the built JAR to the Operator directory in shinyproxy-docker 7. **Set up Docker Buildx**: Configures Docker Buildx for advanced build features -8. **Login to Azure Container Registry**: Authenticates with the Azure Container Registry +8. **Login to Container Registry**: Authenticates with the Container Registry 9. **Extract metadata for Docker**: Generates Docker tags and labels based on Git context 10. **Build and push Docker image**: Builds the Docker image and pushes it to the registry @@ -33,21 +33,21 @@ The workflow is triggered on: The workflow requires the following secrets to be configured in the GitHub repository: -#### `ACR_REGISTRY` -- **Description**: Azure Container Registry URL +#### `CR_REGISTRY` +- **Description**: Container Registry URL - **Type**: Repository Secret -- **Usage**: The registry URL (e.g., `landeranalytics.azurecr.io`) +- **Usage**: The registry URL (e.g., `landeranalytics.azurecr.io`, `ghcr.io`, etc.) - **Example**: `landeranalytics.azurecr.io` -#### `ACR_USERNAME` -- **Description**: Username for Azure Container Registry authentication +#### `CR_USERNAME` +- **Description**: Username for Container Registry authentication - **Type**: Repository Secret -- **Usage**: Used to authenticate with the Azure Container Registry +- **Usage**: Used to authenticate with the Container Registry -#### `ACR_PASSWORD` -- **Description**: Password for Azure Container Registry authentication +#### `CR_PASSWORD` +- **Description**: Password for Container Registry authentication - **Type**: Repository Secret -- **Usage**: Used to authenticate with the Azure Container Registry +- **Usage**: Used to authenticate with the Container Registry ### Setting Up Secrets @@ -57,9 +57,9 @@ To configure these secrets: 2. Navigate to **Settings** > **Secrets and variables** > **Actions** 3. Click **New repository secret** 4. Add each secret with the name and value: - - Name: `ACR_REGISTRY`, Value: Your Azure Container Registry URL (e.g., `landeranalytics.azurecr.io`) - - Name: `ACR_USERNAME`, Value: Your Azure Container Registry username - - Name: `ACR_PASSWORD`, Value: Your Azure Container Registry password + - Name: `CR_REGISTRY`, Value: Your Container Registry URL (e.g., `landeranalytics.azurecr.io`, `ghcr.io`) + - Name: `CR_USERNAME`, Value: Your Container Registry username + - Name: `CR_PASSWORD`, Value: Your Container Registry password ### Docker Image Tags @@ -71,11 +71,15 @@ The workflow automatically generates tags based on the Git context: - `` (e.g., `1.2.3`) - `.` (e.g., `1.2`) - `` (e.g., `1`) -- **SHA-based**: `-` (e.g., `main-abc1234`) +- **Dev branch version tags**: When triggered by the `dev` branch with a version tag + - `dev-` (e.g., `dev-1.2.3`) + - `dev-.` (e.g., `dev-1.2`) + - `dev-` (e.g., `dev-1`) +- **SHA-based**: `` (e.g., `abc1234`) ### Docker Registry -Images are pushed to the Azure Container Registry specified by the `ACR_REGISTRY` secret, under the repository name `shinyproxy-operator`. +Images are pushed to the Container Registry specified by the `CR_REGISTRY` secret, under the repository name `shinyproxy-operator`. ### Build Arguments @@ -112,9 +116,9 @@ If the Maven build fails: #### Docker Push Failures If pushing to the registry fails: -- Verify that `ACR_REGISTRY`, `ACR_USERNAME`, and `ACR_PASSWORD` secrets are correctly set -- Ensure the service principal has push permissions to the registry -- Check that the registry URL in `ACR_REGISTRY` is correct and accessible +- Verify that `CR_REGISTRY`, `CR_USERNAME`, and `CR_PASSWORD` secrets are correctly set +- Ensure the credentials have push permissions to the registry +- Check that the registry URL in `CR_REGISTRY` is correct and accessible #### JAR Not Found From aa857fe6fecac7c5a088aea9bcfe8451cf04ef9a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 19:11:07 +0000 Subject: [PATCH 31/37] Fix dev branch version tags to use extracted version from JAR manifest Co-authored-by: jaredlander <628977+jaredlander@users.noreply.github.com> --- .github/workflows/build-and-push-docker.yaml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-push-docker.yaml b/.github/workflows/build-and-push-docker.yaml index 2700624..9ebf078 100644 --- a/.github/workflows/build-and-push-docker.yaml +++ b/.github/workflows/build-and-push-docker.yaml @@ -45,6 +45,17 @@ jobs: cd shinyproxy-operator mvn -U clean install -DskipTests + - name: Extract version from JAR + id: version + run: | + VERSION=$(unzip -p shinyproxy-operator/target/shinyproxy-operator-jar-with-dependencies.jar META-INF/MANIFEST.MF | grep Implementation-Version | cut -d' ' -f2 | tr -d '\r') + echo "full=$VERSION" >> $GITHUB_OUTPUT + MAJOR=$(echo $VERSION | cut -d. -f1) + echo "major=$MAJOR" >> $GITHUB_OUTPUT + MINOR=$(echo $VERSION | cut -d. -f2) + echo "minor=$MINOR" >> $GITHUB_OUTPUT + echo "Version: $VERSION (Major: $MAJOR, Minor: $MINOR)" + - name: Checkout shinyproxy-docker uses: actions/checkout@v4 with: @@ -76,9 +87,9 @@ jobs: type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} type=semver,pattern={{major}} - type=semver,pattern=dev-{{version}},enable=${{ github.ref == 'refs/heads/dev' }} - type=semver,pattern=dev-{{major}}.{{minor}},enable=${{ github.ref == 'refs/heads/dev' }} - type=semver,pattern=dev-{{major}},enable=${{ github.ref == 'refs/heads/dev' }} + type=raw,value=dev-${{ steps.version.outputs.full }},enable=${{ github.ref == 'refs/heads/dev' }} + type=raw,value=dev-${{ steps.version.outputs.major }}.${{ steps.version.outputs.minor }},enable=${{ github.ref == 'refs/heads/dev' }} + type=raw,value=dev-${{ steps.version.outputs.major }},enable=${{ github.ref == 'refs/heads/dev' }} type=sha,format=short - name: Build and push Docker image From a465ab7f3ac6e76d1241b027c62e637f6dc7882f Mon Sep 17 00:00:00 2001 From: Jared Lander Date: Thu, 4 Dec 2025 14:14:12 -0500 Subject: [PATCH 32/37] Changing gh action to reference master branch as that's what it is still called --- .github/workflows/build-and-push-docker.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-push-docker.yaml b/.github/workflows/build-and-push-docker.yaml index 9ebf078..5d281a4 100644 --- a/.github/workflows/build-and-push-docker.yaml +++ b/.github/workflows/build-and-push-docker.yaml @@ -3,13 +3,13 @@ name: Build and Push Docker Image on: push: branches: - - main + - master - dev tags: - 'v*' pull_request: branches: - - main + - master - dev workflow_dispatch: From 12d454ef49050e2b581fa8ba4a289539c36098e9 Mon Sep 17 00:00:00 2001 From: Jared Lander Date: Thu, 4 Dec 2025 14:29:42 -0500 Subject: [PATCH 33/37] Using setup-java action's builtin caching --- .github/workflows/build-and-push-docker.yaml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-and-push-docker.yaml b/.github/workflows/build-and-push-docker.yaml index 5d281a4..af2b926 100644 --- a/.github/workflows/build-and-push-docker.yaml +++ b/.github/workflows/build-and-push-docker.yaml @@ -32,13 +32,8 @@ jobs: with: java-version: 21 distribution: temurin - - - name: Cache Maven packages - uses: actions/cache@v4 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('shinyproxy-operator/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 + cache: 'maven' + cache-dependency-path: 'shinyproxy-operator/pom.xml' - name: Build shinyproxy-operator JAR run: | From a0b990e476f15a484feddad1b1ebd6c9f240aad1 Mon Sep 17 00:00:00 2001 From: Jared Lander Date: Thu, 4 Dec 2025 15:14:16 -0500 Subject: [PATCH 34/37] Cleaned up readme --- docs/CI-CD.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/CI-CD.md b/docs/CI-CD.md index 10fb511..d787fc3 100644 --- a/docs/CI-CD.md +++ b/docs/CI-CD.md @@ -36,8 +36,8 @@ The workflow requires the following secrets to be configured in the GitHub repos #### `CR_REGISTRY` - **Description**: Container Registry URL - **Type**: Repository Secret -- **Usage**: The registry URL (e.g., `landeranalytics.azurecr.io`, `ghcr.io`, etc.) -- **Example**: `landeranalytics.azurecr.io` +- **Usage**: The registry URL (e.g., `ghcr.io`, etc.) +- **Example**: `ghcr.io` #### `CR_USERNAME` - **Description**: Username for Container Registry authentication @@ -57,7 +57,7 @@ To configure these secrets: 2. Navigate to **Settings** > **Secrets and variables** > **Actions** 3. Click **New repository secret** 4. Add each secret with the name and value: - - Name: `CR_REGISTRY`, Value: Your Container Registry URL (e.g., `landeranalytics.azurecr.io`, `ghcr.io`) + - Name: `CR_REGISTRY`, Value: Your Container Registry URL (e.g., `ghcr.io`) - Name: `CR_USERNAME`, Value: Your Container Registry username - Name: `CR_PASSWORD`, Value: Your Container Registry password From d72024af39f72bb07c3aaf69619b65cfd20244c4 Mon Sep 17 00:00:00 2001 From: Jared Lander Date: Thu, 4 Dec 2025 17:05:33 -0500 Subject: [PATCH 35/37] Made it possible to specify the name of the redis and caddy images. --- BUILD.md | 2 +- devbox.json | 3 ++ docs/environment-variables-docker.md | 34 +++++++++++++++++++ .../impl/docker/CaddyConfig.kt | 6 +++- .../impl/docker/CraneConfig.kt | 2 +- .../impl/docker/DockerOrchestrator.kt | 10 +++++- .../impl/docker/RedisConfig.kt | 8 +++-- .../impl/docker/helpers/DockerAssertions.kt | 10 ++++-- .../docker/helpers/IntegrationTestBase.kt | 10 +++--- 9 files changed, 71 insertions(+), 14 deletions(-) diff --git a/BUILD.md b/BUILD.md index c78a66c..041404e 100644 --- a/BUILD.md +++ b/BUILD.md @@ -85,7 +85,7 @@ Copy the jar and build the image cp ../shinyproxy-operator/shinyproxy-operator-jar-with-dependencies.jar Operator/. # build the docker image -docker build -t shinyproxy-operator-dev:v2.3.1 --build-arg JAR_LOCATION=shinyproxy-operator-jar-with-dependencies.jar Operator +docker build -t shinyproxy-operator-dev:2.3.1 --build-arg JAR_LOCATION=shinyproxy-operator-jar-with-dependencies.jar Operator ``` ## Troubleshooting diff --git a/devbox.json b/devbox.json index 8a9236e..869963d 100644 --- a/devbox.json +++ b/devbox.json @@ -33,6 +33,9 @@ "package-fast": [ "mvn package -DskipTests" ], + "clean-compile": [ + "mvn clean compile -q" + ], "test": [ "mvn test" ] diff --git a/docs/environment-variables-docker.md b/docs/environment-variables-docker.md index 35ab9a5..578b01b 100644 --- a/docs/environment-variables-docker.md +++ b/docs/environment-variables-docker.md @@ -12,6 +12,38 @@ Any environment variable set in the operator's environment that starts with the - Operator environment: `SHINYPROXY_ENV_DATABASE_URL=jdbc:postgresql://db:5432/myapp` - ShinyProxy container receives: `DATABASE_URL=jdbc:postgresql://db:5432/myapp` +## Operator Configuration Variables + +The following environment variables control the ShinyProxy Operator's behavior in Docker mode: + +### SPO_REDIS_CONTAINER_NAME + +**Default:** `sp-redis` + +**Description:** Specifies the name of the Redis container that will be created and used by ShinyProxy for session storage. This allows you to customize the container name if needed (e.g., to avoid naming conflicts in multi-operator setups). + +**Example:** +```bash +SPO_REDIS_CONTAINER_NAME=my-custom-redis +``` + +**Note:** If you change this value, the operator will create a new Redis container with the custom name and a corresponding data directory. Changing this after initial deployment will result in a new Redis instance; the old one will need to be manually cleaned up. + +**Important:** The value of this variable is automatically used by ShinyProxy and Crane services through their Spring Boot configuration, so they will correctly reference the custom Redis container name. + +### SPO_CADDY_CONTAINER_NAME + +**Default:** `sp-caddy` + +**Description:** Specifies the name of the Caddy reverse proxy container that will be created to manage ingress routing for ShinyProxy instances. This allows you to customize the container name if needed (e.g., to avoid naming conflicts in multi-operator setups). + +**Example:** +```bash +SPO_CADDY_CONTAINER_NAME=my-custom-caddy +``` + +**Note:** If you change this value, the operator will create a new Caddy container with the custom name and a corresponding data directory. Changing this after initial deployment will result in a new Caddy instance; the old one will need to be manually cleaned up. + ## Usage with Docker Compose This feature is designed to work seamlessly with Docker Compose and environment files: @@ -27,6 +59,8 @@ services: environment: # Operator configuration - SPO_DOCKER_GID=999 + - SPO_REDIS_CONTAINER_NAME=sp-redis + - SPO_CADDY_CONTAINER_NAME=sp-caddy # Environment variables to pass to ShinyProxy container - SHINYPROXY_ENV_DATABASE_URL=jdbc:postgresql://db:5432/myapp diff --git a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/CaddyConfig.kt b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/CaddyConfig.kt index d845d09..3e87ae8 100644 --- a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/CaddyConfig.kt +++ b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/CaddyConfig.kt @@ -48,7 +48,7 @@ import java.util.concurrent.TimeUnit class CaddyConfig(private val dockerClient: DockerClient, mainDataDir: Path, config: Config) { - private val containerName = "sp-caddy" + private val containerName: String = config.readConfigValue("sp-caddy", "SPO_CADDY_CONTAINER_NAME") { it } private val dataDir: Path = mainDataDir.resolve(containerName) private val shinyProxies = mutableMapOf>() private val craneServers = hashMapOf() @@ -79,6 +79,10 @@ class CaddyConfig(private val dockerClient: DockerClient, mainDataDir: Path, con fileManager.createDirectories(dataDir.resolve("certs")) } + fun getContainerName(): String { + return containerName + } + suspend fun removeRealm(realmId: String) { shinyProxies.remove(realmId) reconcile() diff --git a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/CraneConfig.kt b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/CraneConfig.kt index 6ede8b9..d73a50a 100644 --- a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/CraneConfig.kt +++ b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/CraneConfig.kt @@ -259,7 +259,7 @@ class CraneConfig(private val dockerClient: DockerClient, "data" to mapOf( "redis" to mapOf( "password" to redisConfig.getRedisPassword(), - "host" to "sp-redis" + "host" to redisConfig.getContainerName() ) ) ), diff --git a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt index 00d2e67..2e19efb 100644 --- a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt +++ b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt @@ -137,6 +137,14 @@ class DockerOrchestrator(channel: Channel, const val SHARED_NETWORK_NAME = "sp-shared-network" } + fun getRedisConfig(): RedisConfig { + return redisConfig + } + + fun getCaddyConfig(): CaddyConfig { + return caddyConfig + } + override fun getShinyProxyStatus(shinyProxy: ShinyProxy): ShinyProxyStatus { return state.getOrPut(shinyProxy.realmId) { ShinyProxyStatus(shinyProxy.realmId, shinyProxy.hashOfCurrentSpec) } } @@ -519,7 +527,7 @@ class DockerOrchestrator(channel: Channel, "data" to mapOf( "redis" to mapOf( "password" to redisConfig.getRedisPassword(), - "host" to "sp-redis" + "host" to redisConfig.getContainerName() ) ) )) diff --git a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/RedisConfig.kt b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/RedisConfig.kt index 85fe396..e4d8bde 100644 --- a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/RedisConfig.kt +++ b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/RedisConfig.kt @@ -39,8 +39,8 @@ class RedisConfig(private val dockerClient: DockerClient, private val dataDirUid: Int, config: Config) { - private val containerName = "sp-redis" - private val dataDir: Path = mainDataDir.resolve(containerName) + private val containerName: String = config.readConfigValue("sp-redis", "SPO_REDIS_CONTAINER_NAME") { it } + private val dataDir: Path = mainDataDir.resolve("sp-redis") private lateinit var redisPassword: String private val logger = KotlinLogging.logger {} private val redisImage: String = config.readConfigValue("docker.io/library/redis:8.2.2", "SPO_REDIS_IMAGE") { it } @@ -68,6 +68,10 @@ class RedisConfig(private val dockerClient: DockerClient, return redisPassword } + fun getContainerName(): String { + return containerName + } + suspend fun reconcile() { dockerActions.stopAndRemoveNotRunningContainer(containerName) if (dockerActions.isContainerRunning(containerName, redisImage)) { diff --git a/src/test/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/helpers/DockerAssertions.kt b/src/test/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/helpers/DockerAssertions.kt index 879540f..d302e12 100644 --- a/src/test/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/helpers/DockerAssertions.kt +++ b/src/test/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/helpers/DockerAssertions.kt @@ -25,6 +25,7 @@ import com.fasterxml.jackson.dataformat.yaml.YAMLFactory import com.fasterxml.jackson.module.kotlin.readValue import com.fasterxml.jackson.module.kotlin.registerKotlinModule import eu.openanalytics.shinyproxyoperator.Config +import eu.openanalytics.shinyproxyoperator.impl.docker.DockerOrchestrator import eu.openanalytics.shinyproxyoperator.model.ShinyProxyInstance import org.mandas.docker.client.messages.Container import org.mandas.docker.client.messages.PortBinding @@ -36,14 +37,17 @@ import kotlin.test.assertTrue class DockerAssertions(private val base: IntegrationTestBase, private val dataDir: Path, - private val inputDir: Path) { + private val inputDir: Path, + private val orchestrator: DockerOrchestrator) { private val objectMapper = ObjectMapper(YAMLFactory()).registerKotlinModule() private val dockerGID = Config().readConfigValue(null, "SPO_DOCKER_GID") { it } private val dockerSocket = Config().readConfigValue("/var/run/docker.sock", "SPO_DOCKER_SOCKET") { it } + private val redisContainerName = orchestrator.getRedisConfig().getContainerName() + private val caddyContainerName = orchestrator.getCaddyConfig().getContainerName() fun assertRedisContainer() { - val redisContainer = base.inspectContainer(this.base.getContainerByName("sp-redis")) + val redisContainer = base.inspectContainer(this.base.getContainerByName(redisContainerName)) assertNotNull(redisContainer) assertEquals(true, redisContainer.state().running()) assertEquals("sp-shared-network", redisContainer.hostConfig().networkMode()) @@ -62,7 +66,7 @@ class DockerAssertions(private val base: IntegrationTestBase, } fun assertCaddyContainer(expectedName: String, replacements: Map, tls: Boolean = false) { - val caddyContainer = base.inspectContainer(this.base.getContainerByName("sp-caddy")) + val caddyContainer = base.inspectContainer(this.base.getContainerByName(caddyContainerName)) assertNotNull(caddyContainer) assertEquals(true, caddyContainer.state().running()) assertEquals("sp-shared-network", caddyContainer.hostConfig().networkMode()) diff --git a/src/test/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/helpers/IntegrationTestBase.kt b/src/test/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/helpers/IntegrationTestBase.kt index 3bb9846..cabda55 100644 --- a/src/test/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/helpers/IntegrationTestBase.kt +++ b/src/test/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/helpers/IntegrationTestBase.kt @@ -106,7 +106,7 @@ abstract class IntegrationTestBase { val operator = DockerOperator(mockConfig, eventController, mockRecyclableChecker) eventController.setDelegate(EventController(operator.orchestrator)) - val dockerAssertions = DockerAssertions(this@IntegrationTestBase, dataDir, inputDir) + val dockerAssertions = DockerAssertions(this@IntegrationTestBase, dataDir, inputDir, operator.orchestrator) try { // 3. run test @@ -116,15 +116,15 @@ abstract class IntegrationTestBase { // 4. stop operator operator.stop() // 4. cleanup docker containers - deleteContainers() + deleteContainers(operator.orchestrator.getRedisConfig().getContainerName(), operator.orchestrator.getCaddyConfig().getContainerName()) } } } - private fun deleteContainers() { - stopAndRemoveContainer(getContainerByName("sp-redis")) - stopAndRemoveContainer(getContainerByName("sp-caddy")) + private fun deleteContainers(redisContainerName: String = "sp-redis", caddyContainerName: String = "sp-caddy") { + stopAndRemoveContainer(getContainerByName(redisContainerName)) + stopAndRemoveContainer(getContainerByName(caddyContainerName)) val containers = dockerClient .listContainers(DockerClient.ListContainersParam.allContainers()) From 2d43faf4260b74819f8e4c5029234683607c6a16 Mon Sep 17 00:00:00 2001 From: Jared Lander Date: Fri, 5 Dec 2025 13:39:49 -0500 Subject: [PATCH 36/37] Enabling the user to specify the network name for containers spawned by shinyproxy-orchestraotr. --- .github/copilot-instructions.md | 6 +++--- devbox.json | 5 ++++- .../impl/docker/DockerOrchestrator.kt | 11 +++++++++-- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 79ce75f..b5fcf79 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -18,14 +18,14 @@ ShinyProxy Operator is a Kubernetes/Docker operator for managing ShinyProxy depl ### Building the Project ```bash -mvn -U clean install -DskipTests +devbox run mvn -U clean install -DskipTests ``` The build produces: `target/shinyproxy-operator-jar-with-dependencies.jar` ### Running Tests ```bash -mvn test +devbox run mvn test ``` Tests require: @@ -35,7 +35,7 @@ Tests require: ### License Header Management ```bash -mvn validate license:format +devbox run mvn validate license:format ``` Automatically updates copyright headers. Year updates are handled automatically - don't change year manually. diff --git a/devbox.json b/devbox.json index 869963d..e2d62e4 100644 --- a/devbox.json +++ b/devbox.json @@ -34,7 +34,10 @@ "mvn package -DskipTests" ], "clean-compile": [ - "mvn clean compile -q" + "mvn clean compile" + ], + "clean-compile-fast": [ + "mvn clean compile -DskipTests" ], "test": [ "mvn test" diff --git a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt index 2e19efb..e469231 100644 --- a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt +++ b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt @@ -95,6 +95,8 @@ class DockerOrchestrator(channel: Channel, private val logFilesCleaner: LogFilesCleaner init { + initSharedNetworkName(config) + if (!Files.exists(dataDir)) { throw InternalException("The data directory doesn't exist: '$dataDir'!") } @@ -134,7 +136,12 @@ class DockerOrchestrator(channel: Channel, } companion object { - const val SHARED_NETWORK_NAME = "sp-shared-network" + lateinit var SHARED_NETWORK_NAME: String + private set + + fun initSharedNetworkName(config: Config) { + SHARED_NETWORK_NAME = config.readConfigValue("sp-shared-network", "SPO_SHARED_NETWORK_NAME") { it } + } } fun getRedisConfig(): RedisConfig { @@ -199,7 +206,7 @@ class DockerOrchestrator(channel: Channel, } val containers = dockerActions.getContainers(shinyProxyInstance) if (containers.size < shinyProxy.replicas) { - val networkName = "sp-network-${shinyProxy.realmId}" + val networkName = "${SHARED_NETWORK_NAME}-internal-${shinyProxy.realmId}" if (!dockerActions.networkExists(networkName)) { logger.info { "${logPrefix(shinyProxyInstance)} [Docker] Creating network" } dockerActions.createNetwork(networkName, disableICC) From e4ba3f04ff4a2d4f1e15fe3c47e94390552b7950 Mon Sep 17 00:00:00 2001 From: Jared Lander Date: Tue, 9 Dec 2025 12:08:19 -0500 Subject: [PATCH 37/37] Changed the hard coding of SPRING_CONFIG_IMPORT to SPRING_CONFIG_IMPORT_0 so that we can add additional config imports and spring.config.import is no longer a forbidden configuration. --- .../shinyproxyoperator/impl/docker/DockerOrchestrator.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt index e469231..d7ec23c 100644 --- a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt +++ b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt @@ -316,7 +316,7 @@ class DockerOrchestrator(channel: Channel, val envVars = mutableListOf( "PROXY_VERSION=${version}", "PROXY_REALM_ID=${shinyProxy.realmId}", - "SPRING_CONFIG_IMPORT=/opt/shinyproxy/generated.yml" + "SPRING_CONFIG_IMPORT_0=/opt/shinyproxy/generated.yml" ) // Add environment variables from the operator's environment