diff --git a/.github/workflows/tidy3d-python-client-daily.yml b/.github/workflows/tidy3d-python-client-daily.yml index 9c5ac878aa..3d39974786 100644 --- a/.github/workflows/tidy3d-python-client-daily.yml +++ b/.github/workflows/tidy3d-python-client-daily.yml @@ -27,7 +27,7 @@ jobs: with: release_tag: 'daily-0.0.0' release_type: 'draft' - workflow_control: 'start-tag' + workflow_control: 'only-tag-tests' client_tests: true cli_tests: true submodule_tests: false diff --git a/docs/configuration/index.rst b/docs/configuration/index.rst index 19596fe11c..0b67c2a66a 100644 --- a/docs/configuration/index.rst +++ b/docs/configuration/index.rst @@ -154,3 +154,10 @@ Next Steps - :doc:`reference` - :doc:`migration` - :doc:`../api/configuration` + +.. toctree:: + :hidden: + + reference + migration + nexus diff --git a/docs/configuration/nexus.rst b/docs/configuration/nexus.rst new file mode 100644 index 0000000000..f7f36a1952 --- /dev/null +++ b/docs/configuration/nexus.rst @@ -0,0 +1,594 @@ +.. currentmodule:: tidy3d + +Nexus Environment Configuration +================================ + +Overview +-------- + +The Nexus environment enables Tidy3D to connect to custom on-premises or private cloud deployments for enterprise customers. + +.. note:: + This feature is for enterprise customers with custom Tidy3D deployments. Most users should use the standard API key setup with the default production environment. See the :doc:`../install` guide for standard setup instructions. + +Quick Start +----------- + +**Option 1: Automatic Configuration** (Recommended) + +When you configure Nexus using the CLI, Tidy3D automatically sets it as your default profile: + +.. code-block:: bash + + tidy3d configure --apikey YOUR_KEY \ + --nexus-url http://your-server + +This automatically: + +- Sets API endpoint: ``http://your-server/tidy3d-api`` +- Sets Website endpoint: ``http://your-server/tidy3d`` +- Sets S3 endpoint: ``http://your-server:9000`` +- **Sets Nexus as the default profile** (persisted across sessions) + +After configuration, all your scripts will automatically use Nexus without needing ``config.switch_profile("nexus")``. + +**Option 2: Manual Profile Switching** (for temporary use): + +For local development on localhost without persistent configuration: + +.. code-block:: python + + from tidy3d.config import config + config.switch_profile("nexus") + +This uses the built-in nexus profile with localhost endpoints (127.0.0.1:5000 for API, 127.0.0.1:9000 for S3). + +**Advanced setup** with individual endpoints: + +.. code-block:: bash + + tidy3d configure --apikey YOUR_KEY \ + --api-endpoint http://your-server/tidy3d-api \ + --website-endpoint http://your-server/tidy3d + +Or configure only Nexus endpoints (preserves existing API key): + +.. code-block:: bash + + tidy3d configure --nexus-url http://your-server + +Configuration +------------- + +Command Syntax +~~~~~~~~~~~~~~ + +.. code-block:: bash + + tidy3d configure [--apikey ] [--nexus-url | --api-endpoint --website-endpoint ] [OPTIONS] + +**Options:** + +* ``--apikey ``: API key (prompts if not provided and no Nexus options given) +* ``--nexus-url ``: Nexus base URL (automatically sets api=/tidy3d-api, web=/tidy3d, s3=:9000) +* ``--api-endpoint ``: Nexus API server URL (e.g., http://server/tidy3d-api) +* ``--website-endpoint ``: Nexus web interface URL (e.g., http://server/tidy3d) +* ``--s3-region ``: S3 region (default: us-east-1) +* ``--s3-endpoint ``: S3 storage URL (e.g., http://server:9000) +* ``--ssl-verify`` / ``--no-ssl-verify``: SSL verification (default: enabled for HTTPS) +* ``--enable-caching`` / ``--no-caching``: Server-side result caching + +Examples +~~~~~~~~ + +.. code-block:: bash + + # Simple configuration using nexus-url (recommended) + tidy3d configure --apikey YOUR_KEY \ + --nexus-url http://nexus.company.com + + # Configure individual endpoints + tidy3d configure --apikey YOUR_KEY \ + --api-endpoint http://nexus.company.com/tidy3d-api \ + --website-endpoint http://nexus.company.com/tidy3d + + # Add Nexus to existing configuration (preserves API key) + tidy3d configure --nexus-url http://nexus.company.com + + # Full configuration with all options + tidy3d configure --apikey YOUR_KEY \ + --api-endpoint https://api.company.com/tidy3d-api \ + --website-endpoint https://tidy3d.company.com \ + --s3-region eu-west-1 \ + --s3-endpoint https://s3.company.com:9000 \ + --ssl-verify \ + --enable-caching + +Configuration File +~~~~~~~~~~~~~~~~~~ + +**Location:** + +When you configure Nexus, your custom settings are saved to a profile file: + +- Linux/macOS: ``~/.config/tidy3d/profiles/nexus.toml`` +- Windows: ``C:\Users\username\.config\tidy3d\profiles\nexus.toml`` + +The base configuration file (``~/.config/tidy3d/config.toml``) stores the API key and default profile setting: + +.. code-block:: toml + + # Base config: ~/.config/tidy3d/config.toml + default_profile = "nexus" # Profile to use by default + + [web] + apikey = "your-api-key" + +**Nexus Profile Format** (``profiles/nexus.toml``): + +.. code-block:: toml + + # Nexus profile: ~/.config/tidy3d/profiles/nexus.toml + [web] + api_endpoint = "http://nexus.company.com/tidy3d-api" + website_endpoint = "http://nexus.company.com/tidy3d" + s3_region = "us-east-1" + ssl_verify = false + enable_caching = false + + [web.env_vars] + AWS_ENDPOINT_URL_S3 = "http://nexus.company.com:9000" + +The nexus profile file contains only your custom nexus-specific settings (endpoints, SSL, caching). The API key is stored in the base config and shared across all profiles. When the nexus profile is loaded, these settings override the built-in nexus defaults (localhost:5000). The ``default_profile`` setting in the base config tells Tidy3D to automatically load the nexus profile on startup. + +.. note:: + **Automatic Migration:** If you have an existing configuration in the old flat format (``~/.tidy3d/config``), Tidy3D will automatically convert it to the new structured format on first use. Your old configuration will be backed up as ``config.migrated``. + +Legacy Format (deprecated) +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For reference, the old flat format that is automatically migrated: + +.. code-block:: toml + + # Old format: ~/.tidy3d/config (automatically migrated) + apikey = "your-api-key" + web_api_endpoint = "http://nexus.company.com/tidy3d-api" + website_endpoint = "http://nexus.company.com/tidy3d" + s3_region = "us-east-1" + s3_endpoint = "http://nexus.company.com:9000" + ssl_verify = false + enable_caching = false + +Python Usage +------------ + +After CLI Configuration (Automatic) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Once you've configured Nexus using the CLI, **no code changes are required**. Tidy3D automatically uses your Nexus endpoints: + +.. code-block:: python + + import tidy3d as td + import tidy3d.web as web + + # After running: tidy3d configure --nexus-url http://your-server + # Your scripts automatically use Nexus - no profile switching needed! + + sim = td.Simulation(...) + sim_data = web.run(sim, task_name="my_sim") + +The CLI configuration persists across sessions, so you only need to configure once. + +Manual Profile Switching (Temporary) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For temporary use or local development without persistent configuration: + +.. code-block:: python + + from tidy3d.config import config + + # Temporarily switch to the built-in nexus profile (localhost defaults) + config.switch_profile("nexus") + + # Now run simulations against your local Nexus instance + import tidy3d as td + import tidy3d.web as web + + sim = td.Simulation(...) + sim_data = web.run(sim, task_name="my_sim") + +The built-in ``nexus`` profile uses these default localhost settings: + +- API endpoint: ``http://127.0.0.1:5000`` +- Website endpoint: ``http://127.0.0.1/tidy3d`` +- S3 endpoint: ``http://127.0.0.1:9000`` +- SSL verification: disabled +- Server-side caching: disabled + +Switching Back to Production +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Option 1: Using CLI** (Recommended) + +.. code-block:: bash + + tidy3d configure --restore-defaults + +This will: +- Clear the default_profile setting +- Remove the nexus profile file +- Revert to production endpoints + +**Option 2: Using Python** + +.. code-block:: python + + from tidy3d.config import config + + # Clear the default profile (reverts to production) + config.set_default_profile(None) + config.save() + +**Option 3: Switch profiles per session** + +.. code-block:: python + + from tidy3d.config import config + + # Temporarily switch to production + config.switch_profile("prod") + +Verifying Configuration +~~~~~~~~~~~~~~~~~~~~~~~~ + +Check your current Nexus configuration: + +.. code-block:: python + + from tidy3d import config + + # Check which profile is active and which is default + print(f"Active profile: {config.profile}") + print(f"Default profile: {config.get_default_profile()}") + + # Display current configuration + print(config.format()) + + # Check specific web settings + print(f"API Endpoint: {config.web.api_endpoint}") + print(f"Website: {config.web.website_endpoint}") + print(f"S3 Region: {config.web.s3_region}") + print(f"SSL Verify: {config.web.ssl_verify}") + + # Check S3 endpoint (stored in env_vars) + if "AWS_ENDPOINT_URL_S3" in config.web.env_vars: + print(f"S3 Endpoint: {config.web.env_vars['AWS_ENDPOINT_URL_S3']}") + +Expected output after Nexus configuration: + +.. code-block:: text + + Active profile: nexus + Default profile: nexus + ... + +Programmatic Configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Update Nexus settings from Python: + +.. code-block:: python + + from tidy3d import config + + # Update Nexus endpoints + config.update_section("web", + api_endpoint="http://nexus.company.com/tidy3d-api", + website_endpoint="http://nexus.company.com/tidy3d", + s3_region="us-west-2", + ssl_verify=False, + enable_caching=True + ) + + # Set S3 endpoint + config.update_section("web", + env_vars={"AWS_ENDPOINT_URL_S3": "http://nexus.company.com:9000"} + ) + + # Save changes to disk + config.save() + +Environment Variables +~~~~~~~~~~~~~~~~~~~~~ + +Override any setting temporarily using environment variables: + +.. code-block:: bash + + # Override API endpoint + export TIDY3D_WEB__API_ENDPOINT="http://other-server/tidy3d-api" + + # Override S3 endpoint + export TIDY3D_WEB__ENV_VARS__AWS_ENDPOINT_URL_S3="http://other-s3:9000" + + # Override SSL verification + export TIDY3D_WEB__SSL_VERIFY=false + +Managing Configuration +---------------------- + +Change Default Profile +~~~~~~~~~~~~~~~~~~~~~~~ + +Set or clear the default profile: + +.. code-block:: python + + from tidy3d import config + + # Set nexus as default + config.set_default_profile("nexus") + + # Clear default profile (revert to production) + config.set_default_profile(None) + + # Check current default + print(config.get_default_profile()) + +Reset to Defaults +~~~~~~~~~~~~~~~~~ + +Reset configuration to default values (including clearing default profile): + +.. code-block:: bash + + tidy3d config reset + +Or from Python: + +.. code-block:: python + + from tidy3d import config + config.reset_to_defaults() + config.save() + +View Configuration +~~~~~~~~~~~~~~~~~~ + +Display current configuration: + +.. code-block:: bash + + python -c "from tidy3d import config; print(config.format())" + +Or interactively: + +.. code-block:: python + + from tidy3d import config + + # Print formatted configuration + print(config) + + # Or just web section + print(config.web) + +Migrate Legacy Configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Manually migrate old configuration to new format: + +.. code-block:: bash + + tidy3d config migrate + +Options: + +* ``--delete-legacy``: Remove old ``~/.tidy3d`` directory after migration +* ``--overwrite``: Overwrite existing new-format configuration + +Remove Configuration +~~~~~~~~~~~~~~~~~~~~ + +To completely remove Nexus configuration: + +.. code-block:: bash + + # Linux/macOS + rm ~/.config/tidy3d/config.toml + + # Or reset to defaults + tidy3d config reset + + # Then reconfigure with standard API key + tidy3d configure --apikey YOUR_API_KEY + +Troubleshooting +--------------- + +Verify Configuration +~~~~~~~~~~~~~~~~~~~~ + +Check that your Nexus configuration is active: + +.. code-block:: python + + from tidy3d import config + + print("=== Tidy3D Configuration ===") + print(f"API Endpoint: {config.web.api_endpoint}") + print(f"Website: {config.web.website_endpoint}") + + # Verify it's not using production + if "simulation.cloud" in str(config.web.api_endpoint): + print("?? Using production, not Nexus!") + else: + print("? Using custom Nexus deployment") + +Test Connectivity +~~~~~~~~~~~~~~~~~ + +Verify your Nexus server is accessible: + +.. code-block:: bash + + # Test API endpoint + curl http://your-nexus-server/tidy3d-api/health + + # Or with Python + python -c "import requests; from tidy3d import config; \ + print(requests.get(f'{config.web.api_endpoint}/health').text)" + +Check S3 Configuration +~~~~~~~~~~~~~~~~~~~~~~ + +Verify S3 endpoint configuration: + +.. code-block:: python + + from tidy3d import config + + if config.web.env_vars and "AWS_ENDPOINT_URL_S3" in config.web.env_vars: + s3_endpoint = config.web.env_vars["AWS_ENDPOINT_URL_S3"] + print(f"S3 Endpoint: {s3_endpoint}") + else: + print("Using default AWS S3") + +Legacy Environment Check (Deprecated) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For backward compatibility, the old Environment API still works: + +.. code-block:: python + + # Old API (deprecated, but still functional) + from tidy3d.web.core.environment import Env + print(f"Environment: {Env.current.name}") + print(f"API Endpoint: {Env.current.web_api_endpoint}") + +.. warning:: + The ``Env`` API is deprecated and will be removed in a future version. Use ``tidy3d.config`` instead. + +Common Issues +~~~~~~~~~~~~~ + +**Issue: Still connecting to production instead of Nexus** + +Solution: Check if the default profile is set correctly: + +.. code-block:: python + + from tidy3d import config + print(f"Active profile: {config.profile}") + print(f"Default profile: {config.get_default_profile()}") + print(f"API Endpoint: {config.web.api_endpoint}") + +If the default profile is not ``"nexus"``, reconfigure: + +.. code-block:: bash + + tidy3d configure --nexus-url http://your-server + +**Issue: Configuration not taking effect** + +Solution: Environment variables override the default profile. Check for overrides: + +.. code-block:: bash + + # Check for environment variable overrides + env | grep TIDY3D + +If ``TIDY3D_CONFIG_PROFILE``, ``TIDY3D_PROFILE``, or ``TIDY3D_ENV`` is set, it will override the default profile from config. + +**Issue: Old config file still exists** + +The old flat config file is automatically backed up during migration. It's safe to delete: + +.. code-block:: bash + + # Old config is backed up as config.migrated + ls ~/.tidy3d/ + # You can safely remove: rm ~/.tidy3d/config.migrated + +**Issue: API key validation fails** + +Ensure you're validating against your Nexus endpoint: + +.. code-block:: bash + + tidy3d configure --apikey YOUR_KEY \ + --nexus-url http://your-nexus-server + +Advanced Topics +--------------- + +Profile Selection Priority +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Tidy3D uses the following priority order when determining which profile to load: + +1. **Explicit parameter**: ``ConfigManager(profile="dev")`` +2. **Environment variables**: ``TIDY3D_CONFIG_PROFILE``, ``TIDY3D_PROFILE``, or ``TIDY3D_ENV`` +3. **Default profile in config**: ``default_profile`` in ``config.toml`` +4. **Fallback**: ``"default"`` profile (production) + +This means environment variables always override the default profile setting, allowing temporary overrides without changing the configuration file. + +Multiple Configurations (Profiles) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Switch between different Nexus deployments: + +.. code-block:: python + + from tidy3d import config + + # Switch to a different profile + config.switch_profile("dev") + + # Configure for dev environment + config.update_section("web", + api_endpoint="http://dev-nexus/tidy3d-api", + website_endpoint="http://dev-nexus/tidy3d" + ) + config.save() + + # Switch back to default + config.switch_profile("default") + +Or via environment variable: + +.. code-block:: bash + + # Use dev profile + export TIDY3D_CONFIG_PROFILE=dev + python your_script.py + +Custom S3 Configuration +~~~~~~~~~~~~~~~~~~~~~~~~ + +For Nexus deployments with MinIO or other S3-compatible storage: + +.. code-block:: python + + from tidy3d import config + + config.update_section("web", + s3_region="us-east-1", + env_vars={ + "AWS_ENDPOINT_URL_S3": "http://minio.company.com:9000", + # Optional: Add other AWS config + "AWS_ACCESS_KEY_ID": "your-key", + "AWS_SECRET_ACCESS_KEY": "your-secret" + } + ) + config.save() + +See Also +-------- + +* :doc:`submit_simulations` - Submitting and managing simulations +* :doc:`../install` - Installation and API key setup diff --git a/docs/faq b/docs/faq index ed2256c606..b52e6dacd1 160000 --- a/docs/faq +++ b/docs/faq @@ -1 +1 @@ -Subproject commit ed2256c6061a584f86b97051df6b806b02d8d355 +Subproject commit b52e6dacd1c43929d99c07f920a079206de11f77 diff --git a/docs/notebooks b/docs/notebooks index 0126c1d474..17e845fd1f 160000 --- a/docs/notebooks +++ b/docs/notebooks @@ -1 +1 @@ -Subproject commit 0126c1d47478366f647a72f563409efba6664ba4 +Subproject commit 17e845fd1fb361d1c9ef5d2d850fc6f1d81fb097 diff --git a/tests/config/test_manager.py b/tests/config/test_manager.py index c95430d8fa..5cc656c326 100644 --- a/tests/config/test_manager.py +++ b/tests/config/test_manager.py @@ -130,3 +130,62 @@ def test_as_dict_includes_defaults(config_manager): assert "adjoint" in data assert data["adjoint"]["local_adjoint_dir"] == "adjoint_data" assert "simulation" in data + + +def test_set_default_profile(config_manager): + """Test setting and getting the default profile.""" + # Initially no default profile should be set + assert config_manager.get_default_profile() is None + + # Set nexus as default + config_manager.set_default_profile("nexus") + assert config_manager.get_default_profile() == "nexus" + + # Clear default profile + config_manager.set_default_profile(None) + assert config_manager.get_default_profile() is None + + +def test_default_profile_used_on_init(tmp_path): + """Test that default_profile is used when initializing ConfigManager.""" + from tidy3d.config import ConfigManager + + # Create a manager with a temp config dir + manager = ConfigManager(config_dir=tmp_path) + + # Set nexus as default and save + manager.set_default_profile("nexus") + + # Create a new manager instance - should use nexus profile + new_manager = ConfigManager(config_dir=tmp_path) + assert new_manager.profile == "nexus" + + +def test_env_var_overrides_default_profile(tmp_path, monkeypatch): + """Test that environment variables override default_profile.""" + from tidy3d.config import ConfigManager + + # Create a manager and set nexus as default + manager = ConfigManager(config_dir=tmp_path) + manager.set_default_profile("nexus") + + # Set env var to use dev profile + monkeypatch.setenv("TIDY3D_CONFIG_PROFILE", "dev") + + # Create new manager - should use dev from env var, not nexus from config + new_manager = ConfigManager(config_dir=tmp_path) + assert new_manager.profile == "dev" + + +def test_set_default_profile_normalizes_name(config_manager): + """Test that profile names are normalized.""" + # Set uppercase profile name + config_manager.set_default_profile("NEXUS") + # Should be normalized to lowercase + assert config_manager.get_default_profile() == "nexus" + + +def test_set_default_profile_empty_raises(config_manager): + """Test that empty profile name raises ValueError.""" + with pytest.raises(ValueError, match="Profile name cannot be empty"): + config_manager.set_default_profile("") diff --git a/tests/config/test_nexus_migration.py b/tests/config/test_nexus_migration.py new file mode 100644 index 0000000000..772e5345f4 --- /dev/null +++ b/tests/config/test_nexus_migration.py @@ -0,0 +1,658 @@ +"""Tests for Nexus configuration migration from old to new format.""" + +from __future__ import annotations + +import pytest +import toml + +from tidy3d.config.legacy import load_legacy_flat_config +from tidy3d.config.loader import ConfigLoader +from tidy3d.config.manager import ConfigManager + + +@pytest.fixture +def old_nexus_config(tmp_path): + """Create an old-style Nexus configuration.""" + config_dir = tmp_path / ".tidy3d" + config_dir.mkdir() + old_config = config_dir / "config" + + old_config.write_text(""" +apikey = "test-nexus-key" +web_api_endpoint = "http://nexus.company.com:5000" +website_endpoint = "http://nexus.company.com/tidy3d" +s3_region = "us-west-2" +s3_endpoint = "http://nexus.company.com:9000" +ssl_verify = false +enable_caching = true +""") + + return config_dir, old_config + + +@pytest.fixture +def old_minimal_config(tmp_path): + """Create an old-style config with just API key.""" + config_dir = tmp_path / ".tidy3d" + config_dir.mkdir() + old_config = config_dir / "config" + + old_config.write_text('apikey = "test-key"\n') + + return config_dir, old_config + + +def test_load_legacy_nexus_config(old_nexus_config): + """Test that load_legacy_flat_config parses all Nexus fields correctly.""" + config_dir, old_config = old_nexus_config + + legacy_data = load_legacy_flat_config(config_dir) + + assert "web" in legacy_data + web = legacy_data["web"] + + # Check all fields are migrated + assert web["apikey"] == "test-nexus-key" + assert web["api_endpoint"] == "http://nexus.company.com:5000" + assert web["website_endpoint"] == "http://nexus.company.com/tidy3d" + assert web["s3_region"] == "us-west-2" + assert web["ssl_verify"] is False + assert web["enable_caching"] is True + + # S3 endpoint should be in env_vars + assert "env_vars" in web + assert web["env_vars"]["AWS_ENDPOINT_URL_S3"] == "http://nexus.company.com:9000" + + +def test_load_legacy_minimal_config(old_minimal_config): + """Test that minimal old config (just API key) still works.""" + config_dir, old_config = old_minimal_config + + legacy_data = load_legacy_flat_config(config_dir) + + assert "web" in legacy_data + assert legacy_data["web"]["apikey"] == "test-key" + assert len(legacy_data["web"]) == 1 # Only apikey + + +def test_auto_migration_on_load(old_nexus_config): + """Test that ConfigLoader auto-migrates legacy config on first load.""" + config_dir, old_config = old_nexus_config + + # Verify old config exists + assert old_config.exists() + + # Load via ConfigLoader (should trigger auto-migration) + loader = ConfigLoader(config_dir) + data = loader.load_base() + + # New config.toml should be created + new_config = config_dir / "config.toml" + assert new_config.exists() + + # Old config should be backed up + backup = config_dir / "config.migrated" + assert backup.exists() + assert not old_config.exists() + + # Verify data is correct + assert "web" in data + assert data["web"]["api_endpoint"] == "http://nexus.company.com:5000" + + +def test_migrated_config_format(old_nexus_config): + """Test that migrated config is in correct nested TOML format.""" + config_dir, old_config = old_nexus_config + + # Trigger migration + loader = ConfigLoader(config_dir) + loader.load_base() + + # Read the new config file + new_config = config_dir / "config.toml" + parsed = toml.loads(new_config.read_text()) + + # Should have nested structure + assert "web" in parsed + assert "api_endpoint" in parsed["web"] + assert "website_endpoint" in parsed["web"] + assert "env_vars" in parsed["web"] + assert "AWS_ENDPOINT_URL_S3" in parsed["web"]["env_vars"] + + # Old flat keys should NOT exist + assert "web_api_endpoint" not in parsed + assert "s3_endpoint" not in parsed + + +def test_config_manager_with_legacy_nexus(old_nexus_config): + """Test that ConfigManager can load and use legacy Nexus config.""" + config_dir, old_config = old_nexus_config + + manager = ConfigManager(config_dir=config_dir) + + # Should auto-load Nexus settings + web = manager.get_section("web") + + assert str(web.api_endpoint) == "http://nexus.company.com:5000" + assert str(web.website_endpoint) == "http://nexus.company.com/tidy3d" + assert web.s3_region == "us-west-2" + assert web.ssl_verify is False + assert web.enable_caching is True + + # Check env_vars + assert "AWS_ENDPOINT_URL_S3" in web.env_vars + assert web.env_vars["AWS_ENDPOINT_URL_S3"] == "http://nexus.company.com:9000" + + +def test_no_migration_if_new_config_exists(tmp_path): + """Test that migration doesn't happen if config.toml already exists.""" + config_dir = tmp_path / ".tidy3d" + config_dir.mkdir() + + # Create both old and new config + old_config = config_dir / "config" + old_config.write_text('apikey = "old-key"\n') + + new_config = config_dir / "config.toml" + new_config.write_text('[web]\napikey = "new-key"\n') + + # Load - should use new config, not migrate + loader = ConfigLoader(config_dir) + data = loader.load_base() + + # Should have loaded from new config + assert data["web"]["apikey"] == "new-key" + + # Old config should still exist (not backed up) + assert old_config.exists() + backup = config_dir / "config.migrated" + assert not backup.exists() + + +def test_partial_nexus_config(tmp_path): + """Test migration with partial Nexus settings.""" + config_dir = tmp_path / ".tidy3d" + config_dir.mkdir() + old_config = config_dir / "config" + + # Only some Nexus fields + old_config.write_text(""" +apikey = "test-key" +web_api_endpoint = "http://custom:5000" +website_endpoint = "http://custom/web" +""") + + legacy_data = load_legacy_flat_config(config_dir) + + assert legacy_data["web"]["apikey"] == "test-key" + assert legacy_data["web"]["api_endpoint"] == "http://custom:5000" + assert legacy_data["web"]["website_endpoint"] == "http://custom/web" + + # Fields not provided shouldn't be in the dict + assert "s3_region" not in legacy_data["web"] + assert "env_vars" not in legacy_data["web"] + + +def test_save_after_migration(old_nexus_config): + """Test that saving after migration works correctly. + + After migration, when save() is called on the default profile, + only persisted fields are written to base config. Non-persisted + fields like api_endpoint are filtered out. + """ + config_dir, old_config = old_nexus_config + + manager = ConfigManager(config_dir=config_dir) + + # Verify that before save, the manager has access to migrated values + assert str(manager.web.api_endpoint) == "http://nexus.company.com:5000" + + # Modify a persisted setting + manager.update_section("web", enable_caching=False) + manager.save() + + # Read back the config + new_config = config_dir / "config.toml" + parsed = toml.loads(new_config.read_text()) + + # Only persisted fields remain in base config after save + # apikey and enable_caching are persisted + assert "apikey" in parsed["web"] + assert parsed["web"]["enable_caching"] is False + + # Non-persisted fields like api_endpoint and timeout are filtered out + assert "api_endpoint" not in parsed.get("web", {}) + assert "timeout" not in parsed.get("web", {}) + + +def test_no_migration_if_no_legacy_config(tmp_path): + """Test that loader handles missing legacy config gracefully.""" + config_dir = tmp_path / ".tidy3d" + config_dir.mkdir() + + loader = ConfigLoader(config_dir) + data = loader.load_base() + + # Should return empty dict + assert data == {} + + # No files should be created + assert not (config_dir / "config").exists() + assert not (config_dir / "config.toml").exists() + assert not (config_dir / "config.migrated").exists() + + +def test_migration_preserves_comments_when_possible(old_nexus_config): + """Test that migration creates a clean, well-formatted config.""" + config_dir, old_config = old_nexus_config + + # Trigger migration + loader = ConfigLoader(config_dir) + loader.load_base() + + # Read the new config as text + new_config = config_dir / "config.toml" + content = new_config.read_text() + + # Should have section headers + assert "[web]" in content + + # Should not have old flat format + assert "web_api_endpoint" not in content + + +def test_configure_nexus_saves_to_profile(tmp_path): + """Test that configuring nexus saves to profiles/nexus.toml, not base config.""" + from tidy3d.config import ConfigManager + + # Create a fresh config manager with temp directory + manager = ConfigManager(config_dir=tmp_path) + + # Save API key to base config first (simulating normal configure flow) + manager.update_section("web", apikey="test-key") + manager.save() + + # Switch to nexus profile and configure custom nexus settings + manager.switch_profile("nexus") + manager.update_section( + "web", + api_endpoint="http://custom-nexus.company.com/tidy3d-api", + website_endpoint="http://custom-nexus.company.com/tidy3d", + env_vars={"AWS_ENDPOINT_URL_S3": "http://custom-nexus.company.com:9000"}, + ) + manager.save() + + # Set nexus as default + manager.set_default_profile("nexus") + + # Check that profiles/nexus.toml was created + nexus_profile = tmp_path / "profiles" / "nexus.toml" + assert nexus_profile.exists(), "Nexus profile file should be created" + + # Read the nexus profile + nexus_data = toml.loads(nexus_profile.read_text()) + + # Should contain the custom nexus settings + assert "web" in nexus_data + assert nexus_data["web"]["api_endpoint"] == "http://custom-nexus.company.com/tidy3d-api" + assert nexus_data["web"]["website_endpoint"] == "http://custom-nexus.company.com/tidy3d" + assert ( + nexus_data["web"]["env_vars"]["AWS_ENDPOINT_URL_S3"] + == "http://custom-nexus.company.com:9000" + ) + + # Check base config + base_config = tmp_path / "config.toml" + assert base_config.exists() + base_data = toml.loads(base_config.read_text()) + + # Base config should have apikey and may have default endpoints (not custom ones) + assert "web" in base_data + assert base_data["web"]["apikey"] == "test-key" + # Should NOT have the custom nexus endpoint + if "api_endpoint" in base_data["web"]: + assert base_data["web"]["api_endpoint"] != "http://custom-nexus.company.com/tidy3d-api" + + # Verify default profile is set + assert "default_profile" in base_data + assert base_data["default_profile"] == "nexus" + + +def test_configure_nexus_loads_correctly(tmp_path): + """Test that after configuring nexus, loading the profile works correctly.""" + from tidy3d.config import ConfigManager + + # Create a fresh config manager and configure nexus + manager = ConfigManager(config_dir=tmp_path) + + # Save API key to base + manager.update_section("web", apikey="test-key") + manager.save() + + # Configure nexus settings + manager.switch_profile("nexus") + manager.update_section( + "web", + api_endpoint="http://my-nexus.example.com/tidy3d-api", + website_endpoint="http://my-nexus.example.com/tidy3d", + env_vars={"AWS_ENDPOINT_URL_S3": "http://my-nexus.example.com:9000"}, + ) + manager.save() + manager.set_default_profile("nexus") + + # Create a NEW manager instance to simulate a fresh load + new_manager = ConfigManager(config_dir=tmp_path) + + # Should automatically load nexus profile (because default_profile is set) + assert new_manager.profile == "nexus" + assert str(new_manager.web.api_endpoint) == "http://my-nexus.example.com/tidy3d-api" + assert str(new_manager.web.website_endpoint) == "http://my-nexus.example.com/tidy3d" + assert new_manager.web.env_vars["AWS_ENDPOINT_URL_S3"] == "http://my-nexus.example.com:9000" + + # Verify we can manually switch to production and back + new_manager.switch_profile("prod") + assert str(new_manager.web.api_endpoint) == "https://tidy3d-api.simulation.cloud" + + # Switch back to nexus + new_manager.switch_profile("nexus") + assert str(new_manager.web.api_endpoint) == "http://my-nexus.example.com/tidy3d-api" + + +def test_nexus_url_derivation(): + """Test that nexus URL derivation handles edge cases correctly.""" + from urllib.parse import urlparse, urlunparse + + test_cases = [ + # (input_url, expected_api, expected_website, expected_s3) + ( + "http://localhost", + "http://localhost/tidy3d-api", + "http://localhost/tidy3d", + "http://localhost:9000", + ), + ( + "http://localhost/", + "http://localhost/tidy3d-api", + "http://localhost/tidy3d", + "http://localhost:9000", + ), + ( + "http://localhost:8080", + "http://localhost:8080/tidy3d-api", + "http://localhost:8080/tidy3d", + "http://localhost:9000", + ), + ( + "http://nexus.company.com", + "http://nexus.company.com/tidy3d-api", + "http://nexus.company.com/tidy3d", + "http://nexus.company.com:9000", + ), + ( + "https://nexus.company.com/", + "https://nexus.company.com/tidy3d-api", + "https://nexus.company.com/tidy3d", + "https://nexus.company.com:9000", + ), + ] + + for nexus_url, expected_api, expected_website, expected_s3 in test_cases: + # Replicate the logic from configure_fn + base_url = nexus_url.rstrip("/") + api_endpoint = f"{base_url}/tidy3d-api" + website_endpoint = f"{base_url}/tidy3d" + + parsed = urlparse(nexus_url) + hostname = parsed.hostname or parsed.netloc.split(":")[0] + s3_netloc = f"{hostname}:9000" + s3_endpoint = urlunparse((parsed.scheme, s3_netloc, "", "", "", "")) + + assert api_endpoint == expected_api, f"Failed for {nexus_url}: api_endpoint" + assert website_endpoint == expected_website, f"Failed for {nexus_url}: website_endpoint" + assert s3_endpoint == expected_s3, f"Failed for {nexus_url}: s3_endpoint" + + +def test_configure_fn_with_nexus_url(tmp_path, monkeypatch): + """Test configure_fn with nexus_url parameter.""" + from unittest.mock import Mock + + from tidy3d.config import ConfigManager + from tidy3d.web.cli.app import configure_fn + + # Create a fresh config manager + manager = ConfigManager(config_dir=tmp_path) + + # Monkeypatch the global config and requests + import tidy3d.web.cli.app as cli_app + + monkeypatch.setattr(cli_app, "config", manager) + + # Mock successful API key validation + mock_response = Mock() + mock_response.status_code = 200 + monkeypatch.setattr("requests.get", lambda *args, **kwargs: mock_response) + + # Test with nexus_url + configure_fn( + apikey="test-api-key", + nexus_url="http://localhost:8080", + ) + + # Verify profile was created + nexus_profile = tmp_path / "profiles" / "nexus.toml" + assert nexus_profile.exists() + + # Verify endpoints were derived correctly + nexus_data = toml.loads(nexus_profile.read_text()) + assert nexus_data["web"]["api_endpoint"] == "http://localhost:8080/tidy3d-api" + assert nexus_data["web"]["website_endpoint"] == "http://localhost:8080/tidy3d" + assert nexus_data["web"]["env_vars"]["AWS_ENDPOINT_URL_S3"] == "http://localhost:9000" + + +def test_configure_fn_with_manual_endpoints(tmp_path, monkeypatch): + """Test configure_fn with manual endpoint parameters.""" + from unittest.mock import Mock + + from tidy3d.config import ConfigManager + from tidy3d.web.cli.app import configure_fn + + manager = ConfigManager(config_dir=tmp_path) + + import tidy3d.web.cli.app as cli_app + + monkeypatch.setattr(cli_app, "config", manager) + + mock_response = Mock() + mock_response.status_code = 200 + monkeypatch.setattr("requests.get", lambda *args, **kwargs: mock_response) + + # Test with manual endpoints + # Note: Using ssl_verify=True to differ from builtin nexus default (False) + configure_fn( + apikey="test-key", + api_endpoint="http://custom:5000/api", + website_endpoint="http://custom:5000/web", + s3_endpoint="http://custom:9000", + ssl_verify=True, # Different from builtin nexus default + enable_caching=True, # Different from builtin nexus default + ) + + nexus_profile = tmp_path / "profiles" / "nexus.toml" + assert nexus_profile.exists() + + nexus_data = toml.loads(nexus_profile.read_text()) + assert nexus_data["web"]["api_endpoint"] == "http://custom:5000/api" + assert nexus_data["web"]["website_endpoint"] == "http://custom:5000/web" + # These should be saved since they differ from builtin nexus defaults + assert nexus_data["web"]["ssl_verify"] is True + assert nexus_data["web"]["enable_caching"] is True + + +def test_configure_fn_validation_error(tmp_path, monkeypatch, capsys): + """Test configure_fn with incomplete endpoint specification.""" + from tidy3d.config import ConfigManager + from tidy3d.web.cli.app import configure_fn + + manager = ConfigManager(config_dir=tmp_path) + + import tidy3d.web.cli.app as cli_app + + monkeypatch.setattr(cli_app, "config", manager) + + # Only provide api_endpoint without website_endpoint (should fail) + configure_fn( + apikey="test-key", + api_endpoint="http://custom:5000/api", + ) + + captured = capsys.readouterr() + assert "Both --api-endpoint and --website-endpoint must be provided together" in captured.out + + +def test_configure_fn_restore_defaults(tmp_path, monkeypatch, capsys): + """Test configure_fn with restore_defaults flag.""" + from unittest.mock import Mock + + from tidy3d.config import ConfigManager + from tidy3d.web.cli.app import configure_fn + + manager = ConfigManager(config_dir=tmp_path) + + import tidy3d.web.cli.app as cli_app + + monkeypatch.setattr(cli_app, "config", manager) + + # First configure nexus + mock_response = Mock() + mock_response.status_code = 200 + monkeypatch.setattr("requests.get", lambda *args, **kwargs: mock_response) + + configure_fn(apikey="test-key", nexus_url="http://localhost") + + # Verify profile exists + nexus_profile = tmp_path / "profiles" / "nexus.toml" + assert nexus_profile.exists() + + # Now restore defaults + configure_fn(apikey=None, restore_defaults=True) + + # Verify profile was removed + assert not nexus_profile.exists() + + # Verify message was printed + captured = capsys.readouterr() + assert "Successfully restored production defaults" in captured.out + assert "Cleared default_profile setting" in captured.out + + +def test_get_default_profile_error_handling(tmp_path): + """Test get_default_profile handles corrupted config gracefully.""" + from tidy3d.config.loader import ConfigLoader + + config_dir = tmp_path / ".tidy3d" + config_dir.mkdir() + config_file = config_dir / "config.toml" + + # Write invalid TOML + config_file.write_text("invalid toml {{{") + + loader = ConfigLoader(config_dir) + result = loader.get_default_profile() + + # Should return None instead of crashing + assert result is None + + +def test_legacy_migration_error_handling(tmp_path, monkeypatch): + """Test that legacy migration errors are handled gracefully.""" + from tidy3d.config import ConfigManager + from tidy3d.config.loader import ConfigLoader + + config_dir = tmp_path / ".tidy3d" + config_dir.mkdir() + + # Create a legacy config file + legacy_file = config_dir / "config" + legacy_file.write_text("apikey = test-key") + + # Mock save_base to raise an exception during migration + def mock_save_failure(self, data): + raise RuntimeError("Migration save failed") + + monkeypatch.setattr(ConfigLoader, "save_base", mock_save_failure) + + # This should not crash, even though migration fails + # ConfigManager uses ConfigLoader internally + manager = ConfigManager(config_dir=config_dir) + + # Should still be able to access config (falls back to legacy data) + # The key point is that no exception is raised + assert manager is not None + + +def test_api_key_validation_failure(tmp_path, monkeypatch, capsys): + """Test configure_fn handles API key validation failure.""" + from unittest.mock import Mock + + from tidy3d.config import ConfigManager + from tidy3d.web.cli.app import configure_fn + + manager = ConfigManager(config_dir=tmp_path) + + import tidy3d.web.cli.app as cli_app + + monkeypatch.setattr(cli_app, "config", manager) + + # Mock failed API key validation + mock_response = Mock() + mock_response.status_code = 401 # Unauthorized + monkeypatch.setattr("requests.get", lambda *args, **kwargs: mock_response) + + # Try to configure with invalid API key + configure_fn( + apikey="invalid-key", + nexus_url="http://localhost", + ) + + # Verify error message was printed + captured = capsys.readouterr() + assert "API key validation failed" in captured.out + assert "401" in captured.out + + # Verify no profile was created + nexus_profile = tmp_path / "profiles" / "nexus.toml" + assert not nexus_profile.exists() + + +def test_profile_notification_on_init(tmp_path): + """Test that non-default profile usage is logged on initialization.""" + from tests.utils import AssertLogStr + from tidy3d.config import ConfigManager + + # Test 1: Default profile should not log + with AssertLogStr(log_level_expected="INFO", excludes_str="Using configuration profile"): + manager = ConfigManager(config_dir=tmp_path, profile="default") + + # Test 2: Nexus profile should log + with AssertLogStr( + log_level_expected="INFO", contains_str="Using configuration profile: 'nexus'" + ): + manager = ConfigManager(config_dir=tmp_path, profile="nexus") + + +def test_profile_notification_on_switch(tmp_path): + """Test that profile switching is logged.""" + from tests.utils import AssertLogStr + from tidy3d.config import ConfigManager + + manager = ConfigManager(config_dir=tmp_path) + + # Switch to nexus profile - should log + with AssertLogStr( + log_level_expected="INFO", contains_str="Switched to configuration profile: 'nexus'" + ): + manager.switch_profile("nexus") + + # Switch back to default - should not log + with AssertLogStr(log_level_expected="INFO", excludes_str="Switched to configuration profile"): + manager.switch_profile("default") diff --git a/tests/test_web/test_s3utils.py b/tests/test_web/test_s3utils.py index 175e860b6d..a6552737a4 100644 --- a/tests/test_web/test_s3utils.py +++ b/tests/test_web/test_s3utils.py @@ -120,3 +120,89 @@ def simulate_download_failure(Bucket, Key, Filename, Callback, Config, **kwargs) assert not destination_path.exists() for p in destination_path.parent.iterdir(): assert not p.name.endswith(s3utils.IN_TRANSIT_SUFFIX) # no temporary files are present + + +def test_s3_token_get_client_with_custom_endpoint(tmp_path, monkeypatch): + """Test that S3STSToken.get_client uses custom endpoint from config.""" + from datetime import datetime, timezone + from unittest.mock import MagicMock + + from tidy3d.config import ConfigManager + from tidy3d.web.core.s3utils import _S3STSToken, _UserCredential + + # Create credential using proper Pydantic structure + credential_data = { + "accessKeyId": "test-access-key", + "secretAccessKey": "test-secret-key", + "sessionToken": "test-session-token", + "expiration": datetime.now(timezone.utc), + } + mock_credential = _UserCredential(**credential_data) + + # Create token using proper Pydantic structure + token_data = { + "cloudpath": "s3://test-bucket/test-key", + "userCredentials": credential_data, + } + token = _S3STSToken(**token_data) + + # Mock boto3.client + mock_boto_client = MagicMock() + monkeypatch.setattr("tidy3d.web.core.s3utils.boto3.client", mock_boto_client) + + # Test 1: Without custom endpoint - use fresh config + test_config = ConfigManager(config_dir=tmp_path) + monkeypatch.setattr("tidy3d.web.core.s3utils.config", test_config) + token.get_client() + + # Verify boto3.client was called without endpoint_url + call_kwargs = mock_boto_client.call_args[1] + assert "endpoint_url" not in call_kwargs + assert call_kwargs["service_name"] == "s3" + + # Reset mock + mock_boto_client.reset_mock() + + # Test 2: With custom endpoint - update config and test again + test_config.update_section("web", env_vars={"AWS_ENDPOINT_URL_S3": "http://localhost:9000"}) + token.get_client() + + # Verify boto3.client was called with endpoint_url + call_kwargs = mock_boto_client.call_args[1] + assert call_kwargs["endpoint_url"] == "http://localhost:9000" + assert call_kwargs["service_name"] == "s3" + + +def test_s3_token_get_client_respects_ssl_verify(tmp_path, monkeypatch): + """Test that S3STSToken.get_client respects ssl_verify config.""" + from datetime import datetime, timezone + from unittest.mock import MagicMock + + from tidy3d.config import ConfigManager + from tidy3d.web.core.s3utils import _S3STSToken + + # Create token using proper Pydantic structure + credential_data = { + "accessKeyId": "test-access-key", + "secretAccessKey": "test-secret-key", + "sessionToken": "test-session-token", + "expiration": datetime.now(timezone.utc), + } + token_data = { + "cloudpath": "s3://test-bucket/test-key", + "userCredentials": credential_data, + } + token = _S3STSToken(**token_data) + + mock_boto_client = MagicMock() + monkeypatch.setattr("tidy3d.web.core.s3utils.boto3.client", mock_boto_client) + + # Use fresh config with ssl_verify=False + test_config = ConfigManager(config_dir=tmp_path) + test_config.update_section("web", ssl_verify=False) + monkeypatch.setattr("tidy3d.web.core.s3utils.config", test_config) + + token.get_client() + + call_kwargs = mock_boto_client.call_args[1] + assert call_kwargs["verify"] is False diff --git a/tidy3d/config/legacy.py b/tidy3d/config/legacy.py index 0494964ecb..75d94d20f1 100644 --- a/tidy3d/config/legacy.py +++ b/tidy3d/config/legacy.py @@ -419,7 +419,20 @@ def _maybe_str(value: Any) -> Optional[str]: def load_legacy_flat_config(config_dir: Path) -> dict[str, Any]: - """Load legacy flat configuration file (pre-migration format).""" + """Load legacy flat configuration file (pre-migration format). + + This function now supports both the original flat config format and + Nexus custom deployment settings introduced in later versions. + + Legacy key mappings: + - apikey -> web.apikey + - web_api_endpoint -> web.api_endpoint + - website_endpoint -> web.website_endpoint + - s3_region -> web.s3_region + - s3_endpoint -> web.env_vars.AWS_ENDPOINT_URL_S3 + - ssl_verify -> web.ssl_verify + - enable_caching -> web.enable_caching + """ legacy_path = config_dir / "config" if not legacy_path.exists(): @@ -438,9 +451,43 @@ def load_legacy_flat_config(config_dir: Path) -> dict[str, Any]: return {} legacy_data: dict[str, Any] = {} + + # Migrate API key (original functionality) apikey = parsed.get("apikey") if apikey is not None: legacy_data.setdefault("web", {})["apikey"] = apikey + + # Migrate Nexus API endpoint + web_api = parsed.get("web_api_endpoint") + if web_api is not None: + legacy_data.setdefault("web", {})["api_endpoint"] = web_api + + # Migrate Nexus website endpoint + website = parsed.get("website_endpoint") + if website is not None: + legacy_data.setdefault("web", {})["website_endpoint"] = website + + # Migrate S3 region + s3_region = parsed.get("s3_region") + if s3_region is not None: + legacy_data.setdefault("web", {})["s3_region"] = s3_region + + # Migrate SSL verification setting + ssl_verify = parsed.get("ssl_verify") + if ssl_verify is not None: + legacy_data.setdefault("web", {})["ssl_verify"] = ssl_verify + + # Migrate caching setting + enable_caching = parsed.get("enable_caching") + if enable_caching is not None: + legacy_data.setdefault("web", {})["enable_caching"] = enable_caching + + # Migrate S3 endpoint to env_vars + s3_endpoint = parsed.get("s3_endpoint") + if s3_endpoint is not None: + env_vars = legacy_data.setdefault("web", {}).setdefault("env_vars", {}) + env_vars["AWS_ENDPOINT_URL_S3"] = s3_endpoint + return legacy_data diff --git a/tidy3d/config/loader.py b/tidy3d/config/loader.py index 0e71210d30..21782244c4 100644 --- a/tidy3d/config/loader.py +++ b/tidy3d/config/loader.py @@ -27,15 +27,52 @@ def __init__(self, config_dir: Optional[Path] = None): self._docs: dict[Path, tomlkit.TOMLDocument] = {} def load_base(self) -> dict[str, Any]: - """Load base configuration from config.toml.""" + """Load base configuration from config.toml. + + If config.toml doesn't exist but the legacy flat config does, + automatically migrate to the new format. + """ config_path = self.config_dir / "config.toml" data = self._read_toml(config_path) if data: return data + + # Check for legacy flat config from .legacy import load_legacy_flat_config + legacy_path = self.config_dir / "config" legacy = load_legacy_flat_config(self.config_dir) + + # Auto-migrate if legacy config exists + if legacy and legacy_path.exists(): + log.info( + f"Detected legacy configuration at '{legacy_path}'. " + "Automatically migrating to new format..." + ) + + try: + # Save in new format + self.save_base(legacy) + + # Rename old config to preserve it + backup_path = legacy_path.with_suffix(".migrated") + legacy_path.rename(backup_path) + + log.info( + f"Migration complete. Configuration saved to '{config_path}'. " + f"Legacy config backed up as '{backup_path.name}'." + ) + + # Re-read the newly created config + return self._read_toml(config_path) + except Exception as exc: + log.warning( + f"Failed to auto-migrate legacy configuration: {exc}. " + "Using legacy data without migration." + ) + return legacy + if legacy: return legacy return {} @@ -78,6 +115,49 @@ def profile_path(self, profile: str) -> Path: return self.config_dir / "profiles" / f"{profile}.toml" + def get_default_profile(self) -> Optional[str]: + """Read the default_profile from config.toml. + + Returns + ------- + Optional[str] + The default profile name if set, None otherwise. + """ + + config_path = self.config_dir / "config.toml" + if not config_path.exists(): + return None + + try: + text = config_path.read_text(encoding="utf-8") + data = toml.loads(text) + return data.get("default_profile") + except Exception as exc: + log.warning(f"Failed to read default_profile from '{config_path}': {exc}") + return None + + def set_default_profile(self, profile: Optional[str]) -> None: + """Set the default_profile in config.toml. + + Parameters + ---------- + profile : Optional[str] + The profile name to set as default, or None to remove the setting. + """ + + config_path = self.config_dir / "config.toml" + data = self._read_toml(config_path) + + if profile is None: + # Remove default_profile if it exists + if "default_profile" in data: + del data["default_profile"] + else: + # Set default_profile as a top-level key + data["default_profile"] = profile + + self._atomic_write(config_path, data) + def _read_toml(self, path: Path) -> dict[str, Any]: if not path.exists(): self._docs.pop(path, None) diff --git a/tidy3d/config/manager.py b/tidy3d/config/manager.py index 094188bd95..cbb08a4fdc 100644 --- a/tidy3d/config/manager.py +++ b/tidy3d/config/manager.py @@ -128,6 +128,11 @@ def __init__( attach_manager(self) self._reload() + + # Notify users when using a non-default profile + if self._profile != "default": + log.info(f"Using configuration profile: '{self._profile}'", log_once=True) + self._apply_handlers() @property @@ -174,18 +179,61 @@ def switch_profile(self, profile: str) -> None: raise ValueError("Profile name cannot be empty") self._profile = normalized self._reload() + + # Notify users when switching to a non-default profile + if self._profile != "default": + log.info(f"Switched to configuration profile: '{self._profile}'") + self._apply_handlers() - def save(self, include_defaults: bool = False) -> None: - base_without_env = self._filter_persisted(self._compose_without_env()) - if include_defaults: - defaults = self._filter_persisted(self._default_tree()) - base_without_env = deep_merge(defaults, base_without_env) + def set_default_profile(self, profile: Optional[str]) -> None: + """Set the default profile to be used on startup. + + Parameters + ---------- + profile : Optional[str] + The profile name to use as default, or None to clear the default. + When set, this profile will be automatically loaded unless overridden + by environment variables (TIDY3D_CONFIG_PROFILE, TIDY3D_PROFILE, or TIDY3D_ENV). + + Notes + ----- + This setting is persisted to config.toml and survives across sessions. + Environment variables always take precedence over the default profile. + """ + + if profile is not None: + normalized = normalize_profile_name(profile) + if not normalized: + raise ValueError("Profile name cannot be empty") + self._loader.set_default_profile(normalized) + else: + self._loader.set_default_profile(None) + def get_default_profile(self) -> Optional[str]: + """Get the currently configured default profile. + + Returns + ------- + Optional[str] + The default profile name if set, None otherwise. + """ + + return self._loader.get_default_profile() + + def save(self, include_defaults: bool = False) -> None: if self._profile == "default": + # For base config: only save fields marked with persist=True + base_without_env = self._filter_persisted(self._compose_without_env()) + if include_defaults: + defaults = self._filter_persisted(self._default_tree()) + base_without_env = deep_merge(defaults, base_without_env) self._loader.save_base(base_without_env) else: - baseline = self._filter_persisted(deep_merge(self._builtin_data, self._base_data)) + # For profile overrides: save any field that differs from baseline + # (don't filter by persist flag - profiles should save all customizations) + base_without_env = self._compose_without_env() + baseline = deep_merge(self._builtin_data, self._base_data) diff = deep_diff(baseline, base_without_env) self._loader.save_profile(self._profile, diff) # refresh cached base/profile data after saving @@ -302,13 +350,22 @@ def _resolve_initial_profile(self, profile: Optional[str]) -> str: if profile: return normalize_profile_name(str(profile)) - candidate = ( + # Check environment variables first (highest priority) + env_profile = ( os.getenv("TIDY3D_CONFIG_PROFILE") or os.getenv("TIDY3D_PROFILE") or os.getenv("TIDY3D_ENV") - or "default" ) - return normalize_profile_name(candidate) + if env_profile: + return normalize_profile_name(env_profile) + + # Check for default_profile in config file + config_default = self._loader.get_default_profile() + if config_default: + return normalize_profile_name(config_default) + + # Fall back to "default" profile + return "default" def _reload(self) -> None: self._env_overrides = load_environment_overrides() diff --git a/tidy3d/web/cli/app.py b/tidy3d/web/cli/app.py index 173b962ff5..7b6d015409 100644 --- a/tidy3d/web/cli/app.py +++ b/tidy3d/web/cli/app.py @@ -8,6 +8,7 @@ import shutil import ssl from typing import Any +from urllib.parse import urlparse, urlunparse import click import requests @@ -55,59 +56,246 @@ def tidy3d_cli() -> None: @click.command() -@click.option("--apikey", prompt=False) -def configure(apikey: str) -> None: - """Click command to configure the api key. +@click.option("--apikey", prompt=False, help="Tidy3D API key") +@click.option("--nexus-url", help="Nexus base URL (sets api=/tidy3d-api, web=/tidy3d, s3=:9000)") +@click.option("--api-endpoint", help="Nexus API endpoint URL (e.g., http://server/tidy3d-api)") +@click.option("--website-endpoint", help="Nexus website endpoint URL (e.g., http://server/tidy3d)") +@click.option("--s3-region", help="S3 region (default: us-east-1)") +@click.option("--s3-endpoint", help="S3 endpoint URL (e.g., http://server:9000)") +@click.option("--ssl-verify/--no-ssl-verify", default=None, help="Enable/disable SSL verification") +@click.option( + "--enable-caching/--no-caching", default=None, help="Enable/disable server-side caching" +) +@click.option( + "--restore-defaults", + is_flag=True, + help="Restore production defaults (removes nexus profile and clears default_profile)", +) +def configure( + apikey: str, + nexus_url: str, + api_endpoint: str, + website_endpoint: str, + s3_region: str, + s3_endpoint: str, + ssl_verify: bool, + enable_caching: bool, + restore_defaults: bool, +) -> None: + """Configure API key and optionally Nexus environment settings. Parameters ---------- apikey : str - User input api key. + User API key + nexus_url : str + Nexus base URL (automatically derives api/website/s3 endpoints) + api_endpoint : str + Nexus API endpoint URL + website_endpoint : str + Nexus website endpoint URL + s3_region : str + AWS S3 region + s3_endpoint : str + S3 endpoint URL + ssl_verify : bool + Whether to verify SSL certificates + enable_caching : bool + Whether to enable result caching """ - configure_fn(apikey) + configure_fn( + apikey, + nexus_url, + api_endpoint, + website_endpoint, + s3_region, + s3_endpoint, + ssl_verify, + enable_caching, + restore_defaults, + ) -def configure_fn(apikey: str) -> None: - """Python function that tries to set configuration based on a provided API key. +def configure_fn( + apikey: str | None, + nexus_url: str | None = None, + api_endpoint: str | None = None, + website_endpoint: str | None = None, + s3_region: str | None = None, + s3_endpoint: str | None = None, + ssl_verify: bool | None = None, + enable_caching: bool | None = None, + restore_defaults: bool = False, +) -> None: + """Configure API key and optionally Nexus environment settings. Parameters ---------- - apikey : str - User input api key. + apikey : str | None + User API key + nexus_url : str | None + Nexus base URL (automatically derives api/website/s3 endpoints) + api_endpoint : str | None + Nexus API endpoint URL + website_endpoint : str | None + Nexus website endpoint URL + s3_region : str | None + AWS S3 region + s3_endpoint : str | None + S3 endpoint URL + ssl_verify : bool | None + Whether to verify SSL certificates + enable_caching : bool | None + Whether to enable result caching + restore_defaults : bool + Restore production defaults (clears nexus profile) """ - def auth(req: requests.Request) -> requests.Request: - """Enrich auth information to request. - Parameters - ---------- - req : requests.Request - the request needs to add headers for auth. - Returns - ------- - requests.Request - Enriched request. - """ - req.headers[HEADER_APIKEY] = apikey - return req - - if not apikey: + # Handle restore defaults flag + if restore_defaults: + config.set_default_profile(None) + # Remove nexus profile if it exists + nexus_profile_path = config.config_dir / "profiles" / "nexus.toml" + if nexus_profile_path.exists(): + nexus_profile_path.unlink() + click.echo("Successfully restored production defaults.") + click.echo(" Cleared default_profile setting") + click.echo(" Removed nexus profile") + click.echo("\nTidy3D will now use production endpoints by default.") + return + + # If nexus_url is provided, derive endpoints from it automatically + if nexus_url: + # Strip trailing slashes for clean URLs + base_url = nexus_url.rstrip("/") + api_endpoint = f"{base_url}/tidy3d-api" + website_endpoint = f"{base_url}/tidy3d" + + # For S3 endpoint, replace any existing port with 9000 + parsed = urlparse(nexus_url) + # Reconstruct netloc with port 9000 (remove any existing port) + hostname = parsed.hostname or parsed.netloc.split(":")[0] + s3_netloc = f"{hostname}:9000" + s3_endpoint = urlunparse((parsed.scheme, s3_netloc, "", "", "", "")) + + # Check if any Nexus options are provided + has_nexus_config = any( + [ + api_endpoint, + website_endpoint, + s3_region, + s3_endpoint, + ssl_verify is not None, + enable_caching is not None, + ] + ) + + # Validate that both endpoints are provided if configuring Nexus + if has_nexus_config and (api_endpoint or website_endpoint): + if not (api_endpoint and website_endpoint): + click.echo( + "Error: Both --api-endpoint and --website-endpoint must be provided together " + "(or use --nexus-url to set both automatically)." + ) + return + + # Handle API key prompt if not provided and no Nexus-only config + if not apikey and not has_nexus_config: current_apikey = get_description() message = f"Current API key: [{current_apikey}]\n" if current_apikey else "" apikey = click.prompt(f"{message}Please enter your api key", type=str) - target_url = config.web.build_api_url("apikey") + # Build updates dictionary for web section + web_updates = {} - try: - resp = requests.get(target_url, auth=auth, verify=config.web.ssl_verify) - except (requests.exceptions.SSLError, ssl.SSLError): - resp = requests.get(target_url, auth=auth, verify=False) - - if resp.status_code == 200: - click.echo("Configured successfully.") - config.update_section("web", apikey=apikey) - config.save() - else: - click.echo("API key is invalid.") + if apikey: + web_updates["apikey"] = apikey + + if api_endpoint: + web_updates["api_endpoint"] = api_endpoint + + if website_endpoint: + web_updates["website_endpoint"] = website_endpoint + + if s3_region is not None: + web_updates["s3_region"] = s3_region + + if ssl_verify is not None: + web_updates["ssl_verify"] = ssl_verify + + if enable_caching is not None: + web_updates["enable_caching"] = enable_caching + + # Handle S3 endpoint via env_vars + if s3_endpoint is not None: + current_env_vars = dict(config.web.env_vars) if config.web.env_vars else {} + current_env_vars["AWS_ENDPOINT_URL_S3"] = s3_endpoint + web_updates["env_vars"] = current_env_vars + + # Validate API key if provided + if apikey: + + def auth(req: requests.Request) -> requests.Request: + """Enrich auth information to request.""" + req.headers[HEADER_APIKEY] = apikey + return req + + # Determine validation endpoint + validation_endpoint = api_endpoint if api_endpoint else str(config.web.api_endpoint) + validation_ssl = ssl_verify if ssl_verify is not None else config.web.ssl_verify + + target_url = f"{validation_endpoint.rstrip('/')}/apikey" + + try: + resp = requests.get(target_url, auth=auth, verify=validation_ssl) + except (requests.exceptions.SSLError, ssl.SSLError): + resp = requests.get(target_url, auth=auth, verify=False) + + if resp.status_code != 200: + click.echo( + f"Error: API key validation failed against endpoint: {validation_endpoint}\n" + f"Status code: {resp.status_code}" + ) + return + + # Apply updates if any + if web_updates: + if has_nexus_config: + # For nexus config: save apikey to base, nexus settings to profile + # First save apikey to base config (if provided) + if apikey: + config.update_section("web", apikey=apikey) + config.save() + + # Switch to nexus profile and save nexus-specific settings + config.switch_profile("nexus") + nexus_updates = {k: v for k, v in web_updates.items() if k != "apikey"} + if nexus_updates: + config.update_section("web", **nexus_updates) + config.save() + else: + # Non-nexus config: save everything to base config + config.update_section("web", **web_updates) + config.save() + + if has_nexus_config: + # Set nexus as the default profile when nexus is configured + config.set_default_profile("nexus") + click.echo("Nexus configuration saved successfully to profile: profiles/nexus.toml") + if api_endpoint: + click.echo(f" API endpoint: {api_endpoint}") + if website_endpoint: + click.echo(f" Website endpoint: {website_endpoint}") + if s3_endpoint: + click.echo(f" S3 endpoint: {s3_endpoint}") + click.echo( + "\nDefault profile set to 'nexus'. Tidy3D will now use these endpoints by default." + ) + click.echo("To switch back to production, run: tidy3d configure --restore-defaults") + else: + click.echo("Configuration saved successfully.") + elif not apikey and not has_nexus_config: + click.echo("No configuration changes to apply.") @click.command() diff --git a/tidy3d/web/core/s3utils.py b/tidy3d/web/core/s3utils.py index 6629ba25f2..1dfb3ec1e1 100644 --- a/tidy3d/web/core/s3utils.py +++ b/tidy3d/web/core/s3utils.py @@ -63,16 +63,26 @@ def get_s3_key(self) -> str: return r.path[1:] def get_client(self) -> boto3.client: - """Get the boto client for this token.""" - - return boto3.client( - "s3", - region_name=config.web.s3_region, - aws_access_key_id=self.user_credential.access_key_id, - aws_secret_access_key=self.user_credential.secret_access_key, - aws_session_token=self.user_credential.session_token, - verify=config.web.ssl_verify, - ) + """Get the boto client for this token. + + Automatically configures custom S3 endpoint if specified in web.env_vars. + """ + + client_kwargs = { + "service_name": "s3", + "region_name": config.web.s3_region, + "aws_access_key_id": self.user_credential.access_key_id, + "aws_secret_access_key": self.user_credential.secret_access_key, + "aws_session_token": self.user_credential.session_token, + "verify": config.web.ssl_verify, + } + + # Add custom S3 endpoint if configured (e.g., for Nexus deployments) + if config.web.env_vars and "AWS_ENDPOINT_URL_S3" in config.web.env_vars: + s3_endpoint = config.web.env_vars["AWS_ENDPOINT_URL_S3"] + client_kwargs["endpoint_url"] = s3_endpoint + + return boto3.client(**client_kwargs) def is_expired(self) -> bool: """True if token is expired."""