diff --git a/.github/workflows/auto-assign-author.yml b/.github/workflows/auto-assign-author.yml index a6983e579..5e83b31d9 100644 --- a/.github/workflows/auto-assign-author.yml +++ b/.github/workflows/auto-assign-author.yml @@ -9,16 +9,50 @@ jobs: runs-on: ubuntu-latest permissions: pull-requests: write + issues: write # Required because assignees API works via issues + contents: read # Minimal read access to repository contents steps: - name: Assign PR author uses: actions/github-script@v8 with: script: | - const reporter = context.actor - await github.rest.issues.addAssignees({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.payload.pull_request.number, - assignees: [reporter] - }) + // Repository owner (organization or user who owns the repo) + const owner = context.repo.owner; + + // Repository name + const repo = context.repo.repo; + + // Login of the pull request author + const prAuthor = context.payload.pull_request.user.login; + + try { + // Check the permission level of the PR author + const { data: perm } = + await github.rest.repos.getCollaboratorPermissionLevel({ + owner, + repo, + username: prAuthor + }); + + // If the author has write/maintain/admin rights → assign them to the PR + if (["write", "maintain", "admin"].includes(perm.permission)) { + await github.rest.issues.addAssignees({ + owner, + repo, + issue_number: context.payload.pull_request.number, + assignees: [prAuthor] + }); + console.log(`Assigned PR to ${prAuthor}`); + } else { + // If the author has insufficient rights → skip assignment + console.log( + `Skipping assignment for ${prAuthor}: permission=${perm.permission}` + ); + } + } catch (error) { + // If the author is not a collaborator (e.g., PR from a fork) → skip without failing + console.log( + `Skipping assignment for ${prAuthor}: not a collaborator` + ); + } diff --git a/.gitignore b/.gitignore index 19166677d..cdb9bd6eb 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,7 @@ rest_api/.clickhouse # jira_ui jira_ui/*/.env jira_ui/*/node_modules -jira_ui/*/build/ \ No newline at end of file +jira_ui/*/build/ + +# Vagrant +optscale-deploy/.vagrant/ diff --git a/build.sh b/build.sh index 11c388527..e6e12da6b 100755 --- a/build.sh +++ b/build.sh @@ -110,6 +110,12 @@ do else echo "Building image for ${COMPONENT}, build tag: ${BUILD_TAG}" $BUILD_TOOL build $FLAGS -t ${COMPONENT}:${BUILD_TAG} -f ${DOCKERFILE} . + + # If the build fails, exit with the same status code as the build command + build_status_code="$?" + if [ "$build_status_code" -gt 0 ]; then + exit $build_status_code + fi fi if use_registry; then diff --git a/docker_images/common/install-peer-finder.sh b/docker_images/common/install-peer-finder.sh new file mode 100644 index 000000000..421f7e71b --- /dev/null +++ b/docker_images/common/install-peer-finder.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# TODO: Instead of this script we should use multi stage docker builds +# but this is good enough until we get arround to it +# Or even better -- see if we need this tool at all or if there is a better +# way to install it (e.g. via a package manager) + +set -x + +arch="$(uname -m)" +dest_bin_path="/usr/local/bin/peer-finder" + +apt-get update +apt-get install -y --no-install-recommends openssl ca-certificates wget +rm -rf /var/lib/apt/lists/* + +if [[ "$arch" == "x86_64" || "$arch" == "amd64" ]]; then + wget -O $dest_bin_path https://storage.googleapis.com/kubernetes-release/pets/peer-finder +elif [[ "$arch" == "aarch64" || "$arch" == "arm64" ]]; then + wget https://github.com/kmodules/peer-finder/releases/download/v1.0.2/peer-finder-linux-arm64.tar.gz \ + -O /tmp/peer-finder-linux-arm64.tar.gz + tar -xzf /tmp/peer-finder-linux-arm64.tar.gz -C /tmp + mv /tmp/peer-finder-linux-arm64 $dest_bin_path +else + echo "Unsupported architecture: $arch" + exit 1 +fi + +chmod +x $dest_bin_path +apt-get purge -y --auto-remove ca-certificates wget + diff --git a/docker_images/error_pages/Dockerfile b/docker_images/error_pages/Dockerfile index 4df9b45bb..f2ad6b273 100644 --- a/docker_images/error_pages/Dockerfile +++ b/docker_images/error_pages/Dockerfile @@ -1,3 +1,10 @@ -FROM ingressnginx/custom-error-pages:v1.2.0 +# TODO: The base image doesn't support arm64 yet but shouldn't be too hard to change that, +# though it will require a change in the `kubernetes/ingress-nginx` repo. +# References: +# * Base image's Dockerfile: https://github.com/kubernetes/ingress-nginx/blob/main/images/custom-error-pages/rootfs/Dockerfile +# * Relevant issue on GitHub: https://github.com/kubernetes/ingress-nginx/issues/10245 + +ARG arch=amd64 +FROM --platform="linux/${arch}" ingressnginx/custom-error-pages:v1.2.0 COPY docker_images/error_pages/www /www diff --git a/docker_images/etcd/Dockerfile b/docker_images/etcd/Dockerfile index 1c550c86d..bb432aaa3 100644 --- a/docker_images/etcd/Dockerfile +++ b/docker_images/etcd/Dockerfile @@ -1,7 +1,18 @@ -FROM gcr.io/etcd-development/etcd:v3.2.13 -RUN apk update -# https://github.com/Yelp/dumb-init/issues/73#issuecomment-240439732 -RUN apk add ca-certificates wget && update-ca-certificates -RUN apk --no-cache add curl -RUN wget $(curl -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/nexusriot/etcd-walker/releases/tags/0.0.11 | grep -Eo 'https://(.*linux_x64_static)') -O /bin/etcd-walker +# etcd is a distroless image starting from v3.5, meaning we don't have access to a shell or package manager. +# this is why we use multi-stage builds to build and copy the binary into the final image. +# ref: https://github.com/GoogleContainerTools/distroless?tab=readme-ov-file#docker + +FROM golang:1.24.6 AS build-etcd-walker + +RUN git clone https://github.com/nexusriot/etcd-walker/ /tmp/etcd-walker-src + +WORKDIR /tmp/etcd-walker-src +RUN git checkout 0.2.1 +RUN go build -ldflags "-linkmode external -extldflags -static" -o etcd-walker cmd/etcd-walker/main.go + +RUN mv etcd-walker /bin/etcd-walker RUN chmod +x /bin/etcd-walker + +# NOTE: v3.6+ require significant changes as they removed support for the V2 API, see https://etcd.io/docs/v3.6/upgrades/upgrade_3_6/ +FROM gcr.io/etcd-development/etcd:v3.2.13 +COPY --from=build-etcd-walker /bin/etcd-walker /bin/etcd-walker diff --git a/docker_images/mariadb/Dockerfile b/docker_images/mariadb/Dockerfile index 707ad28cf..5025591f9 100644 --- a/docker_images/mariadb/Dockerfile +++ b/docker_images/mariadb/Dockerfile @@ -1,11 +1,8 @@ FROM mariadb:10.3 -RUN set -x \ - && apt-get update && apt-get install -y --no-install-recommends ca-certificates wget \ - && rm -rf /var/lib/apt/lists/* \ - && wget -O /usr/local/bin/peer-finder https://storage.googleapis.com/kubernetes-release/pets/peer-finder \ - && chmod +x /usr/local/bin/peer-finder \ - && apt-get purge -y --auto-remove ca-certificates wget +COPY docker_images/common/install-peer-finder.sh /tmp/install-peer-finder.sh +RUN chmod +x /tmp/install-peer-finder.sh +RUN /tmp/install-peer-finder.sh COPY docker_images/mariadb/galera /opt/galera/ COPY docker_images/mariadb/docker-entrypoint.sh /usr/local/bin/ diff --git a/docker_images/mongo/Dockerfile b/docker_images/mongo/Dockerfile index 39b1d0d07..683d3ba27 100644 --- a/docker_images/mongo/Dockerfile +++ b/docker_images/mongo/Dockerfile @@ -1,10 +1,7 @@ FROM mongo:3.6 -RUN set -x \ - && apt-get update && apt-get install -y --no-install-recommends openssl ca-certificates wget \ - && rm -rf /var/lib/apt/lists/* \ - && wget -O /usr/local/bin/peer-finder https://storage.googleapis.com/kubernetes-release/pets/peer-finder \ - && chmod +x /usr/local/bin/peer-finder \ - && apt-get purge -y --auto-remove ca-certificates wget +COPY docker_images/common/install-peer-finder.sh /tmp/install-peer-finder.sh +RUN chmod +x /tmp/install-peer-finder.sh +RUN /tmp/install-peer-finder.sh COPY docker_images/mongo/on-start.sh /on-start.sh diff --git a/documentation/setup_dev_vm.md b/documentation/setup_dev_vm.md new file mode 100644 index 000000000..44469debc --- /dev/null +++ b/documentation/setup_dev_vm.md @@ -0,0 +1,365 @@ +# Set up a virtual machine for development / testing of deployment + +With the help of a few tools we now have the capability to run the whole optscale locally by running a few simple commands, +allowing new developers to contribute immediately as well as explore the codebase freely with quick feedback cycle for their +local changes and requiring minimal knowledge about the deployment process. Virtual Machines also make it possible to have +as close to production-level environment running locally helping with testing and making changes to the deployment process +itself and also allowing developers to use different OS and even CPU architecture than Ubuntu 24.04 running on x86 hardware +(which is a hard requirement at the moment for an Optscale deployment), e.g. Apple Silicon Macs. + +The tools which allow us to do that are: + +1. [Vagrant](https://developer.hashicorp.com/vagrant/) to configure and manage the virtual machines +2. [QEMU](https://www.qemu.org/) as the virtualization engine used to run them +3. [`vagrant-qemu`](https://github.com/ppggff/vagrant-qemu) as the bridge between Vagrant and QEMU +4. **VirtualBox** — optional alternative virtualization provider +5. **`vagrant-disksize` plugin** — required when using VirtualBox to control disk size (not required for QEMU, QEMU disk sizing is controlled by the QEMU provider config (qemu.disk_resize) + +> [!WARNING] +> ## Prefer VirtualBox on Linux Kernel 6.14+ +> +> Starting with **Linux kernel 6.14**, changes in KVM/QEMU interaction may cause +> **QEMU virtual machines to fail, lose acceleration, or break after updates**. +> +> Because of this, it is **strongly recommended** to use **VirtualBox** as the +> virtualization provider on Linux hosts running kernel **6.14 or newer**. +> +> Use VirtualBox explicitly: +> +> ```sh +> ./vm.sh --provider virtualbox start +> ``` +> +> Using QEMU on recent Linux kernels may lead to: +> - VM startup failures +> - Missing KVM acceleration +> - Unstable or crashing VM processes +> - Guest OS boot loops +> +> **VirtualBox remains unaffected and provides stable performance.** + +> [!NOTE] +> ## Nested Virtualization (running VM inside another VM) +> +> If you are using **Vagrant/QEMU/VirtualBox inside a virtual machine** (e.g., running on VMware, Proxmox, Hyper-V, or cloud VPS): +> +> - Your host hypervisor **must support nested virtualization**, +> - AND it must be **explicitly enabled** for your VM. +> +> Without nested virtualization: +> +> - QEMU will run **without KVM acceleration** → extremely slow +> - VirtualBox may **fail to start VMs** or run in pure software mode +> - Provisioning times may increase from 20–30 minutes → **several hours** +> +> ### How to enable nested virtualization (quick reference) +> +> **Proxmox:** +> ```sh +> qm set -cpu host +> echo "options kvm-intel nested=Y" >> /etc/modprobe.d/kvm-intel.conf +> modprobe -r kvm_intel && modprobe kvm_intel +> ``` +> +> **VMware ESXi / Workstation / Fusion:** +> ```sh +> vhv.enable = "TRUE" +> ``` +> +> **Hyper-V:** +> ```powershell +> Set-VMProcessor -VMName "MyVM" -ExposeVirtualizationExtensions $true +> ``` +> +> **VirtualBox (running as host hypervisor):** +> ```sh +> VBoxManage modifyvm --nested-hw-virt on +> ``` +> +> ### Recommendation +> For best performance and compatibility: +> +> - Prefer **bare-metal** environments when possible +> - If running inside a VM, ensure **nested virtualization is enabled** before using `vm.sh` with QEMU or VirtualBox + +`Vagrant` already provides a great CLI to manage and run the VMs but it has a few annoying quirks and it still requires +complicated commands to run common operations specific to Optscale, so we built a wrapper script to make it even easier to +set up and use VMs -- `optscale-deploy/vm.sh`. + +--- + +## Install Prerequisites + +1. **Install `vagrant`** using your system package manager. + +### On MacOS: + +```sh +brew tap hashicorp/tap +brew install hashicorp/tap/hashicorp-vagrant +``` + +### On Ubuntu: + +```sh +wget -O - https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg +echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] \ +https://apt.releases.hashicorp.com $(grep -oP '(?<=UBUNTU_CODENAME=).*' /etc/os-release || lsb_release -cs) main" \ +| sudo tee /etc/apt/sources.list.d/hashicorp.list +sudo apt update && sudo apt install vagrant +``` + +2. **Install QEMU** + +### On MacOS: + +```sh +brew install qemu +``` + +### On Ubuntu: +(if decided to use qemu, please note warning above) + +```sh +sudo apt-get install qemu-system +``` + +3. **Install the required Vagrant plugins** + +``` +vagrant plugin install vagrant-qemu +vagrant plugin install vagrant-disksize +``` + +> **IMPORTANT:** +> `vagrant-disksize` is **required** when using VirtualBox, and also used in general to configure VM disk size. + +4. **Optional: Install VirtualBox** + +VirtualBox can be used instead of QEMU. +To force using VirtualBox: + +```sh +./vm.sh --provider virtualbox start +``` + +--- + +## Set up the Virtual Machine + +Use the `vm.sh` script in `optscale-deploy` to manage the VM. There are currently two VMs ready for use: `arm` and `x86`. +Use whichever matches your machine's OS but the other one should also work via emulation (note that it will be +_significantly slower_ though). + +You can either explicitly specify it as the first argument like so: + +```sh +./vm.sh x86 start +``` + +or omit it entirely in which case it will default to your host machines' CPU architecture: + +```sh +./vm.sh start +``` + +(running the command above on an M4 Mac will create the ARM-based virtual machine) + +> Creating a virtual machine also copies your local version of this repo into `~/optscale` meaning it's very easy to +> test local changes not yet pushed to GitHub. + +> Feel free to mess around with the `Vagrantfile` itself whether it's to tweak some of the settings or even create new +> virtual machines, it should be fairly straight-forward to do so :) + +--- + +## VM resource configuration (`--cpus`, `--ram`, `--disk`) + +The `vm.sh` helper now supports dynamic resource overrides: + +| Flag | Meaning | Default | +|------|---------|---------| +| `--cpus N` | Number of virtual CPUs | `4` | +| `--ram N` | VM RAM in GB | `10` | +| `--disk N` | VM disk size in GB | `75` | + +Example: + +```sh +./vm.sh x86 --cpus 8 --ram 16 --disk 120 start +``` + +These settings propagate into the Vagrantfile through environment variables: + +- `VM_CPUS` +- `VM_RAM_GB` +- `VM_DISK_GB` + +They apply both to **QEMU** and **VirtualBox**. + +--- + +#### Preparing virtual environment + +Run the following commands: + +``` +virtualenv -p python3 .venv +source .venv/bin/activate +pip install -r requirements.txt +``` +#### Creating user overlay + +Edit file with overlay - [optscale-deploy/overlay/user_template.yml](optscale-deploy/overlay/user_template.yml); see comments in overlay file for guidance. + +## Optional ELK stack (`--with-elk`) + +The `vm.sh` wrapper also supports enabling the **ELK stack** (Elasticsearch, Logstash, Kibana) in the local deployment. + +Global flag: + +- `--with-elk` — enable ELK support for the current command. + +When `--with-elk` is used: + +1. The `elk` container image is built as part of the provisioning process. +2. The Ansible playbook receives `with_elk=true`. +3. `runkube.py` is executed with the `--with-elk` flag for: + - initial provisioning, + - `deploy-service`, + - `update-only`. + +Examples: + +Provision VM with ELK enabled: + +```sh +./vm.sh --with-elk playbook ansible/provision-vm.yaml +``` + +Deploy a single service with ELK-aware configuration: + +```sh +./vm.sh --with-elk deploy-service rest_api +``` +If ELK stack is required, need to set `--with-elk` directly. + +Update existing OptScale release while keeping ELK enabled: + +```sh +./vm.sh --with-elk update-only +``` + +If `--with-elk` is omitted, the behavior is unchanged from the previous default (no ELK components are built or deployed). + +--- + +## Provider Selection + +By default, QEMU is used (especially useful for Apple Silicon and cross-architecture development). +VirtualBox can also be used and works very well on x86 hosts. + +Explicit provider selection: + +```sh +./vm.sh --provider qemu start +./vm.sh --provider virtualbox start +``` + +> **NOTE:** +> When using VirtualBox, `vagrant-disksize` must be installed or Vagrant will fail to start the VM. + +--- + +## Deploy Optscale on the VM + +There is also an ansible playbook specifically built to allow a single command provisioning of Optscale onto a fresh Virtual +Machine: `optscale-deploy/ansible/provision-vm.yaml`. It will do everything -- from installing dependencies, setting up the +cluster, building all the containers (including `elk` when `--with-elk` is used) and creating a new Kubernetes deployment +using `runkube.py`. + +There is nothing VM-specific this playbook does, it largely simply follows the instructions on the `README.md` page +but it's more automated, so that it can all be done in a single command. + +Execute this (or any other) playbook with: + +```sh +./vm.sh arm playbook ansible/provision-vm.yaml +``` + +Or, with ELK enabled: + +```sh +./vm.sh arm --with-elk playbook ansible/provision-vm.yaml +``` + +There is also a `role` command which allows us to run a specific ansible role against the VM. + +--- + +## Accessing the platform + +If everything goes well, you should be able to access the platform soon. + +Keep in mind that the initial provisioning of the VM takes quite some time (~20–30 mins on an M4 Macbook) +mostly due to all the containers that need to be built from scratch. Also note that the Kubernetes cluster +will need some time (~15 mins) to spin all the pods after the playbook's execution is complete. + +Once ready, open your browser and navigate to: + +- `https://localhost:9444` for **ARM VM** +- `https://localhost:9443` for **x86 VM** + +(additional forwards) +- `http://localhost:41080` for **PhpMyadmin** +- `http://localhost:41081` for **Kibana (if enabled)** +- `http://localhost:41082` for **grafana** + +Port values come from the `Vagrantfile`. + +--- + +## Troubleshooting + +The `./vm.sh` script provides useful commands to debug issues: + +* `info` — shows general information about the VM itself: status, name, process ID etc. +* `ssh` — allows you to ssh into the VM and investigate issues or make persistent changes directly. +* `optscale-info` — shows information specific to the Optscale deployment: frontend access URL, k8s cluster + health, pods which are currently failing etc. + +--- + +## Deploying and testing local changes using the VM + +Once your VM is running, you can easily deploy your local changes using the **experimental** `deploy-service` command. + +Example: + +```sh +./vm.sh deploy-service rest_api +``` + +This will: + +1. Sync your local repo changes into the VM +2. Rebuild the selected service +3. Apply changes using `runkube.py` + +The entire process usually takes ~1 minute. + +If ELK is enabled via `--with-elk`, the same command will keep ELK configuration in sync with your updated deployment. + +--- + +## Other commands + +* `stop` — stop the virtual machine (preserves state) + * use `--force` to forcefully terminate VM process +* `restart` — restarts the VM +* `destroy` — stops and deletes the whole VM including data +* `reset` — a convenience command combining `destroy` + `start` +* `update-only` — rebuilds OptScale containers and redeploys without full reprovisioning + +The resource flags (`--cpus`, `--ram`, `--disk`) work with all lifecycle commands. The ELK flag (`--with-elk`) can be +combined with any command that performs provisioning, deployment or update logic. diff --git a/ngui/package.json b/ngui/package.json index d22bd6891..46da3f9cf 100644 --- a/ngui/package.json +++ b/ngui/package.json @@ -31,7 +31,7 @@ "@types/react-syntax-highlighter": "15.5.13", "@typescript-eslint/eslint-plugin": "8.41.0", "@typescript-eslint/parser": "8.41.0", - "concurrently": "9.2.0", + "concurrently": "9.2.1", "eslint": "9.34.0", "eslint-config-prettier": "9.1.0", "eslint-plugin-import": "2.31.0", diff --git a/ngui/pnpm-lock.yaml b/ngui/pnpm-lock.yaml index 8ececd788..8b3a9f983 100644 --- a/ngui/pnpm-lock.yaml +++ b/ngui/pnpm-lock.yaml @@ -55,8 +55,8 @@ importers: specifier: 8.41.0 version: 8.41.0(eslint@9.34.0(jiti@2.4.2))(typescript@5.7.2) concurrently: - specifier: 9.2.0 - version: 9.2.0 + specifier: 9.2.1 + version: 9.2.1 eslint: specifier: 9.34.0 version: 9.34.0(jiti@2.4.2) @@ -3273,8 +3273,8 @@ packages: resolution: {integrity: sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==} engines: {'0': node >= 0.8} - concurrently@9.2.0: - resolution: {integrity: sha512-IsB/fiXTupmagMW4MNp2lx2cdSN2FfZq78vF90LBB+zZHArbIQZjQtzXCiXnvTxCZSvXanTqFLWBjw2UkLx1SQ==} + concurrently@9.2.1: + resolution: {integrity: sha512-fsfrO0MxV64Znoy8/l1vVIjjHa29SZyyqPgQBwhiDcaW8wJc2W3XWVOGx4M3oJBnv/zdUZIIp1gDeS98GzP8Ng==} engines: {node: '>=18'} hasBin: true @@ -5035,8 +5035,8 @@ packages: resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} engines: {node: '>=10'} - lodash-es@4.17.21: - resolution: {integrity: sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==} + lodash-es@4.17.23: + resolution: {integrity: sha512-kVI48u3PZr38HdYz98UmfPnXl2DXrpdctLrFLCd3kOx1xUkOmpFPx7gCWWM5MPkL/fD8zb+Ph0QzjGFs4+hHWg==} lodash.camelcase@4.3.0: resolution: {integrity: sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==} @@ -5053,8 +5053,8 @@ packages: lodash.sortby@4.7.0: resolution: {integrity: sha512-HDWXG8isMntAyRF5vZ7xKuEvOhT4AhlRt/3czTSjvGUxjYCBVRQY48ViDHyfYz9VIoBkW4TMGQNapx+l3RUwdA==} - lodash@4.17.21: - resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} + lodash@4.17.23: + resolution: {integrity: sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==} log-symbols@4.1.0: resolution: {integrity: sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==} @@ -8367,7 +8367,7 @@ snapshots: common-tags: 1.8.2 graphql: 16.10.0 import-from: 4.0.0 - lodash: 4.17.21 + lodash: 4.17.23 tslib: 2.4.1 '@graphql-codegen/plugin-helpers@5.1.1(graphql@16.10.0)': @@ -8377,7 +8377,7 @@ snapshots: common-tags: 1.8.2 graphql: 16.10.0 import-from: 4.0.0 - lodash: 4.17.21 + lodash: 4.17.23 tslib: 2.6.3 '@graphql-codegen/schema-ast@4.1.0(graphql@16.10.0)': @@ -8733,7 +8733,7 @@ snapshots: https-proxy-agent: 7.0.6 jose: 5.10.0 js-yaml: 4.1.0 - lodash: 4.17.21 + lodash: 4.17.23 scuid: 1.1.0 tslib: 2.8.1 yaml-ast-parser: 0.0.43 @@ -9135,7 +9135,7 @@ snapshots: '@nivo/core': 0.99.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0) '@nivo/theming': 0.99.0(react@18.2.0) '@react-spring/web': 9.7.5(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - lodash: 4.17.21 + lodash: 4.17.23 react: 18.2.0 transitivePeerDependencies: - react-dom @@ -9186,7 +9186,7 @@ snapshots: '@types/d3-shape': 3.1.7 d3-scale: 4.0.2 d3-shape: 3.2.0 - lodash: 4.17.21 + lodash: 4.17.23 react: 18.2.0 transitivePeerDependencies: - react-dom @@ -9203,7 +9203,7 @@ snapshots: d3-color: 3.1.0 d3-scale: 4.0.2 d3-scale-chromatic: 3.1.0 - lodash: 4.17.21 + lodash: 4.17.23 react: 18.2.0 transitivePeerDependencies: - react-dom @@ -9221,7 +9221,7 @@ snapshots: d3-scale-chromatic: 3.1.0 d3-shape: 3.2.0 d3-time-format: 3.0.0 - lodash: 4.17.21 + lodash: 4.17.23 react: 18.2.0 react-virtualized-auto-sizer: 1.0.26(react-dom@18.2.0(react@18.2.0))(react@18.2.0) use-debounce: 10.0.6(react@18.2.0) @@ -9282,7 +9282,7 @@ snapshots: d3-scale: 4.0.2 d3-time: 1.1.0 d3-time-format: 3.0.0 - lodash: 4.17.21 + lodash: 4.17.23 '@nivo/text@0.99.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0)': dependencies: @@ -9295,7 +9295,7 @@ snapshots: '@nivo/theming@0.99.0(react@18.2.0)': dependencies: - lodash: 4.17.21 + lodash: 4.17.23 react: 18.2.0 '@nivo/tooltip@0.99.0(react-dom@18.2.0(react@18.2.0))(react@18.2.0)': @@ -10936,10 +10936,9 @@ snapshots: readable-stream: 2.3.8 typedarray: 0.0.6 - concurrently@9.2.0: + concurrently@9.2.1: dependencies: chalk: 4.1.2 - lodash: 4.17.21 rxjs: 7.8.2 shell-quote: 1.8.3 supports-color: 8.1.1 @@ -12540,7 +12539,7 @@ snapshots: cli-width: 3.0.0 external-editor: 3.1.0 figures: 3.2.0 - lodash: 4.17.21 + lodash: 4.17.23 mute-stream: 0.0.8 ora: 5.4.1 run-async: 2.4.1 @@ -12986,7 +12985,7 @@ snapshots: dependencies: p-locate: 5.0.0 - lodash-es@4.17.21: {} + lodash-es@4.17.23: {} lodash.camelcase@4.3.0: {} @@ -12998,7 +12997,7 @@ snapshots: lodash.sortby@4.7.0: {} - lodash@4.17.21: {} + lodash@4.17.23: {} log-symbols@4.1.0: dependencies: @@ -14237,8 +14236,8 @@ snapshots: dom-helpers: 5.2.1 globalize: 0.1.1 invariant: 2.2.4 - lodash: 4.17.21 - lodash-es: 4.17.21 + lodash: 4.17.23 + lodash-es: 4.17.23 luxon: 2.5.2 memoize-one: 6.0.0 moment: 2.30.1 diff --git a/ngui/ui/src/components/ApolloProvider/ApolloProvider.tsx b/ngui/ui/src/components/ApolloProvider/ApolloProvider.tsx index 42b55e534..fcf0e9db1 100644 --- a/ngui/ui/src/components/ApolloProvider/ApolloProvider.tsx +++ b/ngui/ui/src/components/ApolloProvider/ApolloProvider.tsx @@ -1,3 +1,4 @@ +import type { ReactNode } from "react"; import { ApolloClient, ApolloProvider, InMemoryCache, split, HttpLink, from } from "@apollo/client"; import { onError, type ErrorResponse } from "@apollo/client/link/error"; import { RetryLink } from "@apollo/client/link/retry"; @@ -10,10 +11,14 @@ import { useSignOut } from "hooks/useSignOut"; import { processGraphQLErrorData } from "utils/apollo"; import { getEnvironmentVariable } from "utils/env"; +type ApolloClientProviderProps = { + children: ReactNode; +}; + const httpBase = getEnvironmentVariable("VITE_APOLLO_HTTP_BASE"); const wsBase = getEnvironmentVariable("VITE_APOLLO_WS_BASE"); -const ApolloClientProvider = ({ children }) => { +const ApolloClientProvider = ({ children }: ApolloClientProviderProps) => { const { token } = useGetToken(); const signOut = useSignOut(); @@ -21,7 +26,7 @@ const ApolloClientProvider = ({ children }) => { const cache = new InMemoryCache(); const httpLink = new HttpLink({ - uri: `${httpBase}/api`, + uri: (operation) => `${httpBase}/api?op=${operation.operationName}`, headers: { "x-optscale-token": token } diff --git a/ngui/ui/src/components/ArchivedRecommendationAccordion/ArchivedRecommendationAccordion.tsx b/ngui/ui/src/components/ArchivedRecommendationAccordion/ArchivedRecommendationAccordion.tsx index 8b3a33699..49760409b 100644 --- a/ngui/ui/src/components/ArchivedRecommendationAccordion/ArchivedRecommendationAccordion.tsx +++ b/ngui/ui/src/components/ArchivedRecommendationAccordion/ArchivedRecommendationAccordion.tsx @@ -38,7 +38,7 @@ const ArchivedRecommendationAccordion = ({ reason }); - const allRecommendations = useAllRecommendations(); + const allRecommendations = useAllRecommendations({ withDeprecated: true }); useEffect(() => { if (isExpanded && shouldInvoke) { diff --git a/ngui/ui/src/components/ArtifactsTable/ArtifactsTable.tsx b/ngui/ui/src/components/ArtifactsTable/ArtifactsTable.tsx index fe0cc5807..6472aa23e 100644 --- a/ngui/ui/src/components/ArtifactsTable/ArtifactsTable.tsx +++ b/ngui/ui/src/components/ArtifactsTable/ArtifactsTable.tsx @@ -5,7 +5,7 @@ import { Stack } from "@mui/material"; import { FormattedMessage } from "react-intl"; import { useNavigate } from "react-router-dom"; import LinearSelector from "components/LinearSelector"; -import { TABS } from "components/MlTaskRun/Components/Tabs"; +import { TABS } from "components/MlTaskRun/components/Tabs"; import { MlDeleteArtifactModal } from "components/SideModalManager/SideModals"; import Table from "components/Table"; import TableCellActions from "components/TableCellActions"; diff --git a/ngui/ui/src/components/BIExport/BIExport.tsx b/ngui/ui/src/components/BIExport/BIExport.tsx index cfe340bb4..b32ff3939 100644 --- a/ngui/ui/src/components/BIExport/BIExport.tsx +++ b/ngui/ui/src/components/BIExport/BIExport.tsx @@ -7,8 +7,7 @@ import ActionBar from "components/ActionBar"; import PageContentWrapper from "components/PageContentWrapper"; import { BI_EXPORTS, INTEGRATIONS, getEditBIExportUrl } from "urls"; import { getBIExportActivityStatus, getBIExportStatus } from "utils/biExport"; -import { FilesSummaryList, TargetStorageSummaryList } from "./Components"; -import DetailsSummaryList from "./Components/DetailsSummaryList"; +import { FilesSummaryList, TargetStorageSummaryList, DetailsSummaryList } from "./components"; const BIExport = ({ biExport, isLoading = false }) => { const { diff --git a/ngui/ui/src/components/BIExport/Components/index.ts b/ngui/ui/src/components/BIExport/Components/index.ts deleted file mode 100644 index 12c270520..000000000 --- a/ngui/ui/src/components/BIExport/Components/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -import FilesSummaryList from "./FilesSummaryList"; -import TargetStorageSummaryList from "./TargetStorageSummaryList"; - -export { FilesSummaryList, TargetStorageSummaryList }; diff --git a/ngui/ui/src/components/BIExport/Components/DetailsSummaryList/DetailsSummaryList.tsx b/ngui/ui/src/components/BIExport/components/DetailsSummaryList/DetailsSummaryList.tsx similarity index 100% rename from ngui/ui/src/components/BIExport/Components/DetailsSummaryList/DetailsSummaryList.tsx rename to ngui/ui/src/components/BIExport/components/DetailsSummaryList/DetailsSummaryList.tsx diff --git a/ngui/ui/src/components/BIExport/Components/DetailsSummaryList/index.ts b/ngui/ui/src/components/BIExport/components/DetailsSummaryList/index.ts similarity index 100% rename from ngui/ui/src/components/BIExport/Components/DetailsSummaryList/index.ts rename to ngui/ui/src/components/BIExport/components/DetailsSummaryList/index.ts diff --git a/ngui/ui/src/components/BIExport/Components/FilesSummaryList/FilesSummaryList.tsx b/ngui/ui/src/components/BIExport/components/FilesSummaryList/FilesSummaryList.tsx similarity index 100% rename from ngui/ui/src/components/BIExport/Components/FilesSummaryList/FilesSummaryList.tsx rename to ngui/ui/src/components/BIExport/components/FilesSummaryList/FilesSummaryList.tsx diff --git a/ngui/ui/src/components/BIExport/Components/FilesSummaryList/index.ts b/ngui/ui/src/components/BIExport/components/FilesSummaryList/index.ts similarity index 100% rename from ngui/ui/src/components/BIExport/Components/FilesSummaryList/index.ts rename to ngui/ui/src/components/BIExport/components/FilesSummaryList/index.ts diff --git a/ngui/ui/src/components/BIExport/Components/TargetStorageSummaryList/TargetStorageSummaryList.tsx b/ngui/ui/src/components/BIExport/components/TargetStorageSummaryList/TargetStorageSummaryList.tsx similarity index 100% rename from ngui/ui/src/components/BIExport/Components/TargetStorageSummaryList/TargetStorageSummaryList.tsx rename to ngui/ui/src/components/BIExport/components/TargetStorageSummaryList/TargetStorageSummaryList.tsx diff --git a/ngui/ui/src/components/BIExport/Components/TargetStorageSummaryList/index.ts b/ngui/ui/src/components/BIExport/components/TargetStorageSummaryList/index.ts similarity index 100% rename from ngui/ui/src/components/BIExport/Components/TargetStorageSummaryList/index.ts rename to ngui/ui/src/components/BIExport/components/TargetStorageSummaryList/index.ts diff --git a/ngui/ui/src/components/BIExport/components/index.ts b/ngui/ui/src/components/BIExport/components/index.ts new file mode 100644 index 000000000..aa52765aa --- /dev/null +++ b/ngui/ui/src/components/BIExport/components/index.ts @@ -0,0 +1,5 @@ +import DetailsSummaryList from "./DetailsSummaryList"; +import FilesSummaryList from "./FilesSummaryList"; +import TargetStorageSummaryList from "./TargetStorageSummaryList"; + +export { FilesSummaryList, TargetStorageSummaryList, DetailsSummaryList }; diff --git a/ngui/ui/src/components/JsonView/JsonView.tsx b/ngui/ui/src/components/JsonView/JsonView.tsx index 96cbbe026..095de1757 100644 --- a/ngui/ui/src/components/JsonView/JsonView.tsx +++ b/ngui/ui/src/components/JsonView/JsonView.tsx @@ -13,7 +13,7 @@ const JsonView = ({ value, onChange, id, viewOnly = false }) => { height="auto" width="100%" onChange={onChange} - waitAfterKeyPress={0} + waitAfterKeyPress={1000} confirmGood={false} colors={theme.palette.json} viewOnly={viewOnly} diff --git a/ngui/ui/src/components/MlRunsetOverview/MlRunsetOverview.tsx b/ngui/ui/src/components/MlRunsetOverview/MlRunsetOverview.tsx index 80d2d17b6..9f6d09569 100644 --- a/ngui/ui/src/components/MlRunsetOverview/MlRunsetOverview.tsx +++ b/ngui/ui/src/components/MlRunsetOverview/MlRunsetOverview.tsx @@ -16,8 +16,7 @@ import { ML_RUNSET_TEMPLATES, getMlTaskDetailsUrl, getMlRunsetTemplateUrl } from import { getColorScale } from "utils/charts"; import { SPACING_1 } from "utils/layouts"; import { formatRunFullName } from "utils/ml"; -import { InputParameters, SummaryCards, Tabs } from "./Components"; -import Correlations from "./Components/Correlations"; +import { InputParameters, SummaryCards, Tabs, Correlations } from "./components"; const MlRunsetOverview = ({ runset, diff --git a/ngui/ui/src/components/MlRunsetOverview/Components/Correlations/Correlations.styles.ts b/ngui/ui/src/components/MlRunsetOverview/components/Correlations/Correlations.styles.ts similarity index 100% rename from ngui/ui/src/components/MlRunsetOverview/Components/Correlations/Correlations.styles.ts rename to ngui/ui/src/components/MlRunsetOverview/components/Correlations/Correlations.styles.ts diff --git a/ngui/ui/src/components/MlRunsetOverview/Components/Correlations/Correlations.tsx b/ngui/ui/src/components/MlRunsetOverview/components/Correlations/Correlations.tsx similarity index 100% rename from ngui/ui/src/components/MlRunsetOverview/Components/Correlations/Correlations.tsx rename to ngui/ui/src/components/MlRunsetOverview/components/Correlations/Correlations.tsx diff --git a/ngui/ui/src/components/MlRunsetOverview/Components/Correlations/index.ts b/ngui/ui/src/components/MlRunsetOverview/components/Correlations/index.ts similarity index 100% rename from ngui/ui/src/components/MlRunsetOverview/Components/Correlations/index.ts rename to ngui/ui/src/components/MlRunsetOverview/components/Correlations/index.ts diff --git a/ngui/ui/src/components/MlRunsetOverview/Components/Correlations/utils.ts b/ngui/ui/src/components/MlRunsetOverview/components/Correlations/utils.ts similarity index 100% rename from ngui/ui/src/components/MlRunsetOverview/Components/Correlations/utils.ts rename to ngui/ui/src/components/MlRunsetOverview/components/Correlations/utils.ts diff --git a/ngui/ui/src/components/MlRunsetOverview/Components/GoalsSelector.tsx b/ngui/ui/src/components/MlRunsetOverview/components/GoalsSelector.tsx similarity index 100% rename from ngui/ui/src/components/MlRunsetOverview/Components/GoalsSelector.tsx rename to ngui/ui/src/components/MlRunsetOverview/components/GoalsSelector.tsx diff --git a/ngui/ui/src/components/MlRunsetOverview/Components/InputParameters.tsx b/ngui/ui/src/components/MlRunsetOverview/components/InputParameters.tsx similarity index 100% rename from ngui/ui/src/components/MlRunsetOverview/Components/InputParameters.tsx rename to ngui/ui/src/components/MlRunsetOverview/components/InputParameters.tsx diff --git a/ngui/ui/src/components/MlRunsetOverview/Components/SummaryCards.tsx b/ngui/ui/src/components/MlRunsetOverview/components/SummaryCards.tsx similarity index 100% rename from ngui/ui/src/components/MlRunsetOverview/Components/SummaryCards.tsx rename to ngui/ui/src/components/MlRunsetOverview/components/SummaryCards.tsx diff --git a/ngui/ui/src/components/MlRunsetOverview/Components/Tabs/Executors/Executors.tsx b/ngui/ui/src/components/MlRunsetOverview/components/Tabs/Executors/Executors.tsx similarity index 100% rename from ngui/ui/src/components/MlRunsetOverview/Components/Tabs/Executors/Executors.tsx rename to ngui/ui/src/components/MlRunsetOverview/components/Tabs/Executors/Executors.tsx diff --git a/ngui/ui/src/components/MlRunsetOverview/Components/Tabs/Executors/index.ts b/ngui/ui/src/components/MlRunsetOverview/components/Tabs/Executors/index.ts similarity index 100% rename from ngui/ui/src/components/MlRunsetOverview/Components/Tabs/Executors/index.ts rename to ngui/ui/src/components/MlRunsetOverview/components/Tabs/Executors/index.ts diff --git a/ngui/ui/src/components/MlRunsetOverview/Components/Tabs/Runs/Runs.tsx b/ngui/ui/src/components/MlRunsetOverview/components/Tabs/Runs/Runs.tsx similarity index 100% rename from ngui/ui/src/components/MlRunsetOverview/Components/Tabs/Runs/Runs.tsx rename to ngui/ui/src/components/MlRunsetOverview/components/Tabs/Runs/Runs.tsx diff --git a/ngui/ui/src/components/MlRunsetOverview/Components/Tabs/Runs/RunsFilter.tsx b/ngui/ui/src/components/MlRunsetOverview/components/Tabs/Runs/RunsFilter.tsx similarity index 100% rename from ngui/ui/src/components/MlRunsetOverview/Components/Tabs/Runs/RunsFilter.tsx rename to ngui/ui/src/components/MlRunsetOverview/components/Tabs/Runs/RunsFilter.tsx diff --git a/ngui/ui/src/components/MlRunsetOverview/Components/Tabs/Runs/RunsTable.tsx b/ngui/ui/src/components/MlRunsetOverview/components/Tabs/Runs/RunsTable.tsx similarity index 100% rename from ngui/ui/src/components/MlRunsetOverview/Components/Tabs/Runs/RunsTable.tsx rename to ngui/ui/src/components/MlRunsetOverview/components/Tabs/Runs/RunsTable.tsx diff --git a/ngui/ui/src/components/MlRunsetOverview/Components/Tabs/Runs/index.ts b/ngui/ui/src/components/MlRunsetOverview/components/Tabs/Runs/index.ts similarity index 100% rename from ngui/ui/src/components/MlRunsetOverview/Components/Tabs/Runs/index.ts rename to ngui/ui/src/components/MlRunsetOverview/components/Tabs/Runs/index.ts diff --git a/ngui/ui/src/components/MlRunsetOverview/Components/Tabs/Tabs.tsx b/ngui/ui/src/components/MlRunsetOverview/components/Tabs/Tabs.tsx similarity index 100% rename from ngui/ui/src/components/MlRunsetOverview/Components/Tabs/Tabs.tsx rename to ngui/ui/src/components/MlRunsetOverview/components/Tabs/Tabs.tsx diff --git a/ngui/ui/src/components/MlRunsetOverview/Components/Tabs/index.ts b/ngui/ui/src/components/MlRunsetOverview/components/Tabs/index.ts similarity index 100% rename from ngui/ui/src/components/MlRunsetOverview/Components/Tabs/index.ts rename to ngui/ui/src/components/MlRunsetOverview/components/Tabs/index.ts diff --git a/ngui/ui/src/components/MlRunsetOverview/Components/index.ts b/ngui/ui/src/components/MlRunsetOverview/components/index.ts similarity index 53% rename from ngui/ui/src/components/MlRunsetOverview/Components/index.ts rename to ngui/ui/src/components/MlRunsetOverview/components/index.ts index 86c16003f..74f14afb2 100644 --- a/ngui/ui/src/components/MlRunsetOverview/Components/index.ts +++ b/ngui/ui/src/components/MlRunsetOverview/components/index.ts @@ -1,5 +1,6 @@ +import Correlations from "./Correlations"; import InputParameters from "./InputParameters"; import SummaryCards from "./SummaryCards"; import Tabs from "./Tabs"; -export { Tabs, InputParameters, SummaryCards }; +export { Tabs, InputParameters, SummaryCards, Correlations }; diff --git a/ngui/ui/src/components/MlRunsetTemplate/MlRunsetTemplate.tsx b/ngui/ui/src/components/MlRunsetTemplate/MlRunsetTemplate.tsx index e2096b21f..a9d12ec4e 100644 --- a/ngui/ui/src/components/MlRunsetTemplate/MlRunsetTemplate.tsx +++ b/ngui/ui/src/components/MlRunsetTemplate/MlRunsetTemplate.tsx @@ -10,7 +10,7 @@ import PageContentWrapper from "components/PageContentWrapper"; import { useRefetchApis } from "hooks/useRefetchApis"; import { getMlEditRunsetTemplateUrl, getMlRunsetConfigurationUrl, ML_RUNSET_TEMPLATES } from "urls"; import { SPACING_2 } from "utils/layouts"; -import { RunsetsTable, Summary, Details } from "./Components"; +import { RunsetsTable, Summary, Details } from "./components"; const MlRunsetTemplate = ({ runsetTemplate, diff --git a/ngui/ui/src/components/MlRunsetTemplate/Components/Details.tsx b/ngui/ui/src/components/MlRunsetTemplate/components/Details.tsx similarity index 100% rename from ngui/ui/src/components/MlRunsetTemplate/Components/Details.tsx rename to ngui/ui/src/components/MlRunsetTemplate/components/Details.tsx diff --git a/ngui/ui/src/components/MlRunsetTemplate/Components/RunsetsTable.tsx b/ngui/ui/src/components/MlRunsetTemplate/components/RunsetsTable.tsx similarity index 100% rename from ngui/ui/src/components/MlRunsetTemplate/Components/RunsetsTable.tsx rename to ngui/ui/src/components/MlRunsetTemplate/components/RunsetsTable.tsx diff --git a/ngui/ui/src/components/MlRunsetTemplate/Components/Summary.tsx b/ngui/ui/src/components/MlRunsetTemplate/components/Summary.tsx similarity index 100% rename from ngui/ui/src/components/MlRunsetTemplate/Components/Summary.tsx rename to ngui/ui/src/components/MlRunsetTemplate/components/Summary.tsx diff --git a/ngui/ui/src/components/MlRunsetTemplate/Components/index.ts b/ngui/ui/src/components/MlRunsetTemplate/components/index.ts similarity index 100% rename from ngui/ui/src/components/MlRunsetTemplate/Components/index.ts rename to ngui/ui/src/components/MlRunsetTemplate/components/index.ts diff --git a/ngui/ui/src/components/MlTaskRun/MlTaskRun.tsx b/ngui/ui/src/components/MlTaskRun/MlTaskRun.tsx index 34734ae07..ee4b0ed57 100644 --- a/ngui/ui/src/components/MlTaskRun/MlTaskRun.tsx +++ b/ngui/ui/src/components/MlTaskRun/MlTaskRun.tsx @@ -14,7 +14,7 @@ import { useRefetchApis } from "hooks/useRefetchApis"; import { ML_TASKS, getMlTaskDetailsUrl } from "urls"; import { SPACING_2 } from "utils/layouts"; import { formatRunFullName } from "utils/ml"; -import { Charts, Executors, Overview, Status, Tabs } from "./Components"; +import { Charts, Executors, Overview, Status, Tabs } from "./components"; const MlTaskRun = ({ run, diff --git a/ngui/ui/src/components/MlTaskRun/Components/Charts/Charts.tsx b/ngui/ui/src/components/MlTaskRun/components/Charts/Charts.tsx similarity index 100% rename from ngui/ui/src/components/MlTaskRun/Components/Charts/Charts.tsx rename to ngui/ui/src/components/MlTaskRun/components/Charts/Charts.tsx diff --git a/ngui/ui/src/components/MlTaskRun/Components/Charts/utils.ts b/ngui/ui/src/components/MlTaskRun/components/Charts/utils.ts similarity index 100% rename from ngui/ui/src/components/MlTaskRun/Components/Charts/utils.ts rename to ngui/ui/src/components/MlTaskRun/components/Charts/utils.ts diff --git a/ngui/ui/src/components/MlTaskRun/Components/Executors/Executors.tsx b/ngui/ui/src/components/MlTaskRun/components/Executors/Executors.tsx similarity index 100% rename from ngui/ui/src/components/MlTaskRun/Components/Executors/Executors.tsx rename to ngui/ui/src/components/MlTaskRun/components/Executors/Executors.tsx diff --git a/ngui/ui/src/components/MlTaskRun/Components/Overview/Overview.tsx b/ngui/ui/src/components/MlTaskRun/components/Overview/Overview.tsx similarity index 100% rename from ngui/ui/src/components/MlTaskRun/Components/Overview/Overview.tsx rename to ngui/ui/src/components/MlTaskRun/components/Overview/Overview.tsx diff --git a/ngui/ui/src/components/MlTaskRun/Components/Status/Status.tsx b/ngui/ui/src/components/MlTaskRun/components/Status/Status.tsx similarity index 100% rename from ngui/ui/src/components/MlTaskRun/Components/Status/Status.tsx rename to ngui/ui/src/components/MlTaskRun/components/Status/Status.tsx diff --git a/ngui/ui/src/components/MlTaskRun/Components/Tabs.tsx b/ngui/ui/src/components/MlTaskRun/components/Tabs.tsx similarity index 100% rename from ngui/ui/src/components/MlTaskRun/Components/Tabs.tsx rename to ngui/ui/src/components/MlTaskRun/components/Tabs.tsx diff --git a/ngui/ui/src/components/MlTaskRun/Components/index.ts b/ngui/ui/src/components/MlTaskRun/components/index.ts similarity index 100% rename from ngui/ui/src/components/MlTaskRun/Components/index.ts rename to ngui/ui/src/components/MlTaskRun/components/index.ts diff --git a/ngui/ui/src/components/OrganizationOptions/OrganizationOptions.tsx b/ngui/ui/src/components/OrganizationOptions/OrganizationOptions.tsx index af86d1ec5..c4e887fa8 100644 --- a/ngui/ui/src/components/OrganizationOptions/OrganizationOptions.tsx +++ b/ngui/ui/src/components/OrganizationOptions/OrganizationOptions.tsx @@ -1,4 +1,4 @@ -import { useState } from "react"; +import { useEffect, useState } from "react"; import AddOutlinedIcon from "@mui/icons-material/AddOutlined"; import { CircularProgress } from "@mui/material"; import Box from "@mui/material/Box"; @@ -39,6 +39,10 @@ const OrganizationOptions = ({ const openSideModal = useOpenSideModal(); + useEffect(() => { + setUpdatedValue(value); + }, [value]); + const onJsonChange = ({ error, jsObject }) => { setUpdatedValue(jsObject); setIsValidJson(error === false); diff --git a/ngui/ui/src/components/PowerScheduleSummaryCards/PowerScheduleSummaryCards.tsx b/ngui/ui/src/components/PowerScheduleSummaryCards/PowerScheduleSummaryCards.tsx index 5925729f9..de58b9508 100644 --- a/ngui/ui/src/components/PowerScheduleSummaryCards/PowerScheduleSummaryCards.tsx +++ b/ngui/ui/src/components/PowerScheduleSummaryCards/PowerScheduleSummaryCards.tsx @@ -6,11 +6,12 @@ import PowerScheduleValidityPeriod from "components/PowerScheduleValidityPeriod" import QuestionMark from "components/QuestionMark"; import SummaryGrid from "components/SummaryGrid"; import { SUMMARY_VALUE_COMPONENT_TYPES } from "utils/constants"; +import { isPowerScheduleExpired } from "utils/poweSchedules"; type PowerScheduleSummaryCardsProps = { timeZone: string; - startDate: string; - endDate: string; + startDate: number; + endDate: number; lastRun: number; lastRunError: string; resourcesOnSchedule: number; @@ -83,6 +84,7 @@ const PowerScheduleSummaryCards = ({ dataTestIds: { cardTestId: "card_time_zone" }, + color: isPowerScheduleExpired(endDate) ? "warning" : "primary", isLoading, renderCondition: () => !!startDate || !!endDate } diff --git a/ngui/ui/src/components/PowerScheduleValidityPeriod/PowerScheduleValidityPeriod.tsx b/ngui/ui/src/components/PowerScheduleValidityPeriod/PowerScheduleValidityPeriod.tsx index 4ab597eb5..0a0d84a54 100644 --- a/ngui/ui/src/components/PowerScheduleValidityPeriod/PowerScheduleValidityPeriod.tsx +++ b/ngui/ui/src/components/PowerScheduleValidityPeriod/PowerScheduleValidityPeriod.tsx @@ -1,10 +1,15 @@ +import WarningAmberOutlinedIcon from "@mui/icons-material/WarningAmberOutlined"; +import { Box, SvgIconProps } from "@mui/material"; import { FormattedMessage } from "react-intl"; +import QuestionMark from "components/QuestionMark"; import { intl } from "translations/react-intl-config"; import { EN_FORMAT, unixTimestampToDateTime } from "utils/datetime"; +import { isPowerScheduleExpired } from "utils/poweSchedules"; type PowerScheduleValidityPeriodProps = { startDate: number; endDate: number; + iconFontSize?: SvgIconProps["fontSize"]; }; const stringifiedPowerScheduleValidityPeriod = ({ startDate, endDate }: PowerScheduleValidityPeriodProps) => { @@ -23,19 +28,29 @@ const stringifiedPowerScheduleValidityPeriod = ({ startDate, endDate }: PowerSch ); }; -const PowerScheduleValidityPeriod = ({ startDate, endDate }: PowerScheduleValidityPeriodProps) => { +const PowerScheduleValidityPeriod = ({ startDate, endDate, iconFontSize = "medium" }: PowerScheduleValidityPeriodProps) => { if (!endDate) { return `${intl.formatMessage({ id: "since" })} ${unixTimestampToDateTime(startDate, EN_FORMAT)}`; } return ( - + + + {isPowerScheduleExpired(endDate) && ( + + )} + ); }; diff --git a/ngui/ui/src/components/RecommendationsCard/RecommendationsCard.tsx b/ngui/ui/src/components/RecommendationsCard/RecommendationsCard.tsx index d7db25ea0..e2431df40 100644 --- a/ngui/ui/src/components/RecommendationsCard/RecommendationsCard.tsx +++ b/ngui/ui/src/components/RecommendationsCard/RecommendationsCard.tsx @@ -11,7 +11,7 @@ import { CATEGORY } from "containers/RecommendationsOverviewContainer/recommenda import { ALL_SERVICES } from "hooks/useRecommendationServices"; import { RECOMMENDATIONS, RECOMMENDATION_CATEGORY_QUERY_PARAMETER, RECOMMENDATION_SERVICE_QUERY_PARAMETER } from "urls"; import { SPACING_2 } from "utils/layouts"; -import { InfoCard, PossibleSavingsCard } from "./Components"; +import { InfoCard, PossibleSavingsCard } from "./components"; const RecommendationsCard = ({ isLoading, diff --git a/ngui/ui/src/components/RecommendationsCard/Components/InfoCard/InfoCard.styles.ts b/ngui/ui/src/components/RecommendationsCard/components/InfoCard/InfoCard.styles.ts similarity index 100% rename from ngui/ui/src/components/RecommendationsCard/Components/InfoCard/InfoCard.styles.ts rename to ngui/ui/src/components/RecommendationsCard/components/InfoCard/InfoCard.styles.ts diff --git a/ngui/ui/src/components/RecommendationsCard/Components/InfoCard/InfoCard.tsx b/ngui/ui/src/components/RecommendationsCard/components/InfoCard/InfoCard.tsx similarity index 100% rename from ngui/ui/src/components/RecommendationsCard/Components/InfoCard/InfoCard.tsx rename to ngui/ui/src/components/RecommendationsCard/components/InfoCard/InfoCard.tsx diff --git a/ngui/ui/src/components/RecommendationsCard/Components/InfoCard/index.ts b/ngui/ui/src/components/RecommendationsCard/components/InfoCard/index.ts similarity index 100% rename from ngui/ui/src/components/RecommendationsCard/Components/InfoCard/index.ts rename to ngui/ui/src/components/RecommendationsCard/components/InfoCard/index.ts diff --git a/ngui/ui/src/components/RecommendationsCard/Components/PossibleSavingsCard/PossibleSavingsCard.styles.ts b/ngui/ui/src/components/RecommendationsCard/components/PossibleSavingsCard/PossibleSavingsCard.styles.ts similarity index 100% rename from ngui/ui/src/components/RecommendationsCard/Components/PossibleSavingsCard/PossibleSavingsCard.styles.ts rename to ngui/ui/src/components/RecommendationsCard/components/PossibleSavingsCard/PossibleSavingsCard.styles.ts diff --git a/ngui/ui/src/components/RecommendationsCard/Components/PossibleSavingsCard/PossibleSavingsCard.tsx b/ngui/ui/src/components/RecommendationsCard/components/PossibleSavingsCard/PossibleSavingsCard.tsx similarity index 100% rename from ngui/ui/src/components/RecommendationsCard/Components/PossibleSavingsCard/PossibleSavingsCard.tsx rename to ngui/ui/src/components/RecommendationsCard/components/PossibleSavingsCard/PossibleSavingsCard.tsx diff --git a/ngui/ui/src/components/RecommendationsCard/Components/PossibleSavingsCard/index.ts b/ngui/ui/src/components/RecommendationsCard/components/PossibleSavingsCard/index.ts similarity index 100% rename from ngui/ui/src/components/RecommendationsCard/Components/PossibleSavingsCard/index.ts rename to ngui/ui/src/components/RecommendationsCard/components/PossibleSavingsCard/index.ts diff --git a/ngui/ui/src/components/RecommendationsCard/Components/index.ts b/ngui/ui/src/components/RecommendationsCard/components/index.ts similarity index 100% rename from ngui/ui/src/components/RecommendationsCard/Components/index.ts rename to ngui/ui/src/components/RecommendationsCard/components/index.ts diff --git a/ngui/ui/src/components/Resources/filterConfigs.tsx b/ngui/ui/src/components/Resources/filterConfigs.tsx index 8e0d3bbb8..82c6e70ae 100644 --- a/ngui/ui/src/components/Resources/filterConfigs.tsx +++ b/ngui/ui/src/components/Resources/filterConfigs.tsx @@ -128,6 +128,10 @@ export const FILTER_CONFIGS = { type: { type: "string", enum: CLOUD_ACCOUNT_TYPES_LIST + }, + account_id: { + type: "string", + nullable: true } } } diff --git a/ngui/ui/src/components/SideModalManager/SideModals/recommendations/ObsoleteImagesModal.tsx b/ngui/ui/src/components/SideModalManager/SideModals/recommendations/ObsoleteImagesModal.tsx index 26fe29894..0c58ae309 100644 --- a/ngui/ui/src/components/SideModalManager/SideModals/recommendations/ObsoleteImagesModal.tsx +++ b/ngui/ui/src/components/SideModalManager/SideModals/recommendations/ObsoleteImagesModal.tsx @@ -2,6 +2,11 @@ import BaseSideModal from "../BaseSideModal"; import DaysThreshold from "./components/DaysThreshold"; import InformationWrapper from "./components/InformationWrapper"; +/** + * @deprecated OSN-1266. + * `ObsoleteImagesModal` is deprecated because `ObsoleteImages` was replaced with `SnapshotsWithNonUsedImages`. + */ + class ObsoleteImagesModal extends BaseSideModal { headerProps = { messageId: "obsoleteImagesTitle", diff --git a/ngui/ui/src/containers/MlCreateRunArtifactContainer/MlCreateRunArtifactContainer.tsx b/ngui/ui/src/containers/MlCreateRunArtifactContainer/MlCreateRunArtifactContainer.tsx index fdc7e8a0d..85e86ede3 100644 --- a/ngui/ui/src/containers/MlCreateRunArtifactContainer/MlCreateRunArtifactContainer.tsx +++ b/ngui/ui/src/containers/MlCreateRunArtifactContainer/MlCreateRunArtifactContainer.tsx @@ -3,7 +3,7 @@ import { FormattedMessage } from "react-intl"; import { useParams, Link as RouterLink, useNavigate } from "react-router-dom"; import ActionBar from "components/ActionBar"; import { MlCreateArtifactForm } from "components/forms/MlArtifactForm"; -import { TABS } from "components/MlTaskRun/Components/Tabs"; +import { TABS } from "components/MlTaskRun/components/Tabs"; import PageContentWrapper from "components/PageContentWrapper"; import { useOrganizationInfo } from "hooks/useOrganizationInfo"; import MlArtifactsService from "services/MlArtifactsService"; diff --git a/ngui/ui/src/containers/MlEditRunArtifactContainer/MlEditRunArtifactContainer.tsx b/ngui/ui/src/containers/MlEditRunArtifactContainer/MlEditRunArtifactContainer.tsx index 73c9b7884..83f1c701c 100644 --- a/ngui/ui/src/containers/MlEditRunArtifactContainer/MlEditRunArtifactContainer.tsx +++ b/ngui/ui/src/containers/MlEditRunArtifactContainer/MlEditRunArtifactContainer.tsx @@ -3,7 +3,7 @@ import { FormattedMessage } from "react-intl"; import { useParams, Link as RouterLink, useNavigate } from "react-router-dom"; import ActionBar from "components/ActionBar"; import { MlEditArtifactForm } from "components/forms/MlArtifactForm"; -import { TABS } from "components/MlTaskRun/Components/Tabs"; +import { TABS } from "components/MlTaskRun/components/Tabs"; import PageContentWrapper from "components/PageContentWrapper"; import { useOrganizationInfo } from "hooks/useOrganizationInfo"; import MlArtifactsService from "services/MlArtifactsService"; diff --git a/ngui/ui/src/containers/MlRunsetExecutorsContainer/MlRunsetExecutorsContainer.tsx b/ngui/ui/src/containers/MlRunsetExecutorsContainer/MlRunsetExecutorsContainer.tsx index 0debc3a1b..a4df186d8 100644 --- a/ngui/ui/src/containers/MlRunsetExecutorsContainer/MlRunsetExecutorsContainer.tsx +++ b/ngui/ui/src/containers/MlRunsetExecutorsContainer/MlRunsetExecutorsContainer.tsx @@ -1,5 +1,5 @@ import { useParams } from "react-router-dom"; -import Executors from "components/MlRunsetOverview/Components/Tabs/Executors"; +import Executors from "components/MlRunsetOverview/components/Tabs/Executors"; import MlRunsetsService from "services/MlRunsetsService"; const MlRunsetExecutorsContainer = () => { diff --git a/ngui/ui/src/containers/RecommendationsOverviewContainer/RecommendationsOverview.tsx b/ngui/ui/src/containers/RecommendationsOverviewContainer/RecommendationsOverview.tsx index 54f783237..d6fb70340 100644 --- a/ngui/ui/src/containers/RecommendationsOverviewContainer/RecommendationsOverview.tsx +++ b/ngui/ui/src/containers/RecommendationsOverviewContainer/RecommendationsOverview.tsx @@ -93,9 +93,6 @@ const RecommendationsOverview = ({ .filter(serviceFilter(service)) .filter(searchFilter(search)) .filter(appliedDataSourcesFilter(selectedDataSourceTypes)) - // TODO : Remove obsolete_images recommendation but keep it in archived recommendations - // discuss solutions - .filter((rec) => rec.type !== "obsolete_images") .sort(sortRecommendation); return ( diff --git a/ngui/ui/src/containers/RecommendationsOverviewContainer/recommendations/ObsoleteImages.tsx b/ngui/ui/src/containers/RecommendationsOverviewContainer/recommendations/ObsoleteImages.tsx index ca6c51a89..c5affa212 100644 --- a/ngui/ui/src/containers/RecommendationsOverviewContainer/recommendations/ObsoleteImages.tsx +++ b/ngui/ui/src/containers/RecommendationsOverviewContainer/recommendations/ObsoleteImages.tsx @@ -61,6 +61,12 @@ const columns = [ }) ]; +/** + * @deprecated + * OSN-1266. `ObsoleteImages` is deprecated but still used on the Archived Recommendations page. + * Use `SnapshotsWithNonUsedImages` instead. + */ + class ObsoleteImages extends BaseRecommendation { type = "obsolete_images"; diff --git a/ngui/ui/src/hooks/useAllRecommendations.ts b/ngui/ui/src/hooks/useAllRecommendations.ts index a2484d91a..ba7fb5149 100644 --- a/ngui/ui/src/hooks/useAllRecommendations.ts +++ b/ngui/ui/src/hooks/useAllRecommendations.ts @@ -15,8 +15,8 @@ const ML_RECOMMENDATIONS = Object.fromEntries( ) ); -export const useAllRecommendations = () => { - const optscaleRecommendation = useOptscaleRecommendations(); +export const useAllRecommendations = ({ withDeprecated = false }: { withDeprecated?: boolean } = {}) => { + const optscaleRecommendation = useOptscaleRecommendations({ withDeprecated }); return useMemo(() => ({ ...ML_RECOMMENDATIONS, ...optscaleRecommendation }), [optscaleRecommendation]); }; diff --git a/ngui/ui/src/hooks/useOptscaleRecommendations.ts b/ngui/ui/src/hooks/useOptscaleRecommendations.ts index 277b996bc..944a13f39 100644 --- a/ngui/ui/src/hooks/useOptscaleRecommendations.ts +++ b/ngui/ui/src/hooks/useOptscaleRecommendations.ts @@ -31,9 +31,11 @@ import { useIsNebiusConnectionEnabled } from "hooks/useIsNebiusConnectionEnabled const NEBIUS_RECOMMENDATIONS = [CvocAgreementOpportunities, AbandonedNebiusS3Buckets, NebiusMigration]; +const DEPRECATED_RECOMMENDATIONS = [ObsoleteImages]; + export const NEBIUS_RECOMMENDATION_TYPES = NEBIUS_RECOMMENDATIONS.map((Recommendation) => new Recommendation().type); -export const useOptscaleRecommendations = () => { +export const useOptscaleRecommendations = ({ withDeprecated = false }: { withDeprecated?: boolean } = {}) => { const isNebiusConnectionEnabled = useIsNebiusConnectionEnabled(); return useMemo(() => { @@ -43,7 +45,6 @@ export const useOptscaleRecommendations = () => { RightsizingRdsInstances, RightsizingInstances, ReservedInstances, - ObsoleteImages, ObsoleteSnapshots, ObsoleteSnapshotChains, ObsoleteIps, @@ -63,9 +64,10 @@ export const useOptscaleRecommendations = () => { PublicS3Buckets, SnapshotsWithNonUsedImages, AbandonedImages, - ...(isNebiusConnectionEnabled ? NEBIUS_RECOMMENDATIONS : []) + ...(isNebiusConnectionEnabled ? NEBIUS_RECOMMENDATIONS : []), + ...(withDeprecated ? DEPRECATED_RECOMMENDATIONS : []) ]; return Object.fromEntries(recommendations.map((Rec) => [new Rec().type, Rec])); - }, [isNebiusConnectionEnabled]); + }, [isNebiusConnectionEnabled, withDeprecated]); }; diff --git a/ngui/ui/src/translations/en-US/app.json b/ngui/ui/src/translations/en-US/app.json index f65652f5d..2f845b3ab 100644 --- a/ngui/ui/src/translations/en-US/app.json +++ b/ngui/ui/src/translations/en-US/app.json @@ -1745,6 +1745,7 @@ "powerOff": "Power off", "powerOn": "Power on", "powerSchedule": "Power schedule", + "powerScheduleExpired": "This Power Schedule is expired", "powerSchedulesTitle": "Power Schedules", "preferredCurrency": "Preferred currency", "preparingLiveDemoMessage": "OptScale is preparing your live demo experience. Please stand by.", diff --git a/ngui/ui/src/utils/columns/powerScheduleValidityPeriod.tsx b/ngui/ui/src/utils/columns/powerScheduleValidityPeriod.tsx index c825bebb0..0abffc536 100644 --- a/ngui/ui/src/utils/columns/powerScheduleValidityPeriod.tsx +++ b/ngui/ui/src/utils/columns/powerScheduleValidityPeriod.tsx @@ -20,7 +20,7 @@ const powerScheduleValidityPeriod = ({ startDateAccessor, endDateAccessor }) => cell: ({ row: { original } }) => { const { [startDateAccessor]: startDate, [endDateAccessor]: endDate } = original; - return ; + return ; } }); diff --git a/ngui/ui/src/utils/poweSchedules.ts b/ngui/ui/src/utils/poweSchedules.ts new file mode 100644 index 000000000..deaef832c --- /dev/null +++ b/ngui/ui/src/utils/poweSchedules.ts @@ -0,0 +1,3 @@ +import { isPast, secondsToMilliseconds } from "utils/datetime"; + +export const isPowerScheduleExpired = (endDate: number) => endDate && isPast(secondsToMilliseconds(endDate)); diff --git a/optscale-deploy/Vagrantfile b/optscale-deploy/Vagrantfile new file mode 100644 index 000000000..0602de52a --- /dev/null +++ b/optscale-deploy/Vagrantfile @@ -0,0 +1,177 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +selected_provider = ENV["VAGRANT_DEFAULT_PROVIDER"].to_s.strip +if selected_provider == "qemu" && !Vagrant.has_plugin?("vagrant-qemu") + warn "WARNING: vagrant-qemu plugin is missing. QEMU provider won't be available. " \ + "Install it with 'vagrant plugin install vagrant-qemu' if you want to use the qemu provider." +end + +X86_64_ARCHITECTURES = ["x86_64", "amd64"].freeze +ARM_64_ARCHITECTURES = ["arm64", "aarch64", "aarch64_be", "armv8b", "armv8l"].freeze +VM_CPUS = (ENV["VM_CPUS"] || "4").to_i +VM_RAM_GB = (ENV["VM_RAM_GB"] || "10").to_i +VM_DISK_GB = (ENV["VM_DISK_GB"] || "75").to_i + +def cpu_emulated?(guest_cpu_arch) + host_cpu_arch = `uname -m`.strip + + [X86_64_ARCHITECTURES, ARM_64_ARCHITECTURES].each do |aliases| + if aliases.include?(host_cpu_arch) + return !aliases.include?(guest_cpu_arch) + end + end + + raise RuntimeError, "Unknown host CPU architecture: #{host_cpu_arch}" +end + +def define_vm(config, name:, arch:, base_box:, ports:) + opts = { + primary: !cpu_emulated?(arch), + autostart: !cpu_emulated?(arch), + } + + config.vm.define name, opts do |vm_config| + vm_config.vm.box = base_box + + if X86_64_ARCHITECTURES.include?(arch) + vm_config.vm.box_architecture = "amd64" + elsif ARM_64_ARCHITECTURES.include?(arch) + vm_config.vm.box_architecture = "arm64" + else + raise RuntimeError, "Unsupported architecture: #{arch}" + end + + vm_config.vm.hostname = name + + # Port forwarding + vm_config.vm.network "forwarded_port", guest: 80, host: ports[:http] + vm_config.vm.network "forwarded_port", guest: 443, host: ports[:https] + + # - For QEMU (vagrant-qemu): SSH forwarding is handled by qemu.ssh_port + selected_provider = ENV["VAGRANT_DEFAULT_PROVIDER"].to_s.strip + if selected_provider == "virtualbox" + vm_config.vm.network "forwarded_port", + guest: 22, host: ports[:ssh], + id: "ssh_#{name}" + end + + # DB / Kibana / Grafana + vm_config.vm.network "forwarded_port", guest: 30080, host: 41080, id: "db_#{name}" + vm_config.vm.network "forwarded_port", guest: 30081, host: 41081, id: "kibana_#{name}" + vm_config.vm.network "forwarded_port", guest: 30082, host: 41082, id: "grafana_#{name}" + + # QEMU provider (only if plugin is installed) + if Vagrant.has_plugin?("vagrant-qemu") + vm_config.vagrant.plugins = "vagrant-qemu" + + vm_config.vm.provider "qemu" do |qemu| + qemu.memory = "#{VM_RAM_GB}G" + qemu.net_device = "virtio-net-pci" + qemu.disk_resize = "#{VM_DISK_GB}G" + + # default to OS install, but also support homebrew + if Dir.exist?("/usr/share/qemu") + qemu.qemu_dir = "/usr/share/qemu" + elsif ENV["HOMEBREW_PREFIX"] && Dir.exist?("#{ENV["HOMEBREW_PREFIX"]}/share/qemu") + qemu.qemu_dir = "#{ENV["HOMEBREW_PREFIX"]}/share/qemu" + else + raise RuntimeError, "QEMU directory not found. Please ensure QEMU is installed." + end + + if cpu_emulated?(arch) + qemu.cpu = "max" + accelerator = "tcg,thread=multi,tb-size=512" + else + qemu.cpu = "host" + + if Vagrant::Util::Platform.linux? + accelerator = "kvm" + elsif Vagrant::Util::Platform.darwin? + accelerator = "hvf" + else + raise RuntimeError, "Unsupported platform: #{Vagrant::Util::Platform.platform}" + end + end + + qemu.extra_qemu_args = ["-accel", accelerator] + + qemu.ssh_port = ports[:ssh] + + if X86_64_ARCHITECTURES.include?(arch) + qemu.arch = "x86_64" + qemu.machine = "q35" + qemu.smp = "cpus=#{VM_CPUS},sockets=1,cores=#{VM_CPUS},threads=1" + elsif ARM_64_ARCHITECTURES.include?(arch) + qemu.arch = "aarch64" + qemu.machine = "virt,gic-version=3" + qemu.smp = "cores=#{VM_CPUS},threads=1" + else + raise RuntimeError, "Unsupported architecture: #{arch}" + end + end + end + + # VirtualBox provider + vm_config.vm.provider "virtualbox" do |vb| + vb.name = name + vb.memory = VM_RAM_GB * 1024 + vb.cpus = VM_CPUS + + # ioapic required + vb.customize ["modifyvm", :id, "--ioapic", "on"] + + # Optional tweaks; comment out if not wanted + vb.customize ["modifyvm", :id, "--audio", "none"] + vb.customize ["modifyvm", :id, "--nictype1", "virtio"] + vb.customize ["modifyvm", :id, "--graphicscontroller", "vmsvga"] + end + end +end + + +Vagrant.configure(2) do |config| + + selected_provider = ENV["VAGRANT_DEFAULT_PROVIDER"].to_s.strip + + # Require vagrant-disksize for VirtualBox resizing + if selected_provider == "virtualbox" + if Vagrant.has_plugin?("vagrant-disksize") + config.disksize.size = "#{VM_DISK_GB}GB" + else + raise Vagrant::Errors::VagrantError.new, + "The 'vagrant-disksize' plugin is required for VirtualBox. " \ + "Install it:\n\n vagrant plugin install vagrant-disksize" + end + end + + config.vm.synced_folder "..", "/home/vagrant/optscale", type: "rsync", + rsync__exclude: [ + "*/.venv/", + "optscale-deploy/.vagrant/", + ] + + define_vm( + config, + name: "ubuntu-2404-arm-64", + arch: "arm64", + base_box: "cloud-image/ubuntu-24.04", + ports: { + http: 9081, + https: 9444, + ssh: 50223, + } + ) + + define_vm( + config, + name: "ubuntu-2404-x86-64", + arch: "x86_64", + base_box: "cloud-image/ubuntu-24.04", + ports: { + http: 9080, + https: 9443, + ssh: 50222, + } + ) +end diff --git a/optscale-deploy/ansible/inventories/vm-arm.yaml b/optscale-deploy/ansible/inventories/vm-arm.yaml new file mode 100644 index 000000000..bb556cf08 --- /dev/null +++ b/optscale-deploy/ansible/inventories/vm-arm.yaml @@ -0,0 +1,19 @@ +--- +all: + children: + local: + hosts: + ubuntu-2404-arm-64: + # generated from the output of `vagrant ssh-config` + ansible_host: 127.0.0.1 + ansible_port: "50223" + ansible_user: vagrant + ansible_ssh_user: vagrant + + # Use provider-specific private key (qemu / virtualbox) + ansible_ssh_private_key_file: ".vagrant/machines/ubuntu-2404-arm-64/{{ lookup('env', 'VAGRANT_DEFAULT_PROVIDER') | default('qemu') }}/private_key" + + ansible_ssh_common_args: >- + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + -o LogLevel=FATAL diff --git a/optscale-deploy/ansible/inventories/vm-x86.yaml b/optscale-deploy/ansible/inventories/vm-x86.yaml new file mode 100644 index 000000000..9497796ae --- /dev/null +++ b/optscale-deploy/ansible/inventories/vm-x86.yaml @@ -0,0 +1,19 @@ +--- +all: + children: + local: + hosts: + ubuntu-2404-x86-64: + # generated from the output of `vagrant ssh-config` + ansible_host: 127.0.0.1 + ansible_port: "50222" + ansible_user: vagrant + ansible_ssh_user: vagrant + + # Use provider-specific private key (qemu / virtualbox) + ansible_ssh_private_key_file: ".vagrant/machines/ubuntu-2404-x86-64/{{ lookup('env', 'VAGRANT_DEFAULT_PROVIDER') | default('qemu') }}/private_key" + + ansible_ssh_common_args: >- + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + -o LogLevel=FATAL diff --git a/optscale-deploy/ansible/provision-vm.yaml b/optscale-deploy/ansible/provision-vm.yaml new file mode 100644 index 000000000..ce6a234bd --- /dev/null +++ b/optscale-deploy/ansible/provision-vm.yaml @@ -0,0 +1,137 @@ +--- +- import_playbook: k8s-master.yaml + +- hosts: all + gather_facts: yes + vars: + k8s_deployment_name: optscale + optscale_dir: "{{ ansible_env.HOME }}/optscale" + optscale_deploy_dir: "{{ optscale_dir }}/optscale-deploy" + + user_template_overlay: "{{ optscale_deploy_dir }}/overlay/user_template.yml" + user_thanos_overlay: "{{ optscale_deploy_dir }}/overlay/user_thanos_disabled.yml" + + tasks: + - name: get the container images that need to be rebuilt + command: + argv: + - find + - "{{ optscale_dir }}" + - -mindepth + - 2 + - -maxdepth + - 3 + - -name + - Dockerfile + - -exec + - sh + - -c + - echo $(basename $(dirname $0)) + - {} + - ; + register: containers_to_build + + # Building the containers one by one instead of just using `./build.sh --use-nerdctl` to display the progress + # when running the playbook. Also in case of failure it will be clear exactly which item failed + - name: build the container images (this will take awhile, be patient :)) + command: "{{ optscale_dir }}/build.sh --use-nerdctl {{ item }} local" + args: + chdir: "{{ optscale_dir }}" + environment: + PATH: "{{ ansible_env.HOME }}/.local/bin:{{ ansible_env.HOME }}/bin:{{ ansible_env.PATH }}" + loop: "{{ containers_to_build.stdout_lines }}" + when: item != 'elk' + + - name: build elk container image when enabled + command: "{{ optscale_dir }}/build.sh --use-nerdctl elk local" + args: + chdir: "{{ optscale_dir }}" + environment: + PATH: "{{ ansible_env.HOME }}/.local/bin:{{ ansible_env.HOME }}/bin:{{ ansible_env.PATH }}" + when: with_elk | default(false) | bool + + - name: build runkube extra args + set_fact: + runkube_extra_args: >- + {{ + (with_elk | default(false) | bool) + | ternary(['--with-elk'], []) + }} + + - name: install the system packages to run the deployment + apt: + name: + - python3-pip + - sshpass + - git + - python3-virtualenv + - python3 + - python3-venv + - python3-dev + - wget + - curl + - mc + - htop + - bpytop + - nmap + - bmon + state: present + update_cache: yes + become: true + + - name: install bash-completion package + apt: + name: + - bash-completion + state: present + update_cache: yes + become: true + + - name: configure bash completion and prompt in user .bashrc + blockinfile: + path: "{{ ansible_env.HOME }}/.bashrc" + create: yes + marker: "# {mark} ANSIBLE OPTSCALE BASH SETTINGS" + block: | + # Enable bash-completion if available + if [ -f /usr/share/bash-completion/bash_completion ]; then + . /usr/share/bash-completion/bash_completion + elif [ -f /etc/bash_completion ]; then + . /etc/bash_completion + fi + + # Nice prompt: user@hostname:cwd$ + export PS1='\u@\h:\w\$ ' + # No become here so it edits the actual user's ~/.bashrc (e.g. vagrant) + become: false + + - name: install the python dependancies for Optscale-deploy + pip: + requirements: "{{ optscale_deploy_dir }}/requirements.txt" + virtualenv: "{{ optscale_deploy_dir }}/.venv" + + + - name: build overlay list for runkube + set_fact: + runkube_overlays: >- + {{ + [user_template_overlay] + + (disable_thanos | default(false) | bool | ternary([user_thanos_overlay], [])) + }} + + - name: run the k8s cluster setup script + command: + argv: >- + {{ + [optscale_deploy_dir + '/.venv/bin/python', + 'runkube.py'] + + runkube_extra_args + + ['--no-pull', + '-o'] + + runkube_overlays + + ['--', k8s_deployment_name, 'local'] + }} + args: + chdir: "{{ optscale_deploy_dir }}" + environment: + PATH: "{{ ansible_env.HOME }}/.local/bin:{{ ansible_env.HOME }}/bin:{{ ansible_env.PATH }}" diff --git a/optscale-deploy/ansible/roles/common/defaults/main.yaml b/optscale-deploy/ansible/roles/common/defaults/main.yaml index 89d571506..fa9d4ad0d 100644 --- a/optscale-deploy/ansible/roles/common/defaults/main.yaml +++ b/optscale-deploy/ansible/roles/common/defaults/main.yaml @@ -52,3 +52,13 @@ etcd_restore_tools: open_docker_port: true log_to_elk: true syslog_rotation_size: 100M + +# the architecture returned by `uname -m` (which is the canonical way to get it and it's what +# the `ansible_architecture` variable uses) can return either x86_64 and amd64 even though they +# refer to the same CPU architecture (and same for the other aliases). However when we're installing +# packages only amd64 and arm64 are valid, thus this mapping and the `os_arch` variable bellow +os_arch_aliases: + x86_64: amd64 + aarch64: arm64 + +os_arch: "{{ os_arch_aliases.get(ansible_architecture, ansible_architecture) }}" diff --git a/optscale-deploy/ansible/roles/common/tasks/kubernetes.yaml b/optscale-deploy/ansible/roles/common/tasks/kubernetes.yaml index 46a9f8882..c19e45d88 100644 --- a/optscale-deploy/ansible/roles/common/tasks/kubernetes.yaml +++ b/optscale-deploy/ansible/roles/common/tasks/kubernetes.yaml @@ -30,7 +30,7 @@ - name: add Docker's APT repository apt_repository: - repo: "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker-apt-keyring.asc] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable" + repo: "deb [arch={{ os_arch }} signed-by=/etc/apt/keyrings/docker-apt-keyring.asc] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable" state: present update_cache: yes become: yes diff --git a/optscale-deploy/ansible/roles/common/tasks/nerdctl.yaml b/optscale-deploy/ansible/roles/common/tasks/nerdctl.yaml index 37d2c26c4..71eff4b70 100644 --- a/optscale-deploy/ansible/roles/common/tasks/nerdctl.yaml +++ b/optscale-deploy/ansible/roles/common/tasks/nerdctl.yaml @@ -2,11 +2,11 @@ block: - name: Download nerdctl get_url: - url: https://github.com/containerd/nerdctl/releases/download/v{{ nerdctl_version }}/nerdctl-{{ nerdctl_version }}-linux-amd64.tar.gz - dest: /tmp/nerdctl-{{ nerdctl_version }}-linux-amd64.tar.gz + url: https://github.com/containerd/nerdctl/releases/download/v{{ nerdctl_version }}/nerdctl-{{ nerdctl_version }}-linux-{{ os_arch }}.tar.gz + dest: /tmp/nerdctl-{{ nerdctl_version }}-linux-{{ os_arch }}.tar.gz - name: Extract nerdctl - shell: tar Cxzf /usr/local/bin /tmp/nerdctl-{{ nerdctl_version }}-linux-amd64.tar.gz + shell: tar Cxzf /usr/local/bin /tmp/nerdctl-{{ nerdctl_version }}-linux-{{ os_arch }}.tar.gz become: yes - name: Create bin directory in the user's home @@ -31,7 +31,7 @@ - name: Cleanup temporary file file: - path: /tmp/nerdctl-{{ nerdctl_version }}-linux-amd64.tar.gz + path: /tmp/nerdctl-{{ nerdctl_version }}-linux-{{ os_arch }}.tar.gz state: absent - name: Create conf directory @@ -53,11 +53,11 @@ block: - name: Download buildkit get_url: - url: https://github.com/moby/buildkit/releases/download/v{{ buildkit_version }}/buildkit-v{{ buildkit_version }}.linux-amd64.tar.gz - dest: /tmp/buildkit-{{ buildkit_version }}-linux-amd64.tar.gz + url: https://github.com/moby/buildkit/releases/download/v{{ buildkit_version }}/buildkit-v{{ buildkit_version }}.linux-{{ os_arch }}.tar.gz + dest: /tmp/buildkit-{{ buildkit_version }}-linux-{{ os_arch }}.tar.gz - name: Extract buildkit - shell: tar Cxzf {{ ansible_env.HOME }} /tmp/buildkit-{{ buildkit_version }}-linux-amd64.tar.gz + shell: tar Cxzf {{ ansible_env.HOME }} /tmp/buildkit-{{ buildkit_version }}-linux-{{ os_arch }}.tar.gz - name: Create systemd unit file template: @@ -74,4 +74,4 @@ name: buildkit.service enabled: true state: started - become: yes \ No newline at end of file + become: yes diff --git a/optscale-deploy/overlay/user_thanos_disabled.yml b/optscale-deploy/overlay/user_thanos_disabled.yml new file mode 100644 index 000000000..16c8c7c7f --- /dev/null +++ b/optscale-deploy/overlay/user_thanos_disabled.yml @@ -0,0 +1,16 @@ +# overlay/user_thanos_disabled.yml +thanos_compactor: + cronJob: + suspend: true + +thanos_query: + replicaCount: 0 + +thanos_receive: + replicaCount: 0 + +thanos_storegateway: + replicaCount: 0 + +diproxy: + replicaCount: 0 diff --git a/optscale-deploy/runkube.py b/optscale-deploy/runkube.py index 93800c5c0..5782fc2a5 100755 --- a/optscale-deploy/runkube.py +++ b/optscale-deploy/runkube.py @@ -134,12 +134,18 @@ def get_ctrd_cl(self, node): @property def versions_info(self): if self._versions_info is None: - self._versions_info = {'optscale': self.version} + self._versions_info = {'optscale': self.version, 'images': {}} + with open(COMPONENTS_FILE) as f_comp: components_dict = yaml.safe_load(f_comp) - self._versions_info['images'] = { - component: self.version for component in components_dict - } + + for component in components_dict: + if component == 'elk' and not self.with_elk: + LOG.warning("elk specified in components file (%s) but --with-elk is not set, ignoring it", COMPONENTS_FILE) + continue + + self._versions_info['images'][component] = self.version + return self._versions_info def _pull_image(self, ctrd_cl, image_name, tag, auth_config): diff --git a/optscale-deploy/vm.sh b/optscale-deploy/vm.sh new file mode 100755 index 000000000..3ceb19c4e --- /dev/null +++ b/optscale-deploy/vm.sh @@ -0,0 +1,622 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(dirname "$(realpath "$0")")" + +STATE_DIR="$SCRIPT_DIR/.vagrant/.vm-state" +PROVIDER_FILE="$STATE_DIR/provider" +ARCH_FILE="$STATE_DIR/arch" +CPUS_FILE="$STATE_DIR/cpus" +RAM_FILE="$STATE_DIR/ram_gb" +DISK_FILE="$STATE_DIR/disk_gb" + +_save_state_kv() { + local file="$1" + local val="$2" + mkdir -p "$STATE_DIR" + printf '%s\n' "$val" > "$file" +} + +_load_state_kv() { + local file="$1" + if [[ -f "$file" ]]; then + cat "$file" + else + echo "" + fi +} + +save_cpus() { _save_state_kv "$CPUS_FILE" "$1"; } +load_cpus() { _load_state_kv "$CPUS_FILE"; } + +save_ram_gb() { _save_state_kv "$RAM_FILE" "$1"; } +load_ram_gb() { _load_state_kv "$RAM_FILE"; } + +save_disk_gb() { _save_state_kv "$DISK_FILE" "$1"; } +load_disk_gb() { _load_state_kv "$DISK_FILE"; } + + +save_provider() { + mkdir -p "$STATE_DIR" + printf '%s\n' "$1" > "$PROVIDER_FILE" +} + +load_provider() { + if [[ -f "$PROVIDER_FILE" ]]; then + cat "$PROVIDER_FILE" + else + echo "" + fi +} + +save_arch() { + mkdir -p "$STATE_DIR" + printf '%s\n' "$1" > "$ARCH_FILE" +} + +load_arch() { + if [[ -f "$ARCH_FILE" ]]; then + cat "$ARCH_FILE" + else + echo "" + fi +} + +clear_state() { + rm -rf "$STATE_DIR" +} + + +function host_arch_family { + local host_arch + host_arch="$(uname -m)" + + case "$host_arch" in + x86_64|amd64) echo "x86" ;; + aarch64|arm64) echo "arm" ;; + *) + echo "Error: Unsupported host architecture '$host_arch'." >&2 + exit 1 + ;; + esac +} + +function arch_is_suitable { + local requested="$1" + local host + host="$(host_arch_family)" + + [[ "$requested" == "$host" ]] +} + +print_help() { + cat <] [--no-thanos] [...] + +Available commands: + info show vm status and info + optscale-info show optscale cluster info + start, up start the vm + stop, down stop the vm + destroy destroy the vm + restart stop and start the vm + reset destroy and start the vm + ssh ssh into the vm + playbook run an ansible playbook + role run an ansible role + deploy-service deploy a service inside the vm + update-only rebuild images and update existing optscale release + +Global options: + --provider qemu|virtualbox force specific Vagrant provider + --allow-emulation allow starting non-native architecture VM (emulation) + --cpus N set VM vCPUs (default: 4) + --ram N set VM RAM in GB (default: 10) + --disk N set VM disk size in GB (default: 75) + --no-thanos disable thanos components (passed to ansible & runkube) + --with-elk enable ELK stack (build elk image, pass --with-elk to runkube) + --help, -h show this help message + +EOF +} + +# check if --help or -h is passed as any argument regardless of its position +if [[ "$@" == *"--help"* || "$@" == *"-h"* ]]; then + print_help + exit 0 +fi + +ALLOW_EMULATION=0 +PROVIDER="" +DISABLE_THANOS=0 +WITH_ELK=0 +CPUS="" +RAM_GB="" +DISK_GB="" + +args=("$@") +clean_args=() + +i=0 +while [[ $i -lt $# ]]; do + arg="${args[$i]}" + + case "$arg" in + --provider) + # next arg must contain provider + if [[ $((i+1)) -ge $# ]]; then + echo "Error: --provider must be followed by qemu or virtualbox" + exit 1 + fi + PROVIDER="${args[$((i+1))]}" + i=$((i+2)) + continue + ;; + --provider=*) + PROVIDER="${arg#*=}" + i=$((i+1)) + continue + ;; + --allow-emulation) + ALLOW_EMULATION=1 + i=$((i+1)) + continue + ;; + --no-thanos|--disable-thanos) + DISABLE_THANOS=1 + i=$((i+1)) + continue + ;; + --cpus) + if [[ $((i+1)) -ge $# ]]; then + echo "Error: --cpus must be followed by a number" + exit 1 + fi + CPUS="${args[$((i+1))]}" + i=$((i+2)) + continue + ;; + --cpus=*) + CPUS="${arg#*=}" + i=$((i+1)) + continue + ;; + --ram) + if [[ $((i+1)) -ge $# ]]; then + echo "Error: --ram must be followed by a number (GB)" + exit 1 + fi + RAM_GB="${args[$((i+1))]}" + i=$((i+2)) + continue + ;; + --ram=*) + RAM_GB="${arg#*=}" + i=$((i+1)) + continue + ;; + --disk) + if [[ $((i+1)) -ge $# ]]; then + echo "Error: --disk must be followed by a number (GB)" + exit 1 + fi + DISK_GB="${args[$((i+1))]}" + i=$((i+2)) + continue + ;; + --with-elk|--elk) + WITH_ELK=1 + i=$((i+1)) + continue + ;; + --disk=*) + DISK_GB="${arg#*=}" + i=$((i+1)) + continue + ;; + *) + clean_args+=("$arg") + ;; + esac + + i=$((i+1)) +done + +# Normalize provider if given; otherwise reuse the last selected provider +if [[ "$PROVIDER" == "qemu" || "$PROVIDER" == "virtualbox" ]]; then + export VAGRANT_DEFAULT_PROVIDER="$PROVIDER" + save_provider "$PROVIDER" +elif [[ -n "$PROVIDER" ]]; then + echo "Error: unsupported provider '$PROVIDER'. Expected 'qemu' or 'virtualbox'." + exit 1 +else + last_provider="$(load_provider)" + if [[ -n "$last_provider" ]]; then + export VAGRANT_DEFAULT_PROVIDER="$last_provider" + fi +fi + +if [[ -z "${VAGRANT_DEFAULT_PROVIDER:-}" ]]; then + export VAGRANT_DEFAULT_PROVIDER="qemu" + save_provider "qemu" +fi + +EFFECTIVE_CPUS="" +EFFECTIVE_RAM_GB="" +EFFECTIVE_DISK_GB="" + +if [[ -n "$CPUS" ]]; then + EFFECTIVE_CPUS="$CPUS" + save_cpus "$EFFECTIVE_CPUS" +else + EFFECTIVE_CPUS="$(load_cpus)" +fi + +if [[ -n "$RAM_GB" ]]; then + EFFECTIVE_RAM_GB="$RAM_GB" + save_ram_gb "$EFFECTIVE_RAM_GB" +else + EFFECTIVE_RAM_GB="$(load_ram_gb)" +fi + +if [[ -n "$DISK_GB" ]]; then + EFFECTIVE_DISK_GB="$DISK_GB" + save_disk_gb "$EFFECTIVE_DISK_GB" +else + EFFECTIVE_DISK_GB="$(load_disk_gb)" +fi + +# Export only if we actually have values +if [[ -n "$EFFECTIVE_CPUS" ]]; then + export VM_CPUS="$EFFECTIVE_CPUS" +fi + +if [[ -n "$EFFECTIVE_RAM_GB" ]]; then + export VM_RAM_GB="$EFFECTIVE_RAM_GB" +fi + +if [[ -n "$EFFECTIVE_DISK_GB" ]]; then + export VM_DISK_GB="$EFFECTIVE_DISK_GB" +fi + +# Replace original args with cleaned ones +set -- "${clean_args[@]}" + +if [[ $# -lt 1 ]]; then + echo "Error: Invalid number of arguments." + print_help + exit 1 +fi + +if [[ "${1:-}" == "x86" || "${1:-}" == "arm" ]]; then + COMMAND_PEEK="${2:-}" +else + COMMAND_PEEK="${1:-}" +fi + +if [[ -z "${COMMAND_PEEK:-}" ]]; then + echo "Error: Missing command." + print_help + exit 1 +fi + +if [[ "$1" == "x86" || "$1" == "arm" ]]; then + VM_ARCH="$1" + + if arch_is_suitable "$VM_ARCH"; then + save_arch "$VM_ARCH" + else + if [[ "$COMMAND_PEEK" == "destroy" || "$COMMAND_PEEK" == "stop" || "$COMMAND_PEEK" == "down" ]]; then + echo "Warning: '$VM_ARCH' is not suitable for this host ($(uname -m)), but allowing '$COMMAND_PEEK'." + echo "Arch will NOT be saved." + elif [[ "${ALLOW_EMULATION:-0}" -eq 1 ]]; then + export ALLOW_EMULATION + echo "Warning: '$VM_ARCH' is not native for this host ($(uname -m)), but --allow-emulation is set." + echo "Arch will NOT be saved." + else + echo "Error: '$VM_ARCH' is not suitable for this host ($(uname -m))." + echo "Hint: add --allow-emulation to run non-native architecture VM." + echo "Tip: use the native arch or run 'destroy'/'stop' to clean up that VM." + exit 2 + fi + fi + + shift +else + last_arch="$(load_arch)" + if [[ -n "$last_arch" ]]; then + VM_ARCH="$last_arch" + echo "VM architecture not provided, using saved: $VM_ARCH" + else + host_os_arch=$(uname -m) + if [[ "$host_os_arch" == "x86_64" || "$host_os_arch" == "amd64" ]]; then + VM_ARCH="x86" + elif [[ "$host_os_arch" == "aarch64" || "$host_os_arch" == "arm64" ]]; then + VM_ARCH="arm" + else + echo "Error: Unsupported host architecture '$host_os_arch'. Provide 'x86' or 'arm'." + exit 1 + fi + save_arch "$VM_ARCH" + echo "VM architecture not provided, using host: $VM_ARCH" + fi +fi + + + +COMMAND="$1" +shift + +if [[ "$VM_ARCH" == "x86" ]]; then + VM_NAME="ubuntu-2404-x86-64" +else + VM_NAME="ubuntu-2404-arm-64" +fi + +INVENTORY_FILE="$SCRIPT_DIR/ansible/inventories/vm-$VM_ARCH.yaml" + +function require_arch_compatible_or_die { + local cmd="$1" + case "$cmd" in + start|up|restart|reset) ;; + *) return 0 ;; + esac + + local host_arch + host_arch="$(uname -m)" + + local host_family="" + case "$host_arch" in + x86_64|amd64) host_family="x86" ;; + aarch64|arm64) host_family="arm" ;; + *) + echo "Error: Unsupported host architecture '$host_arch'." + exit 1 + ;; + esac + + if [[ "$VM_ARCH" != "$host_family" ]]; then + if [[ "${ALLOW_EMULATION:-0}" -eq 1 ]]; then + echo "Warning: starting '$VM_ARCH' VM on '$host_arch' host (non-native), because --allow-emulation is set." + return 0 + fi + + echo "Error: You're trying to start '$VM_ARCH' VM on '$host_arch' host." + echo "This script is configured to fail-fast on non-native architecture." + echo + echo "Fix: run with the native arch:" + echo + echo "Hint: add --allow-emulation to run non-native architecture VM (at own risk)" + + if [[ "$host_family" == "x86" ]]; then + echo " $0 x86 --provider ${VAGRANT_DEFAULT_PROVIDER} up" + else + echo " $0 arm --provider ${VAGRANT_DEFAULT_PROVIDER} up" + fi + exit 2 + fi +} + +function _venv_run { + if [[ -n "${VIRTUAL_ENV:-}" ]]; then + exec "$@" + elif type uv >/dev/null 2>&1; then + uv run --directory "$SCRIPT_DIR" "$@" + else + default_venv_dir="$SCRIPT_DIR/.venv" + if [[ ! -d "$default_venv_dir" ]]; then + echo "Error: No virtualenv found." + exit 1 + fi + exec "$default_venv_dir/bin/$@" + fi +} + +function vm_info { + vm_state=$(vagrant status $VM_NAME --machine-readable | grep ',state,' | cut -d ',' -f4) + vm_qemu_id=$(cat ".vagrant/machines/$VM_NAME/qemu/id" 2>/dev/null || echo "") + vm_vagrant_id=$(cat ".vagrant/machines/$VM_NAME/qemu/index_uuid" 2>/dev/null || echo "") + + vm_process_info=$(ps -eo pid,command | grep "qemu-system-.*$VM_NAME" | grep -v grep || echo "") + vm_qemu_pid=$(echo "$vm_process_info" | cut -d ' ' -f1 || echo "") + vm_qemu_accelerator=$(echo "$vm_process_info" | sed 's/.*-accel \([^ ]*\).*/\1/' || echo "") + + echo "Name: ${VM_NAME}" + echo "State: ${vm_state:-unknown}" + echo "QEMU Machine ID: ${vm_qemu_id:-N/A}" + echo "Vagrant ID: ${vm_vagrant_id:-N/A}" + echo "QEMU Process ID: ${vm_qemu_pid:-N/A}" + echo "QEMU Accelerator: ${vm_qemu_accelerator:-N/A}" + echo "VM_CPUS: ${VM_CPUS:-$(load_cpus):-default}" + echo "VM_RAM_GB: ${VM_RAM_GB:-$(load_ram_gb):-default}" + echo "VM_DISK_GB: ${VM_DISK_GB:-$(load_disk_gb):-default}" +} + +function vm_start { + vagrant up $VM_NAME +} + +function vm_stop { + vagrant halt $VM_NAME "$@" + + if [[ "$@" == *"--force"* ]]; then + pid=$(pgrep -f "qemu-system-.*$VM_NAME" || true) + [[ -n "$pid" ]] && kill "$pid" + fi + + while pgrep -f "qemu-system-.*$VM_NAME" >/dev/null; do + echo "$(date +%X) - Waiting until VM stops" + sleep 1 + done +} + +function vm_destroy { + vm_stop --force + vagrant destroy --force $VM_NAME + clear_state +} + +function vm_ssh { + cmd_to_run="${1:-/bin/bash}" + + if [[ "$TERM" == "xterm-kitty" ]]; then + TERM="xterm-256color" vagrant ssh $VM_NAME -c "$cmd_to_run" + else + vagrant ssh $VM_NAME -c "$cmd_to_run" + fi +} + +function vm_run_ansible_playbook { + local playbook="$1" + shift + + if [[ -f "$playbook" ]]; then + playbook_path="$playbook" + elif [[ -f "$SCRIPT_DIR/ansible/$playbook.yaml" ]]; then + playbook_path="$SCRIPT_DIR/ansible/$playbook.yaml" + else + echo "Error: playbook '$playbook' not found." + exit 1 + fi + + extra_vars=() + if [[ "$DISABLE_THANOS" -eq 1 ]]; then + extra_vars+=(-e disable_thanos=true) + fi + if [[ "$WITH_ELK" -eq 1 ]]; then + extra_vars+=(-e with_elk=true) + fi + + _venv_run ansible-playbook --inventory-file "$INVENTORY_FILE" \ + "${extra_vars[@]}" \ + "$playbook_path" "$@" +} + + +function vm_run_ansible_role { + local role="$1" + shift + + if [[ -d "$SCRIPT_DIR/ansible/roles/$role" ]]; then + role_path="$SCRIPT_DIR/ansible/roles/$role" + else + echo "Error: role '$role' not found." + exit 1 + fi + + _venv_run ansible all \ + --inventory-file "$INVENTORY_FILE" \ + --module-name include_role \ + --args "name=$role_path" \ + "$@" +} + +function vm_show_optscale_info { + cluster_secret=$(vm_ssh "kubectl get secret cluster-secret -o jsonpath='{.data.cluster_secret}' | base64 --decode; echo") + cluster_ip_addr=$(vm_ssh "kubectl get svc ngingress-ingress-nginx-controller -o jsonpath='{.spec.clusterIP}'") + + forwarded_https_port=$(grep -A10 "define_vm" "$SCRIPT_DIR/Vagrantfile" | + grep -A10 "$VM_NAME" | + grep -o "https: [0-9]*" | + awk '{print $2}') + + frontend_url="https://localhost:$forwarded_https_port/" + status_code=$(curl --insecure -L -I "$frontend_url" -sw '%{http_code}\n' -o /dev/null || echo "") + + echo "Frontend URL: $frontend_url [$status_code]" + echo "Cluster IP Address: $cluster_ip_addr" + echo "Cluster Secret: $cluster_secret" + + k8s_pods=$(vm_ssh "kubectl get pods --all-namespaces") + echo "================================" + echo "K8S Pods" + echo "$k8s_pods" +} + +function vm_deploy_service { + vagrant rsync "$VM_NAME" + + if [[ $# -eq 1 ]]; then + service_name="$1" + vm_ssh "cd optscale && ./build.sh --use-nerdctl $service_name local" + fi + + # Build overlay args for runkube + local overlay_args="-o overlay/user_template.yml" + if [[ "$DISABLE_THANOS" -eq 1 ]]; then + overlay_args="$overlay_args overlay/user_thanos_disabled.yml" + fi + + local elk_flag="" + if [[ "$WITH_ELK" -eq 1 ]]; then + elk_flag="--with-elk" + fi + + vm_ssh "cd optscale/optscale-deploy && .venv/bin/python runkube.py --no-pull $elk_flag $overlay_args -- optscale local" +} + +function vm_update_only { + # Find running OptScale release based on chart/name containing "optscale" + local releases release_name + + releases=$(vm_ssh "helm list -o json 2>/dev/null | python3 -c 'import sys, json; data = json.load(sys.stdin); print(\"\\n\".join(r[\"name\"] for r in data if \"optscale\" in r.get(\"chart\", \"\") or \"optscale\" in r.get(\"name\", \"\")))' || true") + releases=$(echo "$releases" | sed '/^$/d') + + if [[ -z "$releases" ]]; then + echo "Error: No running OptScale release with chart/name containing 'optscale' found." + echo "helm list output from VM:" + vm_ssh "helm list 2>/dev/null || true" + exit 1 + fi + + if [[ $(echo "$releases" | wc -l) -ne 1 ]]; then + echo "Error: Multiple OptScale-like releases found:" + echo "$releases" + echo "Please clean up or update the script to select one." + exit 1 + fi + + release_name=$(echo "$releases" | head -n1 | tr -d '\r\n') + echo "Using OptScale release: ${release_name}" + + # Rebuild services (all or specific) + if [[ $# -eq 0 ]]; then + echo "Rebuilding all OptScale images inside VM..." + vm_ssh "cd optscale && ./build.sh --use-nerdctl" + else + for svc in "$@"; do + echo "Rebuilding service: ${svc}" + vm_ssh "cd optscale && ./build.sh --use-nerdctl ${svc} local" + done + fi + + local elk_flag="" + if [[ "$WITH_ELK" -eq 1 ]]; then + elk_flag="--with-elk" + fi + + echo "Running runkube update-only for release '${release_name}'..." + vm_ssh "cd optscale/optscale-deploy && .venv/bin/python runkube.py --update-only --no-pull $elk_flag -- ${release_name} local" +} + +require_arch_compatible_or_die "$COMMAND" +case $COMMAND in + info) vm_info ;; + optscale-info) vm_show_optscale_info ;; + start|up) vm_start ;; + stop|down) vm_stop "$@" ;; + destroy) vm_destroy ;; + restart) vm_stop; vm_start ;; + reset) vm_destroy; vm_start ;; + ssh) vm_ssh "$@" ;; + playbook) vm_run_ansible_playbook "$@" ;; + role) vm_run_ansible_role "$@" ;; + deploy-service) vm_deploy_service "$@" ;; + update-only) vm_update_only "$@" ;; + *) + echo "Error: Invalid command: $COMMAND" + print_help + exit 1 + ;; +esac diff --git a/rest_api/live_demo.tar.xz b/rest_api/live_demo.tar.xz index 276d0e5be..eec46580c 100644 Binary files a/rest_api/live_demo.tar.xz and b/rest_api/live_demo.tar.xz differ diff --git a/tools/cloud_adapter/clouds/alibaba.py b/tools/cloud_adapter/clouds/alibaba.py index a44f66833..907e72d9f 100644 --- a/tools/cloud_adapter/clouds/alibaba.py +++ b/tools/cloud_adapter/clouds/alibaba.py @@ -678,10 +678,10 @@ def _discover_region_lbs(self, region_details, lb_type): for lb in response: lb_id = lb['LoadBalancerId'] if attr_request: - attr_request = attr_request() - attr_request.set_LoadBalancerId(lb_id) + req = attr_request() + req.set_LoadBalancerId(lb_id) attrs = handle_discovery_client_exc( - self._send_request, attr_request, + self._send_request, req, region_id=region_details['RegionId']) security_groups = attrs.get('SecurityGroupIds', []) if 'Tags' in lb and isinstance(lb['Tags'], dict):