From 2f68812b6c27e24addad8c7847bf1f42eb46303b Mon Sep 17 00:00:00 2001 From: Eli Date: Fri, 11 Apr 2025 06:53:26 +0200 Subject: [PATCH 01/11] feat: add sqlmodel import to alembic script template --- backend/alembic/script.py.mako | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/alembic/script.py.mako b/backend/alembic/script.py.mako index fbc4b07..6ce3351 100644 --- a/backend/alembic/script.py.mako +++ b/backend/alembic/script.py.mako @@ -9,6 +9,7 @@ from typing import Sequence, Union from alembic import op import sqlalchemy as sa +import sqlmodel ${imports if imports else ""} # revision identifiers, used by Alembic. From 590d01f07eaea57d80b479dac728261b7263b8ec Mon Sep 17 00:00:00 2001 From: Eli Date: Fri, 11 Apr 2025 06:56:48 +0200 Subject: [PATCH 02/11] feat: refactor MYSQL_PASSWORD validation to use shared method --- backend/core/config.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/backend/core/config.py b/backend/core/config.py index 3fef689..1ad175d 100644 --- a/backend/core/config.py +++ b/backend/core/config.py @@ -180,33 +180,39 @@ def validate_mysql_port(cls, v: int) -> int: raise ValueError("MYSQL_PORT must be between 1 and 65535") return v - @classmethod - @field_validator("MYSQL_PASSWORD", mode="before") - def validate_mysql_password(cls, v: SecretStr) -> SecretStr: + @staticmethod + def _validate_password(field_name: str, secret: SecretStr) -> SecretStr: """ - Validate the MYSQL_PASSWORD environment variable. - Ensure it meets the password policy. + Shared validator for password fields. """ - raw = v.get_secret_value() - # Enforce a password policy similar to FIRST_SUPERUSER_PASSWORD + raw = secret.get_secret_value() if not ValidationConstants.PASSWORD_REGEX.match(raw): raise ValueError( - "MYSQL_PASSWORD must be at least 8 characters " + f"{field_name} must be at least 8 characters " "and include at least one lowercase letter, " "one uppercase letter, one digit, and one special character." ) - return v + return secret + + @classmethod + @field_validator("MYSQL_PASSWORD", mode="before") + def validate_mysql_password(cls, v: SecretStr) -> SecretStr: + """ + Validate the MYSQL_PASSWORD environment variable. + Ensure it meets the password policy. + """ + return cls._validate_password("MYSQL_PASSWORD", v) - # type: ignore[prop-decorator, C0103] @computed_field @property # Union[PostgresDsn, MySQLDsn]: def SQLALCHEMY_DATABASE_URI(self) -> MySQLDsn: """Set SqlAlchemy db url""" - encoded_password = quote_plus(self.MYSQL_PASSWORD) + validated_password = self._validate_password("MYSQL_PASSWORD", self.MYSQL_PASSWORD) + encoded_password = quote_plus(validated_password) return ( f"mysql+pymysql://{self.MYSQL_USER}:{encoded_password}@" - f"{self.MYSQL_SERVER}:{self.MYSQL_PORT}/{self.MYSQL_DB}" + f"{self.MYSQL_HOST}:{self.MYSQL_PORT}/{self.MYSQL_DB}" ) @computed_field # type: ignore[prop-decorator] From d468aeebec60ae0b552fb2015e274c43eb4aef1e Mon Sep 17 00:00:00 2001 From: Eli Date: Fri, 11 Apr 2025 06:57:02 +0200 Subject: [PATCH 03/11] fix: correct MYSQL_SERVER variable to MYSQL_HOST in docker_start.sh --- backend/scripts/docker_start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/scripts/docker_start.sh b/backend/scripts/docker_start.sh index 8ae736f..6c5ff8e 100644 --- a/backend/scripts/docker_start.sh +++ b/backend/scripts/docker_start.sh @@ -2,7 +2,7 @@ set -e # wait for the database service to be available -./backend/scripts/wait_for_db.sh "${MYSQL_SERVER}" "${MYSQL_PORT}" "${MYSQL_USER}" "${MYSQL_PASSWORD}" +./backend/scripts/wait_for_db.sh "${MYSQL_HOST}" "${MYSQL_PORT}" "${MYSQL_USER}" "${MYSQL_PASSWORD}" # Run migrations # Check if alembic/versions is empty From be042d51b923dcb74c2e242d836c5c0275086541 Mon Sep 17 00:00:00 2001 From: Eli Date: Fri, 11 Apr 2025 06:57:35 +0200 Subject: [PATCH 04/11] refactor: update docstring format for utility functions in paths.py --- backend/utils/paths.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/utils/paths.py b/backend/utils/paths.py index f6b727f..4cfb1de 100644 --- a/backend/utils/paths.py +++ b/backend/utils/paths.py @@ -1,9 +1,10 @@ +""" +Utility functions for handling file paths in the backend. +""" # backend/utils/paths.py - from pathlib import Path from os.path import join as join_path -# This assumes `backend/` is your top-level package ROOT_DIR = Path(__file__).resolve() From 9e22b94634f7f501da5c7ca2ae970efa1345e40a Mon Sep 17 00:00:00 2001 From: Eli Date: Fri, 11 Apr 2025 06:57:44 +0200 Subject: [PATCH 05/11] feat: add .env* to .dockerignore to exclude environment variable files --- backend/.dockerignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/.dockerignore b/backend/.dockerignore index 3b93f1f..a76dd7b 100644 --- a/backend/.dockerignore +++ b/backend/.dockerignore @@ -1,2 +1,3 @@ Dockerfile -/alembic/versions/* \ No newline at end of file +/alembic/versions/* +.env* \ No newline at end of file From 033641e91d37335a13acafbbbcf2649d23277465 Mon Sep 17 00:00:00 2001 From: Eli Date: Fri, 11 Apr 2025 06:57:50 +0200 Subject: [PATCH 06/11] refactor: remove Docker-specific environment file copy from Dockerfile --- backend/Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index c8186d7..ad09975 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -43,9 +43,6 @@ RUN pip install --no-cache-dir --user --upgrade pip && \ # Copy the backend source code COPY --chown=appuser:appuser . ./backend -# Replace env file with the one for Docker -COPY --chown=appuser:appuser .env.docker ./backend/.env - RUN dos2unix /opt/emon_tools/backend/scripts/docker_start.sh RUN dos2unix /opt/emon_tools/backend/scripts/pre_start.sh RUN dos2unix /opt/emon_tools/backend/scripts/wait_for_db.sh From fffa352d5c6b58a0e190005c40676ac7cbd93030 Mon Sep 17 00:00:00 2001 From: Eli Date: Fri, 11 Apr 2025 06:58:08 +0200 Subject: [PATCH 07/11] feat: add env_file configuration to emon_api service in docker-compose.yml --- docker-compose/dev/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose/dev/docker-compose.yml b/docker-compose/dev/docker-compose.yml index 1f05b70..f89f244 100644 --- a/docker-compose/dev/docker-compose.yml +++ b/docker-compose/dev/docker-compose.yml @@ -17,6 +17,8 @@ services: build: context: ../../backend dockerfile: Dockerfile + env_file: + - ./.env volumes: - ../../backend:/opt/emon_tools/backend - ./alembic:/opt/emon_tools/backend/alembic/versions From 2d59a513ada010dd99a65e7e70072626c683c5b8 Mon Sep 17 00:00:00 2001 From: Eli Date: Fri, 11 Apr 2025 07:02:34 +0200 Subject: [PATCH 08/11] fix: retrieve secret value for MYSQL_PASSWORD in SQLALCHEMY_DATABASE_URI method --- backend/core/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/core/config.py b/backend/core/config.py index 1ad175d..679b63c 100644 --- a/backend/core/config.py +++ b/backend/core/config.py @@ -209,7 +209,7 @@ def validate_mysql_password(cls, v: SecretStr) -> SecretStr: def SQLALCHEMY_DATABASE_URI(self) -> MySQLDsn: """Set SqlAlchemy db url""" validated_password = self._validate_password("MYSQL_PASSWORD", self.MYSQL_PASSWORD) - encoded_password = quote_plus(validated_password) + encoded_password = quote_plus(validated_password.get_secret_value()) return ( f"mysql+pymysql://{self.MYSQL_USER}:{encoded_password}@" f"{self.MYSQL_HOST}:{self.MYSQL_PORT}/{self.MYSQL_DB}" From db6a9e7f1d4a2005e16baa5a96204c963ae43e73 Mon Sep 17 00:00:00 2001 From: Eli Date: Fri, 11 Apr 2025 07:09:17 +0200 Subject: [PATCH 09/11] fix: retrieve secret value for FIRST_SUPERUSER_PASSWORD in init_db function --- backend/core/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/core/db.py b/backend/core/db.py index f1055fe..68fe453 100644 --- a/backend/core/db.py +++ b/backend/core/db.py @@ -47,7 +47,7 @@ def init_db(session: Session) -> None: if not user: user_in = UserCreate( email=settings.FIRST_SUPERUSER, - password=settings.FIRST_SUPERUSER_PASSWORD, + password=settings.FIRST_SUPERUSER_PASSWORD.get_secret_value(), is_superuser=True, ) user = UserController.create_user(session=session, user_create=user_in) From 7c3237b6ecf078e90e9c3e5fad17d8ee798bf8f5 Mon Sep 17 00:00:00 2001 From: Eli Date: Fri, 11 Apr 2025 07:10:32 +0200 Subject: [PATCH 10/11] feat: add example environment configuration file for local development --- docker-compose/dev/.example_env | 36 +++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 docker-compose/dev/.example_env diff --git a/docker-compose/dev/.example_env b/docker-compose/dev/.example_env new file mode 100644 index 0000000..bfc5e2f --- /dev/null +++ b/docker-compose/dev/.example_env @@ -0,0 +1,36 @@ +# project envs +# Domain +# This would be set to the production domain with an env var on deployment +# used by Traefik to transmit traffic and aqcuire TLS certificates +DOMAIN="localhost" +# To test the local Traefik config +# DOMAIN=localhost.tiangolo.com + +# Environment: local, staging, production +ENVIRONMENT="local" + +API_V1_STR="/api/v1" +PROJECT_NAME="EmonTools" +STACK_NAME="full-stack-emontools-project" + +# Data Path +DATA_BASE_PATH="/opt/emon_tools/data" +STATIC_BASE_PATH="/opt/emon_tools/static" + +# Backend +# Used by the backend to generate links in emails to the frontend +FRONTEND_HOST="http://localhost:5173" +# In staging and production, set this env var to the frontend host, e.g. +# FRONTEND_HOST=https://dashboard.example.com +BACKEND_CORS_ORIGINS="http://localhost,http://localhost:5173,https://localhost,https://localhost:5173" + +SECRET_KEY=changethis +FIRST_SUPERUSER=changethis +FIRST_SUPERUSER_PASSWORD=changethis + +# Mysql +MYSQL_HOST="127.0.0.1" +MYSQL_PORT=3306 +MYSQL_DB="emontools" +MYSQL_USER="emontools" +MYSQL_PASSWORD=changethis From 176699f0cc19eb1cdf89fad2eff8c1a4a8d37a6e Mon Sep 17 00:00:00 2001 From: mano8 Date: Fri, 11 Apr 2025 09:08:41 +0200 Subject: [PATCH 11/11] docs: update README to include full-stack visualization app deployment instructions and local development setup --- README.md | 122 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 110 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 9645ca5..9c1f888 100644 --- a/README.md +++ b/README.md @@ -25,11 +25,13 @@ Explore the project Wiki for detailed documentation and guides: [emon_tools Wiki - [emon_api](https://github.com/vemonitor/emon_tools/blob/main/README.md#emon_api) -3. [Running Tests](https://github.com/vemonitor/emon_tools/blob/main/README.md#running-tests) - -4. [Contributing](https://github.com/vemonitor/emon_tools/blob/main/README.md#contributing) - -5. [License](https://github.com/vemonitor/emon_tools/blob/main/README.md#license) +3. [Optional Full-Stack Visualization App](#optional-full-stack-visualization-app) + - [Overview](#overview) + - [Local Development Setup](#local-development-setup) + - [Docker Compose Deployment](#docker-compose-deployment) +4. [Running Tests](#running-tests) +5. [Contributing](#contributing) +6. [License](#license) ## Installation @@ -44,10 +46,10 @@ The `emon-tools` package offers flexible installation options tailored to variou 3. **Check Requirements**: Review the module's requirements in the `setup.cfg` file or on the PyPI page to ensure compatibility with your system. ### Global Installation -To install the entire emon-tools package along with all dependencies, run the following command: +To install the entire emon-tools package along with all dependencies—and to ensure you get the latest version—run: ``` -pip install emon-tools["all"] +pip install emon-tools["all"] --upgrade ``` Included Dependencies: @@ -64,31 +66,31 @@ You can install specific modules and their dependencies as needed. For example: - To enable `emon_fina` module: ``` -pip install emon-tools["fina"] +pip install emon-tools["fina"] --upgrade ``` - To enable pandas time-series output functionality: ``` -pip install emon-tools["fina, time_series"] +pip install emon-tools["fina, time_series"] --upgrade ``` - To include graph plotting capabilities: ``` -pip install emon-tools["fina, plot"] +pip install emon-tools["fina, plot"] --upgrade ``` - To enable `emon_api` module: ``` -pip install emon-tools["api"] +pip install emon-tools["api"] --upgrade ``` - To enable `emon_fina` and `emon_api` modules: ``` -pip install emon-tools["api, fina"] +pip install emon-tools["api, fina"] --upgrade ``` ## Modules @@ -166,6 +168,102 @@ print("Feeds: ", feeds) - **Wiki**: See `emon_api` module [wiki](https://github.com/vemonitor/emon_tools/wiki/emon_api) section. - **Examples**: Explore [api_bulk_structure](https://github.com/vemonitor/emon_tools/blob/main/examples/emon_api.py) for input and feed supervision, as well as posting bulk dummy data. +## 🚀 Full-Stack Visualization App Deployment + +This optional full-stack application offers a graphical interface to visualize data from any EmonCMS instance and explore archived PhpFina file backups. It comprises: + +- FastAPI Backend: Exposes the `emon-tools` functionalities via a RESTful API. +- Vite React Frontend: A modern dashboard built using React, TypeScript, and Tailwind CSS. + +### Overview + +The full-stack app enables users to: + +- Interactively browse and monitor live data from EmonCMS instances. +- Visualize historical time-series data from archived PhpFina files. +- Manage EmonCMS feeds and inputs via an intuitive web interface. + +### 🧰 Prerequisites + +Ensure the following are installed on your system: +- Docker +- Docker Compose + +### 📦 Deployment Steps +This repository now includes a Docker Compose example that leverages the provided Dockerfiles for both backend and frontend. This approach is ideal for quickly deploying the full-stack app in a containerized environment. + +1. Clone the Repository + +```bash +git clone https://github.com/vemonitor/emon_tools.git +cd emon_tools +``` + +2. Set Up Environment Variables + +Navigate to the Docker Compose development directory and create a `.env` file by copying the provided example: + +```bash +cd docker-compose\dev +copy .example_env .env +``` + +Edit the .env file to replace placeholder values (changethis) with your actual configuration: + +```env +MYSQL_HOST=your_mysql_host +MYSQL_PORT=3306 +MYSQL_DB=your_database_name +MYSQL_USER=your_username +MYSQL_PASSWORD=your_password +MYSQL_ROOT_PASSWORD=your_root_password +``` + +> Note: The `.env` file is utilized by Docker Compose for variable substitution in the `docker-compose.yml` file. Ensure all required variables are defined to prevent runtime errors. + +3. Build containers and start the containers: + +From the `docker-compose/dev` directory, execute: + +```bash +docker-compose up --build +``` + +This command builds the Docker images and starts the containers as defined in the `docker-compose.yml` file. + +4. Access the Application + +Once the containers are running: + +- Frontend: Access the React dashboard at http://localhost:3000 +- Backend: Access the FastAPI backend at http://localhost:8000 + - Swagger UI: http://localhost:8000/docs + - ReDoc: http://localhost:8000/redoc + +### 🪟 Accessing from Windows Host (WSL2 Users) + +If you're running Docker within WSL2 and need to access the application from your Windows host: + + 1. Determine the IP address of your WSL2 instance: + + ```bash + ip addr show eth0 | grep inet + ``` + + Look for an output similar to: + + ```ccp + inet 172.20.39.89/20 brd 172.20.47.255 scope global eth0 + ``` + + 2. Use the extracted IP address to access the application from your Windows browser: + + - Frontend: http://172.20.39.89:3000​ + + - Backend: http://172.20.39.89:8000 + + > Note: WSL2 has a separate network interface, so `localhost` on Windows does not directly map to `localhost` within WSL2. Using the WSL2 IP address bridges this gap. ​ + ## Running Tests To ensure everything is functioning correctly, run the test suite: