diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c5bce54 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,25 @@ +name: CI + +on: + pull_request: + branches: + - main + - dev + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Java + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '17' + cache: maven + + - name: Build and test + run: mvn -B clean verify diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..8692e12 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,88 @@ +name: Release to Maven Central + +on: + push: + branches: + - main + +jobs: + release: + runs-on: ubuntu-latest + + permissions: + contents: write + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Java + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '17' + server-id: central + server-username: CENTRAL_USERNAME + server-password: CENTRAL_PASSWORD + gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }} + gpg-passphrase: GPG_PASSPHRASE + + - name: Configure Git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Get current version and increment + id: version + run: | + CURRENT_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) + echo "current=$CURRENT_VERSION" >> $GITHUB_OUTPUT + + # Remove -SNAPSHOT if present + RELEASE_VERSION=${CURRENT_VERSION%-SNAPSHOT} + echo "release=$RELEASE_VERSION" >> $GITHUB_OUTPUT + + # Calculate next version (increment patch) + IFS='.' read -r major minor patch <<< "$RELEASE_VERSION" + NEXT_VERSION="$major.$minor.$((patch + 1))-SNAPSHOT" + echo "next=$NEXT_VERSION" >> $GITHUB_OUTPUT + + - name: Set release version + run: | + mvn versions:set -DnewVersion=${{ steps.version.outputs.release }} -DgenerateBackupPoms=false + + - name: Build and verify + run: mvn -B clean verify -Prelease + env: + GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + + - name: Deploy to Maven Central + run: mvn -B deploy -Prelease -DskipTests + env: + CENTRAL_USERNAME: ${{ secrets.CENTRAL_USERNAME }} + CENTRAL_PASSWORD: ${{ secrets.CENTRAL_PASSWORD }} + GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + + - name: Create Git tag + run: | + git tag -a "v${{ steps.version.outputs.release }}" -m "Release v${{ steps.version.outputs.release }}" + git push origin "v${{ steps.version.outputs.release }}" + + - name: Create GitHub Release + uses: softprops/action-gh-release@v1 + with: + tag_name: v${{ steps.version.outputs.release }} + name: Release v${{ steps.version.outputs.release }} + generate_release_notes: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set next development version + run: | + mvn versions:set -DnewVersion=${{ steps.version.outputs.next }} -DgenerateBackupPoms=false + git add pom.xml + git commit -m "chore: bump version to ${{ steps.version.outputs.next }} [skip ci]" + git push origin main diff --git a/.github/workflows/snapshot.yml b/.github/workflows/snapshot.yml new file mode 100644 index 0000000..23288cc --- /dev/null +++ b/.github/workflows/snapshot.yml @@ -0,0 +1,31 @@ +name: Dev Build + +on: + push: + branches: + - dev + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Java + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '17' + cache: maven + + - name: Build and test + run: mvn -B clean verify + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: endee-java-client-dev + path: target/*.jar + retention-days: 7 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ab47174 --- /dev/null +++ b/.gitignore @@ -0,0 +1,95 @@ +# ==================== Build Output ==================== +target/ +build/ +out/ +bin/ +*.class +*.jar +*.war +*.ear +*.nar + +# ==================== Maven ==================== +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +pom.xml.next +release.properties +dependency-reduced-pom.xml +buildNumber.properties +.mvn/timing.properties +.mvn/wrapper/maven-wrapper.jar + +# ==================== IDE - IntelliJ IDEA ==================== +.idea/ +*.iml +*.ipr +*.iws +.idea_modules/ +atlassian-ide-plugin.xml + +# ==================== IDE - Eclipse ==================== +.classpath +.project +.settings/ +.metadata/ +*.launch +.loadpath +.recommenders/ +.externalToolBuilders/ + +# ==================== IDE - NetBeans ==================== +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ + +# ==================== IDE - VS Code ==================== +.vscode/ +*.code-workspace + +# ==================== OS Files ==================== +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db +desktop.ini + +# ==================== Logs ==================== +*.log +logs/ + +# ==================== Environment & Secrets ==================== +.env +.env.* +*.env +*.pem +*.key +credentials.json +secrets.json + +# ==================== Test Files (Manual Tests) ==================== +src/test/ + +# ==================== Temporary Files ==================== +*.tmp +*.temp +*.swp +*.swo +*~ +\#*\# + +# ==================== Package Manager ==================== +node_modules/ + +# ==================== Coverage Reports ==================== +coverage/ +*.lcov +jacoco.exec + +# AI Tools +.claude \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..09dfabc --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Endee Labs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE \ No newline at end of file diff --git a/README.md b/README.md index f678f97..8ef72ea 100644 --- a/README.md +++ b/README.md @@ -1 +1,597 @@ -# endee-java-client +# Endee - Java Vector Database Client + +Endee is a Java client for a local vector database designed for maximum speed and efficiency. This package provides type-safe operations, modern Java features, and optimized code for rapid Approximate Nearest Neighbor (ANN) searches on vector data. + +## Key Features + +- **Type Safe**: Full compile-time type checking with builder patterns +- **Fast ANN Searches**: Efficient similarity searches on vector data +- **Multiple Distance Metrics**: Support for cosine, L2, and inner product distance metrics +- **Hybrid Indexes**: Support for dense vectors, sparse vectors, and hybrid (dense + sparse) searches +- **Metadata Support**: Attach and search with metadata and filters +- **High Performance**: HTTP/2, MessagePack serialization, and DEFLATE compression +- **Modern Java**: Requires Java 17+, uses modern APIs + +## Requirements + +- Java 17 or higher +- Endee Local server running (see [Quick Start](https://docs.endee.io/quick-start)) + +## Installation + +### Maven + +```xml + + io.endee + endee-java-client + 1.0.0 + +``` + +### Gradle + +```groovy +implementation 'io.endee:endee-java-client:1.0.0' +``` + +## Quick Start + +### Initialize the Client + +The Endee client connects to your local server (defaults to `http://127.0.0.1:8080/api/v1`): + +```java +import io.endee.client.Endee; +import io.endee.client.Index; +import io.endee.client.types.*; + +// Connect to local Endee server (defaults to localhost:8080) +Endee client = new Endee(); +``` + +**Using Authentication?** If your server has `NDD_AUTH_TOKEN` set, pass the token when initializing: + +```java +// With Auth Token +Endee client = new Endee("auth-token"); +``` + +### Setting a Custom Base URL + +If your server runs on a different port, use `setBaseUrl()`: + +```java +Endee client = new Endee(); + +// Set custom base URL for non-default port +client.setBaseUrl("http://0.0.0.0:8081/api/v1"); +``` + +### Create a Dense Index + +```java +import io.endee.client.types.CreateIndexOptions; +import io.endee.client.types.Precision; +import io.endee.client.types.SpaceType; + +CreateIndexOptions options = CreateIndexOptions.builder("my_vectors", 384) + .spaceType(SpaceType.COSINE) + .precision(Precision.INT8D) + .build(); + +client.createIndex(options); +``` + +**Dense Index Parameters:** + +| Parameter | Description | Default | +| ----------- | ---------------------------------------------------------------------------- | -------- | +| `name` | Unique name for your index (alphanumeric + underscore, max 48 chars) | Required | +| `dimension` | Vector dimensionality (must match your embedding model's output, max 10,000) | Required | +| `spaceType` | Distance metric - `COSINE`, `L2`, or `IP` (inner product) | `COSINE` | +| `m` | Graph connectivity - higher values increase recall but use more memory | 16 | +| `efCon` | Construction-time parameter - higher values improve index quality | 128 | +| `precision` | Quantization precision | `INT8D` | + +### Create a Hybrid Index + +Hybrid indexes combine dense vector search with sparse vector search. Add the `sparseDimension` parameter: + +```java +CreateIndexOptions options = CreateIndexOptions.builder("hybrid_index", 384) + .sparseDimension(30000) // Sparse vector dimension (vocabulary size) + .spaceType(SpaceType.COSINE) + .precision(Precision.INT8D) + .build(); + +client.createIndex(options); +``` + +### List and Access Indexes + +```java +// List all indexes (returns JSON string) +String indexes = client.listIndexes(); + +// Get reference to an existing index +Index index = client.getIndex("my_vectors"); + +// Delete an index +client.deleteIndex("my_vectors"); +``` + +## Upserting Vectors + +The `index.upsert()` method adds or updates vectors in an existing index. + +```java +import io.endee.client.types.VectorItem; +import java.util.List; +import java.util.Map; + +Index index = client.getIndex("my_index"); + +List vectors = List.of( + VectorItem.builder("vec1", new double[] {0.1, 0.2, 0.3 /* ... */}) + .meta(Map.of("title", "First document", "group" , 10)) // meta : {"title" : "First Document", "label" : 10 } + .filter(Map.of("category", "tech", "group" , 10)) // filter : {"category" : "tech" , "group" : 10} + .build(), + + VectorItem.builder("vec2", new double[] {0.3, 0.4, 0.5 /* ... */}) + .meta(Map.of("title", "Second document")) + .filter(Map.of("category", "science")) + .build() +); + +index.upsert(vectors); +``` + +**VectorItem Fields:** + +| Field | Required | Description | +| -------- | -------- | --------------------------------------------------- | +| `id` | Yes | Unique identifier for the vector (non-empty string) | +| `vector` | Yes | Array of doubles representing the embedding | +| `meta` | No | Arbitrary metadata map | +| `filter` | No | Key-value pairs for filtering during queries | + +**Limits:** + +- Maximum 1,000 vectors per upsert call +- Vector dimension must match index dimension +- IDs must be unique within a single upsert batch + +## Querying the Index + +The `index.query()` method performs a similarity search. + +```java +import io.endee.client.types.QueryOptions; +import io.endee.client.types.QueryResult; + +List results = index.query( + QueryOptions.builder() + .vector(new double[] {0.15, 0.25 /* ... */}) + .topK(5) + .ef(128) + .includeVectors(true) + .build() +); + +for (QueryResult item : results) { + System.out.println("ID: " + item.getId()); + System.out.println("Similarity: " + item.getSimilarity()); + System.out.println("Distance: " + item.getDistance()); + System.out.println("Meta: " + item.getMeta()); +} +``` + +**Query Parameters:** + +| Parameter | Description | Default | Max | +| ---------------- | ------------------------------------------------------- | -------- | ---- | +| `vector` | Query vector (must match index dimension) | Required | - | +| `topK` | Number of results to return | 10 | 512 | +| `ef` | Search quality parameter - higher values improve recall | 128 | 1024 | +| `includeVectors` | Include vector data in results | false | - | + +## Filtered Querying + +Use the `filter` parameter to restrict results. All filters are combined with **logical AND**. + +```java +List results = index.query( + QueryOptions.builder() + .vector(new double[] {0.15, 0.25 /* ... */}) + .topK(5) + .filter(List.of( + Map.of("category", Map.of("$eq", "tech")), + Map.of("score", Map.of("$range", List.of(80, 100))) + )) + .build() +); +``` + +### Filtering Operators + +| Operator | Description | Example | +| -------- | ------------------------- | ---------------------------------------------------- | +| `$eq` | Exact match | `Map.of("status", Map.of("$eq", "published"))` | +| `$in` | Match any in list | `Map.of("tags", Map.of("$in", List.of("ai", "ml")))` | +| `$range` | Numeric range (inclusive) | `Map.of("score", Map.of("$range", List.of(70, 95)))` | + +> **Note:** The `$range` operator supports values within **[0 - 999]**. Normalize larger values before upserting. + +## Hybrid Search + +### Upserting Hybrid Vectors + +Provide both dense vectors and sparse representations: + +```java +Index index = client.getIndex("hybrid_index"); + +List vectors = List.of( + VectorItem.builder("doc1", new double[] {0.1, 0.2 /* ... */}) + .sparseIndices(new int[] {10, 50, 200}) // Non-zero term positions + .sparseValues(new double[] {0.8, 0.5, 0.3}) // Weights for each position + .meta(Map.of("title", "Document 1")) + .build(), + + VectorItem.builder("doc2", new double[] {0.3, 0.4 /* ... */}) + .sparseIndices(new int[] {15, 100, 500}) + .sparseValues(new double[] {0.9, 0.4, 0.6}) + .meta(Map.of("title", "Document 2")) + .build() +); + +index.upsert(vectors); +``` + +**Hybrid Vector Fields:** + +| Field | Required | Description | +| --------------- | ------------ | ---------------------------------------- | +| `id` | Yes | Unique identifier | +| `vector` | Yes | Dense embedding vector | +| `sparseIndices` | Yes (hybrid) | Non-zero term positions in sparse vector | +| `sparseValues` | Yes (hybrid) | Weights for each sparse index | +| `meta` | No | Metadata map | +| `filter` | No | Filter fields | + +> **Important:** `sparseIndices` and `sparseValues` must have the same length. Values in `sparseIndices` must be within `[0, sparseDimension)`. + +### Querying Hybrid Index + +Provide both dense and sparse query vectors: + +```java +List results = index.query( + QueryOptions.builder() + .vector(new double[] {0.15, 0.25 /* ... */}) // Dense query + .sparseIndices(new int[] {10, 100, 300}) // Sparse query positions + .sparseValues(new double[] {0.7, 0.5, 0.4}) // Sparse query weights + .topK(5) + .build() +); + +for (QueryResult item : results) { + System.out.println("ID: " + item.getId() + ", Similarity: " + item.getSimilarity()); +} +``` + +You can also query with: + +- **Dense only**: Provide only `vector` +- **Sparse only**: Provide only `sparseIndices` and `sparseValues` +- **Hybrid**: Provide all three for combined results + +## Deletion Methods + +### Delete by ID + +Delete a vector with a specific vector ID. + +```java +index.deleteVector("vec1"); +``` + +### Delete by Filter + +Delete all vectors matching specific filters. + +```java +index.deleteWithFilter(List.of( + Map.of("category", Map.of("$eq", "tech")) +)); +``` + +### Delete Index + +Delete an entire index. + +```java +client.deleteIndex("my_index"); +``` + +> **Warning:** Deletion operations are **irreversible**. + +## Additional Operations + +### Get Vector by ID + +```java +import io.endee.client.types.VectorInfo; + +VectorInfo vector = index.getVector("vec1"); +System.out.println("ID: " + vector.getId()); +System.out.println("Vector: " + Arrays.toString(vector.getVector())); +System.out.println("Meta: " + vector.getMeta()); +System.out.println("Norm: " + vector.getNorm()); +``` + +### Describe Index + +```java +import io.endee.client.types.IndexDescription; + +IndexDescription info = index.describe(); +System.out.println(info); +// IndexDescription{name='my_index', spaceType=COSINE, dimension=384, +// sparseDimension=0, isHybrid=false, count=1000, +// precision=INT8D, m=16} +``` + +### Check if Index is Hybrid + +```java +boolean isHybrid = index.isHybrid(); +``` + +## Precision Options + +Endee supports different quantization precision levels: + +```java +import io.endee.client.types.Precision; + +Precision.BINARY // Binary quantization (1-bit) - smallest storage, fastest search +Precision.INT8D // 8-bit integer quantization (default) - balanced performance +Precision.INT16D // 16-bit integer quantization - higher precision +Precision.FLOAT16 // 16-bit floating point - good balance +Precision.FLOAT32 // 32-bit floating point - highest precision +``` + +**Choosing Precision:** + +| Precision | Use Case | +| --------- | ------------------------------------------------------------------------- | +| `BINARY` | Very large datasets where speed and storage are critical | +| `INT8D` | Recommended for most use cases - good balance of accuracy and performance | +| `INT16D` | Better accuracy than INT8D but less storage than FLOAT32 | +| `FLOAT16` | Good compromise between precision and storage for embeddings | +| `FLOAT32` | Maximum precision when storage is not a concern | + +## Space Types (Distance Metrics) + +```java +import io.endee.client.types.SpaceType; + +SpaceType.COSINE // Cosine similarity (default) - best for normalized embeddings +SpaceType.L2 // Euclidean distance - best for spatial data +SpaceType.IP // Inner product - best for unnormalized embeddings +``` + +## Error Handling + +The client throws specific exceptions for different error scenarios: + +```java +import io.endee.client.exception.EndeeException; +import io.endee.client.exception.EndeeApiException; + +try { + client.createIndex(options); +} catch (EndeeApiException e) { + // API-specific errors (e.g., 400, 401, 404, 409, 500) + System.err.println("Status Code: " + e.getStatusCode()); + System.err.println("Error Body: " + e.getErrorBody()); +} catch (EndeeException e) { + // Client errors (network, serialization, etc.) + System.err.println("Client Error: " + e.getMessage()); +} catch (IllegalArgumentException e) { + // Validation errors + System.err.println("Validation Error: " + e.getMessage()); +} +``` + +**HTTP Status Codes:** + +| Code | Description | +| ---- | ------------------------------------------------ | +| 400 | Bad Request - Invalid parameters | +| 401 | Unauthorized - Invalid or missing authentication | +| 403 | Forbidden - Insufficient permissions | +| 404 | Not Found - Index or vector doesn't exist | +| 409 | Conflict - Index already exists | +| 500 | Internal Server Error | + +## Complete Example + +```java +import io.endee.client.Endee; +import io.endee.client.Index; +import io.endee.client.types.*; + +import java.util.List; +import java.util.Map; + +public class EndeeExample { + public static void main(String[] args) { + // Initialize client + Endee client = new Endee(); + + // Create a dense index + CreateIndexOptions createOptions = CreateIndexOptions.builder("documents", 384) + .spaceType(SpaceType.COSINE) + .precision(Precision.INT8D) + .build(); + + client.createIndex(createOptions); + + // Get the index + Index index = client.getIndex("documents"); + + // Add vectors + List vectors = List.of( + VectorItem.builder("doc1", new double[384]{/*...*/}) // 384 dimensions + .meta(Map.of("title", "First Document")) + .filter(Map.of("category", "tech")) + .build(), + + VectorItem.builder("doc2", new double[384]) + .meta(Map.of("title", "Second Document")) + .filter(Map.of("category", "science")) + .build() + ); + + index.upsert(vectors); + + // Query the index + List results = index.query( + QueryOptions.builder() + .vector(new double[384]) // Query vector + .topK(5) + .build() + ); + + for (QueryResult item : results) { + System.out.println("ID: " + item.getId() + ", Similarity: " + item.getSimilarity()); + } + + // Describe the index + IndexDescription description = index.describe(); + System.out.println(description); + + // Clean up + client.deleteIndex("documents"); + } +} +``` + +## API Reference + +### Endee Class + +| Method | Parameters | Return Type | Description | +| --------------------------------- | ---------- | ----------- | ----------------------------- | +| `Endee()` | - | - | Create client without auth | +| `Endee(String token)` | `token` | - | Create client with auth token | +| `setBaseUrl(String url)` | `url` | `String` | Set custom base URL | +| `createIndex(CreateIndexOptions)` | `options` | `String` | Create a new index | +| `listIndexes()` | - | `String` | List all indexes (JSON) | +| `deleteIndex(String name)` | `name` | `String` | Delete an index | +| `getIndex(String name)` | `name` | `Index` | Get reference to an index | + +### Index Class + +| Method | Parameters | Return Type | Description | +| ----------------------------- | ---------- | ------------------- | -------------------------- | +| `upsert(List)` | `vectors` | `String` | Insert or update vectors | +| `query(QueryOptions)` | `options` | `List` | Search for similar vectors | +| `deleteVector(String id)` | `id` | `String` | Delete a vector by ID | +| `deleteWithFilter(List)` | `filter` | `String` | Delete vectors by filter | +| `getVector(String id)` | `id` | `VectorInfo` | Get a vector by ID | +| `describe()` | - | `IndexDescription` | Get index metadata | +| `isHybrid()` | - | `boolean` | Check if index is hybrid | + +### Builder Classes + +#### CreateIndexOptions.Builder + +```java +CreateIndexOptions.builder(String name, int dimension) + .spaceType(SpaceType) // Default: COSINE + .m(int) // Default: 16 + .efCon(int) // Default: 128 + .precision(Precision) // Default: INT8D + .sparseDimension(Integer) // Optional, for hybrid indexes + .build() +``` + +#### QueryOptions.Builder + +```java +QueryOptions.builder() + .vector(double[]) // Required for dense search + .topK(int) // Required + .ef(int) // Default: 128 + .filter(List>) // Optional + .includeVectors(boolean) // Default: false + .sparseIndices(int[]) // Optional, for hybrid search + .sparseValues(double[]) // Optional, for hybrid search + .build() +``` + +#### VectorItem.Builder + +```java +VectorItem.builder(String id, double[] vector) + .meta(Map) // Optional + .filter(Map) // Optional + .sparseIndices(int[]) // Optional, for hybrid + .sparseValues(double[]) // Optional, for hybrid + .build() +``` + +## Data Types + +### QueryResult + +| Field | Type | Description | +| ------------ | --------------------- | -------------------------- | +| `id` | `String` | Vector identifier | +| `similarity` | `double` | Similarity score | +| `distance` | `double` | Distance (1 - similarity) | +| `meta` | `Map` | Metadata | +| `filter` | `Map` | Filter values | +| `norm` | `double` | Normalization factor | +| `vector` | `double[]` | Vector data (if requested) | + +### VectorInfo + +| Field | Type | Description | +| -------- | --------------------- | -------------------- | +| `id` | `String` | Vector identifier | +| `vector` | `double[]` | Vector data | +| `meta` | `Map` | Metadata | +| `filter` | `Map` | Filter values | +| `norm` | `double` | Normalization factor | + +### IndexDescription + +| Field | Type | Description | +| ----------------- | ----------- | ------------------------------------ | +| `name` | `String` | Index name | +| `spaceType` | `SpaceType` | Distance metric | +| `dimension` | `int` | Vector dimension | +| `sparseDimension` | `int` | Sparse vector dimension | +| `isHybrid` | `boolean` | Whether index supports hybrid search | +| `count` | `long` | Number of vectors | +| `precision` | `Precision` | Quantization precision | +| `m` | `int` | Graph connectivity | + +## Dependencies + +- Jackson (JSON serialization) +- MessagePack (Binary serialization) +- SLF4J (Logging) + +## License + +MIT + +## Author + +Pankaj Singh diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..cca5e72 --- /dev/null +++ b/pom.xml @@ -0,0 +1,204 @@ + + + 4.0.0 + + io.endee + endee-java-client + 0.1.1-SNAPSHOT + jar + + Endee Java Client + Java client library for Endee vector database + + https://github.com/EndeeLabs/endee-java-client + + + scm:git:https://github.com/EndeeLabs/endee-java-client.git + scm:git:ssh://git@github.com:EndeeLabs/endee-java-client.git + https://github.com/EndeeLabs/endee-java-client + HEAD + + + + + + UTF-8 + 17 + 17 + 2.17.0 + 0.9.8 + 2.0.12 + 5.10.2 + + + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + + + org.msgpack + msgpack-core + ${msgpack.version} + + + + + org.slf4j + slf4j-api + ${slf4j.version} + + + + + org.junit.jupiter + junit-jupiter + ${junit.version} + test + + + org.slf4j + slf4j-simple + ${slf4j.version} + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.12.1 + + 17 + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.2.5 + + + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + + io.endee.client.ManualTest + test + + + + + org.apache.maven.plugins + maven-source-plugin + 3.3.0 + + + attach-sources + + jar + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.6.3 + + false + UTF-8 + UTF-8 + UTF-8 + + + + attach-javadocs + + jar + + + + + + + + + + release + + + + org.apache.maven.plugins + maven-gpg-plugin + 3.1.0 + + + --pinentry-mode + loopback + + + + + sign-artifacts + verify + + sign + + + + + + org.sonatype.central + central-publishing-maven-plugin + 0.6.0 + true + + central + true + + + + + + + + + + central-snapshots + https://s01.oss.sonatype.org/content/repositories/snapshots/ + + + central + https://central.sonatype.com/api/v1/publisher + + + + + + pankaj + Pankaj Singh + pankaj@endee.io + Endee Labs + https://endee.io + + + + + + + MIT License + https://opensource.org/licenses/MIT + repo + + + diff --git a/src/main/java/io/endee/client/Endee.java b/src/main/java/io/endee/client/Endee.java new file mode 100644 index 0000000..901b7a7 --- /dev/null +++ b/src/main/java/io/endee/client/Endee.java @@ -0,0 +1,288 @@ +package io.endee.client; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.endee.client.exception.EndeeApiException; +import io.endee.client.exception.EndeeException; +import io.endee.client.types.CreateIndexOptions; +import io.endee.client.types.IndexInfo; +import io.endee.client.types.Precision; +import io.endee.client.types.SpaceType; +import io.endee.client.util.JsonUtils; +import io.endee.client.util.ValidationUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.*; + +/** + * Main Endee client for Endee-DB. + * + *

+ * Example usage: + *

+ * + *
{@code
+ * Endee client = new Endee("auth-token");
+ *
+ * // Create an index
+ * CreateIndexOptions options = CreateIndexOptions.builder("my_index", 128)
+ *         .spaceType(SpaceType.COSINE)
+ *         .precision(Precision.INT8D)
+ *         .build();
+ * client.createIndex(options);
+ *
+ * // Get an index and perform operations
+ * Index index = client.getIndex("my_index");
+ * }
+ */ +public class Endee { + private static final Logger logger = LoggerFactory.getLogger(Endee.class); + private static final Duration DEFAULT_TIMEOUT = Duration.ofSeconds(30); + private static final int MAX_DIMENSION = 10000; + + private String token; + private String baseUrl; + private final int version; + private final HttpClient httpClient; + private final ObjectMapper objectMapper; + + /** + * Creates a new Endee client without authentication. + * Uses local server at http://127.0.0.1:8080/api/v1 + */ + public Endee() { + this(null); + this.baseUrl = "http://127.0.0.1:8080/api/v1"; + } + + /** + * Creates a new Endee client. + * + * @param token the Auth token (optional) + */ + public Endee(String token) { + this.token = token; + this.baseUrl = "http://127.0.0.1:8080/api/v1"; + this.version = 1; + this.objectMapper = new ObjectMapper(); + + if (token != null && !token.isEmpty()) { + String[] tokenParts = token.split(":"); + if (tokenParts.length > 2) { + this.baseUrl = "https://" + tokenParts[2] + ".endee.io/api/v1"; + this.token = tokenParts[0] + ":" + tokenParts[1]; + } + } + + this.httpClient = HttpClient.newBuilder() + .version(HttpClient.Version.HTTP_2) + .connectTimeout(DEFAULT_TIMEOUT) + .build(); + } + + /** + * Sets a custom base URL for the API. + * + * @param url the base URL + * @return the URL that was set + */ + public String setBaseUrl(String url) { + this.baseUrl = url; + return url; + } + + /** + * Creates a new index. + * + * @param options the index creation options + * @return success message + * @throws EndeeException if the operation fails + */ + public String createIndex(CreateIndexOptions options) { + if (!ValidationUtils.isValidIndexName(options.getName())) { + throw new IllegalArgumentException( + "Invalid index name. Index name must be alphanumeric and can contain underscores and less than 48 characters"); + } + if (options.getDimension() > MAX_DIMENSION) { + throw new IllegalArgumentException("Dimension cannot be greater than " + MAX_DIMENSION); + } + if (options.getSparseDimension() != null && options.getSparseDimension() < 0) { + throw new IllegalArgumentException("Sparse dimension cannot be less than 0"); + } + + String normalizedSpaceType = options.getSpaceType().getValue().toLowerCase(); + if (!List.of("cosine", "l2", "ip").contains(normalizedSpaceType)) { + throw new IllegalArgumentException("Invalid space type: " + options.getSpaceType()); + } + + Map data = new HashMap<>(); + data.put("index_name", options.getName()); + data.put("dim", options.getDimension()); + data.put("space_type", normalizedSpaceType); + data.put("M", options.getM()); + data.put("ef_con", options.getEfCon()); + data.put("checksum", -1); + data.put("precision", options.getPrecision().getValue()); + + if (options.getSparseDimension() != null) { + data.put("sparse_dim", options.getSparseDimension()); + } + if (options.getVersion() != null) { + data.put("version", options.getVersion()); + } + + try { + HttpRequest request = buildPostRequest("/index/create", data); + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() != 200) { + logger.error("Error: {}", response.body()); + EndeeApiException.raiseException(response.statusCode(), response.body()); + } + + return "Index created successfully"; + } catch (IOException | InterruptedException e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new EndeeException("Failed to create index", e); + } + } + + /** + * Lists all indexes. + * + * @return list of index information + * @throws EndeeException if the operation fails + */ + public String listIndexes() { + try { + HttpRequest request = buildGetRequest("/index/list"); + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + + return response.body(); + } catch (IOException | InterruptedException e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new EndeeException("Failed to list indexes", e); + } + } + + /** + * Deletes an index. + * + * @param name the index name to delete + * @return success message + * @throws EndeeException if the operation fails + */ + public String deleteIndex(String name) { + try { + HttpRequest request = buildDeleteRequest("/index/" + name + "/delete"); + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() != 200) { + logger.error("Error: {}", response.body()); + EndeeApiException.raiseException(response.statusCode(), response.body()); + } + + return "Index " + name + " deleted successfully"; + } catch (IOException | InterruptedException e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new EndeeException("Failed to delete index", e); + } + } + + /** + * Gets an index by name. + * + * @param name the index name + * @return the Index object for performing vector operations + * @throws EndeeException if the operation fails + */ + public Index getIndex(String name) { + try { + HttpRequest request = buildGetRequest("/index/" + name + "/info"); + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() != 200) { + EndeeApiException.raiseException(response.statusCode(), response.body()); + } + + JsonNode data = objectMapper.readTree(response.body()); + + IndexInfo indexInfo = new IndexInfo(); + indexInfo.setSpaceType(SpaceType.fromValue(data.get("space_type").asText())); + indexInfo.setDimension(data.get("dimension").asInt()); + indexInfo.setTotalElements(data.get("total_elements").asLong()); + indexInfo.setPrecision(Precision.fromValue(data.get("precision").asText())); + indexInfo.setM(data.get("M").asInt()); + indexInfo.setChecksum(data.get("checksum").asLong()); + + if (data.has("version") && !data.get("version").isNull()) { + indexInfo.setVersion(data.get("version").asInt()); + } + if (data.has("sparse_dim") && !data.get("sparse_dim").isNull()) { + indexInfo.setSparseDimension(data.get("sparse_dim").asInt()); + } + + return new Index(name, token, baseUrl, version, indexInfo); + } catch (IOException | InterruptedException e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new EndeeException("Failed to get index", e); + } + } + + private HttpRequest buildGetRequest(String path) { + HttpRequest.Builder builder = HttpRequest.newBuilder() + .uri(URI.create(baseUrl + path)) + .header("Content-Type", "application/json") + .timeout(DEFAULT_TIMEOUT) + .GET(); + + if (token != null && !token.isEmpty()) { + builder.header("Authorization", token); + } + + return builder.build(); + } + + private HttpRequest buildPostRequest(String path, Map data) { + String json = JsonUtils.toJson(data); + HttpRequest.Builder builder = HttpRequest.newBuilder() + .uri(URI.create(baseUrl + path)) + .header("Content-Type", "application/json") + .timeout(DEFAULT_TIMEOUT) + .POST(HttpRequest.BodyPublishers.ofString(json)); + + if (token != null && !token.isEmpty()) { + builder.header("Authorization", token); + } + + return builder.build(); + } + + private HttpRequest buildDeleteRequest(String path) { + HttpRequest.Builder builder = HttpRequest.newBuilder() + .uri(URI.create(baseUrl + path)) + .timeout(DEFAULT_TIMEOUT) + .DELETE(); + + if (token != null && !token.isEmpty()) { + builder.header("Authorization", token); + } + + return builder.build(); + } +} diff --git a/src/main/java/io/endee/client/Index.java b/src/main/java/io/endee/client/Index.java new file mode 100644 index 0000000..4335be2 --- /dev/null +++ b/src/main/java/io/endee/client/Index.java @@ -0,0 +1,490 @@ +package io.endee.client; + +import io.endee.client.exception.EndeeApiException; +import io.endee.client.exception.EndeeException; +import io.endee.client.types.*; +import io.endee.client.util.CryptoUtils; +import io.endee.client.util.JsonUtils; +import io.endee.client.util.MessagePackUtils; +import io.endee.client.util.ValidationUtils; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.*; +import java.util.stream.Collectors; + +/** + * Index client for Endee-DB vector operations. + * + *

+ * Example usage: + *

+ * + *
{@code
+ * Index index = client.getIndex("my_index");
+ *
+ * // Upsert vectors
+ * List vectors = List.of(
+ *         VectorItem.builder("vec1", new double[] { 0.1, 0.2, 0.3 })
+ *                 .meta(Map.of("label", "example"))
+ *                 .build());
+ * index.upsert(vectors);
+ *
+ * // Query
+ * List results = index.query(
+ *         QueryOptions.builder()
+ *                 .vector(new double[] { 0.1, 0.2, 0.3 })
+ *                 .topK(10)
+ *                 .build());
+ * }
+ */ +public class Index { + private static final Duration DEFAULT_TIMEOUT = Duration.ofSeconds(30); + private static final int MAX_BATCH_SIZE = 1000; + private static final int MAX_TOP_K = 512; + private static final int MAX_EF = 1024; + + private final String name; + private final String token; + private final String url; + private final HttpClient httpClient; + + private long count; + private SpaceType spaceType; + private int dimension; + private Precision precision; + private int m; + private int sparseDimension; + + /** + * Creates a new Index instance. + */ + public Index(String name, String token, String url, int version, IndexInfo params) { + this.name = name; + this.token = token; + this.url = url; + + this.count = params != null ? params.getTotalElements() : 0; + this.spaceType = params != null && params.getSpaceType() != null ? params.getSpaceType() : SpaceType.COSINE; + this.dimension = params != null ? params.getDimension() : 0; + this.precision = params != null && params.getPrecision() != null ? params.getPrecision() : Precision.INT8D; + this.m = params != null ? params.getM() : 16; + this.sparseDimension = params != null && params.getSparseDimension() != null ? params.getSparseDimension() : 0; + + this.httpClient = HttpClient.newBuilder() + .version(HttpClient.Version.HTTP_2) + .connectTimeout(DEFAULT_TIMEOUT) + .build(); + } + + @Override + public String toString() { + return name; + } + + /** + * Checks if this index supports hybrid (sparse + dense) vectors. + */ + public boolean isHybrid() { + return sparseDimension > 0; + } + + /** + * Normalizes a vector for cosine similarity. + * Returns [normalizedVector, norm]. + */ + private double[][] normalizeVector(double[] vector) { + if (vector.length != dimension) { + throw new IllegalArgumentException( + "Vector dimension mismatch: expected " + dimension + ", got " + vector.length); + } + + if (spaceType != SpaceType.COSINE) { + return new double[][] { vector, { 1.0 } }; + } + + double sumSquares = 0; + for (double v : vector) { + sumSquares += v * v; + } + double norm = Math.sqrt(sumSquares); + + if (norm == 0) { + return new double[][] { vector, { 1.0 } }; + } + + double[] normalized = new double[vector.length]; + for (int i = 0; i < vector.length; i++) { + normalized[i] = vector[i] / norm; + } + + return new double[][] { normalized, { norm } }; + } + + /** + * Upserts vectors into the index. + * + * @param inputArray list of vector items to upsert + * @return success message + */ + public String upsert(List inputArray) { + if (inputArray.size() > MAX_BATCH_SIZE) { + throw new IllegalArgumentException("Cannot insert more than " + MAX_BATCH_SIZE + " vectors at a time"); + } + + List ids = inputArray.stream() + .map(item -> item.getId() != null ? item.getId() : "") + .collect(Collectors.toList()); + ValidationUtils.validateVectorIds(ids); + + List vectorBatch = new ArrayList<>(); + + for (VectorItem item : inputArray) { + double[][] result = normalizeVector(item.getVector()); + double[] normalizedVector = result[0]; + double norm = result[1][0]; + + byte[] metaData = CryptoUtils.jsonZip(item.getMeta() != null ? item.getMeta() : Map.of()); + + int[] sparseIndices = item.getSparseIndices() != null ? item.getSparseIndices() : new int[0]; + double[] sparseValues = item.getSparseValues() != null ? item.getSparseValues() : new double[0]; + + if (!isHybrid() && (sparseIndices.length > 0 || sparseValues.length > 0)) { + throw new IllegalArgumentException( + "Cannot insert sparse data into a dense-only index. Create index with sparseDimension > 0 for hybrid support."); + } + + if (isHybrid()) { + if (sparseIndices.length == 0 || sparseValues.length == 0) { + throw new IllegalArgumentException( + "Both sparse_indices and sparse_values must be provided for hybrid vectors."); + } + if (sparseIndices.length != sparseValues.length) { + throw new IllegalArgumentException( + "sparseIndices and sparseValues must have the same length. Got " + + sparseIndices.length + " indices and " + sparseValues.length + " values."); + } + for (int idx : sparseIndices) { + if (idx < 0 || idx >= sparseDimension) { + throw new IllegalArgumentException( + "Sparse index " + idx + " is out of bounds. Must be in range [0," + sparseDimension + + ")."); + } + } + } + + String filterJson = JsonUtils.toJson(item.getFilter() != null ? item.getFilter() : Map.of()); + + if (isHybrid()) { + vectorBatch.add(new Object[] { item.getId(), metaData, filterJson, norm, normalizedVector, + sparseIndices, sparseValues }); + } else { + vectorBatch.add(new Object[] { item.getId(), metaData, filterJson, norm, normalizedVector }); + } + } + + byte[] serializedData = MessagePackUtils.packVectors(vectorBatch); + + try { + HttpRequest request = buildPostMsgpackRequest("/index/" + name + "/vector/insert", serializedData); + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray()); + + if (response.statusCode() != 200) { + EndeeApiException.raiseException(response.statusCode(), new String(response.body())); + } + + return "Vectors inserted successfully"; + } catch (IOException | InterruptedException e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new EndeeException("Failed to upsert vectors", e); + } + } + + /** + * Queries the index for similar vectors. + * + * @param options the query options + * @return list of query results + */ + public List query(QueryOptions options) { + if (options.getTopK() > MAX_TOP_K || options.getTopK() < 0) { + throw new IllegalArgumentException("top_k cannot be greater than " + MAX_TOP_K + " and less than 0"); + } + if (options.getEf() > MAX_EF) { + throw new IllegalArgumentException("ef search cannot be greater than " + MAX_EF); + } + + boolean hasSparse = options.getSparseIndices() != null && options.getSparseIndices().length > 0 + && options.getSparseValues() != null && options.getSparseValues().length > 0; + boolean hasDense = options.getVector() != null; + + if (!hasDense && !hasSparse) { + throw new IllegalArgumentException( + "At least one of 'vector' or 'sparseIndices'/'sparseValues' must be provided."); + } + + if (hasSparse && !isHybrid()) { + throw new IllegalArgumentException( + "Cannot perform sparse search on a dense-only index. Create index with sparseDimension > 0 for hybrid support."); + } + + if (hasSparse && options.getSparseIndices().length != options.getSparseValues().length) { + throw new IllegalArgumentException( + "sparseIndices and sparseValues must have the same length."); + } + + Map data = new HashMap<>(); + data.put("k", options.getTopK()); + data.put("ef", options.getEf()); + data.put("include_vectors", options.isIncludeVectors()); + + if (hasDense) { + double[][] result = normalizeVector(options.getVector()); + data.put("vector", result[0]); + } + + if (hasSparse) { + data.put("sparse_indices", options.getSparseIndices()); + data.put("sparse_values", options.getSparseValues()); + } + + if (options.getFilter() != null) { + data.put("filter", JsonUtils.toJson(options.getFilter())); + } + + try { + String jsonBody = JsonUtils.toJson(data); + HttpRequest request = buildPostJsonRequest("/index/" + name + "/search", jsonBody); + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray()); + + if (response.statusCode() != 200) { + EndeeApiException.raiseException(response.statusCode(), new String(response.body())); + } + + List decoded = MessagePackUtils.unpackQueryResults(response.body()); + List results = new ArrayList<>(); + + for (Object[] tuple : decoded) { + double similarity = (Double) tuple[0]; + String vectorId = (String) tuple[1]; + byte[] metaData = (byte[]) tuple[2]; + String filterStr = (String) tuple[3]; + double normValue = (Double) tuple[4]; + + Map meta = CryptoUtils.jsonUnzip(metaData); + + QueryResult result = new QueryResult(); + result.setId(vectorId); + result.setSimilarity(similarity); + result.setDistance(1 - similarity); + result.setMeta(meta); + result.setNorm(normValue); + + if (filterStr != null && !filterStr.isEmpty() && !filterStr.equals("{}")) { + @SuppressWarnings("unchecked") + Map parsedFilter = JsonUtils.fromJson(filterStr, Map.class); + result.setFilter(parsedFilter); + } + + if (options.isIncludeVectors() && tuple.length > 5) { + result.setVector((double[]) tuple[5]); + } + + results.add(result); + } + + return results; + } catch (IOException | InterruptedException e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new EndeeException("Failed to query index", e); + } + } + + /** + * Deletes a vector by ID. + * + * @param id the vector ID to delete + * @return success message + */ + public String deleteVector(String id) { + try { + HttpRequest request = buildDeleteRequest("/index/" + name + "/vector/" + id + "/delete"); + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() != 200) { + EndeeApiException.raiseException(response.statusCode(), response.body()); + } + + return response.body(); + } catch (IOException | InterruptedException e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new EndeeException("Failed to delete vector", e); + } + } + + /** + * Deletes vectors matching a filter. + * + * @param filter the filter criteria + * @return the API response + */ + public String deleteWithFilter(List> filter) { + try { + Map data = Map.of("filter", filter); + String jsonBody = JsonUtils.toJson(data); + + HttpRequest request = buildDeleteJsonRequest("/index/" + name + "/vectors/delete", jsonBody); + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() != 200) { + EndeeApiException.raiseException(response.statusCode(), response.body()); + } + + return response.body(); + } catch (IOException | InterruptedException e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new EndeeException("Failed to delete vectors with filter", e); + } + } + + /** + * Gets a vector by ID. + * + * @param id the vector ID + * @return the vector information + */ + public VectorInfo getVector(String id) { + try { + Map data = Map.of("id", id); + String jsonBody = JsonUtils.toJson(data); + + HttpRequest request = buildPostJsonRequest("/index/" + name + "/vector/get", jsonBody); + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray()); + + if (response.statusCode() != 200) { + EndeeApiException.raiseException(response.statusCode(), new String(response.body())); + } + + Object[] vectorObj = MessagePackUtils.unpackVector(response.body()); + + VectorInfo info = new VectorInfo(); + info.setId((String) vectorObj[0]); + info.setMeta(CryptoUtils.jsonUnzip((byte[]) vectorObj[1])); + + String filterStr = (String) vectorObj[2]; + if (filterStr != null && !filterStr.isEmpty() && !filterStr.equals("{}")) { + @SuppressWarnings("unchecked") + Map parsedFilter = JsonUtils.fromJson(filterStr, Map.class); + info.setFilter(parsedFilter); + } + + info.setNorm((Double) vectorObj[3]); + info.setVector((double[]) vectorObj[4]); + + return info; + } catch (IOException | InterruptedException e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new EndeeException("Failed to get vector", e); + } + } + + /** + * Returns a description of this index. + * + * @return the index description + */ + public IndexDescription describe() { + return new IndexDescription( + name, + spaceType, + dimension, + sparseDimension, + isHybrid(), + count, + precision, + m); + } + + // ==================== HTTP Request Helper Methods ==================== + + /** + * Builds a POST request with JSON body. + */ + private HttpRequest buildPostJsonRequest(String path, String jsonBody) { + HttpRequest.Builder builder = HttpRequest.newBuilder() + .uri(URI.create(url + path)) + .header("Content-Type", "application/json") + .timeout(DEFAULT_TIMEOUT) + .POST(HttpRequest.BodyPublishers.ofString(jsonBody)); + + if (token != null && !token.isBlank()) { + builder.header("Authorization", token); + } + + return builder.build(); + } + + /** + * Builds a POST request with MessagePack body. + */ + private HttpRequest buildPostMsgpackRequest(String path, byte[] body) { + HttpRequest.Builder builder = HttpRequest.newBuilder() + .uri(URI.create(url + path)) + .header("Content-Type", "application/msgpack") + .timeout(DEFAULT_TIMEOUT) + .POST(HttpRequest.BodyPublishers.ofByteArray(body)); + + if (token != null && !token.isBlank()) { + builder.header("Authorization", token); + } + + return builder.build(); + } + + /** + * Builds a DELETE request. + */ + private HttpRequest buildDeleteRequest(String path) { + HttpRequest.Builder builder = HttpRequest.newBuilder() + .uri(URI.create(url + path)) + .timeout(DEFAULT_TIMEOUT) + .DELETE(); + + if (token != null && !token.isBlank()) { + builder.header("Authorization", token); + } + + return builder.build(); + } + + /** + * Builds a DELETE request with JSON body. + */ + private HttpRequest buildDeleteJsonRequest(String path, String jsonBody) { + HttpRequest.Builder builder = HttpRequest.newBuilder() + .uri(URI.create(url + path)) + .header("Content-Type", "application/json") + .timeout(DEFAULT_TIMEOUT) + .method("DELETE", HttpRequest.BodyPublishers.ofString(jsonBody)); + + if (token != null && !token.isBlank()) { + builder.header("Authorization", token); + } + + return builder.build(); + } +} diff --git a/src/main/java/io/endee/client/exception/EndeeApiException.java b/src/main/java/io/endee/client/exception/EndeeApiException.java new file mode 100644 index 0000000..1a6ab68 --- /dev/null +++ b/src/main/java/io/endee/client/exception/EndeeApiException.java @@ -0,0 +1,40 @@ +package io.endee.client.exception; + +/** + * Exception thrown when the Endee API returns an error response. + */ +public class EndeeApiException extends EndeeException { + + private final int statusCode; + private final String errorBody; + + public EndeeApiException(String message, int statusCode, String errorBody) { + super(message); + this.statusCode = statusCode; + this.errorBody = errorBody; + } + + public int getStatusCode() { + return statusCode; + } + + public String getErrorBody() { + return errorBody; + } + + /** + * Raises the appropriate exception based on status code. + */ + public static void raiseException(int statusCode, String errorBody) { + String message = switch (statusCode) { + case 400 -> "Bad Request: " + errorBody; + case 401 -> "Unauthorized: " + errorBody; + case 403 -> "Forbidden: " + errorBody; + case 404 -> "Not Found: " + errorBody; + case 409 -> "Conflict: " + errorBody; + case 500 -> "Internal Server Error: " + errorBody; + default -> "API Error (" + statusCode + "): " + errorBody; + }; + throw new EndeeApiException(message, statusCode, errorBody); + } +} diff --git a/src/main/java/io/endee/client/exception/EndeeException.java b/src/main/java/io/endee/client/exception/EndeeException.java new file mode 100644 index 0000000..ea56c96 --- /dev/null +++ b/src/main/java/io/endee/client/exception/EndeeException.java @@ -0,0 +1,19 @@ +package io.endee.client.exception; + +/** + * Base exception for all Endee client errors. + */ +public class EndeeException extends RuntimeException { + + public EndeeException(String message) { + super(message); + } + + public EndeeException(String message, Throwable cause) { + super(message, cause); + } + + public EndeeException(Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/io/endee/client/types/CreateIndexOptions.java b/src/main/java/io/endee/client/types/CreateIndexOptions.java new file mode 100644 index 0000000..30ac60c --- /dev/null +++ b/src/main/java/io/endee/client/types/CreateIndexOptions.java @@ -0,0 +1,75 @@ +package io.endee.client.types; + +/** + * Options for creating an Endee index. + */ +public class CreateIndexOptions { + private final String name; + private final int dimension; + private SpaceType spaceType = SpaceType.COSINE; + private int m = 16; + private int efCon = 128; + private Precision precision = Precision.INT8D; + private Integer version = null; + private Integer sparseDimension = null; + + private CreateIndexOptions(String name, int dimension) { + this.name = name; + this.dimension = dimension; + } + + public static Builder builder(String name, int dimension) { + return new Builder(name, dimension); + } + + public String getName() { return name; } + public int getDimension() { return dimension; } + public SpaceType getSpaceType() { return spaceType; } + public int getM() { return m; } + public int getEfCon() { return efCon; } + public Precision getPrecision() { return precision; } + public Integer getVersion() { return version; } + public Integer getSparseDimension() { return sparseDimension; } + + public static class Builder { + private final CreateIndexOptions options; + + private Builder(String name, int dimension) { + this.options = new CreateIndexOptions(name, dimension); + } + + public Builder spaceType(SpaceType spaceType) { + options.spaceType = spaceType; + return this; + } + + public Builder m(int m) { + options.m = m; + return this; + } + + public Builder efCon(int efCon) { + options.efCon = efCon; + return this; + } + + public Builder precision(Precision precision) { + options.precision = precision; + return this; + } + + public Builder version(Integer version) { + options.version = version; + return this; + } + + public Builder sparseDimension(Integer sparseDimension) { + options.sparseDimension = sparseDimension; + return this; + } + + public CreateIndexOptions build() { + return options; + } + } +} diff --git a/src/main/java/io/endee/client/types/IndexDescription.java b/src/main/java/io/endee/client/types/IndexDescription.java new file mode 100644 index 0000000..045b96d --- /dev/null +++ b/src/main/java/io/endee/client/types/IndexDescription.java @@ -0,0 +1,43 @@ +package io.endee.client.types; + +/** + * Description of an Endee index. + */ +public class IndexDescription { + private final String name; + private final SpaceType spaceType; + private final int dimension; + private final int sparseDimension; + private final boolean isHybrid; + private final long count; + private final Precision precision; + private final int m; + + public IndexDescription(String name, SpaceType spaceType, int dimension, + int sparseDimension, boolean isHybrid, long count, + Precision precision, int m) { + this.name = name; + this.spaceType = spaceType; + this.dimension = dimension; + this.sparseDimension = sparseDimension; + this.isHybrid = isHybrid; + this.count = count; + this.precision = precision; + this.m = m; + } + + public String getName() { return name; } + public SpaceType getSpaceType() { return spaceType; } + public int getDimension() { return dimension; } + public int getSparseDimension() { return sparseDimension; } + public boolean isHybrid() { return isHybrid; } + public long getCount() { return count; } + public Precision getPrecision() { return precision; } + public int getM() { return m; } + + @Override + public String toString() { + return "{name='" + name + "', spaceType= " + spaceType + + ", dimension=" + dimension + ", precision=" + precision + ", count=" + count + ", isHybrid=" + isHybrid +", sparseDimension=" + sparseDimension + ", M=" + m +"}"; + } +} diff --git a/src/main/java/io/endee/client/types/IndexInfo.java b/src/main/java/io/endee/client/types/IndexInfo.java new file mode 100644 index 0000000..f732c44 --- /dev/null +++ b/src/main/java/io/endee/client/types/IndexInfo.java @@ -0,0 +1,45 @@ +package io.endee.client.types; + +/** + * Information about an Endee index from the server. + */ +public class IndexInfo { + private String name; + private SpaceType spaceType; + private int dimension; + private long totalElements; + private Precision precision; + private int m; + private long checksum; + private Integer version; + private Integer sparseDimension; + + public IndexInfo() {} + + public String getName() { return name; } + public void setName(String name) { this.name = name; } + + public SpaceType getSpaceType() { return spaceType; } + public void setSpaceType(SpaceType spaceType) { this.spaceType = spaceType; } + + public int getDimension() { return dimension; } + public void setDimension(int dimension) { this.dimension = dimension; } + + public long getTotalElements() { return totalElements; } + public void setTotalElements(long totalElements) { this.totalElements = totalElements; } + + public Precision getPrecision() { return precision; } + public void setPrecision(Precision precision) { this.precision = precision; } + + public int getM() { return m; } + public void setM(int m) { this.m = m; } + + public long getChecksum() { return checksum; } + public void setChecksum(long checksum) { this.checksum = checksum; } + + public Integer getVersion() { return version; } + public void setVersion(Integer version) { this.version = version; } + + public Integer getSparseDimension() { return sparseDimension; } + public void setSparseDimension(Integer sparseDimension) { this.sparseDimension = sparseDimension; } +} diff --git a/src/main/java/io/endee/client/types/Precision.java b/src/main/java/io/endee/client/types/Precision.java new file mode 100644 index 0000000..1f0569f --- /dev/null +++ b/src/main/java/io/endee/client/types/Precision.java @@ -0,0 +1,36 @@ +package io.endee.client.types; + +/** + * Precision types for vector quantization. + */ +public enum Precision { + BINARY("binary"), + INT8D("int8d"), + INT16D("int16d"), + FLOAT32("float32"), + FLOAT16("float16"); + + private final String value; + + Precision(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + public static Precision fromValue(String value) { + for (Precision p : values()) { + if (p.value.equalsIgnoreCase(value)) { + return p; + } + } + throw new IllegalArgumentException("Unknown precision: " + value); + } + + @Override + public String toString() { + return value; + } +} diff --git a/src/main/java/io/endee/client/types/QueryOptions.java b/src/main/java/io/endee/client/types/QueryOptions.java new file mode 100644 index 0000000..002d70d --- /dev/null +++ b/src/main/java/io/endee/client/types/QueryOptions.java @@ -0,0 +1,93 @@ +package io.endee.client.types; + +import java.util.List; +import java.util.Map; + +/** + * Options for querying an Endee index. + * + *

Example usage with filters:

+ *
{@code
+ * QueryOptions options = QueryOptions.builder()
+ *     .vector(new double[]{0.1, 0.2, 0.3})
+ *     .topK(10)
+ *     .filter(List.of(
+ *         Map.of("category", Map.of("$eq", "tech")),
+ *         Map.of("score", Map.of("$range", List.of(80, 100)))
+ *     ))
+ *     .build();
+ * }
+ */ +public class QueryOptions { + private double[] vector; + private int topK; + private List> filter; + private int ef = 128; + private boolean includeVectors = false; + private int[] sparseIndices; + private double[] sparseValues; + + private QueryOptions() {} + + public static Builder builder() { + return new Builder(); + } + + public double[] getVector() { return vector; } + public int getTopK() { return topK; } + public List> getFilter() { return filter; } + public int getEf() { return ef; } + public boolean isIncludeVectors() { return includeVectors; } + public int[] getSparseIndices() { return sparseIndices; } + public double[] getSparseValues() { return sparseValues; } + + public static class Builder { + private final QueryOptions options = new QueryOptions(); + + public Builder vector(double[] vector) { + options.vector = vector; + return this; + } + + public Builder topK(int topK) { + options.topK = topK; + return this; + } + + /** + * Sets the filter conditions as an array of filter objects. + * + * @param filter list of filter conditions, e.g.: + * [{"category": {"$eq": "tech"}}, {"score": {"$range": [80, 100]}}] + * @return this builder + */ + public Builder filter(List> filter) { + options.filter = filter; + return this; + } + + public Builder ef(int ef) { + options.ef = ef; + return this; + } + + public Builder includeVectors(boolean includeVectors) { + options.includeVectors = includeVectors; + return this; + } + + public Builder sparseIndices(int[] sparseIndices) { + options.sparseIndices = sparseIndices; + return this; + } + + public Builder sparseValues(double[] sparseValues) { + options.sparseValues = sparseValues; + return this; + } + + public QueryOptions build() { + return options; + } + } +} diff --git a/src/main/java/io/endee/client/types/QueryResult.java b/src/main/java/io/endee/client/types/QueryResult.java new file mode 100644 index 0000000..9b27d68 --- /dev/null +++ b/src/main/java/io/endee/client/types/QueryResult.java @@ -0,0 +1,49 @@ +package io.endee.client.types; + +import java.util.Arrays; +import java.util.Map; + +/** + * Result from a query operation. + */ +public class QueryResult { + private String id; + private double similarity; + private double distance; + private Map meta; + private double norm; + private Map filter; + private double[] vector; + + public QueryResult() {} + + public String getId() { return id; } + public void setId(String id) { this.id = id; } + + public double getSimilarity() { return similarity; } + public void setSimilarity(double similarity) { this.similarity = similarity; } + + public double getDistance() { return distance; } + public void setDistance(double distance) { this.distance = distance; } + + public Map getMeta() { return meta; } + public void setMeta(Map meta) { this.meta = meta; } + + public double getNorm() { return norm; } + public void setNorm(double norm) { this.norm = norm; } + + public Map getFilter() { return filter; } + public void setFilter(Map filter) { this.filter = filter; } + + public double[] getVector() { return vector; } + public void setVector(double[] vector) { this.vector = vector; } + + @Override + public String toString() { + String result = "QueryResult{id='" + id + "', similarity=" + similarity + ", distance=" + distance; + if (vector != null) { + result += ", vector=" + Arrays.toString(vector); + } + return result + "}"; + } +} diff --git a/src/main/java/io/endee/client/types/SpaceType.java b/src/main/java/io/endee/client/types/SpaceType.java new file mode 100644 index 0000000..fb96607 --- /dev/null +++ b/src/main/java/io/endee/client/types/SpaceType.java @@ -0,0 +1,34 @@ +package io.endee.client.types; + +/** + * Space types for distance calculation. + */ +public enum SpaceType { + COSINE("cosine"), + L2("l2"), + IP("ip"); + + private final String value; + + SpaceType(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + public static SpaceType fromValue(String value) { + for (SpaceType t : values()) { + if (t.value.equalsIgnoreCase(value)) { + return t; + } + } + throw new IllegalArgumentException("Unknown space type: " + value); + } + + @Override + public String toString() { + return value; + } +} diff --git a/src/main/java/io/endee/client/types/VectorInfo.java b/src/main/java/io/endee/client/types/VectorInfo.java new file mode 100644 index 0000000..77b832b --- /dev/null +++ b/src/main/java/io/endee/client/types/VectorInfo.java @@ -0,0 +1,36 @@ +package io.endee.client.types; + +import java.util.Map; + +/** + * Information about a vector retrieved from an index. + */ +public class VectorInfo { + private String id; + private Map meta; + private Map filter; + private double norm; + private double[] vector; + + public VectorInfo() {} + + public String getId() { return id; } + public void setId(String id) { this.id = id; } + + public Map getMeta() { return meta; } + public void setMeta(Map meta) { this.meta = meta; } + + public Map getFilter() { return filter; } + public void setFilter(Map filter) { this.filter = filter; } + + public double getNorm() { return norm; } + public void setNorm(double norm) { this.norm = norm; } + + public double[] getVector() { return vector; } + public void setVector(double[] vector) { this.vector = vector; } + + @Override + public String toString() { + return "VectorInfo{id='" + id + "', norm=" + norm + ", vectorLength=" + (vector != null ? vector.length : 0) + "}"; + } +} diff --git a/src/main/java/io/endee/client/types/VectorItem.java b/src/main/java/io/endee/client/types/VectorItem.java new file mode 100644 index 0000000..f773405 --- /dev/null +++ b/src/main/java/io/endee/client/types/VectorItem.java @@ -0,0 +1,63 @@ +package io.endee.client.types; + +import java.util.Map; + +/** + * A vector item for upsert operations. + */ +public class VectorItem { + private final String id; + private final double[] vector; + private Map meta; + private Map filter; + private int[] sparseIndices; + private double[] sparseValues; + + private VectorItem(String id, double[] vector) { + this.id = id; + this.vector = vector; + } + + public static Builder builder(String id, double[] vector) { + return new Builder(id, vector); + } + + public String getId() { return id; } + public double[] getVector() { return vector; } + public Map getMeta() { return meta; } + public Map getFilter() { return filter; } + public int[] getSparseIndices() { return sparseIndices; } + public double[] getSparseValues() { return sparseValues; } + + public static class Builder { + private final VectorItem item; + + private Builder(String id, double[] vector) { + this.item = new VectorItem(id, vector); + } + + public Builder meta(Map meta) { + item.meta = meta; + return this; + } + + public Builder filter(Map filter) { + item.filter = filter; + return this; + } + + public Builder sparseIndices(int[] sparseIndices) { + item.sparseIndices = sparseIndices; + return this; + } + + public Builder sparseValues(double[] sparseValues) { + item.sparseValues = sparseValues; + return this; + } + + public VectorItem build() { + return item; + } + } +} diff --git a/src/main/java/io/endee/client/util/CryptoUtils.java b/src/main/java/io/endee/client/util/CryptoUtils.java new file mode 100644 index 0000000..832b402 --- /dev/null +++ b/src/main/java/io/endee/client/util/CryptoUtils.java @@ -0,0 +1,267 @@ +package io.endee.client.util; + +import io.endee.client.exception.EndeeException; + +import javax.crypto.Cipher; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.SecretKeySpec; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.security.SecureRandom; +import java.util.Arrays; +import java.util.Map; +import java.util.zip.Deflater; +import java.util.zip.Inflater; + +/** + * Cryptographic utilities for Endee-DB. + * Provides compression/decompression and AES encryption/decryption for metadata. + */ +public final class CryptoUtils { + + private static final int AES_KEY_SIZE = 32; // 256 bits + private static final int IV_SIZE = 16; // 128 bits + private static final int BLOCK_SIZE = 16; + + private CryptoUtils() {} + + /** + * Gets checksum from key by converting last two hex characters to integer. + * + * @param key the key string + * @return checksum value or -1 if key is null + */ + public static int getChecksum(String key) { + if (key == null || key.length() < 2) { + return -1; + } + try { + String lastTwo = key.substring(key.length() - 2); + return Integer.parseInt(lastTwo, 16); + } catch (NumberFormatException e) { + return -1; + } + } + + /** + * Compresses a map to deflated JSON bytes, optionally encrypting with AES. + * + * @param data the map to compress + * @return compressed (and optionally encrypted) bytes + */ + public static byte[] jsonZip(Map data) { + if (data == null || data.isEmpty()) { + return new byte[0]; + } + try { + String json = JsonUtils.toJson(data); + byte[] compressed = deflateCompress(json.getBytes(StandardCharsets.UTF_8)); + + return compressed; + } catch (Exception e) { + throw new EndeeException("Failed to compress metadata", e); + } + } + + /** + * Decompresses deflated JSON bytes to a map. + * + * @param data the compressed bytes + * @return the decompressed map + */ + public static Map jsonUnzip(byte[] data) { + return jsonUnzip(data, null); + } + + /** + * Decompresses deflated JSON bytes to a map, optionally decrypting with AES. + * + * @param data the compressed (and optionally encrypted) bytes + * @param key optional hex key for AES decryption (64 hex chars = 256 bits) + * @return the decompressed map + */ + @SuppressWarnings("unchecked") + public static Map jsonUnzip(byte[] data, String key) { + if (data == null || data.length == 0) { + return Map.of(); + } + try { + byte[] buffer = data; + + // If key is provided, decrypt first + if (key != null && !key.isEmpty()) { + buffer = aesDecrypt(buffer, key); + } + + byte[] decompressed = deflateDecompress(buffer); + String json = new String(decompressed, StandardCharsets.UTF_8); + return JsonUtils.fromJson(json, Map.class); + } catch (Exception e) { + // Return empty map on failure (matches TypeScript behavior) + return Map.of(); + } + } + + /** + * Encrypts data using AES-256-CBC. + * + * @param data the data to encrypt + * @param keyHex a 256-bit hex key (64 hex characters) + * @return IV + ciphertext (IV is prepended to the ciphertext) + */ + public static byte[] aesEncrypt(byte[] data, String keyHex) { + try { + byte[] key = hexToBytes(keyHex); + + if (key.length != AES_KEY_SIZE) { + throw new IllegalArgumentException("Key must be 256 bits (64 hex characters)"); + } + + // Generate random IV + byte[] iv = new byte[IV_SIZE]; + new SecureRandom().nextBytes(iv); + + // Pad data with PKCS7 + byte[] paddedData = pkcs7Pad(data); + + // Create cipher and encrypt + Cipher cipher = Cipher.getInstance("AES/CBC/NoPadding"); + SecretKeySpec keySpec = new SecretKeySpec(key, "AES"); + IvParameterSpec ivSpec = new IvParameterSpec(iv); + cipher.init(Cipher.ENCRYPT_MODE, keySpec, ivSpec); + + byte[] ciphertext = cipher.doFinal(paddedData); + + // Return IV + ciphertext + byte[] result = new byte[iv.length + ciphertext.length]; + System.arraycopy(iv, 0, result, 0, iv.length); + System.arraycopy(ciphertext, 0, result, iv.length, ciphertext.length); + + return result; + } catch (IllegalArgumentException e) { + throw e; + } catch (Exception e) { + throw new EndeeException("AES encryption failed", e); + } + } + + /** + * Decrypts data using AES-256-CBC. + * + * @param data the encrypted data (IV + ciphertext) + * @param keyHex a 256-bit hex key (64 hex characters) + * @return the decrypted data + */ + public static byte[] aesDecrypt(byte[] data, String keyHex) { + try { + byte[] key = hexToBytes(keyHex); + + if (key.length != AES_KEY_SIZE) { + throw new IllegalArgumentException("Key must be 256 bits (64 hex characters)"); + } + + if (data.length < IV_SIZE) { + throw new IllegalArgumentException("Data too short to contain IV"); + } + + // Extract IV and ciphertext + byte[] iv = Arrays.copyOfRange(data, 0, IV_SIZE); + byte[] ciphertext = Arrays.copyOfRange(data, IV_SIZE, data.length); + + // Create cipher and decrypt + Cipher cipher = Cipher.getInstance("AES/CBC/NoPadding"); + SecretKeySpec keySpec = new SecretKeySpec(key, "AES"); + IvParameterSpec ivSpec = new IvParameterSpec(iv); + cipher.init(Cipher.DECRYPT_MODE, keySpec, ivSpec); + + byte[] paddedData = cipher.doFinal(ciphertext); + + // Remove PKCS7 padding + return pkcs7Unpad(paddedData); + } catch (IllegalArgumentException e) { + throw e; + } catch (Exception e) { + throw new EndeeException("AES decryption failed", e); + } + } + + /** + * Compresses data using DEFLATE algorithm (raw, no headers). + */ + private static byte[] deflateCompress(byte[] data) { + Deflater deflater = new Deflater(); + deflater.setInput(data); + deflater.finish(); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(data.length); + byte[] buffer = new byte[1024]; + + while (!deflater.finished()) { + int count = deflater.deflate(buffer); + bos.write(buffer, 0, count); + } + + deflater.end(); + return bos.toByteArray(); + } + + /** + * Decompresses DEFLATE compressed data. + */ + private static byte[] deflateDecompress(byte[] data) throws Exception { + Inflater inflater = new Inflater(); + inflater.setInput(data); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(data.length); + byte[] buffer = new byte[1024]; + + while (!inflater.finished()) { + int count = inflater.inflate(buffer); + if (count == 0 && inflater.needsInput()) { + break; + } + bos.write(buffer, 0, count); + } + + inflater.end(); + return bos.toByteArray(); + } + + /** + * Adds PKCS7 padding to data. + */ + private static byte[] pkcs7Pad(byte[] data) { + int paddingLength = BLOCK_SIZE - (data.length % BLOCK_SIZE); + byte[] padded = new byte[data.length + paddingLength]; + System.arraycopy(data, 0, padded, 0, data.length); + Arrays.fill(padded, data.length, padded.length, (byte) paddingLength); + return padded; + } + + /** + * Removes PKCS7 padding from data. + */ + private static byte[] pkcs7Unpad(byte[] data) { + if (data.length == 0) { + return data; + } + int paddingLength = data[data.length - 1] & 0xFF; + if (paddingLength > BLOCK_SIZE || paddingLength > data.length) { + return data; // Invalid padding, return as-is + } + return Arrays.copyOfRange(data, 0, data.length - paddingLength); + } + + /** + * Converts a hex string to byte array. + */ + private static byte[] hexToBytes(String hex) { + int len = hex.length(); + byte[] data = new byte[len / 2]; + for (int i = 0; i < len; i += 2) { + data[i / 2] = (byte) ((Character.digit(hex.charAt(i), 16) << 4) + + Character.digit(hex.charAt(i + 1), 16)); + } + return data; + } +} diff --git a/src/main/java/io/endee/client/util/JsonUtils.java b/src/main/java/io/endee/client/util/JsonUtils.java new file mode 100644 index 0000000..4d5aa2a --- /dev/null +++ b/src/main/java/io/endee/client/util/JsonUtils.java @@ -0,0 +1,37 @@ +package io.endee.client.util; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.endee.client.exception.EndeeException; + +/** + * JSON serialization utilities. + */ +public final class JsonUtils { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + private JsonUtils() {} + + public static ObjectMapper getObjectMapper() { + return OBJECT_MAPPER; + } + + public static String toJson(Object object) { + try { + return OBJECT_MAPPER.writeValueAsString(object); + } catch (JsonProcessingException e) { + throw new EndeeException("Failed to serialize to JSON", e); + } + } + + public static T fromJson(String json, Class type) { + try { + return OBJECT_MAPPER.readValue(json, type); + } catch (JsonProcessingException e) { + throw new EndeeException("Failed to deserialize JSON", e); + } + } +} diff --git a/src/main/java/io/endee/client/util/MessagePackUtils.java b/src/main/java/io/endee/client/util/MessagePackUtils.java new file mode 100644 index 0000000..bb1faed --- /dev/null +++ b/src/main/java/io/endee/client/util/MessagePackUtils.java @@ -0,0 +1,164 @@ +package io.endee.client.util; + +import io.endee.client.exception.EndeeException; +import org.msgpack.core.MessageBufferPacker; +import org.msgpack.core.MessagePack; +import org.msgpack.core.MessageUnpacker; +import org.msgpack.value.Value; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * MessagePack serialization utilities. + */ +public final class MessagePackUtils { + + private MessagePackUtils() { + } + + /** + * Packs vector data for upsert operations. + */ + public static byte[] packVectors(List vectors) { + try (MessageBufferPacker packer = MessagePack.newDefaultBufferPacker()) { + packer.packArrayHeader(vectors.size()); + + for (Object[] vector : vectors) { + packVectorTuple(packer, vector); + } + + return packer.toByteArray(); + } catch (IOException e) { + throw new EndeeException("Failed to pack vectors", e); + } + } + + private static void packVectorTuple(MessageBufferPacker packer, Object[] vector) throws IOException { + packer.packArrayHeader(vector.length); + + // id (string) + packer.packString((String) vector[0]); + + // metadata (bytes) + byte[] meta = (byte[]) vector[1]; + packer.packBinaryHeader(meta.length); + packer.writePayload(meta); + + // filter (string) + packer.packString((String) vector[2]); + + // norm (double) + packer.packDouble((Double) vector[3]); + + // vector (double[]) + double[] vec = (double[]) vector[4]; + packer.packArrayHeader(vec.length); + for (double v : vec) { + packer.packDouble(v); + } + + // Optional sparse data + if (vector.length > 5) { + int[] sparseIndices = (int[]) vector[5]; + packer.packArrayHeader(sparseIndices.length); + for (int idx : sparseIndices) { + packer.packInt(idx); + } + + double[] sparseValues = (double[]) vector[6]; + packer.packArrayHeader(sparseValues.length); + for (double val : sparseValues) { + packer.packDouble(val); + } + } + } + + /** + * Unpacks query results from MessagePack bytes. + */ + public static List unpackQueryResults(byte[] data) { + List results = new ArrayList<>(); + + try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(data)) { + int arraySize = unpacker.unpackArrayHeader(); + + for (int i = 0; i < arraySize; i++) { + int tupleSize = unpacker.unpackArrayHeader(); + Object[] tuple = new Object[tupleSize]; + + tuple[0] = unpackNumberAsDouble(unpacker); // similarity + tuple[1] = unpacker.unpackString(); // vectorId + int metaLen = unpacker.unpackBinaryHeader(); + tuple[2] = unpacker.readPayload(metaLen); // metadata + tuple[3] = unpacker.unpackString(); // filter + tuple[4] = unpackNumberAsDouble(unpacker); // norm + + if (tupleSize > 5) { + int vecLen = unpacker.unpackArrayHeader(); + double[] vec = new double[vecLen]; + for (int j = 0; j < vecLen; j++) { + vec[j] = unpackNumberAsDouble(unpacker); + } + tuple[5] = vec; + } + + results.add(tuple); + } + } catch (IOException e) { + throw new EndeeException("Failed to unpack query results", e); + } + + return results; + } + + /** + * Unpacks a single vector from MessagePack bytes. + */ + public static Object[] unpackVector(byte[] data) { + try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(data)) { + int tupleSize = unpacker.unpackArrayHeader(); + Object[] tuple = new Object[tupleSize]; + + tuple[0] = unpacker.unpackString(); // id + int metaLen = unpacker.unpackBinaryHeader(); + tuple[1] = unpacker.readPayload(metaLen); // metadata + tuple[2] = unpacker.unpackString(); // filter + tuple[3] = unpackNumberAsDouble(unpacker); // norm + + int vecLen = unpacker.unpackArrayHeader(); + double[] vec = new double[vecLen]; + for (int i = 0; i < vecLen; i++) { + vec[i] = unpackNumberAsDouble(unpacker); + } + tuple[4] = vec; + + return tuple; + } catch (IOException e) { + throw new EndeeException("Failed to unpack vector", e); + } + } + + /** + * Helper function to unpackNumber as double even though it + * can be integer + * @param unpacker + * @return + * @throws IOException + */ + private static double unpackNumberAsDouble(MessageUnpacker unpacker) throws IOException { + Value value = unpacker.unpackValue(); + + if (value.isFloatValue()) { + return value.asFloatValue().toDouble(); + } + + if (value.isIntegerValue()) { + return value.asIntegerValue().toDouble(); + } + + throw new IllegalStateException( + "Expected numeric value (int/float), got " + value.getValueType()); + } +} diff --git a/src/main/java/io/endee/client/util/ValidationUtils.java b/src/main/java/io/endee/client/util/ValidationUtils.java new file mode 100644 index 0000000..17afc16 --- /dev/null +++ b/src/main/java/io/endee/client/util/ValidationUtils.java @@ -0,0 +1,54 @@ +package io.endee.client.util; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; + +/** + * Validation utilities. + */ +public final class ValidationUtils { + + private static final Pattern INDEX_NAME_PATTERN = Pattern.compile("^[a-zA-Z0-9_]+$"); + private static final int MAX_INDEX_NAME_LENGTH = 48; + + private ValidationUtils() {} + + /** + * Validates an index name. + * Must be alphanumeric with underscores, less than 48 characters. + */ + public static boolean isValidIndexName(String name) { + if (name == null || name.isEmpty()) { + return false; + } + if (name.length() >= MAX_INDEX_NAME_LENGTH) { + return false; + } + return INDEX_NAME_PATTERN.matcher(name).matches(); + } + + /** + * Validates that all vector IDs are non-empty and unique. + */ + public static void validateVectorIds(List ids) { + Set seenIds = new HashSet<>(); + Set duplicateIds = new HashSet<>(); + + for (String id : ids) { + if (id == null || id.isEmpty()) { + throw new IllegalArgumentException("All vectors must have a non-empty ID"); + } + if (seenIds.contains(id)) { + duplicateIds.add(id); + } else { + seenIds.add(id); + } + } + + if (!duplicateIds.isEmpty()) { + throw new IllegalArgumentException("Duplicate IDs found: " + String.join(", ", duplicateIds)); + } + } +}