fix: tensor_split should be optional list

abetlen · abetlen · commit 118b7f6d5c2c · 2023-07-14T16:52:48.000-04:00
diff --git a/llama_cpp/server/app.py b/llama_cpp/server/app.py
@@ -31,7 +31,7 @@ class Settings(BaseSettings):
         ge=0,
         description="The number of layers to put on the GPU. The rest will be on the CPU.",
     )
-    tensor_split: List[float] = Field(
+    tensor_split: Optional[List[float]] = Field(
         default=None,
         description="Split layers across multiple GPUs in proportion.",
     )

Original file line number	Diff line number	Diff line change
`@@ -31,7 +31,7 @@ class Settings(BaseSettings):`
`31`	`31`	`ge=0,`
`32`	`32`	`description="The number of layers to put on the GPU. The rest will be on the CPU.",`
`33`	`33`	`)`
`34`		`- tensor_split: List[float] = Field(`
	`34`	`+ tensor_split: Optional[List[float]] = Field(`
`35`	`35`	`default=None,`
`36`	`36`	`description="Split layers across multiple GPUs in proportion.",`
`37`	`37`	`)`