Skip to content

Commit 032a491

Browse files
committed
Update parse_node.py
1 parent 07f1e23 commit 032a491

File tree

1 file changed

+3
-3
lines changed

1 file changed

+3
-3
lines changed

scrapegraphai/nodes/parse_node.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -74,21 +74,21 @@ def execute(self, state: dict) -> dict:
7474
docs_transformed = docs_transformed[0]
7575

7676
chunks = chunk(text=docs_transformed.page_content,
77-
chunk_size= self.node_config.get("chunk_size", 4096),
77+
chunk_size= self.node_config.get("chunk_size", 4096)-250,
7878
token_counter=lambda x: len(x),
7979
memoize=False)
8080
else:
8181
docs_transformed = docs_transformed[0]
8282

8383
if type(docs_transformed) == Document:
8484
chunks = chunk(text=docs_transformed.page_content,
85-
chunk_size= self.node_config.get("chunk_size", 4096),
85+
chunk_size= self.node_config.get("chunk_size", 4096)-250,
8686
token_counter=lambda x: len(x),
8787
memoize=False)
8888
else:
8989

9090
chunks = chunk(text=docs_transformed,
91-
chunk_size= self.node_config.get("chunk_size", 4096),
91+
chunk_size= self.node_config.get("chunk_size", 4096)-250,
9292
token_counter=lambda x: len(x),
9393
memoize=False)
9494

0 commit comments

Comments
 (0)