Skip to content

Commit 4dc6049

Browse files
epage480epage480
committed
Simplified create graph functions using common params
1 parent 444a13a commit 4dc6049

File tree

8 files changed

+5
-107
lines changed

8 files changed

+5
-107
lines changed

‎scrapegraphai/graphs/abstract_graph.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,11 @@ def __init__(self, prompt: str, config: dict, source: Optional[str] = None):
5555
self.verbose = True if config is None else config.get("verbose", False)
5656
self.headless = True if config is None else config.get(
5757
"headless", True)
58-
common_params = {"verbose": self.verbose,
59-
"headless": self.headless}
60-
self.set_common_params(common_params)
58+
common_params = {"headless": self.headless,
59+
"verbose": self.verbose,
60+
"llm_model": self.llm_model,
61+
"embedder_model": self.embedder_model}
62+
self.set_common_params(common_params, overwrite=False)
6163

6264

6365
def set_common_params(self, params: dict):

‎scrapegraphai/graphs/csv_scraper_graph.py‎

Lines changed: 0 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -32,35 +32,18 @@ def _create_graph(self):
3232
fetch_node = FetchNode(
3333
input="csv_dir",
3434
output=["doc"],
35-
node_config={
36-
"headless": self.headless,
37-
"verbose": self.verbose
38-
}
3935
)
4036
parse_node = ParseNode(
4137
input="doc",
4238
output=["parsed_doc"],
43-
node_config={
44-
"chunk_size": self.model_token,
45-
"verbose": self.verbose
46-
}
4739
)
4840
rag_node = RAGNode(
4941
input="user_prompt & (parsed_doc | doc)",
5042
output=["relevant_chunks"],
51-
node_config={
52-
"llm_model": self.llm_model,
53-
"embedder_model": self.embedder_model,
54-
"verbose": self.verbose
55-
}
5643
)
5744
generate_answer_node = GenerateAnswerCSVNode(
5845
input="user_prompt & (relevant_chunks | parsed_doc | doc)",
5946
output=["answer"],
60-
node_config={
61-
"llm_model": self.llm_model,
62-
"verbose": self.verbose
63-
}
6447
)
6548

6649
return BaseGraph(

‎scrapegraphai/graphs/json_scraper_graph.py‎

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -56,35 +56,21 @@ def _create_graph(self) -> BaseGraph:
5656
fetch_node = FetchNode(
5757
input="json_dir",
5858
output=["doc"],
59-
node_config={
60-
"headless": self.headless,
61-
"verbose": self.verbose
62-
}
6359
)
6460
parse_node = ParseNode(
6561
input="doc",
6662
output=["parsed_doc"],
6763
node_config={
6864
"chunk_size": self.model_token,
69-
"verbose": self.verbose
7065
}
7166
)
7267
rag_node = RAGNode(
7368
input="user_prompt & (parsed_doc | doc)",
7469
output=["relevant_chunks"],
75-
node_config={
76-
"llm_model": self.llm_model,
77-
"embedder_model": self.embedder_model,
78-
"verbose": self.verbose
79-
}
8070
)
8171
generate_answer_node = GenerateAnswerNode(
8272
input="user_prompt & (relevant_chunks | parsed_doc | doc)",
8373
output=["answer"],
84-
node_config={
85-
"llm_model": self.llm_model,
86-
"verbose": self.verbose
87-
}
8874
)
8975

9076
return BaseGraph(

‎scrapegraphai/graphs/script_creator_graph.py‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -61,32 +61,20 @@ def _create_graph(self) -> BaseGraph:
6161
fetch_node = FetchNode(
6262
input="url | local_dir",
6363
output=["doc"],
64-
node_config={
65-
"headless": self.headless,
66-
"verbose": self.verbose
67-
}
6864
)
6965
parse_node = ParseNode(
7066
input="doc",
7167
output=["parsed_doc"],
7268
node_config={"chunk_size": self.model_token,
73-
"verbose": self.verbose
7469
}
7570
)
7671
rag_node = RAGNode(
7772
input="user_prompt & (parsed_doc | doc)",
7873
output=["relevant_chunks"],
79-
node_config={
80-
"llm_model": self.llm_model,
81-
"embedder_model": self.embedder_model,
82-
"verbose": self.verbose
83-
}
8474
)
8575
generate_scraper_node = GenerateScraperNode(
8676
input="user_prompt & (relevant_chunks | parsed_doc | doc)",
8777
output=["answer"],
88-
node_config={"llm_model": self.llm_model,
89-
"verbose": self.verbose},
9078
library=self.library,
9179
website=self.source
9280
)

‎scrapegraphai/graphs/search_graph.py‎

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -49,43 +49,25 @@ def _create_graph(self) -> BaseGraph:
4949
search_internet_node = SearchInternetNode(
5050
input="user_prompt",
5151
output=["url"],
52-
node_config={
53-
"llm_model": self.llm_model,
54-
"verbose": self.verbose
55-
}
5652
)
5753
fetch_node = FetchNode(
5854
input="url | local_dir",
5955
output=["doc"],
60-
node_config={
61-
"headless": self.headless,
62-
"verbose": self.verbose
63-
}
6456
)
6557
parse_node = ParseNode(
6658
input="doc",
6759
output=["parsed_doc"],
6860
node_config={
6961
"chunk_size": self.model_token,
70-
"verbose": self.verbose
7162
}
7263
)
7364
rag_node = RAGNode(
7465
input="user_prompt & (parsed_doc | doc)",
7566
output=["relevant_chunks"],
76-
node_config={
77-
"llm_model": self.llm_model,
78-
"embedder_model": self.embedder_model,
79-
"verbose": self.verbose
80-
}
8167
)
8268
generate_answer_node = GenerateAnswerNode(
8369
input="user_prompt & (relevant_chunks | parsed_doc | doc)",
8470
output=["answer"],
85-
node_config={
86-
"llm_model": self.llm_model,
87-
"verbose": self.verbose
88-
}
8971
)
9072

9173
return BaseGraph(

‎scrapegraphai/graphs/smart_scraper_graph.py‎

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -58,35 +58,21 @@ def _create_graph(self) -> BaseGraph:
5858
fetch_node = FetchNode(
5959
input="url | local_dir",
6060
output=["doc"],
61-
node_config={
62-
"headless": self.headless,
63-
"verbose": self.verbose
64-
}
6561
)
6662
parse_node = ParseNode(
6763
input="doc",
6864
output=["parsed_doc"],
6965
node_config={
7066
"chunk_size": self.model_token,
71-
"verbose": self.verbose
7267
}
7368
)
7469
rag_node = RAGNode(
7570
input="user_prompt & (parsed_doc | doc)",
7671
output=["relevant_chunks"],
77-
node_config={
78-
"llm_model": self.llm_model,
79-
"embedder_model": self.embedder_model,
80-
"verbose": self.verbose
81-
}
8272
)
8373
generate_answer_node = GenerateAnswerNode(
8474
input="user_prompt & (relevant_chunks | parsed_doc | doc)",
8575
output=["answer"],
86-
node_config={
87-
"llm_model": self.llm_model,
88-
"verbose": self.verbose
89-
}
9076
)
9177

9278
return BaseGraph(

‎scrapegraphai/graphs/speech_graph.py‎

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -57,42 +57,27 @@ def _create_graph(self) -> BaseGraph:
5757
fetch_node = FetchNode(
5858
input="url | local_dir",
5959
output=["doc"],
60-
node_config={
61-
"headless": self.headless,
62-
"verbose": self.verbose
63-
}
6460
)
6561
parse_node = ParseNode(
6662
input="doc",
6763
output=["parsed_doc"],
6864
node_config={
6965
"chunk_size": self.model_token,
70-
"verbose": self.verbose
7166
}
7267
)
7368
rag_node = RAGNode(
7469
input="user_prompt & (parsed_doc | doc)",
7570
output=["relevant_chunks"],
76-
node_config={
77-
"llm_model": self.llm_model,
78-
"embedder_model": self.embedder_model,
79-
"verbose": self.verbose
80-
}
8171
)
8272
generate_answer_node = GenerateAnswerNode(
8373
input="user_prompt & (relevant_chunks | parsed_doc | doc)",
8474
output=["answer"],
85-
node_config={
86-
"llm_model": self.llm_model,
87-
"verbose": self.verbose
88-
}
8975
)
9076
text_to_speech_node = TextToSpeechNode(
9177
input="answer",
9278
output=["audio"],
9379
node_config={
9480
"tts_model": OpenAITextToSpeech(self.config["tts_model"]),
95-
"verbose": self.verbose
9681
}
9782
)
9883

‎scrapegraphai/graphs/xml_scraper_graph.py‎

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -58,35 +58,21 @@ def _create_graph(self) -> BaseGraph:
5858
fetch_node = FetchNode(
5959
input="xml_dir",
6060
output=["doc"],
61-
node_config={
62-
"headless": self.headless,
63-
"verbose": self.verbose
64-
}
6561
)
6662
parse_node = ParseNode(
6763
input="doc",
6864
output=["parsed_doc"],
6965
node_config={
7066
"chunk_size": self.model_token,
71-
"verbose": self.verbose
7267
}
7368
)
7469
rag_node = RAGNode(
7570
input="user_prompt & (parsed_doc | doc)",
7671
output=["relevant_chunks"],
77-
node_config={
78-
"llm_model": self.llm_model,
79-
"embedder_model": self.embedder_model,
80-
"verbose": self.verbose
81-
}
8272
)
8373
generate_answer_node = GenerateAnswerNode(
8474
input="user_prompt & (relevant_chunks | parsed_doc | doc)",
8575
output=["answer"],
86-
node_config={
87-
"llm_model": self.llm_model,
88-
"verbose": self.verbose
89-
}
9076
)
9177

9278
return BaseGraph(

0 commit comments

Comments
 (0)