fix(proxy-rotation): removed duplicated arg and passed the loader_kwarhs correctly to the node

Author: PeriniM

examples/openai/proxy.py

@@ -0,0 +1,11 @@
+from scrapegraphai.utils import search_proxy_servers
+
+proxies = search_proxy_servers(
+    anonymous=True,
+    countryset={"IT"},
+    # secure=True,
+    timeout=1.0,
+    max_shape=2
+)
+
+print(proxies)

examples/openai/smart_scraper_openai.py

@@ -22,6 +22,19 @@
         "model": "gpt-3.5-turbo",
     },
     "verbose": True,
+    "headless": False,
+    "loader_kwargs": {
+        "proxy" : {
+            "server": "broker",
+            "criteria": {
+                "anonymous": True,
+                # "secure": True,
+                "countryset": {"IT"},
+                "timeout": 5.0,
+                "max_shape": 2
+            },
+        },
+    }
 }
 
 # ************************************************

scrapegraphai/graphs/abstract_graph.py

@@ -58,8 +58,11 @@ def __init__(self, prompt: str, config: dict, source: Optional[str] = None):
             "verbose", False)
         self.headless = True if config is None else config.get(
             "headless", True)
+        self.loader_kwargs = config.get("loader_kwargs", {})
+
         common_params = {"headless": self.headless,
                          "verbose": self.verbose,
+                         "loader_kwargs": self.loader_kwargs,
                          "llm_model": self.llm_model,
                          "embedder_model": self.embedder_model}
         self.set_common_params(common_params, overwrite=False)

scrapegraphai/graphs/smart_scraper_graph.py

@@ -57,7 +57,10 @@ def _create_graph(self) -> BaseGraph:
         """
         fetch_node = FetchNode(
             input="url | local_dir",
-            output=["doc"]
+            output=["doc"],
+            node_config={
+                "loader_kwargs": self.config.get("loader_kwargs", {}),
+            }
         )
         parse_node = ParseNode(
             input="doc",

scrapegraphai/nodes/fetch_node.py

@@ -49,6 +49,9 @@ def __init__(
         self.verbose = (
             False if node_config is None else node_config.get("verbose", False)
         )
+        self.loader_kwargs = (
+            {} if node_config is None else node_config.get("loader_kwargs", {})
+        )
 
     def execute(self, state):
         """

scrapegraphai/utils/proxy_rotation.py

@@ -161,7 +161,7 @@ def _search_proxy(proxy: Proxy) -> ProxySettings:
     Returns:
         A 'playwright' compliant proxy configuration.
     """
-    server = search_proxy_servers(max_shape=1, **proxy.get("criteria", {}))[0]
+    server = search_proxy_servers(**proxy.get("criteria", {}))[0]
 
     return {"server": server}