fixed request

Author: VinciGit00

requirements-dev.lock

@@ -40,7 +40,6 @@ babel==2.15.0
 beautifulsoup4==4.12.3
     # via furo
     # via google
-    # via markdownify
     # via scrapegraphai
 blinker==1.8.2
     # via streamlit
@@ -246,18 +245,13 @@ lxml==5.2.2
 lxml-html-clean==0.1.1
     # via lxml
 markdown-it-py==3.0.0
-    # via mdformat
     # via rich
-markdownify==0.12.1
-    # via scrapegraphai
 markupsafe==2.1.5
     # via jinja2
 marshmallow==3.21.3
     # via dataclasses-json
 matplotlib==3.9.0
     # via burr
-mdformat==0.7.17
-    # via scrapegraphai
 mdurl==0.1.2
     # via markdown-it-py
 minify-html==0.15.0
@@ -347,8 +341,6 @@ pygments==2.18.0
     # via furo
     # via rich
     # via sphinx
-pyhtml2md==1.6.0
-    # via scrapegraphai
 pyparsing==3.1.2
     # via httplib2
     # via matplotlib
@@ -409,7 +401,6 @@ sf-hamilton==1.66.1
 shellingham==1.5.4
     # via typer
 six==1.16.0
-    # via markdownify
     # via python-dateutil
 smmap==5.0.1
     # via gitdb

requirements.lock

@@ -28,7 +28,6 @@ babel==2.15.0
     # via courlan
 beautifulsoup4==4.12.3
     # via google
-    # via markdownify
     # via scrapegraphai
 boto3==1.34.129
     # via langchain-aws
@@ -175,16 +174,8 @@ lxml==5.2.2
     # via trafilatura
 lxml-html-clean==0.1.1
     # via lxml
-markdown-it-py==3.0.0
-    # via mdformat
-markdownify==0.12.1
-    # via scrapegraphai
 marshmallow==3.21.3
     # via dataclasses-json
-mdformat==0.7.17
-    # via scrapegraphai
-mdurl==0.1.2
-    # via markdown-it-py
 minify-html==0.15.0
     # via scrapegraphai
 multidict==6.0.5
@@ -238,8 +229,6 @@ pydantic-core==2.18.4
     # via pydantic
 pyee==11.1.0
     # via playwright
-pyhtml2md==1.6.0
-    # via scrapegraphai
 pyparsing==3.1.2
     # via httplib2
 python-dateutil==2.9.0.post0
@@ -275,7 +264,6 @@ s3transfer==0.10.1
 semchunk==1.0.1
     # via scrapegraphai
 six==1.16.0
-    # via markdownify
     # via python-dateutil
 sniffio==1.3.1
     # via anthropic

scrapegraphai/nodes/fetch_node.py

@@ -9,7 +9,7 @@
 import requests
 from langchain_community.document_loaders import PyPDFLoader
 from langchain_core.documents import Document
-
+from ..utils.cleanup_html import cleanup_html
 from ..docloaders import ChromiumLoader
 from ..utils.convert_to_md import convert_to_md
 from ..utils.logging import get_logger
@@ -164,7 +164,7 @@ def execute(self, state):
                 if not response.text.strip():
                     raise ValueError("No HTML body content found in the response.")
 
-                parsed_content = source
+                parsed_content = cleanup_html(response, source)
 
                 if  isinstance(self.llm_model, OpenAI) and not self.script_creator or self.force and not self.script_creator:
                     parsed_content = convert_to_md(source)

scrapegraphai/utils/convert_to_md.py

@@ -2,7 +2,6 @@
 convert_to_md modul
 """
 import html2text
-import mdformat
 from trafilatura import extract