--

pyproject.toml

@@ -7,12 +7,12 @@ name = "nercone-website"
 version = "1.0.0"
 requires-python = ">=3.8"
 dependencies = [
-    "psutil",
     "httpx",
     "websockets",
     "markitdown",
-    "fastapi",
+    "beautifulsoup4",
     "jinja2",
+    "fastapi",
     "uvicorn[standard]"
 ]
 

src/nercone_website/server.py

@@ -2,6 +2,7 @@ import io
 import json
 import random
 from pathlib import Path
+from bs4 import BeautifulSoup
 from markitdown import MarkItDown
 from datetime import datetime, timezone
 from fastapi import FastAPI, Request, Response
@@ -111,7 +112,9 @@ async def default_response(request: Request, full_path: str) -> Response:
         try:
             if "curl" in request.headers.get("user-agent", "").lower():
                 content = templates.env.get_template(name).render(request=request)
-                markdown = markitdown.convert_stream(io.BytesIO(content.encode("utf-8")), file_extension=".html")
+                soup = BeautifulSoup(content, "html.parser")
+                main = str(soup.find("main")) if soup.find("main") else content
+                markdown = markitdown.convert_stream(io.BytesIO(main.encode("utf-8")), file_extension=".html")
                 accesscounter.increase()
                 return PlainTextResponse(markdown.text_content, status_code=200)
             else: