diff --git a/pyproject.toml b/pyproject.toml index 8ba1c5e..6a6d4b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,12 +7,12 @@ name = "nercone-website" version = "1.0.0" requires-python = ">=3.8" dependencies = [ - "psutil", "httpx", "websockets", "markitdown", - "fastapi", + "beautifulsoup4", "jinja2", + "fastapi", "uvicorn[standard]" ] diff --git a/src/nercone_website/server.py b/src/nercone_website/server.py index e06d4ca..4f84a2c 100644 --- a/src/nercone_website/server.py +++ b/src/nercone_website/server.py @@ -2,6 +2,7 @@ import io import json import random from pathlib import Path +from bs4 import BeautifulSoup from markitdown import MarkItDown from datetime import datetime, timezone from fastapi import FastAPI, Request, Response @@ -111,7 +112,9 @@ async def default_response(request: Request, full_path: str) -> Response: try: if "curl" in request.headers.get("user-agent", "").lower(): content = templates.env.get_template(name).render(request=request) - markdown = markitdown.convert_stream(io.BytesIO(content.encode("utf-8")), file_extension=".html") + soup = BeautifulSoup(content, "html.parser") + main = str(soup.find("main")) if soup.find("main") else content + markdown = markitdown.convert_stream(io.BytesIO(main.encode("utf-8")), file_extension=".html") accesscounter.increase() return PlainTextResponse(markdown.text_content, status_code=200) else: