84 lines
3.0 KiB
Python
84 lines
3.0 KiB
Python
import os
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
from urllib.parse import urljoin, urlparse, urlunparse
|
|
from markdownify import markdownify as md
|
|
|
|
BASE_URL = "https://docs.cloudferro.com/en/latest/"
|
|
OUTPUT_DIR = "docs"
|
|
IMAGES_DIR = os.path.join(OUTPUT_DIR, "_images")
|
|
visited = set()
|
|
|
|
def normalize_url(url):
|
|
parsed = urlparse(url)
|
|
# Remove query and fragment, normalize path
|
|
path = parsed.path.rstrip("/")
|
|
if path == "":
|
|
path = "/"
|
|
normalized = urlunparse((parsed.scheme, parsed.netloc, path, '', '', ''))
|
|
return normalized
|
|
|
|
def download_image(img_url):
|
|
os.makedirs(IMAGES_DIR, exist_ok=True)
|
|
filename = os.path.basename(urlparse(img_url).path)
|
|
local_path = os.path.join(IMAGES_DIR, filename)
|
|
if not os.path.exists(local_path):
|
|
try:
|
|
r = requests.get(img_url, timeout=10)
|
|
r.raise_for_status()
|
|
with open(local_path, "wb") as f:
|
|
f.write(r.content)
|
|
except Exception as e:
|
|
print(f"Failed to download {img_url}: {e}")
|
|
return f"_images/{filename}"
|
|
|
|
def process_page(url, rel_path):
|
|
print(f"Processing: {url}")
|
|
r = requests.get(url)
|
|
soup = BeautifulSoup(r.text, "html.parser")
|
|
# Download images and update src
|
|
for img in soup.find_all("img"):
|
|
img_url = urljoin(url, img["src"])
|
|
local_img = download_image(img_url)
|
|
img["src"] = local_img
|
|
# Convert to Markdown
|
|
main = soup.find("main") or soup.body
|
|
md_content = md(str(main))
|
|
# Save Markdown
|
|
out_path = os.path.join(OUTPUT_DIR, rel_path)
|
|
os.makedirs(os.path.dirname(out_path), exist_ok=True)
|
|
with open(out_path, "w", encoding="utf-8") as f:
|
|
f.write(md_content)
|
|
|
|
def crawl_iterative(start_url, start_rel_path="index.md"):
|
|
stack = [(start_url, start_rel_path)]
|
|
while stack:
|
|
url, rel_path = stack.pop()
|
|
norm_url = normalize_url(url)
|
|
if norm_url in visited:
|
|
continue
|
|
visited.add(norm_url)
|
|
process_page(url, rel_path)
|
|
try:
|
|
r = requests.get(url)
|
|
soup = BeautifulSoup(r.text, "html.parser")
|
|
for a in soup.find_all("a", href=True):
|
|
href = a["href"]
|
|
if href.startswith("http") and not href.startswith(BASE_URL):
|
|
continue
|
|
abs_url = urljoin(url, href)
|
|
norm_abs_url = normalize_url(abs_url)
|
|
if norm_abs_url.startswith(BASE_URL.rstrip("/")) and norm_abs_url not in visited:
|
|
path = urlparse(abs_url).path.replace("/en/latest/", "")
|
|
if path.endswith("/") or path == "":
|
|
rel_md = os.path.join(path, "index.md")
|
|
else:
|
|
rel_md = path.rstrip("/") + ".md"
|
|
stack.append((abs_url, rel_md.lstrip("/")))
|
|
except Exception as e:
|
|
print(f"Failed to process links on {url}: {e}")
|
|
|
|
if __name__ == "__main__":
|
|
crawl_iterative(BASE_URL)
|
|
print("Scraping complete. All Markdown and images are in the 'docs/' folder.")
|