170 lines
6.1 KiB
Python
170 lines
6.1 KiB
Python
import requests
|
|
from bs4 import BeautifulSoup
|
|
import os
|
|
from urllib.parse import urljoin, urlparse
|
|
import time
|
|
import yaml
|
|
import markdownify
|
|
|
|
def scrape_cloudferro_docs(base_url):
|
|
"""Scrape CloudFerro documentation and save as markdown files."""
|
|
|
|
# Create project directory
|
|
project_dir = "cloudferro-docs"
|
|
docs_dir = os.path.join(project_dir, "docs")
|
|
os.makedirs(docs_dir, exist_ok=True)
|
|
|
|
# Keep track of visited URLs and navigation structure
|
|
visited_urls = set()
|
|
nav_structure = []
|
|
|
|
def clean_filename(text):
|
|
"""Convert text to a clean filename"""
|
|
return text.strip().replace(' ', '_').replace('/', '_').replace('?', '').lower()
|
|
|
|
def scrape_page(url, parent_path=""):
|
|
if url in visited_urls:
|
|
return
|
|
|
|
visited_urls.add(url)
|
|
print(f"Scraping: {url}")
|
|
|
|
try:
|
|
# Add delay to be respectful to the server
|
|
time.sleep(1)
|
|
response = requests.get(url)
|
|
response.raise_for_status()
|
|
|
|
soup = BeautifulSoup(response.content, 'html.parser')
|
|
|
|
# Extract main content
|
|
content = soup.select_one('.rst-content .document') or soup.select_one('.document')
|
|
if not content:
|
|
print(f"No content found at {url}")
|
|
return
|
|
|
|
# Get title
|
|
title = soup.select_one('h1')
|
|
if not title:
|
|
title = soup.select_one('title')
|
|
page_title = title.text.strip() if title else url.split('/')[-1]
|
|
|
|
# Convert content to markdown
|
|
md_content = markdownify.markdownify(str(content), heading_style="ATX")
|
|
|
|
# Create file path
|
|
filename = clean_filename(page_title) + '.md'
|
|
relative_path = os.path.join(parent_path, filename)
|
|
file_path = os.path.join(docs_dir, relative_path)
|
|
|
|
# Ensure directory exists
|
|
os.makedirs(os.path.dirname(file_path), exist_ok=True)
|
|
|
|
# Save content
|
|
with open(file_path, 'w', encoding='utf-8') as f:
|
|
f.write(f"# {page_title}\n\n")
|
|
f.write(md_content)
|
|
|
|
print(f"Saved: {file_path}")
|
|
|
|
# Update navigation structure
|
|
current_section = nav_structure
|
|
if parent_path:
|
|
for section in parent_path.split('/'):
|
|
if section:
|
|
section_dict = next((item for item in current_section
|
|
if isinstance(item, dict) and section in item), None)
|
|
if not section_dict:
|
|
section_dict = {section: []}
|
|
current_section.append(section_dict)
|
|
current_section = section_dict[section]
|
|
|
|
current_section.append({page_title: relative_path})
|
|
|
|
# Find and process links
|
|
menu = soup.select_one('.wy-menu-vertical')
|
|
if menu:
|
|
for link in menu.select('a'):
|
|
href = link.get('href')
|
|
if not href or href.startswith(('#', 'mailto:', 'javascript:', 'tel:')):
|
|
continue
|
|
|
|
next_url = urljoin(url, href)
|
|
if urlparse(next_url).netloc == urlparse(base_url).netloc:
|
|
parent = link.find_parent('li', class_='toctree-l2')
|
|
next_parent_path = parent_path
|
|
if parent:
|
|
section = parent.find_previous_sibling('li', class_='toctree-l1')
|
|
if section:
|
|
section_name = clean_filename(section.get_text().strip())
|
|
next_parent_path = os.path.join(parent_path, section_name)
|
|
scrape_page(next_url, next_parent_path)
|
|
|
|
except Exception as e:
|
|
print(f"Error processing {url}: {str(e)}")
|
|
|
|
try:
|
|
scrape_page(base_url)
|
|
except Exception as e:
|
|
print(f"Error during scraping: {str(e)}")
|
|
|
|
# Generate mkdocs.yml
|
|
mkdocs_config = {
|
|
'site_name': 'CloudFerro Documentation',
|
|
'site_description': 'CloudFerro Documentation Mirror',
|
|
'theme': {
|
|
'name': 'material',
|
|
'palette': {
|
|
'primary': 'blue',
|
|
'accent': 'blue'
|
|
},
|
|
'features': [
|
|
'navigation.instant',
|
|
'navigation.tracking',
|
|
'navigation.tabs',
|
|
'navigation.sections',
|
|
'navigation.expand',
|
|
'navigation.indexes',
|
|
'toc.integrate',
|
|
'content.code.copy'
|
|
]
|
|
},
|
|
'nav': nav_structure,
|
|
'markdown_extensions': [
|
|
'admonition',
|
|
'pymdownx.details',
|
|
'pymdownx.superfences',
|
|
'pymdownx.highlight',
|
|
'pymdownx.inlinehilite',
|
|
'pymdownx.snippets',
|
|
'pymdownx.tabbed',
|
|
'footnotes',
|
|
'toc',
|
|
'tables',
|
|
'attr_list'
|
|
]
|
|
}
|
|
|
|
with open(os.path.join(project_dir, 'mkdocs.yml'), 'w', encoding='utf-8') as f:
|
|
yaml.dump(mkdocs_config, f, allow_unicode=True, sort_keys=False)
|
|
|
|
# Create requirements.txt
|
|
requirements = [
|
|
"mkdocs>=1.5.0",
|
|
"mkdocs-material>=9.6.0",
|
|
"pymdown-extensions>=10.7",
|
|
]
|
|
|
|
with open(os.path.join(project_dir, 'requirements.txt'), 'w', encoding='utf-8') as f:
|
|
f.write('\n'.join(requirements))
|
|
|
|
print("\nScraping completed!")
|
|
print(f"Documentation has been generated in the {project_dir} directory")
|
|
print("\nTo preview the documentation locally:")
|
|
print("1. Install requirements: pip install -r cloudferro-docs/requirements.txt")
|
|
print("2. Run local server: cd cloudferro-docs && mkdocs serve")
|
|
|
|
if __name__ == "__main__":
|
|
url = 'https://docs.cloudferro.com/en/latest/'
|
|
scrape_cloudferro_docs(url)
|