Files
50ohm/src/build.py
T
2026-04-06 22:20:47 +02:00

670 lines
27 KiB
Python

import json
import os
import random
import re
import shutil
import zipfile
from pathlib import Path
from jinja2 import Environment, FileSystemLoader
from mistletoe import Document
from rich.progress import BarColumn, Progress, TaskProgressColumn, TextColumn, TimeRemainingColumn
from tqdm import tqdm
from renderer.fifty_ohm_html_renderer import FiftyOhmHtmlRenderer
from renderer.fifty_ohm_html_slide_renderer import FiftyOhmHtmlSlideRenderer
from renderer.index import Index
from .config import Config
class Build:
def __init__(self, config: Config):
self.config = config
self.env = Environment(loader=FileSystemLoader(self.config.p_templates))
self.env.filters["shuffle_answers"] = self.__filter_shuffle_answers
self.questions = self.__parse_katalog()
self.question_index = {}
self.question_token_pattern = re.compile(r"^\s*\[question:([\w\d]+)\]", re.MULTILINE)
self.index_entries = {}
self.index_token_pattern = Index.pattern
def __parse_katalog(self):
with self.config.p_data_fragenkatalog.open() as file:
fragenkatalog = json.load(file)
questions = {}
for exampart in fragenkatalog["sections"]:
for chapter in exampart["sections"]:
if "questions" in chapter:
for question in chapter["questions"]:
questions[question["number"]] = question
if "sections" in chapter:
for section in chapter["sections"]:
for question in section["questions"]:
questions[question["number"]] = question
return questions
def __build_question(self, number, template_file="html/question.html"):
"""Combines the original question dataset from BNetzA with our internal metadata"""
question_template = self.env.get_template(template_file)
with (self.config.p_data_metadata).open() as file:
metadata_json = json.load(file)
question = None
metadata = None
if number in self.questions:
question = self.questions[number]
if number in metadata_json:
metadata = metadata_json[number]
if question is None or metadata is None:
tqdm.write(
f"\033[31mQuestion #{number} is missing"
+ (" (Question not in question pool)" if question is None else "")
+ (" (Question not in metadata)" if metadata is None else "")
+ "\033[0m"
)
metadata = {"layout": "not-found", "picture_a": ""}
number = 404
question = {"question": f"Frage {input} nicht gefunden"}
if "answer_a" in question:
answers = [question["answer_a"], question["answer_b"], question["answer_c"], question["answer_d"]]
else:
answers = []
if metadata["picture_a"] != "":
alt_text_a = self.__picture_handler(metadata["picture_a"])
alt_text_b = self.__picture_handler(metadata["picture_b"])
alt_text_c = self.__picture_handler(metadata["picture_c"])
alt_text_d = self.__picture_handler(metadata["picture_d"])
answer_pictures = [
metadata["picture_a"],
metadata["picture_b"],
metadata["picture_c"],
metadata["picture_d"],
]
alt_text_answers = [
alt_text_a,
alt_text_b,
alt_text_c,
alt_text_d,
]
else:
answer_pictures = []
alt_text_answers = []
if "picture_question" in question and metadata["picture_question"] != "":
picture_question = metadata["picture_question"]
alt_text_question = self.__picture_handler(picture_question)
else:
picture_question = ""
alt_text_question = ""
solution_file = self.config.p_data_solutions / f"{number}.md"
return question_template.render(
question=question["question"],
number=number,
layout=metadata["layout"],
picture_question=picture_question,
answers=answers,
answer_pictures=answer_pictures,
alt_text_answers=alt_text_answers,
alt_text_question=alt_text_question,
has_solution=solution_file.exists(),
)
def __build_question_slide(self, input):
return self.__build_question(input, template_file="slide/question.html")
def __build_page(self, content, course_wrapper=False, sidebar=None):
page_template = self.env.get_template("html/page.html")
return page_template.render(content=content, course_wrapper=course_wrapper, sidebar=sidebar)
def __copy_picture(self, id):
file = f"{id}.svg"
try:
shutil.copyfile(self.config.p_data_pictures / file, self.config.p_build_pictures / file)
if (self.config.p_data_pictures / f"{id}.txt").exists():
return (self.config.p_data_pictures / f"{id}.txt").read_text()
else:
return "Bildbeschreibung noch nicht verfügbar"
except FileNotFoundError:
tqdm.write(f"\033[31mPicture #{id} not found\033[0m")
def __picture_handler(self, id):
self.config.p_build_pictures.mkdir(parents=True, exist_ok=True)
return self.__copy_picture(id)
def __photo_handler(self, id):
self.config.p_build_photos.mkdir(parents=True, exist_ok=True)
file = f"{id}.png"
try:
shutil.copyfile(self.config.p_data_photos / file, self.config.p_build_photos / file)
if (self.config.p_data_photos / f"{id}.txt").exists():
return (self.config.p_data_photos / f"{id}.txt").read_text()
else:
return "Bildbeschreibung noch nicht verfügbar"
except FileNotFoundError:
tqdm.write(f"\033[31mPhoto #{id} not found\033[0m")
def __build_chapter_index(self, edition, edition_name, number, chapter, next_chapter=None):
chapter_template = self.env.get_template("html/chapter.html")
next_chapter_template = self.env.get_template("html/next_chapter.html")
with (self.config.p_build / f"{edition}_chapter_{chapter['ident']}.html").open("w") as file:
result = chapter_template.render(
edition=edition,
name=edition_name,
number=number,
chapter=chapter,
)
if next_chapter is not None:
result += next_chapter_template.render(
url=f"{edition}_chapter_{next_chapter['ident']}.html",
title=next_chapter["title"],
)
result = self.__build_page(result, course_wrapper=True)
file.write(result)
def __include_handler(self, include):
with (self.config.p_data_html / f"{include}.html").open() as file:
code = file.read()
svg_list = re.findall(r"(\d+)\.svg", code or "")
for id in svg_list:
self.__copy_picture(id)
return code
def __build_section(
self,
edition,
edition_name,
section,
section_id,
chapter,
next_section=None,
next_chapter=None,
chapter_number=None,
):
section_template = self.env.get_template("html/section.html")
next_section_template = self.env.get_template("html/next_section.html")
next_chapter_template = self.env.get_template("html/next_chapter.html")
with (self.config.p_build / f"{edition}_{section['ident']}.html").open("w") as file:
# Use provided chapter_number or fall back to chapter dict
chapter_num = str(chapter_number) if chapter_number is not None else chapter.get("number", "0")
section_num = str(section_id)
# Set the section URL for references
section_filename = f"{edition}_{section['ident']}.html"
with FiftyOhmHtmlRenderer(
question_renderer=self.__build_question,
picture_handler=self.__picture_handler,
photo_handler=self.__photo_handler,
include_handler=self.__include_handler,
edition=edition,
chapter=chapter_num,
section=section_num,
section_url=section_filename,
) as renderer:
# First pass: collect all figures and assign hierarchical numbers
doc = Document(section["content"])
renderer.collect_figures(doc)
# Second pass: render with hierarchical numbers
section["content"] = renderer.render(doc)
result = section_template.render(
edition=edition,
name=edition_name,
section=section,
section_id=section_id,
chapter=chapter,
)
if next_section is not None:
result += next_section_template.render(
url=f"{edition}_{next_section['ident']}.html",
title=next_section["title"],
)
elif next_chapter is not None:
result += next_chapter_template.render(
url=f"{edition}_chapter_{next_chapter['ident']}.html",
title=next_chapter["title"],
)
result = self.__build_page(result, course_wrapper=True)
file.write(result)
def __build_chapter_slidedeck(
self, edition, chapter, sections, next_chapter, chapter_number=None, progress: Progress = None
):
with (self.config.p_build / f"{edition}_slide_{chapter['ident']}.html").open("w") as file:
slide_template = self.env.get_template("slide/slide.html")
help_template = self.env.get_template("slide/help.html")
next_template = self.env.get_template("slide/next.html")
# Use provided chapter_number or fall back to chapter dict
chapter_num = str(chapter_number) if chapter_number is not None else chapter.get("number", "0")
# Set the slide URL for references (all sections in one file)
# Note: filename uses original edition (e.g., "N"), not modified edition with "S" (e.g., "NS")
slide_filename = f"{edition}_slide_{chapter['ident']}.html"
with FiftyOhmHtmlSlideRenderer(
question_renderer=self.__build_question_slide,
picture_handler=self.__picture_handler,
photo_handler=self.__photo_handler,
include_handler=self.__include_handler,
edition=edition,
chapter=chapter_num,
section="0", # Will be updated per section
section_url=slide_filename,
) as renderer:
result = "<section>\n"
result += f'<section data-background="#DAEEFA">\n<h1>{chapter["title"]}</h1>\n</section>\n'
result += help_template.render()
result += "</section>\n"
section_counter = 0
slides_task = progress.add_task("Rendering slides ...")
for section in progress.track(sections, task_id=slides_task):
progress.update(slides_task, description=f"Rendering slides of {section['title']}")
if section["slide"] is None:
continue
section_counter += 1
# Update section number for this section
renderer.section = str(section_counter)
# Reset figure counter for each section
renderer.figure_counter = 0
renderer.figure_map = {}
if not section["slide"].startswith("---"):
section["slide"] = "---\n" + section["slide"]
# First pass: collect figures
doc = Document(section["slide"])
renderer.collect_figures(doc)
# Second pass: render with hierarchical numbers
tmp = f'<section data-background="#DAEEFA">\n<h1>{section["title"]}</h1>\n</section>\n'
tmp += renderer.render(doc)
result += f"<section>{tmp}</section>\n"
progress.remove_task(slides_task)
result += next_template.render(
edition=edition,
next_chapter=next_chapter,
chapter=chapter,
)
result = slide_template.render(content=result)
file.write(result)
def __filter_shuffle_answers(self, seq):
answers = []
firstrun = True
for answer in seq:
if firstrun:
answers.append({"content": answer, "correct": True})
firstrun = False
else:
answers.append({"content": answer, "correct": False})
random.shuffle(answers)
return answers
def __build_book_index(self, book):
template = self.env.get_template("html/course_index.html")
with (self.config.p_build / f"{book['edition']}_course_index.html").open("w") as file:
result = template.render(
book=book,
)
result = self.__build_page(result)
file.write(result)
def __build_slide_index(self, book):
template = self.env.get_template("slide/slide_index.html")
with (self.config.p_build / f"{book['edition']}_slide_index.html").open("w") as file:
result = template.render(
book=book,
)
result = self.__build_page(result)
file.write(result)
def build_edition(self, edition: str):
self.config.p_build.mkdir(exist_ok=True)
edition = edition.upper()
with (
(self.config.p_data_toc / f"{edition}.json").open() as file,
Progress(
TaskProgressColumn(),
BarColumn(),
TimeRemainingColumn(),
TextColumn("[progress.description]{task.description}"),
transient=True,
) as progress,
):
chapter_task = progress.add_task(f"Building edition {edition} ...")
book = json.load(file)
edition_name = book["title"]
# Create index pages for chapters and slides.
self.__build_book_index(book)
self.__build_slide_index(book)
chapters = book["chapters"]
for chapter_number, chapter in enumerate(progress.track(chapters, task_id=chapter_task), 1):
progress.update(chapter_task, description=f"Building edition {edition}: Chapter {chapter['title']}")
# Determine next chapter for navigation (None if this is the last chapter)
next_chapter = chapters[chapter_number] if chapter_number < len(chapters) else None
self.__build_chapter_index(edition, edition_name, chapter_number, chapter, next_chapter)
# Open, parse and render each section.
section_task = progress.add_task(description="Rendering sections ...")
for section_number, section in enumerate(progress.track(chapter["sections"], task_id=section_task), 1):
progress.update(section_task, description=f"Rendering section {section['title']}")
# Read section and slide content from the corresponding files.
ident = section["ident"]
section["content"] = None
section["slide"] = None
with (self.config.p_data_sections / f"{ident}.md").open() as sfile:
section_content = sfile.read()
section["content"] = section_content
with (self.config.p_data_slides / f"{ident}.md").open() as sfile:
section_content = sfile.read()
section["slide"] = section_content
if section["content"] is not None:
self.__collect_question_occurrences(
edition,
chapter["title"],
ident,
section["title"],
section["content"],
)
self.__collect_index_occurrences(
edition,
chapter["title"],
ident,
section["title"],
section["content"],
)
next_section = (
chapter["sections"][section_number] if section_number < len(chapter["sections"]) else None
)
self.__build_section(
edition,
edition_name,
section,
section_number,
chapter,
next_section,
next_chapter,
chapter_number,
)
progress.remove_task(section_task)
# Build the slidedeck after all sections have been processed, as they require the files to be read.
self.__build_chapter_slidedeck(
edition, chapter, chapter["sections"], next_chapter, chapter_number, progress
)
progress.remove_task(chapter_task)
def build_assets(self):
self.config.p_build.mkdir(exist_ok=True)
shutil.copytree(
self.config.p_assets, self.config.p_build_assets, dirs_exist_ok=True, ignore=shutil.ignore_patterns(".git")
)
def __parse_snippets(self):
snippets = {}
for md_file in Path(self.config.p_data_snippets).glob("*.md"):
with md_file.open() as file:
snippets[md_file.stem] = file.read()
with FiftyOhmHtmlRenderer(
question_renderer=self.__build_question,
picture_handler=self.__picture_handler,
photo_handler=self.__photo_handler,
include_handler=self.__include_handler,
) as renderer:
for key, value in snippets.items():
snippets[key] = renderer.render_inner(Document(value))
# Remove leading <p> and trailing </p>:
snippets[key] = snippets[key][3:-4]
return snippets
def __parse_contents(self):
static = []
for static_file in Path(self.config.p_data_static).glob("*.html"):
if "sidebar" not in static_file.stem:
with static_file.open() as file:
content = file.read()
sidebar_file = self.config.p_data_static / f"{static_file.stem}_sidebar.html"
if not sidebar_file.exists():
sidebar = ""
else:
with sidebar_file.open() as file:
sidebar = file.read()
static.append(
{
"url_part": static_file.stem,
"content": content,
"sidebar": sidebar if sidebar != "" else None,
}
)
return static
def __build_index(self, snippets):
template = self.env.get_template("html/index.html")
result = template.render({"snippets": snippets})
with (self.config.p_build / "index.html").open("w") as file:
result = self.__build_page(result)
file.write(result)
def __build_course_page(self, snippets, template, page):
template = self.env.get_template(f"html/{template}.html")
result = template.render({"snippets": snippets})
with (self.config.p_build / f"{page}.html").open("w") as file:
result = self.__build_page(result)
file.write(result)
def __build_html_page(self, contents, page):
for content in contents:
if content["url_part"] == page:
with (self.config.p_build / f"{page}.html").open("w") as file:
result = self.__build_page(content=content["content"], sidebar=content["sidebar"])
file.write(result)
def build_website(self):
self.config.p_build.mkdir(exist_ok=True)
snippets = self.__parse_snippets()
contents = self.__parse_contents()
self.__build_index(snippets)
self.__build_course_page(snippets, "kurse-karte", "kurse_vor_ort_karte")
self.__build_course_page(snippets, "kurse-liste", "kurse_vor_ort_liste")
self.__build_course_page(snippets, "patenkarte", "patenkarte")
self.__build_html_page(contents, "pruefung")
self.__build_html_page(contents, "infos")
self.__build_html_page(contents, "suche")
def build_solutions(self):
for solution_file in self.config.p_data_solutions.glob("*.md"):
with solution_file.open() as file:
content = file.read()
with (self.config.p_build / f"{solution_file.stem}.html").open("w") as file:
with FiftyOhmHtmlRenderer(
question_renderer=self.__build_question,
picture_handler=self.__picture_handler,
photo_handler=self.__photo_handler,
include_handler=self.__include_handler,
) as renderer:
question = self.__build_question(
solution_file.stem, template_file="html/solution_question.html"
)
solution_template = self.env.get_template("html/solution.html")
solution = renderer.render(Document(content))
page = solution_template.render(question=question, solution=solution, number=solution_file.stem)
page = self.__build_page(page, course_wrapper=False)
file.write(page)
def __collect_question_occurrences(
self,
edition: str,
chapter_title: str,
section_ident: str,
section_title: str,
section_content: str,
):
for question_number in self.question_token_pattern.findall(section_content or ""):
question_entry = self.question_index.setdefault(
question_number,
{
"has_solution": self.__has_solution(question_number),
"section": section_ident,
"chapter_title": chapter_title,
"section_title": section_title,
"editions": [],
},
)
if edition not in question_entry["editions"]:
question_entry["editions"].append(edition)
def __has_solution(self, question_number: str) -> bool:
return (self.config.p_data_solutions / f"{question_number}.md").exists()
def __collect_index_occurrences(
self,
edition: str,
chapter_title: str,
section_ident: str,
section_title: str,
section_content: str,
):
for match in self.index_token_pattern.finditer(section_content or ""):
first = match.group(1).strip()
second = match.group(2).strip() if match.group(2) else None
normalized_first = FiftyOhmHtmlRenderer._normalize_index_part(first)
if second:
normalized_second = FiftyOhmHtmlRenderer._normalize_index_part(second)
anchor_id = f"index_{normalized_first}__{normalized_second}"
else:
anchor_id = f"index_{normalized_first}"
entry_key = f"{section_ident}#{anchor_id}"
index_entry = self.index_entries.setdefault(
entry_key,
{
"term": first,
"subterm": second,
"anchor_id": anchor_id,
"section": section_ident,
"chapter_title": chapter_title,
"section_title": section_title,
"editions": [],
},
)
if edition not in index_entry["editions"]:
index_entry["editions"].append(edition)
def build_question_index(self):
tqdm.write("Creating question index")
for question_data in self.question_index.values():
question_data["editions"] = sorted(question_data["editions"])
path = self.config.p_build_assets / "question_index.json"
with (path).open("w", encoding="utf-8") as file:
json.dump(self.question_index, file, ensure_ascii=False, indent=2, sort_keys=True)
file.write("\n")
def build_index(self):
tqdm.write("Creating index")
for index_data in self.index_entries.values():
index_data["editions"] = sorted(index_data["editions"])
entries = sorted(
self.index_entries.values(),
key=lambda item: (
item["term"].casefold(),
(item["subterm"] or "").casefold(),
item["section"],
item["anchor_id"],
),
)
path = self.config.p_build_assets / "index.json"
with (path).open("w", encoding="utf-8") as file:
json.dump(entries, file, ensure_ascii=False, indent=2)
file.write("\n")
def build_zip(self, zip_name: str | None = None) -> Path:
"""Create a zip archive of the complete build output directory.
The zip file is written into the build directory itself and is excluded
from the archive contents.
"""
build_dir = self.config.p_build
build_dir.mkdir(parents=True, exist_ok=True)
zip_path = build_dir / (zip_name if zip_name is not None else f"{build_dir.name}.zip")
zip_path = zip_path.resolve()
# If a previous archive exists, remove it first to avoid zipping stale data.
if zip_path.exists():
zip_path.unlink()
tqdm.write(f"Creating zip archive: {zip_path}")
with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as zf:
# os.walk includes dotfiles; sort for stable archives.
for root, dirs, files in os.walk(build_dir):
dirs.sort()
files.sort()
root_path = Path(root)
for filename in files:
file_path = (root_path / filename).resolve()
if file_path == zip_path:
continue
arcname = file_path.relative_to(build_dir.resolve())
zf.write(file_path, arcname=arcname)
return zip_path