import json import os import random import re import shutil import zipfile from pathlib import Path from jinja2 import Environment, FileSystemLoader from mistletoe import Document from rich.progress import BarColumn, Progress, TaskProgressColumn, TextColumn, TimeRemainingColumn from tqdm import tqdm from renderer.fifty_ohm_html_renderer import FiftyOhmHtmlRenderer from renderer.fifty_ohm_html_slide_renderer import FiftyOhmHtmlSlideRenderer from renderer.index import Index from .config import Config class Build: def __init__(self, config: Config): self.config = config self.env = Environment(loader=FileSystemLoader(self.config.p_templates)) self.env.filters["shuffle_answers"] = self.__filter_shuffle_answers self.questions = self.__parse_katalog() self.question_index = {} self.question_token_pattern = re.compile(r"^\s*\[question:([\w\d]+)\]", re.MULTILINE) self.index_entries = {} self.index_token_pattern = Index.pattern def __parse_katalog(self): with self.config.p_data_fragenkatalog.open(encoding="utf-8") as file: fragenkatalog = json.load(file) questions = {} for exampart in fragenkatalog["sections"]: for chapter in exampart["sections"]: if "questions" in chapter: for question in chapter["questions"]: questions[question["number"]] = question if "sections" in chapter: for section in chapter["sections"]: for question in section["questions"]: questions[question["number"]] = question return questions def __build_question(self, number, template_file="html/question.html"): """Combines the original question dataset from BNetzA with our internal metadata""" question_template = self.env.get_template(template_file) with (self.config.p_data_metadata).open(encoding="utf-8") as file: metadata_json = json.load(file) question = None metadata = None if number in self.questions: question = self.questions[number] if number in metadata_json: metadata = metadata_json[number] if question is None or metadata is None: tqdm.write( f"\033[31mQuestion #{number} is missing" + (" (Question not in question pool)" if question is None else "") + (" (Question not in metadata)" if metadata is None else "") + "\033[0m" ) metadata = {"layout": "not-found", "picture_a": ""} number = 404 question = {"question": f"Frage {input} nicht gefunden"} if "answer_a" in question: answers = [question["answer_a"], question["answer_b"], question["answer_c"], question["answer_d"]] else: answers = [] if metadata["picture_a"] != "": alt_text_a = self.__picture_handler(metadata["picture_a"]) alt_text_b = self.__picture_handler(metadata["picture_b"]) alt_text_c = self.__picture_handler(metadata["picture_c"]) alt_text_d = self.__picture_handler(metadata["picture_d"]) answer_pictures = [ metadata["picture_a"], metadata["picture_b"], metadata["picture_c"], metadata["picture_d"], ] alt_text_answers = [ alt_text_a, alt_text_b, alt_text_c, alt_text_d, ] else: answer_pictures = [] alt_text_answers = [] if "picture_question" in question and metadata["picture_question"] != "": picture_question = metadata["picture_question"] alt_text_question = self.__picture_handler(picture_question) else: picture_question = "" alt_text_question = "" solution_file = self.config.p_data_solutions / f"{number}.md" return question_template.render( question=question["question"], number=number, layout=metadata["layout"], picture_question=picture_question, answers=answers, answer_pictures=answer_pictures, alt_text_answers=alt_text_answers, alt_text_question=alt_text_question, has_solution=solution_file.exists(), ) def __build_question_slide(self, input): return self.__build_question(input, template_file="slide/question.html") def __build_page(self, content, course_wrapper=False, sidebar=None): page_template = self.env.get_template("html/page.html") return page_template.render(content=content, course_wrapper=course_wrapper, sidebar=sidebar) def __copy_picture(self, id): file = f"{id}.svg" try: shutil.copyfile(self.config.p_data_pictures / file, self.config.p_build_pictures / file) if (self.config.p_data_pictures / f"{id}.txt").exists(): return (self.config.p_data_pictures / f"{id}.txt").read_text(encoding="utf-8") else: return "Bildbeschreibung noch nicht verfügbar" except FileNotFoundError: tqdm.write(f"\033[31mPicture #{id} not found\033[0m") def __picture_handler(self, id): self.config.p_build_pictures.mkdir(parents=True, exist_ok=True) return self.__copy_picture(id) def __photo_handler(self, id): self.config.p_build_photos.mkdir(parents=True, exist_ok=True) file = f"{id}.png" try: shutil.copyfile(self.config.p_data_photos / file, self.config.p_build_photos / file) if (self.config.p_data_photos / f"{id}.txt").exists(): return (self.config.p_data_photos / f"{id}.txt").read_text(encoding="utf-8") else: return "Bildbeschreibung noch nicht verfügbar" except FileNotFoundError: tqdm.write(f"\033[31mPhoto #{id} not found\033[0m") def __build_chapter_index(self, edition, edition_name, number, chapter, next_chapter=None): chapter_template = self.env.get_template("html/chapter.html") next_chapter_template = self.env.get_template("html/next_chapter.html") with (self.config.p_build / f"{edition}_chapter_{chapter['ident']}.html").open("w", encoding="utf-8") as file: result = chapter_template.render( edition=edition, name=edition_name, number=number, chapter=chapter, ) if next_chapter is not None: result += next_chapter_template.render( url=f"{edition}_chapter_{next_chapter['ident']}.html", title=next_chapter["title"], ) result = self.__build_page(result, course_wrapper=True) file.write(result) def __include_handler(self, include): with (self.config.p_data_html / f"{include}.html").open(encoding="utf-8") as file: code = file.read() svg_list = re.findall(r"(\d+)\.svg", code or "") for id in svg_list: self.__copy_picture(id) return code def __build_section( self, edition, edition_name, section, section_id, chapter, next_section=None, next_chapter=None, chapter_number=None, ): section_template = self.env.get_template("html/section.html") next_section_template = self.env.get_template("html/next_section.html") next_chapter_template = self.env.get_template("html/next_chapter.html") with (self.config.p_build / f"{edition}_{section['ident']}.html").open("w", encoding="utf-8") as file: # Use provided chapter_number or fall back to chapter dict chapter_num = str(chapter_number) if chapter_number is not None else chapter.get("number", "0") section_num = str(section_id) # Set the section URL for references section_filename = f"{edition}_{section['ident']}.html" with FiftyOhmHtmlRenderer( question_renderer=self.__build_question, picture_handler=self.__picture_handler, photo_handler=self.__photo_handler, include_handler=self.__include_handler, edition=edition, chapter=chapter_num, section=section_num, section_url=section_filename, ) as renderer: # First pass: collect all figures and assign hierarchical numbers doc = Document(section["content"]) renderer.collect_figures(doc) # Second pass: render with hierarchical numbers section["content"] = renderer.render(doc) result = section_template.render( edition=edition, name=edition_name, section=section, section_id=section_id, chapter=chapter, ) if next_section is not None: result += next_section_template.render( url=f"{edition}_{next_section['ident']}.html", title=next_section["title"], ) elif next_chapter is not None: result += next_chapter_template.render( url=f"{edition}_chapter_{next_chapter['ident']}.html", title=next_chapter["title"], ) result = self.__build_page(result, course_wrapper=True) file.write(result) def __build_chapter_slidedeck( self, edition, chapter, sections, next_chapter, chapter_number=None, progress: Progress = None ): with (self.config.p_build / f"{edition}_slide_{chapter['ident']}.html").open("w", encoding="utf-8") as file: slide_template = self.env.get_template("slide/slide.html") help_template = self.env.get_template("slide/help.html") next_template = self.env.get_template("slide/next.html") # Use provided chapter_number or fall back to chapter dict chapter_num = str(chapter_number) if chapter_number is not None else chapter.get("number", "0") # Set the slide URL for references (all sections in one file) # Note: filename uses original edition (e.g., "N"), not modified edition with "S" (e.g., "NS") slide_filename = f"{edition}_slide_{chapter['ident']}.html" with FiftyOhmHtmlSlideRenderer( question_renderer=self.__build_question_slide, picture_handler=self.__picture_handler, photo_handler=self.__photo_handler, include_handler=self.__include_handler, edition=edition, chapter=chapter_num, section="0", # Will be updated per section section_url=slide_filename, ) as renderer: result = "
\n" result += f'
\n

{chapter["title"]}

\n
\n' result += help_template.render() result += "
\n" section_counter = 0 slides_task = progress.add_task("Rendering slides ...") for section in progress.track(sections, task_id=slides_task): progress.update(slides_task, description=f"Rendering slides of {section['title']}") if section["slide"] is None: continue section_counter += 1 # Update section number for this section renderer.section = str(section_counter) # Reset figure counter for each section renderer.figure_counter = 0 renderer.figure_map = {} if not section["slide"].startswith("---"): section["slide"] = "---\n" + section["slide"] # First pass: collect figures doc = Document(section["slide"]) renderer.collect_figures(doc) # Second pass: render with hierarchical numbers tmp = f'
\n

{section["title"]}

\n
\n' tmp += renderer.render(doc) result += f"
{tmp}
\n" progress.remove_task(slides_task) result += next_template.render( edition=edition, next_chapter=next_chapter, chapter=chapter, ) result = slide_template.render(content=result) file.write(result) def __filter_shuffle_answers(self, seq): answers = [] firstrun = True for answer in seq: if firstrun: answers.append({"content": answer, "correct": True}) firstrun = False else: answers.append({"content": answer, "correct": False}) random.shuffle(answers) return answers def __build_book_index(self, book): template = self.env.get_template("html/course_index.html") with (self.config.p_build / f"{book['edition']}_course_index.html").open("w", encoding="utf-8") as file: result = template.render( book=book, ) result = self.__build_page(result) file.write(result) def __build_slide_index(self, book): template = self.env.get_template("slide/slide_index.html") with (self.config.p_build / f"{book['edition']}_slide_index.html").open("w", encoding="utf-8") as file: result = template.render( book=book, ) result = self.__build_page(result) file.write(result) def build_edition(self, edition: str): self.config.p_build.mkdir(exist_ok=True) edition = edition.upper() with ( (self.config.p_data_toc / f"{edition}.json").open(encoding="utf-8") as file, Progress( TaskProgressColumn(), BarColumn(), TimeRemainingColumn(), TextColumn("[progress.description]{task.description}"), transient=True, ) as progress, ): chapter_task = progress.add_task(f"Building edition {edition} ...") book = json.load(file) edition_name = book["title"] # Create index pages for chapters and slides. self.__build_book_index(book) self.__build_slide_index(book) chapters = book["chapters"] for chapter_number, chapter in enumerate(progress.track(chapters, task_id=chapter_task), 1): progress.update(chapter_task, description=f"Building edition {edition}: Chapter {chapter['title']}") # Determine next chapter for navigation (None if this is the last chapter) next_chapter = chapters[chapter_number] if chapter_number < len(chapters) else None self.__build_chapter_index(edition, edition_name, chapter_number, chapter, next_chapter) # Open, parse and render each section. section_task = progress.add_task(description="Rendering sections ...") for section_number, section in enumerate(progress.track(chapter["sections"], task_id=section_task), 1): progress.update(section_task, description=f"Rendering section {section['title']}") # Read section and slide content from the corresponding files. ident = section["ident"] section["content"] = None section["slide"] = None with (self.config.p_data_sections / f"{ident}.md").open(encoding="utf-8") as sfile: section_content = sfile.read() section["content"] = section_content with (self.config.p_data_slides / f"{ident}.md").open(encoding="utf-8") as sfile: section_content = sfile.read() section["slide"] = section_content if section["content"] is not None: self.__collect_question_occurrences( edition, chapter["title"], ident, section["title"], section["content"], ) self.__collect_index_occurrences( edition, chapter["title"], ident, section["title"], section["content"], ) next_section = ( chapter["sections"][section_number] if section_number < len(chapter["sections"]) else None ) self.__build_section( edition, edition_name, section, section_number, chapter, next_section, next_chapter, chapter_number, ) progress.remove_task(section_task) # Build the slidedeck after all sections have been processed, as they require the files to be read. self.__build_chapter_slidedeck( edition, chapter, chapter["sections"], next_chapter, chapter_number, progress ) progress.remove_task(chapter_task) def build_assets(self): self.config.p_build.mkdir(exist_ok=True) shutil.copytree( self.config.p_assets, self.config.p_build_assets, dirs_exist_ok=True, ignore=shutil.ignore_patterns(".git") ) def __parse_snippets(self): snippets = {} for md_file in Path(self.config.p_data_snippets).glob("*.md"): with md_file.open(encoding="utf-8") as file: snippets[md_file.stem] = file.read() with FiftyOhmHtmlRenderer( question_renderer=self.__build_question, picture_handler=self.__picture_handler, photo_handler=self.__photo_handler, include_handler=self.__include_handler, ) as renderer: for key, value in snippets.items(): snippets[key] = renderer.render_inner(Document(value)) # Remove leading

and trailing

: snippets[key] = snippets[key][3:-4] return snippets def __parse_contents(self): static = [] for static_file in Path(self.config.p_data_static).glob("*.html"): if "sidebar" not in static_file.stem: with static_file.open(encoding="utf-8") as file: content = file.read() sidebar_file = self.config.p_data_static / f"{static_file.stem}_sidebar.html" if not sidebar_file.exists(): sidebar = "" else: with sidebar_file.open(encoding="utf-8") as file: sidebar = file.read() static.append( { "url_part": static_file.stem, "content": content, "sidebar": sidebar if sidebar != "" else None, } ) return static def __build_index(self, snippets): template = self.env.get_template("html/index.html") result = template.render({"snippets": snippets}) with (self.config.p_build / "index.html").open("w", encoding="utf-8") as file: result = self.__build_page(result) file.write(result) def __build_course_page(self, snippets, template, page): template = self.env.get_template(f"html/{template}.html") result = template.render({"snippets": snippets}) with (self.config.p_build / f"{page}.html").open("w", encoding="utf-8") as file: result = self.__build_page(result) file.write(result) def __build_html_page(self, contents, page): for content in contents: if content["url_part"] == page: with (self.config.p_build / f"{page}.html").open("w", encoding="utf-8") as file: result = self.__build_page(content=content["content"], sidebar=content["sidebar"]) file.write(result) def build_website(self): self.config.p_build.mkdir(exist_ok=True) snippets = self.__parse_snippets() contents = self.__parse_contents() self.__build_index(snippets) self.__build_course_page(snippets, "kurse-karte", "kurse_vor_ort_karte") self.__build_course_page(snippets, "kurse-liste", "kurse_vor_ort_liste") self.__build_course_page(snippets, "patenkarte", "patenkarte") self.__build_html_page(contents, "pruefung") self.__build_html_page(contents, "infos") self.__build_html_page(contents, "suche") def build_solutions(self): for solution_file in self.config.p_data_solutions.glob("*.md"): with solution_file.open(encoding="utf-8") as file: content = file.read() with (self.config.p_build / f"{solution_file.stem}.html").open("w", encoding="utf-8") as file: with FiftyOhmHtmlRenderer( question_renderer=self.__build_question, picture_handler=self.__picture_handler, photo_handler=self.__photo_handler, include_handler=self.__include_handler, ) as renderer: question = self.__build_question( solution_file.stem, template_file="html/solution_question.html" ) solution_template = self.env.get_template("html/solution.html") solution = renderer.render(Document(content)) page = solution_template.render(question=question, solution=solution, number=solution_file.stem) page = self.__build_page(page, course_wrapper=False) file.write(page) def __collect_question_occurrences( self, edition: str, chapter_title: str, section_ident: str, section_title: str, section_content: str, ): for question_number in self.question_token_pattern.findall(section_content or ""): question_entry = self.question_index.setdefault( question_number, { "has_solution": self.__has_solution(question_number), "section": section_ident, "chapter_title": chapter_title, "section_title": section_title, "editions": [], }, ) if edition not in question_entry["editions"]: question_entry["editions"].append(edition) def __has_solution(self, question_number: str) -> bool: return (self.config.p_data_solutions / f"{question_number}.md").exists() def __collect_index_occurrences( self, edition: str, chapter_title: str, section_ident: str, section_title: str, section_content: str, ): for match in self.index_token_pattern.finditer(section_content or ""): first = match.group(1).strip() second = match.group(2).strip() if match.group(2) else None normalized_first = FiftyOhmHtmlRenderer._normalize_index_part(first) if second: normalized_second = FiftyOhmHtmlRenderer._normalize_index_part(second) anchor_id = f"index_{normalized_first}__{normalized_second}" else: anchor_id = f"index_{normalized_first}" entry_key = f"{section_ident}#{anchor_id}" index_entry = self.index_entries.setdefault( entry_key, { "term": first, "subterm": second, "anchor_id": anchor_id, "section": section_ident, "chapter_title": chapter_title, "section_title": section_title, "editions": [], }, ) if edition not in index_entry["editions"]: index_entry["editions"].append(edition) def build_question_index(self): tqdm.write("Creating question index") for question_data in self.question_index.values(): question_data["editions"] = sorted(question_data["editions"]) path = self.config.p_build_assets / "question_index.json" with (path).open("w", encoding="utf-8") as file: json.dump(self.question_index, file, ensure_ascii=False, indent=2, sort_keys=True) file.write("\n") def build_index(self): tqdm.write("Creating index") for index_data in self.index_entries.values(): index_data["editions"] = sorted(index_data["editions"]) entries = sorted( self.index_entries.values(), key=lambda item: ( item["term"].casefold(), (item["subterm"] or "").casefold(), item["section"], item["anchor_id"], ), ) path = self.config.p_build_assets / "index.json" with (path).open("w", encoding="utf-8") as file: json.dump(entries, file, ensure_ascii=False, indent=2) file.write("\n") def build_zip(self, zip_name: str | None = None) -> Path: """Create a zip archive of the complete build output directory. The zip file is written into the build directory itself and is excluded from the archive contents. """ build_dir = self.config.p_build build_dir.mkdir(parents=True, exist_ok=True) zip_path = build_dir / (zip_name if zip_name is not None else f"{build_dir.name}.zip") zip_path = zip_path.resolve() # If a previous archive exists, remove it first to avoid zipping stale data. if zip_path.exists(): zip_path.unlink() tqdm.write(f"Creating zip archive: {zip_path}") with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as zf: # os.walk includes dotfiles; sort for stable archives. for root, dirs, files in os.walk(build_dir): dirs.sort() files.sort() root_path = Path(root) for filename in files: file_path = (root_path / filename).resolve() if file_path == zip_path: continue arcname = file_path.relative_to(build_dir.resolve()) zf.write(file_path, arcname=arcname) return zip_path