Superminaren/50ohm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into fix-svg-include

Browse Source
This commit is contained in:
Matthias Jung
2026-02-14 01:42:43 +01:00
committed by GitHub
parent f5be12555c c591afb6e9
commit 4131f6ae9f
30 changed files with 248 additions and 33262 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/build.py
+70 -46
View File
@@ -2,9 +2,9 @@ import json
import random
import re
import shutil
from pathlib import Path
from jinja2 import Environment, FileSystemLoader
from joblib import Memory
from mistletoe import Document
from tqdm import tqdm
@@ -18,20 +18,12 @@ class Build:
def __init__(self, config: Config):
self.config = config
memory = Memory("./cache", verbose=0)
self.env = Environment(loader=FileSystemLoader("templates/"))
self.env.filters["shuffle_answers"] = self.__filter_shuffle_answers
self.questions = self.__parse_katalog()
# Decorate the method with memory.cache
self.__build_question = memory.cache(self.__build_question)
self.__build_question_slide = memory.cache(self.__build_question_slide)
self.__build_chapter = memory.cache(self.__build_chapter)
self.__build_section = memory.cache(self.__build_section)
self.__build_chapter_slidedeck = memory.cache(self.__build_chapter_slidedeck)
def __parse_katalog(self):
with self.config.p_fragenkatalog.open() as file:
with self.config.p_data_fragenkatalog.open() as file:
fragenkatalog = json.load(file)
questions = {}
@@ -48,27 +40,28 @@ class Build:
return questions
# cached
def __build_question(self, input, template_file="html/question.html"):
def __build_question(self, number, template_file="html/question.html"):
"""Combines the original question dataset from BNetzA with our internal metadata"""
question_template = self.env.get_template(template_file)
with (self.config.p_data / "metadata3b.json").open() as file:
metadata = json.load(file)
with (self.config.p_data_metadata).open() as file:
metadata_json = json.load(file)
question = None
number = None
if f"{input}" in metadata:
metadata = metadata[f"{input}"]
number = metadata["number"] # Fragennummer z.B. AB123
if number in self.questions:
question = self.questions[number]
metadata = None
if question is None:
if number in self.questions:
question = self.questions[number]
if number in metadata_json:
metadata = metadata_json[number]
if question is None or metadata is None:
tqdm.write(
f"\033[31mQuestion #{input} is missing"
+ (f" (but found number: {number})" if number is not None else "")
f"\033[31mQuestion #{number} is missing"
+ (" (Question not in question pool)" if question is None else "")
+ (" (Question not in metadata)" if metadata is None else "")
+ "\033[0m"
)
metadata = {"layout": "not-found", "picture_a": ""}
@@ -125,7 +118,6 @@ class Build:
def __build_question_slide(self, input):
return self.__build_question(input, template_file="slide/question.html")
# cached
def __build_page(self, content, course_wrapper=False, sidebar=None):
page_template = self.env.get_template("html/page.html")
return page_template.render(content=content, course_wrapper=course_wrapper, sidebar=sidebar)
@@ -147,7 +139,7 @@ class Build:
def __photo_handler(self, id):
self.config.p_build_photos.mkdir(parents=True, exist_ok=True)
file = f"{id}.jpg"
file = f"{id}.png"
try:
shutil.copyfile(self.config.p_data_photos / file, self.config.p_build_photos / file)
if (self.config.p_data_photos / f"{id}.txt").exists():
@@ -157,7 +149,6 @@ class Build:
except FileNotFoundError:
tqdm.write(f"\033[31mPhoto #{id} not found\033[0m")
# cached
def __build_chapter(self, edition, edition_name, number, chapter, next_chapter=None):
chapter_template = self.env.get_template("html/chapter.html")
next_chapter_template = self.env.get_template("html/next_chapter.html")
@@ -179,15 +170,13 @@ class Build:
file.write(result)
def __include_handler(self, include):
with (self.config.p_data / "includes.json").open() as file:
includes = json.load(file)
code = includes.get(include)
with (self.config.p_data_html / f"{include}.html").open() as file:
code = file.read()
svg_list = re.findall(r"(\d+)\.svg", code or "")
for id in svg_list:
self.__copy_picture(id)
return code
# cached
def __build_section(
self,
edition,
@@ -345,9 +334,22 @@ class Build:
edition = edition.upper()
with (self.config.p_data / f"book_{edition}.json").open() as file:
with (self.config.p_data_toc / f"{edition}.json").open() as file:
book = json.load(file)
chapters = book["chapters"]
for chapter in chapters:
for section in chapter["sections"]:
ident = section["ident"]
section["content"] = None
section["slide"] = None
with (self.config.p_data_sections / f"{ident}.md").open() as sfile:
section_content = sfile.read()
section["content"] = section_content
with (self.config.p_data_slides / f"{ident}.md").open() as sfile:
section_content = sfile.read()
section["slide"] = section_content
edition_name = book["title"]
self.__build_book_index(book)
self.__build_slide_index(book)
@@ -379,25 +381,47 @@ class Build:
)
def __parse_snippets(self):
with (self.config.p_data / "snippets.json").open() as file:
snippets = json.load(file)
snippets = {}
for md_file in Path(self.config.p_data_snippets).glob("*.md"):
with md_file.open() as file:
snippets[md_file.stem] = file.read()
with FiftyOhmHtmlRenderer(
question_renderer=self.__build_question,
picture_handler=self.__picture_handler,
photo_handler=self.__photo_handler,
include_handler=self.__include_handler,
) as renderer:
for key, value in snippets.items():
snippets[key] = renderer.render_inner(Document(value))
# Remove leading <p> and trailing </p>:
snippets[key] = snippets[key][3:-4]
with FiftyOhmHtmlRenderer(
question_renderer=self.__build_question,
picture_handler=self.__picture_handler,
photo_handler=self.__photo_handler,
include_handler=self.__include_handler,
) as renderer:
for key, value in snippets.items():
snippets[key] = renderer.render_inner(Document(value))
# Remove leading <p> and trailing </p>:
snippets[key] = snippets[key][3:-4]
return snippets
def __parse_contents(self):
with (self.config.p_data / "content.json").open() as file:
contents = json.load(file)
return contents
static = []
for static_file in Path(self.config.p_data_static).glob("*.html"):
if "sidebar" not in static_file.stem:
with static_file.open() as file:
content = file.read()
sidebar_file = self.config.p_data_static / f"{static_file.stem}_sidebar.html"
if not sidebar_file.exists():
sidebar = ""
else:
with sidebar_file.open() as file:
sidebar = file.read()
static.append(
{
"url_part": static_file.stem,
"content": content,
"sidebar": sidebar if sidebar != "" else None,
}
)
return static
def __build_index(self, snippets):
template = self.env.get_template("html/index.html")
src/config.py
+14 -9
View File
@@ -13,17 +13,22 @@ class Config:
else:
self.config = {}
self.question_access_token = self.get_config_value("directus_question_api_key")
self.question_base_url = self.get_config_value("directus_question_base_url", "https://fragenkatalog.darc.de/")
self.content_access_token = self.get_config_value("directus_content_api_key")
self.content_base_url = self.get_config_value("directus_content_base_url", "https://redaktion.50ohm.de/")
self.no_latex = self.get_config_value("no_latex", False)
self.p_data = Path(self.get_config_value("content_path", "/var/www/content"))
self.p_fragenkatalog = Path(self.get_config_value("path_fragenkatalog", "./data/fragenkatalog3b.json"))
self.p_data_toc = self.p_data / "toc"
self.p_data = Path("./data")
self.p_data_photos = self.p_data / "photos"
self.p_data_pictures = self.p_data / "pictures"
self.p_data_contents = self.p_data / "contents"
self.p_data_questions = self.p_data_contents / "questions"
self.p_data_html = self.p_data_contents / "html"
self.p_data_photos = self.p_data_contents / "photos"
self.p_data_pictures = self.p_data_contents / "drawings"
self.p_data_sections = self.p_data_contents / "sections"
self.p_data_slides = self.p_data_contents / "slides"
self.p_data_snippets = self.p_data_contents / "snippets"
self.p_data_static = self.p_data_contents / "static"
self.p_data_fragenkatalog = self.p_data_questions / "fragenkatalog3b.json"
self.p_data_metadata = self.p_data_questions / "metadata3b.json"
self.p_build = Path("./build")
self.p_build_photos = self.p_build / "photos"
src/download.py
-162
View File
@@ -1,162 +0,0 @@
import json
from tqdm import tqdm
from api.directus import DirectusAPI
from .config import Config
class Download:
def __init__(self, api: DirectusAPI, content_api: DirectusAPI, config: Config):
self.api = api
self.config = config
self.content_api = content_api
def download_edition(self, edition):
in_edition = self.api.get_one("items/edition", {"filter": {"ident": {"_eq": edition}}})
in_chapters = self.api.get(
"items/chapter",
{"filter": {"edition": {"edition_id": {"_eq": in_edition["id"]}}}},
)
out_chapters = []
for in_chapter in tqdm(in_chapters, desc="Downloading edition"):
in_sections = self.api.get(
"items/section",
{"filter": {"chapter": {"chapter_id": {"_eq": in_chapter["id"]}}}},
)
out_chapters.append(
{
"title": in_chapter["title"],
"abstract": in_chapter["abstract"],
"ident": in_chapter["ident"],
"video_url": in_chapter["video_url"],
"sections": [],
}
)
for in_section in in_sections:
out_chapters[-1]["sections"].append(
{
"title": in_section["title"],
"ident": in_section["ident"],
"status": in_section["status"],
"class": in_section["class"],
"content": in_section["content"],
"slide": in_section["slide"],
"video_url": in_section["video_url"],
"questions": [],
}
)
out_edition = {
"title": in_edition["title"],
"abstract": in_edition["abstract"],
"chapters": out_chapters,
"edition": edition,
}
with (self.config.p_data / f"book_{edition}.json").open("w", encoding="utf-8") as file:
json.dump(out_edition, file, ensure_ascii=False, indent=4)
def download_question_metadata(self):
questions = self.api.get(
"items/questions", params={"filter": {"class_3": {"_in": [1, 2, 3]}}, "sort": "position", "limit": -1}
)
result = {}
for question in tqdm(questions, desc="Downloading metadata"):
result[question["id"]] = {
"number": question["number"],
"picture_question": question["picture_question"] if question["picture_question"] is not None else "",
"picture_a": question["picture_a"] if question["picture_a"] is not None else "",
"picture_b": question["picture_b"] if question["picture_b"] is not None else "",
"picture_c": question["picture_c"] if question["picture_c"] is not None else "",
"picture_d": question["picture_d"] if question["picture_d"] is not None else "",
"layout": question["layout"] if question["layout"] is not None else "",
}
# Serializing json:
with (self.config.p_data / "metadata.json").open("w", encoding="utf-8") as f:
json.dump(result, f, indent=4)
f.close()
def download_photos(self):
self.config.p_data_photos.mkdir(parents=True, exist_ok=True)
photos = self.api.get("items/Fotos", params={"limit": -1})
for photo in tqdm(photos, desc="Downloading photos"):
# Save Photo to file
data = self.api.get_file("assets/" + photo["photo"])
with (self.config.p_data_photos / f"{photo['id']}.jpg").open("wb") as file:
file.write(data)
# Save Alt-Text to file
prefix = ""
if photo["alt_text"] is not None:
if photo["alt_text_reviewed"] is None:
prefix = "Der folgende Alt-Text wurde noch nicht geprüft: "
with (self.config.p_data_photos / f"{photo['id']}.txt").open("w") as file:
file.write(prefix + photo["alt_text"])
def download_pictures(self):
self.config.p_data_pictures.mkdir(parents=True, exist_ok=True)
pictures = self.api.get("items/pictures", {"limit": -1})
for picture in tqdm(pictures, desc="Downloading pictures"):
# Save LaTeX sources to file
with (self.config.p_data_pictures / f"{picture['id']}include.tex").open("w") as file:
file.write(picture["latex"])
# Save Alt-Text to file
prefix = ""
if picture["alt_text"] is not None:
if picture["alt_text_reviewed"] is None:
prefix = "Der folgende Alt-Text wurde noch nicht geprüft: "
with (self.config.p_data_pictures / f"{picture['id']}.txt").open("w") as file:
file.write(prefix + picture["alt_text"])
# Download SVG
data = self.api.get_file(f"assets/{picture['picture']}")
with (self.config.p_data_pictures / f"{picture['id']}.svg").open("wb") as file:
file.write(data)
def download_includes(self):
includes = self.api.get("items/include", params={"limit": -1})
result = {}
for include in tqdm(includes, desc="Downloading includes"):
result[include["ident"]] = include["content"]
with (self.config.p_data / "includes.json").open("w", encoding="utf-8") as file:
json.dump(result, file, indent=4)
def download_snippets(self):
snippets = {}
for snippet in tqdm(self.content_api.get("items/snippet"), desc="Downloading snippets"):
snippets[snippet["ident"]] = snippet["content"]
with (self.config.p_data / "snippets.json").open("w", encoding="utf-8") as file:
json.dump(snippets, file, ensure_ascii=False, indent=4)
def download_content(self):
contents = []
for content in self.content_api.get(
"items/content"
): # tqdm(self.content_api.get("items/content"), desc="Downloading content"):
contents.append(
{
"url_part": content["url_part"],
"content": content["content"],
"sidebar": content["sidebar"],
}
)
with (self.config.p_data / "content.json").open("w", encoding="utf-8") as file:
json.dump(contents, file, ensure_ascii=False, indent=4)