From e5cb1527a9c4ac26b6af6395a0a50c2eda7d99a4 Mon Sep 17 00:00:00 2001 From: Tim Grelka Date: Sat, 24 May 2025 22:25:18 +0200 Subject: [PATCH] feat: allow to skip latex builds --- assets/images/50ohm_gray.svg | 1 + build.py | 8 +-- src/download.py | 120 +++++++++++++++-------------------- 3 files changed, 56 insertions(+), 73 deletions(-) create mode 100644 assets/images/50ohm_gray.svg diff --git a/assets/images/50ohm_gray.svg b/assets/images/50ohm_gray.svg new file mode 100644 index 0000000..84c6872 --- /dev/null +++ b/assets/images/50ohm_gray.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/build.py b/build.py index ed68bb6..ee88e46 100644 --- a/build.py +++ b/build.py @@ -1,13 +1,12 @@ - import api.directus as directus import src.build as build import src.config as config import src.download as download conf = config.Config() -api = directus.DirectusAPI(conf) -dl = download.Download(api) -bd = build.Build() +api = directus.DirectusAPI(conf) +dl = download.Download(api, conf) +bd = build.Build() # Download the text files: dl.download_edition("N") @@ -22,4 +21,3 @@ bd.build_edition("N") # Build assets: bd.build_assets() - diff --git a/src/download.py b/src/download.py index 5be20e9..39aaa11 100644 --- a/src/download.py +++ b/src/download.py @@ -1,21 +1,19 @@ import json import multiprocessing import os +import shutil from joblib import Memory, Parallel, delayed from tqdm import tqdm class Download: - def __init__(self, api): + def __init__(self, api, config): self.api = api - - def download_edition(self, edition) : + self.config = config - in_edition = self.api.get_one( - "items/edition", - {"filter": {"ident": {"_eq": edition}}} - ) + def download_edition(self, edition): + in_edition = self.api.get_one("items/edition", {"filter": {"ident": {"_eq": edition}}}) in_chapters = self.api.get( "items/chapter", @@ -24,8 +22,7 @@ class Download: out_chapters = [] - for in_chapter in tqdm(in_chapters, desc="Downloading edition"): - + for in_chapter in tqdm(in_chapters, desc="Downloading edition"): in_sections = self.api.get( "items/section", {"filter": {"chapter": {"chapter_id": {"_eq": in_chapter["id"]}}}}, @@ -33,49 +30,47 @@ class Download: out_chapters.append( { - "title" : in_chapter["title"], - "abstract" : in_chapter["abstract"], - "ident" : in_chapter["ident"], - "video_url" : in_chapter["video_url"], - "sections" : [], + "title": in_chapter["title"], + "abstract": in_chapter["abstract"], + "ident": in_chapter["ident"], + "video_url": in_chapter["video_url"], + "sections": [], } ) for in_section in in_sections: out_chapters[-1]["sections"].append( { - "title" : in_section["title"], - "ident" : in_section["ident"], - "status" : in_section["status"], - "class" : in_section["class"], - "content" : in_section["content"], - "slide" : in_section["slide"], - "video" : in_section["video_url"], - "questions" : [], + "title": in_section["title"], + "ident": in_section["ident"], + "status": in_section["status"], + "class": in_section["class"], + "content": in_section["content"], + "slide": in_section["slide"], + "video": in_section["video_url"], + "questions": [], } ) - with open("data/book_"+edition+".json", "w", encoding="utf-8") as file: + with open("data/book_" + edition + ".json", "w", encoding="utf-8") as file: json.dump(out_chapters, file, ensure_ascii=False, indent=4) - def download_question_metadata(self) : - + def download_question_metadata(self): questions = self.api.get( - "items/questions", - params={"filter": {"class_3": {"_in": [1,2,3]}}, "sort": "position", "limit": -1} + "items/questions", params={"filter": {"class_3": {"_in": [1, 2, 3]}}, "sort": "position", "limit": -1} ) result = {} for question in tqdm(questions, desc="Downloading metadata"): result[question["id"]] = { - "number": question["number"], + "number": question["number"], "picture_question": question["picture_question"] if question["picture_question"] is not None else "", - "picture_a": question["picture_a"] if question["picture_a"] is not None else "", - "picture_b": question["picture_b"] if question["picture_b"] is not None else "", - "picture_c": question["picture_c"] if question["picture_c"] is not None else "", - "picture_d": question["picture_d"] if question["picture_d"] is not None else "", - "layout": question["layout"] if question["layout"] is not None else "", + "picture_a": question["picture_a"] if question["picture_a"] is not None else "", + "picture_b": question["picture_b"] if question["picture_b"] is not None else "", + "picture_c": question["picture_c"] if question["picture_c"] is not None else "", + "picture_d": question["picture_d"] if question["picture_d"] is not None else "", + "layout": question["layout"] if question["layout"] is not None else "", } # Serializing json: @@ -83,60 +78,49 @@ class Download: json.dump(result, f, indent=4) f.close() - def download_photos(self) : - - photos = self.api.get( - "items/Fotos", - params={ "limit": -1 } - ) + def download_photos(self): + photos = self.api.get("items/Fotos", params={"limit": -1}) for photo in tqdm(photos, desc="Downloading photos"): data = self.api.get_file("assets/" + photo["photo"]) - file = open("./data/photos/" + str(photo["id"]) + ".jpg", "wb") - file.write(data) - file.close() + with open(f"./data/photos/{photo['id']}.jpg", "wb") as file: + file.write(data) - def __symlink_pictures(self) : + def __symlink_pictures(self): os.chdir("./data/pictures") if not os.path.islink("foto"): os.system("ln -s ../photos foto") os.chdir("../..") - def download_pictures(self) : + def download_pictures(self): self.__symlink_pictures() - - memory=Memory("./cache", verbose=0) + + memory = Memory("./cache", verbose=0) @memory.cache - def build_picture(picture) : + def build_picture(picture): pid = picture["id"] - - if "\\begin{document}" in picture["latex"] or "\\begin{table}" in picture["latex"]: # Workaround, if the picture contains a document class it should be ignored! + + if ( + "\\begin{document}" in picture["latex"] or "\\begin{table}" in picture["latex"] + ): # Workaround, if the picture contains a document class it should be ignored! return - - with open(f"./data/pictures/{pid}.tex", 'w', encoding='utf-8') as file: - # Write the auxilary LaTeX file: - file.write(f"\\documentclass{{BNetzA-Fragenkatalog}}\\DARCimageOnly{{9cm}}{{{picture['id']}include}}") - file.close() + + if self.config.no_latex: + shutil.copyfile("./assets/images/50ohm_gray.svg", f"./data/pictures/{pid}.svg") + else: + with open(f"./data/pictures/img/{pid}include.tex", "w") as file: + file.write(picture["latex"]) + with open(f"./data/pictures/{pid}.tex", "w", encoding="utf-8") as file: + # Write the auxilary LaTeX file: + file.write(f"\\documentclass{{BNetzA-Fragenkatalog}}\\DARCimageOnly{{9cm}}{{{pid}include}}") # Build the picture: os.system(f"latexmk -lualatex -f -interaction=nonstopmode ./data/pictures/{pid}.tex > /dev/null 2>&1") # Convert the PDF to SVG: - os.system(f"pdftocairo -svg data/pictures/{picture['id']}.pdf - > data/pictures/{picture['id']}.svg") + os.system(f"pdftocairo -svg data/pictures/{pid}.pdf - > data/pictures/{pid}.svg") - pictures = self.api.get( - "items/pictures", - params={ "limit": -1 } - ) - - for picture in tqdm(pictures, desc="Download all pictures"): - latex = picture["latex"] - with open("./data/pictures/img/" + str(picture["id"]) + "include.tex", "w") as file: - file.write(latex) - file.close() + pictures = self.api.get("items/pictures", params={"limit": -1}) Parallel(n_jobs=multiprocessing.cpu_count())( delayed(build_picture)(picture) for picture in tqdm(pictures, desc="Build All Pictures") ) - - - \ No newline at end of file