diff --git a/build.py b/build.py index f86c6e7..e5143dc 100644 --- a/build.py +++ b/build.py @@ -7,7 +7,7 @@ conf = config.Config() api = directus.DirectusAPI(conf.question_base_url, conf.question_access_token) content_api = directus.DirectusAPI(conf.content_base_url, conf.content_access_token) dl = download.Download(api, content_api, conf) -bd = build.Build() +bd = build.Build(conf) # Download the text files: dl.download_edition("N") diff --git a/src/build.py b/src/build.py index d915ce9..9e0d9a9 100644 --- a/src/build.py +++ b/src/build.py @@ -1,5 +1,4 @@ import json -import os import random import shutil @@ -10,26 +9,29 @@ from tqdm import tqdm from renderer.fifty_ohm_html_renderer import FiftyOhmHtmlRenderer from renderer.fifty_ohm_html_slide_renderer import FiftyOhmHtmlSlideRenderer +from src.config import Config class Build: - def __init__(self): + def __init__(self, config: Config): + self.config = config + memory = Memory("./cache", verbose=0) self.env = Environment(loader=FileSystemLoader("templates/")) self.env.filters["shuffle_answers"] = self.__filter_shuffle_answers self.questions = self.__parse_katalog() # Decorate the method with memory.cache - self.__build_question = memory.cache(self.__build_question) - self.__build_question_slide = memory.cache(self.__build_question_slide) - self.__build_chapter = memory.cache(self.__build_chapter) - self.__build_section = memory.cache(self.__build_section) - self.__build_page = memory.cache(self.__build_page) + self.__build_question = memory.cache(self.__build_question) + self.__build_question_slide = memory.cache(self.__build_question_slide) + self.__build_chapter = memory.cache(self.__build_chapter) + self.__build_section = memory.cache(self.__build_section) + self.__build_page = memory.cache(self.__build_page) self.__build_chapter_slidedeck = memory.cache(self.__build_chapter_slidedeck) def __parse_katalog(self): - with open("data/fragenkatalog3b.json") as fragenkatalog_file: - fragenkatalog = json.load(fragenkatalog_file) + with self.config.p_fragenkatalog.open() as file: + fragenkatalog = json.load(file) questions = {} @@ -51,8 +53,8 @@ class Build: question_template = self.env.get_template(template_file) - with open("data/metadata.json") as metadata_file: - metadata = json.load(metadata_file) + with (self.config.p_data / "metadata.json").open() as file: + metadata = json.load(file) number = metadata[f"{input}"]["number"] # Fragennummer z.B. AB123 question = self.questions[number] @@ -97,31 +99,25 @@ class Build: return self.__build_question(input, template_file="slide/question.html") # cached - def __build_page(self, content, course_wrapper=False, sidebar=None) : + def __build_page(self, content, course_wrapper=False, sidebar=None): page_template = self.env.get_template("html/page.html") - return page_template.render( - content=content, - course_wrapper=course_wrapper, - sidebar=sidebar - ) - + return page_template.render(content=content, course_wrapper=course_wrapper, sidebar=sidebar) + def __picture_handler(self, id): - os.makedirs("build/pictures", exist_ok=True) - src = f"data/pictures/{id}.svg" - dst = f"build/pictures/{id}.svg" - shutil.copyfile(src, dst) + self.config.p_build_pictures.mkdir(parents=True, exist_ok=True) + file = f"{id}.svg" + shutil.copyfile(self.config.p_data_pictures / file, self.config.p_build_pictures / file) def __photo_handler(self, id): - os.makedirs("build/photos", exist_ok=True) - src = f"data/photos/{id}.jpg" - dst = f"build/photos/{id}.jpg" - shutil.copyfile(src, dst) + self.config.p_build_photos.mkdir(parents=True, exist_ok=True) + file = f"{id}.jpg" + shutil.copyfile(self.config.p_data_photos / file, self.config.p_build_photos / file) # cached def __build_chapter(self, edition, edition_name, number, chapter, next_chapter=None): chapter_template = self.env.get_template("html/chapter.html") next_chapter_template = self.env.get_template("html/next_chapter.html") - with open(f'build/{edition}_chapter_{chapter["ident"]}.html', 'w') as file: + with (self.config.p_build / f"{edition}_chapter_{chapter['ident']}.html").open("w") as file: result = chapter_template.render( edition=edition, name=edition_name, @@ -131,7 +127,7 @@ class Build: if next_chapter is not None: result += next_chapter_template.render( - url=f"{edition}_chapter_{next_chapter["ident"]}.html", + url=f"{edition}_chapter_{next_chapter['ident']}.html", title=next_chapter["title"], ) @@ -139,22 +135,23 @@ class Build: file.write(result) def __include_handler(self, include): - with open("data/includes.json") as file: + with (self.config.p_data / "includes.json").open() as file: includes = json.load(file) return includes.get(include) # cached - def __build_section(self, edition, edition_name, section, section_id, chapter, next_section=None, next_chapter=None): + def __build_section( + self, edition, edition_name, section, section_id, chapter, next_section=None, next_chapter=None + ): section_template = self.env.get_template("html/section.html") next_section_template = self.env.get_template("html/next_section.html") next_chapter_template = self.env.get_template("html/next_chapter.html") - with open(f'build/{edition}_{section["ident"]}.html', 'w') as file: - + with (self.config.p_build / f"{edition}_{section['ident']}.html").open("w") as file: with FiftyOhmHtmlRenderer( question_renderer=self.__build_question, picture_handler=self.__picture_handler, photo_handler=self.__photo_handler, - include_handler=self.__include_handler + include_handler=self.__include_handler, ) as renderer: section["content"] = renderer.render(Document(section["content"])) @@ -168,12 +165,12 @@ class Build: if next_section is not None: result += next_section_template.render( - url=f"{edition}_{next_section["ident"]}.html", + url=f"{edition}_{next_section['ident']}.html", title=next_section["title"], ) elif next_chapter is not None: result += next_chapter_template.render( - url=f"{edition}_chapter_{next_chapter["ident"]}.html", + url=f"{edition}_chapter_{next_chapter['ident']}.html", title=next_chapter["title"], ) @@ -181,7 +178,7 @@ class Build: file.write(result) def __build_chapter_slidedeck(self, edition, chapter, sections, next_chapter): - with open(f'build/{edition}_slide_{chapter["ident"]}.html', 'w') as file: + with (self.config.p_build / f"{edition}_slide_{chapter['ident']}.html").open("w") as file: slide_template = self.env.get_template("slide/slide.html") help_template = self.env.get_template("slide/help.html") next_template = self.env.get_template("slide/next.html") @@ -189,7 +186,7 @@ class Build: question_renderer=self.__build_question_slide, picture_handler=self.__picture_handler, photo_handler=self.__photo_handler, - include_handler=self.__include_handler + include_handler=self.__include_handler, ) as renderer: result = "
\n" result += f'
\n

{chapter["title"]}

\n
\n' @@ -204,7 +201,7 @@ class Build: next_chapter=next_chapter, chapter=chapter, ) - + result = slide_template.render(content=result) file.write(result) @@ -219,55 +216,55 @@ class Build: answers.append({"content": answer, "correct": False}) random.shuffle(answers) return answers - + def __build_book_index(self, book): template = self.env.get_template("html/course_index.html") - with open(f"build/{book['edition']}_course_index.html", "w") as file: + with (self.config.p_build / f"{book['edition']}_course_index.html").open("w") as file: result = template.render( book=book, ) result = self.__build_page(result) - file.write(result) - + file.write(result) + def __build_slide_index(self, book): template = self.env.get_template("slide/slide_index.html") - with open(f"build/{book['edition']}_slide_index.html", "w") as file: + with (self.config.p_build / f"{book['edition']}_slide_index.html").open("w") as file: result = template.render( book=book, ) result = self.__build_page(result) - file.write(result) + file.write(result) def build_edition(self, edition): edition = edition.upper() - with open(f'data/book_{edition}.json') as f: - book = json.load(f) + with (self.config.p_data / f"book_{edition}.json").open() as file: + book = json.load(file) chapters = book["chapters"] edition_name = book["title"] self.__build_book_index(book) self.__build_slide_index(book) - for number, chapter in enumerate(tqdm(chapters, desc=f"Build Edition: {edition}"),1): + for number, chapter in enumerate(tqdm(chapters, desc=f"Build Edition: {edition}"), 1): next_chapter = chapters[number] if number < len(chapters) else None self.__build_chapter(edition, edition_name, number, chapter, next_chapter) self.__build_chapter_slidedeck(edition, chapter, chapter["sections"], next_chapter) - for i, section in enumerate(chapter["sections"],1) : + for i, section in enumerate(chapter["sections"], 1): next_section = chapter["sections"][i] if i < len(chapter["sections"]) else None self.__build_section(edition, edition_name, section, i, chapter, next_section, next_chapter) def build_assets(self): - shutil.copytree("assets", "build/assets", dirs_exist_ok=True) + shutil.copytree(self.config.p_assets, self.config.p_build_assets, dirs_exist_ok=True) def __parse_snippets(self): - with open("data/snippets.json") as f: - snippets = json.load(f) + with (self.config.p_data / "snippets.json").open() as file: + snippets = json.load(file) with FiftyOhmHtmlRenderer( question_renderer=self.__build_question, picture_handler=self.__picture_handler, photo_handler=self.__photo_handler, - include_handler=self.__include_handler + include_handler=self.__include_handler, ) as renderer: for key, value in snippets.items(): snippets[key] = renderer.render_inner(Document(value)) @@ -276,35 +273,32 @@ class Build: return snippets def __parse_contents(self): - with open("data/content.json") as f: - contents = json.load(f) + with (self.config.p_data / "content.json").open() as file: + contents = json.load(file) return contents def __build_index(self, snippets): template = self.env.get_template("html/index.html") result = template.render({"snippets": snippets}) - with open("build/index.html", "w") as file: + with (self.config.p_build / "index.html").open("w") as file: result = self.__build_page(result) file.write(result) def __build_course_page(self, snippets, template, page): template = self.env.get_template(f"html/{template}.html") - result=template.render({"snippets": snippets}) + result = template.render({"snippets": snippets}) - with open(f"build/{page}.html", "w") as file: + with (self.config.p_build / f"{page}.html").open("w") as file: result = self.__build_page(result) file.write(result) def __build_html_page(self, contents, page): for content in contents: if content["url_part"] == page: - with open(f"build/{page}.html", "w") as file: - result = self.__build_page( - content=content["content"], - sidebar=content["sidebar"] - ) + with (self.config.p_build / f"{page}.html").open("w") as file: + result = self.__build_page(content=content["content"], sidebar=content["sidebar"]) file.write(result) def build_website(self): @@ -315,4 +309,4 @@ class Build: self.__build_course_page(snippets, "kurse-liste", "kurse_vor_ort_liste") self.__build_course_page(snippets, "patenkarte", "patenkarte") self.__build_html_page(contents, "pruefung") - self.__build_html_page(contents, "infos") \ No newline at end of file + self.__build_html_page(contents, "infos") diff --git a/src/config.py b/src/config.py index ef1f6c2..fe381b9 100644 --- a/src/config.py +++ b/src/config.py @@ -1,12 +1,10 @@ import json import os +from pathlib import Path class Config: - env_prefix = "OHM" - access_token: str - base_url: str - no_latex: bool + __env_prefix = "OHM" def __init__(self): if os.path.isfile("config/config.json"): @@ -19,14 +17,28 @@ class Config: self.content_base_url = self.get_config_value("directus_content_base_url", "https://redaktion.50ohm.de/") self.no_latex = self.get_config_value("no_latex", False) + self.p_fragenkatalog = Path(self.get_config_value("path_fragenkatalog", "./data/fragenkatalog3b.json")) + + self.p_data = Path("./data") + self.p_data_photos = self.p_data / "photos" + self.p_data_pictures = self.p_data / "pictures" + + self.p_build = Path("./build") + self.p_build_photos = self.p_build / "photos" + self.p_build_pictures = self.p_build / "pictures" + self.p_build_assets = self.p_build / "assets" + + self.p_assets = Path("./assets") + def get_config_value(self, key: str, default=None): if key in self.config: return self.config[key] - elif f"{self.env_prefix}_{key.upper()}" in os.environ.keys(): - return os.environ[f"{self.env_prefix}_{key.upper()}"] + elif f"{self.__env_prefix}_{key.upper()}" in os.environ.keys(): + return os.environ[f"{self.__env_prefix}_{key.upper()}"] elif default is not None: return default else: raise Exception( - f"ERROR: Required value for '{key}' not found. Add to config.json or env as '${self.env_prefix}_{key.upper()}'" + f"ERROR: Required value for '{key}' not found." + f"Add to config.json or env as '${self.__env_prefix}_{key.upper()}'" ) diff --git a/src/download.py b/src/download.py index 9295825..3b0165c 100644 --- a/src/download.py +++ b/src/download.py @@ -6,9 +6,12 @@ import shutil from joblib import Memory, Parallel, delayed from tqdm import tqdm +from api.directus import DirectusAPI +from src.config import Config + class Download: - def __init__(self, api, content_api, config): + def __init__(self, api: DirectusAPI, content_api: DirectusAPI, config: Config): self.api = api self.config = config self.content_api = content_api @@ -60,7 +63,7 @@ class Download: "edition": edition, } - with open("data/book_" + edition + ".json", "w", encoding="utf-8") as file: + with (self.config.p_data / f"book_{edition}.json").open("w", encoding="utf-8") as file: json.dump(out_edition, file, ensure_ascii=False, indent=4) def download_question_metadata(self): @@ -82,26 +85,26 @@ class Download: } # Serializing json: - with open("data/metadata.json", "w") as f: + with (self.config.p_data / "metadata.json").open("w", encoding="utf-8") as f: json.dump(result, f, indent=4) f.close() def download_photos(self): + self.config.p_data_photos.mkdir(parents=True, exist_ok=True) + photos = self.api.get("items/Fotos", params={"limit": -1}) for photo in tqdm(photos, desc="Downloading photos"): data = self.api.get_file("assets/" + photo["photo"]) - with open(f"./data/photos/{photo['id']}.jpg", "wb") as file: + with (self.config.p_data_photos / f"{photo['id']}.jpg").open("wb") as file: file.write(data) - def __symlink_pictures(self): - os.chdir("./data/pictures") - if not os.path.islink("foto"): - os.system("ln -s ../photos foto") - os.chdir("../..") - def download_pictures(self): - self.__symlink_pictures() + self.config.p_data_pictures.mkdir(parents=True, exist_ok=True) + # Symlink photos, so LaTeX can access these assets to render into graphics. + foto_link = self.config.p_data_pictures / "foto" + if not foto_link.is_symlink(): + foto_link.symlink_to(self.config.p_data_photos) memory = Memory("./cache", verbose=0) @@ -115,17 +118,22 @@ class Download: return if self.config.no_latex: - shutil.copyfile("./assets/images/50ohm_gray.svg", f"./data/pictures/{pid}.svg") + shutil.copyfile("./assets/images/50ohm_gray.svg", self.config.p_data_pictures / f"{pid}.svg") else: - with open(f"./data/pictures/img/{pid}include.tex", "w") as file: + with (self.config.p_data_pictures / f"img/{pid}include.tex").open("w") as file: file.write(picture["latex"]) - with open(f"./data/pictures/{pid}.tex", "w", encoding="utf-8") as file: + + aux_file = self.config.p_data_photos / f"{pid}.tex" + with aux_file.open("w", encoding="utf-8") as file: # Write the auxilary LaTeX file: file.write(f"\\documentclass{{BNetzA-Fragenkatalog}}\\DARCimageOnly{{9cm}}{{{pid}include}}") # Build the picture: - os.system(f"latexmk -lualatex -f -interaction=nonstopmode ./data/pictures/{pid}.tex > /dev/null 2>&1") + os.system(f"latexmk -lualatex -f -interaction=nonstopmode {aux_file} > /dev/null 2>&1") # Convert the PDF to SVG: - os.system(f"pdftocairo -svg data/pictures/{pid}.pdf - > data/pictures/{pid}.svg") + os.system( + "pdftocairo -svg" + f"{self.config.p_data_pictures / f'{pid}.pdf'} - > {self.config.p_data_pictures / f'{pid}.svg'}" + ) pictures = self.api.get("items/pictures", params={"limit": -1}) @@ -139,20 +147,22 @@ class Download: result = {} for include in tqdm(includes, desc="Downloading includes"): result[include["ident"]] = include["content"] - - with open("data/includes.json", "w") as f: - json.dump(result, f, indent=4) + + with (self.config.p_data / "includes.json").open("w", encoding="utf-8") as file: + json.dump(result, file, indent=4) def download_snippets(self): snippets = {} for snippet in tqdm(self.content_api.get("items/snippet"), desc="Downloading snippets"): snippets[snippet["ident"]] = snippet["content"] - with open("data/snippets.json", "w", encoding="utf-8") as file: + with (self.config.p_data / "snippets.json").open("w", encoding="utf-8") as file: json.dump(snippets, file, ensure_ascii=False, indent=4) def download_content(self): contents = [] - for content in self.content_api.get("items/content"): #tqdm(self.content_api.get("items/content"), desc="Downloading content"): + for content in self.content_api.get( + "items/content" + ): # tqdm(self.content_api.get("items/content"), desc="Downloading content"): contents.append( { "url_part": content["url_part"], @@ -160,6 +170,6 @@ class Download: "sidebar": content["sidebar"], } ) - - with open("data/content.json", "w", encoding="utf-8") as file: + + with (self.config.p_data / "content.json").open("w", encoding="utf-8") as file: json.dump(contents, file, ensure_ascii=False, indent=4)