From d5399e30f9123e25cb51b1a0e9b46a056ec98f96 Mon Sep 17 00:00:00 2001 From: Matthias Jung Date: Sun, 1 Feb 2026 16:28:04 +0100 Subject: [PATCH] refactor: removed question-directus download --- build.py | 36 ++++------ config/config.json.example | 4 +- src/config.py | 12 ++-- src/download.py | 141 ++++--------------------------------- 4 files changed, 34 insertions(+), 159 deletions(-) diff --git a/build.py b/build.py index d43e22b..cbd48f1 100644 --- a/build.py +++ b/build.py @@ -4,33 +4,23 @@ import src.config as config import src.download as download conf = config.Config() -api = directus.DirectusAPI(conf.question_base_url, conf.question_access_token) content_api = directus.DirectusAPI(conf.content_base_url, conf.content_access_token) -dl = download.Download(api, content_api, conf) -bd = build.Build(conf) +dl = download.Download(content_api, conf) -# Download the text files: -dl.download_edition("N") -dl.download_edition("E") -dl.download_edition("A") -dl.download_edition("NE") -dl.download_edition("EA") -dl.download_edition("NEA") +# Download the content files: +dl.download_git_content() # Download additional files: -# dl.download_question_metadata() # (now constant file metadata3b.json) -dl.download_photos() -dl.download_pictures() -dl.download_includes() dl.download_snippets() dl.download_content() -# Build Everything: -bd.build_website() -bd.build_edition("N") -bd.build_edition("E") -bd.build_edition("A") -bd.build_edition("NE") -bd.build_edition("EA") -bd.build_edition("NEA") -bd.build_assets() +## Build Everything: +bd = build.Build(conf) +# bd.build_website() +# bd.build_edition("N") +# bd.build_edition("E") +# bd.build_edition("A") +# bd.build_edition("NE") +# bd.build_edition("EA") +# bd.build_edition("NEA") +# bd.build_assets() diff --git a/config/config.json.example b/config/config.json.example index 9392a7c..d93bed8 100644 --- a/config/config.json.example +++ b/config/config.json.example @@ -1,7 +1,5 @@ { - "directus_question_base_url" : "https://fragenkatalog.darc.de/", - "directus_question_api_key" : "", "directus_content_base_url" : "https://redaktion.50ohm.de/", "directus_content_api_key" : "", - "no_latex": true + "git_url" : "git@github.com:DARC-e-V/50ohm-contents-dl.git", } diff --git a/src/config.py b/src/config.py index a7444c6..7ce9eb8 100644 --- a/src/config.py +++ b/src/config.py @@ -13,17 +13,17 @@ class Config: else: self.config = {} - self.question_access_token = self.get_config_value("directus_question_api_key") - self.question_base_url = self.get_config_value("directus_question_base_url", "https://fragenkatalog.darc.de/") self.content_access_token = self.get_config_value("directus_content_api_key") self.content_base_url = self.get_config_value("directus_content_base_url", "https://redaktion.50ohm.de/") - self.no_latex = self.get_config_value("no_latex", False) + self.git_url = self.get_config_value("git_url", "git@github.com:DARC-e-V/50ohm-contents-dl.git") - self.p_fragenkatalog = Path(self.get_config_value("path_fragenkatalog", "./data/fragenkatalog3b.json")) + self.p_fragenkatalog = Path( + self.get_config_value("path_fragenkatalog", "./data/git_content/contents/questions/fragenkatalog3b.json") + ) self.p_data = Path("./data") - self.p_data_photos = self.p_data / "photos" - self.p_data_pictures = self.p_data / "pictures" + self.p_data_photos = self.p_data / "git_content" / "photos" + self.p_data_pictures = self.p_data / "git_content" / "pictures" self.p_build = Path("./build") self.p_build_photos = self.p_build / "photos" diff --git a/src/download.py b/src/download.py index fd443f6..9e9dab4 100644 --- a/src/download.py +++ b/src/download.py @@ -1,5 +1,6 @@ import json +from git import Repo from tqdm import tqdm from api.directus import DirectusAPI @@ -8,136 +9,10 @@ from .config import Config class Download: - def __init__(self, api: DirectusAPI, content_api: DirectusAPI, config: Config): - self.api = api + def __init__(self, content_api: DirectusAPI, config: Config): self.config = config self.content_api = content_api - def download_edition(self, edition): - in_edition = self.api.get_one("items/edition", {"filter": {"ident": {"_eq": edition}}}) - - in_chapters = self.api.get( - "items/chapter", - {"filter": {"edition": {"edition_id": {"_eq": in_edition["id"]}}}}, - ) - - out_chapters = [] - - for in_chapter in tqdm(in_chapters, desc="Downloading edition"): - in_sections = self.api.get( - "items/section", - {"filter": {"chapter": {"chapter_id": {"_eq": in_chapter["id"]}}}}, - ) - - out_chapters.append( - { - "title": in_chapter["title"], - "abstract": in_chapter["abstract"], - "ident": in_chapter["ident"], - "video_url": in_chapter["video_url"], - "sections": [], - } - ) - - for in_section in in_sections: - out_chapters[-1]["sections"].append( - { - "title": in_section["title"], - "ident": in_section["ident"], - "status": in_section["status"], - "class": in_section["class"], - "content": in_section["content"], - "slide": in_section["slide"], - "video_url": in_section["video_url"], - "questions": [], - } - ) - - out_edition = { - "title": in_edition["title"], - "abstract": in_edition["abstract"], - "chapters": out_chapters, - "edition": edition, - } - - with (self.config.p_data / f"book_{edition}.json").open("w", encoding="utf-8") as file: - json.dump(out_edition, file, ensure_ascii=False, indent=4) - - def download_question_metadata(self): - questions = self.api.get( - "items/questions", params={"filter": {"class_3": {"_in": [1, 2, 3]}}, "sort": "position", "limit": -1} - ) - - result = {} - - for question in tqdm(questions, desc="Downloading metadata"): - result[question["id"]] = { - "number": question["number"], - "picture_question": question["picture_question"] if question["picture_question"] is not None else "", - "picture_a": question["picture_a"] if question["picture_a"] is not None else "", - "picture_b": question["picture_b"] if question["picture_b"] is not None else "", - "picture_c": question["picture_c"] if question["picture_c"] is not None else "", - "picture_d": question["picture_d"] if question["picture_d"] is not None else "", - "layout": question["layout"] if question["layout"] is not None else "", - } - - # Serializing json: - with (self.config.p_data / "metadata.json").open("w", encoding="utf-8") as f: - json.dump(result, f, indent=4) - f.close() - - def download_photos(self): - self.config.p_data_photos.mkdir(parents=True, exist_ok=True) - - photos = self.api.get("items/Fotos", params={"limit": -1}) - - for photo in tqdm(photos, desc="Downloading photos"): - # Save Photo to file - data = self.api.get_file("assets/" + photo["photo"]) - with (self.config.p_data_photos / f"{photo['id']}.jpg").open("wb") as file: - file.write(data) - - # Save Alt-Text to file - prefix = "" - if photo["alt_text"] is not None: - if photo["alt_text_reviewed"] is None: - prefix = "Der folgende Alt-Text wurde noch nicht geprüft: " - with (self.config.p_data_photos / f"{photo['id']}.txt").open("w") as file: - file.write(prefix + photo["alt_text"]) - - def download_pictures(self): - self.config.p_data_pictures.mkdir(parents=True, exist_ok=True) - - pictures = self.api.get("items/pictures", {"limit": -1}) - - for picture in tqdm(pictures, desc="Downloading pictures"): - # Save LaTeX sources to file - with (self.config.p_data_pictures / f"{picture['id']}include.tex").open("w") as file: - file.write(picture["latex"]) - - # Save Alt-Text to file - prefix = "" - if picture["alt_text"] is not None: - if picture["alt_text_reviewed"] is None: - prefix = "Der folgende Alt-Text wurde noch nicht geprüft: " - with (self.config.p_data_pictures / f"{picture['id']}.txt").open("w") as file: - file.write(prefix + picture["alt_text"]) - - # Download SVG - data = self.api.get_file(f"assets/{picture['picture']}") - with (self.config.p_data_pictures / f"{picture['id']}.svg").open("wb") as file: - file.write(data) - - def download_includes(self): - includes = self.api.get("items/include", params={"limit": -1}) - - result = {} - for include in tqdm(includes, desc="Downloading includes"): - result[include["ident"]] = include["content"] - - with (self.config.p_data / "includes.json").open("w", encoding="utf-8") as file: - json.dump(result, file, indent=4) - def download_snippets(self): snippets = {} for snippet in tqdm(self.content_api.get("items/snippet"), desc="Downloading snippets"): @@ -160,3 +35,15 @@ class Download: with (self.config.p_data / "content.json").open("w", encoding="utf-8") as file: json.dump(contents, file, ensure_ascii=False, indent=4) + + def download_git_content(self): + repo_path = self.config.p_data / "git_content" + if not repo_path.exists(): + print("Cloning git repository, this may take a while ...") + Repo.clone_from(self.config.git_url, repo_path) + print("... done.") + else: + repo = Repo(repo_path) + origin = repo.remotes.origin + origin.pull() + print("Repository updated.")