From b61efacbbce4299cff12e240a1d5c141abdbcdcf Mon Sep 17 00:00:00 2001
From: "Prof. Dr. Matthias Jung DL9MJ" <dl9mj@darc.de>
Date: Sun, 18 May 2025 10:17:41 +0200
Subject: [PATCH] feature: added code to provide first data download

---
 api/download.py | 86 ++++++++++++++++++++++++++++++++++++++-----------
 build.py        |  4 +++
 2 files changed, 72 insertions(+), 18 deletions(-)

diff --git a/api/download.py b/api/download.py
index 3e26d11..05da09b 100644
--- a/api/download.py
+++ b/api/download.py
@@ -1,5 +1,8 @@
 import json
+import multiprocessing
+import os
 
+from joblib import Memory, Parallel, delayed
 from tqdm import tqdm
 
 
@@ -7,24 +10,23 @@ class Download:
     def __init__(self, api):
         self.api = api
     
-    def download_edition(cls, edition) :
-        print("Downloading edition " + edition)
+    def download_edition(self, edition) :
 
-        in_edition = cls.api.get_one(
+        in_edition = self.api.get_one(
             "items/edition",
             {"filter": {"ident": {"_eq": edition}}}
         )
 
-        in_chapters = cls.api.get(
+        in_chapters = self.api.get(
             "items/chapter",
             {"filter": {"edition": {"edition_id": {"_eq": in_edition["id"]}}}},
         )
 
         out_chapters = []
 
-        for in_chapter in tqdm(in_chapters):
+        for in_chapter in tqdm(in_chapters,  desc="Downloading edition"):
 
-            in_sections = cls.api.get(
+            in_sections = self.api.get(
                 "items/section",
                 {"filter": {"chapter": {"chapter_id": {"_eq": in_chapter["id"]}}}},
             )
@@ -53,17 +55,16 @@ class Download:
             with open("data/book_"+edition+".json", "w", encoding="utf-8") as file:
                 json.dump(out_chapters, file, ensure_ascii=False, indent=4)
 
-    def download_question_metadata(cls) :
-        print("Downloading question metadata")
+    def download_question_metadata(self) :
 
-        questions = cls.api.get(
+        questions = self.api.get(
             "items/questions",
             params={"filter": {"class_3": {"_in": [1,2,3]}}, "sort": "position"}
         )
 
         result = {}
 
-        for question in questions:
+        for question in tqdm(questions, desc="Downloading question metadata"):
             result[question["id"]] = {
                 "number":           question["number"],
                 "picture_question": question["picture_question"] if question["picture_question"] is not None else "",
@@ -79,16 +80,65 @@ class Download:
             json.dump(result, f, indent=4)
         f.close()
 
-    def download_photos(cls) :
-        print("Downloading photos")
+    def download_photos(self) :
 
-        photos = cls.api.get(
+        photos = self.api.get(
             "items/Fotos",
-            params=None
+            params={ "limit": -1 }
         )
 
-        for photo in tqdm(photos):
-            data = cls.api.get_file("assets/" + photo["photo"])
-            file = open("./data/photo/" + str(photo["id"]) + ".jpg", "wb")
+        for photo in tqdm(photos, desc="Downloading photos"):
+            data = self.api.get_file("assets/" + photo["photo"])
+            file = open("./data/photos/" + str(photo["id"]) + ".jpg", "wb")
             file.write(data)
-            file.close()
\ No newline at end of file
+            file.close()
+
+    def __symlink_pictures(self) :
+        os.chdir("./data/pictures")
+        if not os.path.islink("foto"):
+            os.system("ln -s ../photos foto")
+        os.chdir("../..")
+
+    def download_pictures(self) :
+        self.__symlink_pictures()
+        
+        memory=Memory("./cache", verbose=0)
+
+        @memory.cache
+        def build_picture(picture) :
+            pid = picture["id"]
+            
+            if "\\begin{document}" in picture["latex"] or "\\begin{table}" in picture["latex"]: # Workaround, if the picture contains a document class it should be ignored!
+                return
+            
+            with open(f"./data/pictures/{pid}.tex", 'w', encoding='utf-8') as file:
+                # Write the auxilary LaTeX file:
+                file.write(f"\\documentclass{{BNetzA-Fragenkatalog}}\\DARCimageOnly{{9cm}}{{{picture['id']}include}}")
+                file.close()
+                # Build the picture:
+                os.system(f"latexmk -lualatex -f -interaction=nonstopmode ./data/pictures/{pid}.tex > /dev/null 2>&1")
+                # Convert the PDF to SVG:
+                os.system(f"pdftocairo -svg data/pictures/{picture['id']}.pdf - > data/pictures/{picture['id']}.svg > /dev/null 2>&1")
+                # Remove all build files:
+                #os.system(f"latexmk -C ./data/pictures/{pid}.tex > /dev/null 2>&1")
+                #os.system(f"rm ./data/pictures/{pid}.tex > /dev/null 2>&1")
+
+        pictures = self.api.get(
+            "items/pictures",
+            params={ "limit": -1 }
+        )
+
+        for picture in tqdm(pictures, desc="Download all pictures"):
+            latex = picture["latex"]
+            with open("./data/pictures/img/" + str(picture["id"]) + "include.tex", "w") as file:
+                file.write(latex)
+            file.close()
+
+        #for picture in tqdm(pictures, desc="Build all pictures"):
+        #    build_picture(picture)
+        Parallel(n_jobs=multiprocessing.cpu_count())(
+            delayed(build_picture)(picture) for picture in tqdm(pictures, desc="Build All Pictures")
+        )
+
+
+                
\ No newline at end of file
diff --git a/build.py b/build.py
index 88b5a89..84a77b1 100644
--- a/build.py
+++ b/build.py
@@ -7,6 +7,10 @@ conf = config.Config()
 api  = directus.DirectusAPI(conf)
 dl   = download.Download(api)
 
+# Download the text files:
 dl.download_edition("N")
+
+# Download additional files:
 dl.download_question_metadata()
 dl.download_photos()
+dl.download_pictures()