144 lines
5.3 KiB
Python
144 lines
5.3 KiB
Python
import json
|
|
import multiprocessing
|
|
import os
|
|
|
|
from joblib import Memory, Parallel, delayed
|
|
from tqdm import tqdm
|
|
|
|
|
|
class Download:
|
|
def __init__(self, api):
|
|
self.api = api
|
|
|
|
def download_edition(self, edition) :
|
|
|
|
in_edition = self.api.get_one(
|
|
"items/edition",
|
|
{"filter": {"ident": {"_eq": edition}}}
|
|
)
|
|
|
|
in_chapters = self.api.get(
|
|
"items/chapter",
|
|
{"filter": {"edition": {"edition_id": {"_eq": in_edition["id"]}}}},
|
|
)
|
|
|
|
out_chapters = []
|
|
|
|
for in_chapter in tqdm(in_chapters, desc="Downloading edition"):
|
|
|
|
in_sections = self.api.get(
|
|
"items/section",
|
|
{"filter": {"chapter": {"chapter_id": {"_eq": in_chapter["id"]}}}},
|
|
)
|
|
|
|
out_chapters.append(
|
|
{
|
|
"title": in_chapter["title"],
|
|
"abstract": in_chapter["abstract"],
|
|
"sections": [],
|
|
}
|
|
)
|
|
|
|
for in_section in in_sections:
|
|
out_chapters[-1]["sections"].append(
|
|
{
|
|
"title": in_section["title"],
|
|
"ident": in_section["ident"],
|
|
"status": in_section["status"],
|
|
"content": in_section["content"],
|
|
"slide": in_section["slide"],
|
|
"video": in_section["video_url"],
|
|
"questions": [],
|
|
}
|
|
)
|
|
|
|
with open("data/book_"+edition+".json", "w", encoding="utf-8") as file:
|
|
json.dump(out_chapters, file, ensure_ascii=False, indent=4)
|
|
|
|
def download_question_metadata(self) :
|
|
|
|
questions = self.api.get(
|
|
"items/questions",
|
|
params={"filter": {"class_3": {"_in": [1,2,3]}}, "sort": "position"}
|
|
)
|
|
|
|
result = {}
|
|
|
|
for question in tqdm(questions, desc="Downloading question metadata"):
|
|
result[question["id"]] = {
|
|
"number": question["number"],
|
|
"picture_question": question["picture_question"] if question["picture_question"] is not None else "",
|
|
"picture_a": question["picture_a"] if question["picture_a"] is not None else "",
|
|
"picture_b": question["picture_b"] if question["picture_b"] is not None else "",
|
|
"picture_c": question["picture_c"] if question["picture_c"] is not None else "",
|
|
"picture_d": question["picture_d"] if question["picture_d"] is not None else "",
|
|
"layout": question["layout"] if question["layout"] is not None else "",
|
|
}
|
|
|
|
# Serializing json:
|
|
with open("data/metadata.json", "w") as f:
|
|
json.dump(result, f, indent=4)
|
|
f.close()
|
|
|
|
def download_photos(self) :
|
|
|
|
photos = self.api.get(
|
|
"items/Fotos",
|
|
params={ "limit": -1 }
|
|
)
|
|
|
|
for photo in tqdm(photos, desc="Downloading photos"):
|
|
data = self.api.get_file("assets/" + photo["photo"])
|
|
file = open("./data/photos/" + str(photo["id"]) + ".jpg", "wb")
|
|
file.write(data)
|
|
file.close()
|
|
|
|
def __symlink_pictures(self) :
|
|
os.chdir("./data/pictures")
|
|
if not os.path.islink("foto"):
|
|
os.system("ln -s ../photos foto")
|
|
os.chdir("../..")
|
|
|
|
def download_pictures(self) :
|
|
self.__symlink_pictures()
|
|
|
|
memory=Memory("./cache", verbose=0)
|
|
|
|
@memory.cache
|
|
def build_picture(picture) :
|
|
pid = picture["id"]
|
|
|
|
if "\\begin{document}" in picture["latex"] or "\\begin{table}" in picture["latex"]: # Workaround, if the picture contains a document class it should be ignored!
|
|
return
|
|
|
|
with open(f"./data/pictures/{pid}.tex", 'w', encoding='utf-8') as file:
|
|
# Write the auxilary LaTeX file:
|
|
file.write(f"\\documentclass{{BNetzA-Fragenkatalog}}\\DARCimageOnly{{9cm}}{{{picture['id']}include}}")
|
|
file.close()
|
|
# Build the picture:
|
|
os.system(f"latexmk -lualatex -f -interaction=nonstopmode ./data/pictures/{pid}.tex > /dev/null 2>&1")
|
|
# Convert the PDF to SVG:
|
|
os.system(f"pdftocairo -svg data/pictures/{picture['id']}.pdf - > data/pictures/{picture['id']}.svg > /dev/null 2>&1")
|
|
# Remove all build files:
|
|
#os.system(f"latexmk -C ./data/pictures/{pid}.tex > /dev/null 2>&1")
|
|
#os.system(f"rm ./data/pictures/{pid}.tex > /dev/null 2>&1")
|
|
|
|
pictures = self.api.get(
|
|
"items/pictures",
|
|
params={ "limit": -1 }
|
|
)
|
|
|
|
for picture in tqdm(pictures, desc="Download all pictures"):
|
|
latex = picture["latex"]
|
|
with open("./data/pictures/img/" + str(picture["id"]) + "include.tex", "w") as file:
|
|
file.write(latex)
|
|
file.close()
|
|
|
|
#for picture in tqdm(pictures, desc="Build all pictures"):
|
|
# build_picture(picture)
|
|
Parallel(n_jobs=multiprocessing.cpu_count())(
|
|
delayed(build_picture)(picture) for picture in tqdm(pictures, desc="Build All Pictures")
|
|
)
|
|
|
|
|
|
|