refactor: reorganized code
This commit is contained in:
Prof. Dr. Matthias Jung DL9MJ
@@ -1,30 +0,0 @@
|
||||
import json
|
||||
import os
|
||||
|
||||
|
||||
class Config:
|
||||
|
||||
def __init__(self):
|
||||
|
||||
self.access_token = None
|
||||
|
||||
if os.path.isfile("config/config.json") :
|
||||
with open ("config/config.json") as fh:
|
||||
config_doc = json.load(fh)
|
||||
self.access_token = config_doc["key"]
|
||||
elif 'DIRECTUS_API_KEY' in os.environ.keys():
|
||||
self.access_token = os.environ['DIRECTUS_API_KEY']
|
||||
else:
|
||||
raise Exception("ERROR: Cannot find API key in config or environment!")
|
||||
|
||||
if 'DIRECTUS_BASEURL' in os.environ.keys():
|
||||
self.base_url = os.environ['DIRECTUS_BASEURL']
|
||||
else:
|
||||
self.base_url = "https://fragenkatalog.darc.de/"
|
||||
|
||||
if not self.access_token:
|
||||
raise Exception("ERROR: Cannot find API key in config or environment!")
|
||||
|
||||
|
||||
|
||||
|
||||
-144
@@ -1,144 +0,0 @@
|
||||
import json
|
||||
import multiprocessing
|
||||
import os
|
||||
|
||||
from joblib import Memory, Parallel, delayed
|
||||
from tqdm import tqdm
|
||||
|
||||
|
||||
class Download:
|
||||
def __init__(self, api):
|
||||
self.api = api
|
||||
|
||||
def download_edition(self, edition) :
|
||||
|
||||
in_edition = self.api.get_one(
|
||||
"items/edition",
|
||||
{"filter": {"ident": {"_eq": edition}}}
|
||||
)
|
||||
|
||||
in_chapters = self.api.get(
|
||||
"items/chapter",
|
||||
{"filter": {"edition": {"edition_id": {"_eq": in_edition["id"]}}}},
|
||||
)
|
||||
|
||||
out_chapters = []
|
||||
|
||||
for in_chapter in tqdm(in_chapters, desc="Downloading edition"):
|
||||
|
||||
in_sections = self.api.get(
|
||||
"items/section",
|
||||
{"filter": {"chapter": {"chapter_id": {"_eq": in_chapter["id"]}}}},
|
||||
)
|
||||
|
||||
out_chapters.append(
|
||||
{
|
||||
"title": in_chapter["title"],
|
||||
"abstract": in_chapter["abstract"],
|
||||
"sections": [],
|
||||
}
|
||||
)
|
||||
|
||||
for in_section in in_sections:
|
||||
out_chapters[-1]["sections"].append(
|
||||
{
|
||||
"title": in_section["title"],
|
||||
"ident": in_section["ident"],
|
||||
"status": in_section["status"],
|
||||
"content": in_section["content"],
|
||||
"slide": in_section["slide"],
|
||||
"video": in_section["video_url"],
|
||||
"questions": [],
|
||||
}
|
||||
)
|
||||
|
||||
with open("data/book_"+edition+".json", "w", encoding="utf-8") as file:
|
||||
json.dump(out_chapters, file, ensure_ascii=False, indent=4)
|
||||
|
||||
def download_question_metadata(self) :
|
||||
|
||||
questions = self.api.get(
|
||||
"items/questions",
|
||||
params={"filter": {"class_3": {"_in": [1,2,3]}}, "sort": "position"}
|
||||
)
|
||||
|
||||
result = {}
|
||||
|
||||
for question in tqdm(questions, desc="Downloading question metadata"):
|
||||
result[question["id"]] = {
|
||||
"number": question["number"],
|
||||
"picture_question": question["picture_question"] if question["picture_question"] is not None else "",
|
||||
"picture_a": question["picture_a"] if question["picture_a"] is not None else "",
|
||||
"picture_b": question["picture_b"] if question["picture_b"] is not None else "",
|
||||
"picture_c": question["picture_c"] if question["picture_c"] is not None else "",
|
||||
"picture_d": question["picture_d"] if question["picture_d"] is not None else "",
|
||||
"layout": question["layout"] if question["layout"] is not None else "",
|
||||
}
|
||||
|
||||
# Serializing json:
|
||||
with open("data/metadata.json", "w") as f:
|
||||
json.dump(result, f, indent=4)
|
||||
f.close()
|
||||
|
||||
def download_photos(self) :
|
||||
|
||||
photos = self.api.get(
|
||||
"items/Fotos",
|
||||
params={ "limit": -1 }
|
||||
)
|
||||
|
||||
for photo in tqdm(photos, desc="Downloading photos"):
|
||||
data = self.api.get_file("assets/" + photo["photo"])
|
||||
file = open("./data/photos/" + str(photo["id"]) + ".jpg", "wb")
|
||||
file.write(data)
|
||||
file.close()
|
||||
|
||||
def __symlink_pictures(self) :
|
||||
os.chdir("./data/pictures")
|
||||
if not os.path.islink("foto"):
|
||||
os.system("ln -s ../photos foto")
|
||||
os.chdir("../..")
|
||||
|
||||
def download_pictures(self) :
|
||||
self.__symlink_pictures()
|
||||
|
||||
memory=Memory("./cache", verbose=0)
|
||||
|
||||
@memory.cache
|
||||
def build_picture(picture) :
|
||||
pid = picture["id"]
|
||||
|
||||
if "\\begin{document}" in picture["latex"] or "\\begin{table}" in picture["latex"]: # Workaround, if the picture contains a document class it should be ignored!
|
||||
return
|
||||
|
||||
with open(f"./data/pictures/{pid}.tex", 'w', encoding='utf-8') as file:
|
||||
# Write the auxilary LaTeX file:
|
||||
file.write(f"\\documentclass{{BNetzA-Fragenkatalog}}\\DARCimageOnly{{9cm}}{{{picture['id']}include}}")
|
||||
file.close()
|
||||
# Build the picture:
|
||||
os.system(f"latexmk -lualatex -f -interaction=nonstopmode ./data/pictures/{pid}.tex > /dev/null 2>&1")
|
||||
# Convert the PDF to SVG:
|
||||
os.system(f"pdftocairo -svg data/pictures/{picture['id']}.pdf - > data/pictures/{picture['id']}.svg > /dev/null 2>&1")
|
||||
# Remove all build files:
|
||||
#os.system(f"latexmk -C ./data/pictures/{pid}.tex > /dev/null 2>&1")
|
||||
#os.system(f"rm ./data/pictures/{pid}.tex > /dev/null 2>&1")
|
||||
|
||||
pictures = self.api.get(
|
||||
"items/pictures",
|
||||
params={ "limit": -1 }
|
||||
)
|
||||
|
||||
for picture in tqdm(pictures, desc="Download all pictures"):
|
||||
latex = picture["latex"]
|
||||
with open("./data/pictures/img/" + str(picture["id"]) + "include.tex", "w") as file:
|
||||
file.write(latex)
|
||||
file.close()
|
||||
|
||||
#for picture in tqdm(pictures, desc="Build all pictures"):
|
||||
# build_picture(picture)
|
||||
Parallel(n_jobs=multiprocessing.cpu_count())(
|
||||
delayed(build_picture)(picture) for picture in tqdm(pictures, desc="Build All Pictures")
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user