Add noindex pages, robots.txt and operator footer

- Page.noindex: marked pages get a robots meta tag and a Disallow line in the
  generated robots.txt (single source of truth in src/pages.py)
- templates/robots.txt: public pages indexable, noindex pages and data files not
- Footer shows an optional operator line from contents/snippets/footer_operator.md
- Tests for robots.txt, noindex meta and footer

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

src/build.py

@@ -17,7 +17,7 @@ from renderer.index import Index
 
 from .config import Config
 from .edition import Edition
-from .pages import EXAM_RESULT, EXAM_SIMULATOR, HOMEWORK, PAGES, page_urls
+from .pages import EXAM_RESULT, EXAM_SIMULATOR, HOMEWORK, NOINDEX_DIRECTIVES, PAGES, page_urls
 
 
 class Build:
@@ -149,6 +149,7 @@ class Build:
         page_stylesheets=None,
         page_scripts=None,
         social_image=None,
+        robots=None,
     ):
         page_template = self.env.get_template("html/page.html")
         return page_template.render(
@@ -158,6 +159,7 @@ class Build:
             page_stylesheets=page_stylesheets or [],
             page_scripts=page_scripts or [],
             social_image=social_image,
+            robots=robots,
         )
 
     def __build_exam_simulator(self):
@@ -586,11 +588,15 @@ class Build:
             result = self.__build_page(result)
             file.write(result)
 
-    def __build_html_page(self, contents, page):
+    def __build_html_page(self, contents, page, noindex=False):
         for content in contents:
             if content["url_part"] == page:
                 with (self.config.p_build / f"{page}.html").open("w", encoding="utf-8") as file:
-                    result = self.__build_page(content=content["content"], sidebar=content["sidebar"])
+                    result = self.__build_page(
+                        content=content["content"],
+                        sidebar=content["sidebar"],
+                        robots=NOINDEX_DIRECTIVES if noindex else None,
+                    )
                     file.write(result)
                 return True
         return False
@@ -600,14 +606,24 @@ class Build:
 
         snippets = self.__parse_snippets()
         contents = self.__parse_contents()
+        # Operator line in the footer of every page (contents/snippets/footer_operator.md, optional).
+        self.env.globals["footer_operator"] = snippets.get("footer_operator", "")
         self.__build_index(snippets)
         for page in PAGES:
             # Prefer the static page of the content repository, fall back to the generic template.
-            if not self.__build_html_page(contents, page.name) and page.template is not None:
+            if not self.__build_html_page(contents, page.name, page.noindex) and page.template is not None:
                 self.__build_course_page(snippets, page.template, page.name)
+        self.__build_robots()
         self.__build_exam_simulator()
         self.__build_homework_tool()
 
+    def __build_robots(self):
+        """robots.txt: the public site may be indexed, pages marked noindex (contact details) may not be crawled."""
+        template = self.env.get_template("robots.txt")
+        noindex_pages = [f"{page.name}.html" for page in PAGES if page.noindex]
+        with (self.config.p_build / "robots.txt").open("w", encoding="utf-8") as file:
+            file.write(template.render(noindex_pages=noindex_pages))
+
     def build_solutions(self):
         if not self.has_question_pool:
             return

src/pages.py

@@ -14,6 +14,8 @@ class Page:
     # Template in templates/html/ used as fallback if the content repository
     # has no static page with this name.
     template: str | None = None
+    # Never index this page: robots meta tag + Disallow in robots.txt (e.g. contact details).
+    noindex: bool = False
 
 
 # Pages whose body comes from contents/static/<name>.html.
@@ -24,8 +26,12 @@ PAGES = (
     Page("kurser_karta", template="kurser-karta"),
     Page("kurser_lista", template="kurser-lista"),
     Page("mentorer", template="mentorer"),
+    Page("kontakt", noindex=True),
 )
 
+# Robots directives for pages marked noindex (meta tag; the nginx configs send the same as X-Robots-Tag).
+NOINDEX_DIRECTIVES = "noindex, nofollow, noarchive, nosnippet"
+
 # Pages generated by the generator itself. The last two only exist if the
 # content repository contains a question pool.
 INDEX = "index"

templates/html/page.html

@@ -4,6 +4,9 @@
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1">
     <title>50Ω – lär dig amatörradio</title>
+    {% if robots %}
+    <meta name="robots" content="{{ robots }}">
+    {% endif %}
     {% if social_image %}
     <meta property="og:type" content="website">
     <meta property="og:site_name" content="50Ω">
@@ -153,8 +156,7 @@
                     Inte godkänd av DARC eller SSA.
                   </div>
                   <div class="fiftyohm-footer-links d-flex gap-2">
-                    {# TODO: The operator must add their own imprint / contact details and privacy information here. #}
-                    <span class="text-secondary">[Utgivare, kontakt och integritetspolicy &ndash; ska fyllas i av driftansvarig]</span>
+                    {% if footer_operator %}<span class="text-secondary">{{ footer_operator }}</span>{% endif %}
                   </div>
                 </div>
               </div>

templates/robots.txt

@@ -0,0 +1,11 @@
+# robots.txt – generated by the site generator, do not edit by hand.
+# The course pages may be crawled and indexed. Pages with contact details may not.
+
+User-agent: *
+Allow: /
+{% for page in noindex_pages -%}
+Disallow: /{{ page }}
+{% endfor -%}
+# Data files (search indexes etc.) are not pages
+Disallow: /*.json$
+Disallow: /*.zip$

test/test_robots.py

@@ -0,0 +1,48 @@
+from src.build import Build
+from src.config import Config
+from src.pages import NOINDEX_DIRECTIVES
+
+
+def write_site(tmp_path):
+    (tmp_path / "toc").mkdir()
+    (tmp_path / "contents" / "static").mkdir(parents=True)
+    (tmp_path / "contents" / "snippets").mkdir(parents=True)
+    (tmp_path / "contents" / "static" / "kontakt.html").write_text(
+        "<h1>Kontakt</h1> hemlig@example.se", encoding="utf-8"
+    )
+    (tmp_path / "contents" / "static" / "info.html").write_text("<h1>Info</h1>", encoding="utf-8")
+    (tmp_path / "contents" / "snippets" / "footer_operator.md").write_text(
+        'Ansvarig: Test (SA0TST) &middot; <a href="kontakt.html" rel="nofollow">Kontakt</a>\n', encoding="utf-8"
+    )
+
+
+def test_contact_page_is_noindex_and_listed_in_robots_txt(tmp_path):
+    write_site(tmp_path)
+    Build(Config(content_path=tmp_path, build_path=tmp_path / "build")).build_website()
+    out = tmp_path / "build"
+
+    robots = (out / "robots.txt").read_text(encoding="utf-8")
+    assert "User-agent: *" in robots
+    assert "Allow: /\n" in robots
+    assert "Disallow: /kontakt.html" in robots
+    assert "Disallow: /info.html" not in robots  # normal pages stay indexable
+
+    assert f'<meta name="robots" content="{NOINDEX_DIRECTIVES}">' in (out / "kontakt.html").read_text(encoding="utf-8")
+    assert 'name="robots"' not in (out / "info.html").read_text(encoding="utf-8")
+
+
+def test_footer_names_operator_without_contact_details(tmp_path):
+    write_site(tmp_path)
+    Build(Config(content_path=tmp_path, build_path=tmp_path / "build")).build_website()
+    info = (tmp_path / "build" / "info.html").read_text(encoding="utf-8")
+
+    assert "Ansvarig: Test (SA0TST)" in info
+    assert 'href="kontakt.html"' in info
+    assert "hemlig@example.se" not in info
+
+
+def test_no_footer_operator_without_snippet(tmp_path):
+    write_site(tmp_path)
+    (tmp_path / "contents" / "snippets" / "footer_operator.md").unlink()
+    Build(Config(content_path=tmp_path, build_path=tmp_path / "build")).build_website()
+    assert "Ansvarig" not in (tmp_path / "build" / "info.html").read_text(encoding="utf-8")