From bfa965f399df79a3373ffef270b08efd60e38182 Mon Sep 17 00:00:00 2001 From: omicron Date: Wed, 12 Aug 2026 11:23:45 +0200 Subject: [PATCH] Refactor and split up Content code, introduce overarching Output class In order to handle static files in the content generation pipeline while still being able to easily validate that no output overwrites any other output without adding special cases we introduce a level above Content that contains all Output. The idea is that all output files derive from Output so that they can be checked against the by_uri map. All files will then follow the discover -> write loop. --- omicron/ssg/content.py | 170 ----------------------- omicron/ssg/markdown.py | 53 +++++++ omicron/ssg/output/__init__.py | 11 ++ omicron/ssg/output/article.py | 41 ++++++ omicron/ssg/output/content.py | 49 +++++++ omicron/ssg/output/factory.py | 45 ++++++ omicron/ssg/output/output.py | 16 +++ omicron/ssg/site.py | 14 +- omicron/ssg/templates/plain/article.html | 2 +- 9 files changed, 221 insertions(+), 180 deletions(-) delete mode 100644 omicron/ssg/content.py create mode 100644 omicron/ssg/markdown.py create mode 100644 omicron/ssg/output/__init__.py create mode 100644 omicron/ssg/output/article.py create mode 100644 omicron/ssg/output/content.py create mode 100644 omicron/ssg/output/factory.py create mode 100644 omicron/ssg/output/output.py diff --git a/omicron/ssg/content.py b/omicron/ssg/content.py deleted file mode 100644 index 8c982d3..0000000 --- a/omicron/ssg/content.py +++ /dev/null @@ -1,170 +0,0 @@ -import logging -import re -from abc import ABC, abstractmethod -from pathlib import Path -from datetime import date -from typing import TYPE_CHECKING, Any, cast - -if TYPE_CHECKING: - from omicron.ssg.site import Site -else: - Site = "omicron.ssg.site.Site" - -import yaml -from markupsafe import Markup -from markdown_it import MarkdownIt -from markdown_it.token import Token -from pygments import highlight as pygments_highlight -from pygments.formatters import HtmlFormatter -from pygments.lexers import get_lexer_by_name -from pygments.lexers.special import TextLexer -from pygments.util import ClassNotFound - -log = logging.getLogger(__name__) - -FRONTMATTER_CONTENT = {".md"} -CONTENT_EXTENSIONS = FRONTMATTER_CONTENT | {".yml"} - - -class ContentError(RuntimeError): - pass - - -text_lexer = TextLexer() -html_formatter = HtmlFormatter() - - -def highlight_style() -> str: - style = html_formatter.get_style_defs(".highlight") # type: ignore[no-untyped-call] - return cast(str, style) - - -def highlight(code: str, lang: str, attrs: str) -> str: - try: - lexer = get_lexer_by_name(lang) if lang else text_lexer - except ClassNotFound: - lexer = text_lexer - return pygments_highlight(code, lexer, html_formatter) - - -md = MarkdownIt(options_update={"highlight": highlight}) - - -def read_frontmatter(path: Path) -> dict[str, Any]: - lines = [] - with path.open("r", encoding="utf-8") as f: - if f.readline() != "---\n": - raise ContentError(f"Frontmatter opening delimiter not found for {path}") - while (line := f.readline()) != "---\n": - if line == "": - raise ContentError( - f"Frontmatter closing delimiter not found for {path}" - ) - lines.append(line) - data = yaml.safe_load("".join(lines)) - if not isinstance(data, dict) or not all(isinstance(k, str) for k in data): - raise ContentError( - f"Frontmatter must be a YAML mapping with string keys in {path}" - ) - return cast(dict[str, Any], data) - - -def create_content(path: Path) -> Content: - if path.suffix in FRONTMATTER_CONTENT: - meta = read_frontmatter(path) - elif path.suffix == ".yml": - meta = yaml.safe_load(path.read_text()) - else: - raise ContentError(f"Unhandled file type for '{path}'") - - if meta["type"] == "article": - return Article(path, meta) - else: - raise ContentError(f"Invalid type '{meta['type']}' for '{path}'") - - -class Content(ABC): - _SECTION_RE = re.compile(r"^[a-z0-9_-]+(/[a-z0-9_-]+)*$") - - def __init__( - self, - path: Path, - section: str, - slug: str, - created: date, - updated: date | None = None, - tags: set[str] | None = None, - summary: str | None = None, - ): - if tags is None: - tags = set() - self.source = path - self.section: str = section - self.created = created - self.updated = updated - self.tags: set[str] = tags - self.summary: str | None = summary - self.uri: str = self.build_uri(section, slug) - - @staticmethod - def build_uri(section: str, slug: str) -> str: - if not Content._SECTION_RE.match(section): - raise ContentError(f"Invalid section '{section}'") - uri = f"{section}/{slug}.html" - return uri - - def read_body(self) -> str: - with self.source.open("r", encoding="utf-8") as f: - f.readline() # opening --- - while (line := f.readline()) != "---\n": - if line == "": - raise ContentError( - f"Frontmatter closing delimiter not found for {self.source}" - ) - return f.read() - - @abstractmethod - def render(self, site: Site) -> None: - pass - - -class Article(Content): - def __init__(self, path: Path, meta: dict[str, Any]): - if "tags" not in meta: - log.warning("no tags for article in '%s'", path) - super().__init__( - path, - meta["section"], - meta["slug"], - meta["date"], - meta.get("updated", None), - set(meta["tags"]) if "tags" in meta else None, - meta.get("summary", None), - ) - self.title: str = meta["title"] - self.content: Markup = Markup("") - - @staticmethod - def extract_summary(tokens: list[Token]) -> str: - if not tokens or tokens[0].type != "paragraph_open" or tokens[0].hidden: - raise ContentError( - "Body does not start with a paragraph; set 'summary' in frontmatter" - ) - for j in range(1, len(tokens)): - if tokens[j].type == "paragraph_close": - return cast(str, md.renderer.render(tokens[1:j], md.options, {})) - raise ContentError("Content has no paragraph to use as summary") - - def render(self, site: Site) -> None: - tokens = md.parse(self.read_body()) - if self.summary is None: - self.summary = Article.extract_summary(tokens) - self.content = Markup(md.renderer.render(tokens, md.options, {})) - - template = site.jinja_env.get_template("article.html") - html = template.render(article=self, site=site) - - out_path = site.output_path / self.uri - out_path.parent.mkdir(parents=True, exist_ok=True) - out_path.write_text(html, encoding="utf-8") - log.debug("Rendered %s -> %s", self.source, out_path) diff --git a/omicron/ssg/markdown.py b/omicron/ssg/markdown.py new file mode 100644 index 0000000..dc9ab38 --- /dev/null +++ b/omicron/ssg/markdown.py @@ -0,0 +1,53 @@ +from typing import cast +from markdown_it import MarkdownIt +from markdown_it.token import Token +from pygments import highlight as pygments_highlight +from pygments.formatters import HtmlFormatter +from pygments.lexers import get_lexer_by_name +from pygments.lexers.special import TextLexer +from pygments.util import ClassNotFound +from markupsafe import Markup + +text_lexer = TextLexer() +html_formatter = HtmlFormatter() + + +class MarkdownError(RuntimeError): + pass + + +def highlight_style() -> str: + style = html_formatter.get_style_defs(".highlight") # type: ignore[no-untyped-call] + return cast(str, style) + + +def highlight(code: str, lang: str, attrs: str) -> str: + try: + lexer = get_lexer_by_name(lang) if lang else text_lexer + except ClassNotFound: + lexer = text_lexer + return pygments_highlight(code, lexer, html_formatter) + + +md = MarkdownIt(options_update={"highlight": highlight}) + + +def extract_summary(tokens: list[Token]) -> Markup: + if not tokens or tokens[0].type != "paragraph_open" or tokens[0].hidden: + raise MarkdownError( + "Body does not start with a paragraph; set 'summary' in frontmatter" + ) + for j in range(1, len(tokens)): + if tokens[j].type == "paragraph_close": + return Markup(md.renderer.render(tokens[1:j], md.options, {})) + raise MarkdownError("Content has no paragraph to use as summary") + + +def parse(body: str, include_summary: bool = True) -> tuple[Markup, Markup | None]: + tokens = md.parse(body) + if include_summary: + summary = extract_summary(tokens) + else: + summary = None + content = Markup(md.renderer.render(tokens, md.options, {})) + return content, summary diff --git a/omicron/ssg/output/__init__.py b/omicron/ssg/output/__init__.py new file mode 100644 index 0000000..3acffc8 --- /dev/null +++ b/omicron/ssg/output/__init__.py @@ -0,0 +1,11 @@ +from omicron.ssg.output.output import Output +from omicron.ssg.output.content import Content, ContentError +from omicron.ssg.output.factory import create_content, is_content + +__all__ = [ + "Output", + "Content", + "ContentError", + "create_content", + "is_content", +] diff --git a/omicron/ssg/output/article.py b/omicron/ssg/output/article.py new file mode 100644 index 0000000..993a126 --- /dev/null +++ b/omicron/ssg/output/article.py @@ -0,0 +1,41 @@ +import logging +from typing import TYPE_CHECKING, Any +from omicron.ssg.output.content import Content +from pathlib import Path +import omicron.ssg.markdown as md + +log = logging.getLogger(__name__) + +if TYPE_CHECKING: + from omicron.ssg.site import Site # for static checking with mypy +else: + Site = "omicron.ssg.site.Site" # for runtime checking with beartype + + +class Article(Content): + def __init__(self, path: Path, meta: dict[str, Any]): + if "tags" not in meta: + log.warning("no tags for article in '%s'", path) + super().__init__( + path, + meta["section"], + meta["slug"], + meta["date"], + meta.get("updated", None), + set(meta["tags"]) if "tags" in meta else None, + meta.get("summary", None), + ) + self.title: str = meta["title"] + + def write(self, site: Site) -> None: + content, summary = md.parse(self.read_body(), self.summary is None) + if summary is not None: + self.summary = summary + + template = site.jinja_env.get_template("article.html") + html = template.render(article=self, content=content, site=site) + + out_path = site.output_path / self.uri + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(html, encoding="utf-8") + log.debug("Rendered %s -> %s", self.source, out_path) diff --git a/omicron/ssg/output/content.py b/omicron/ssg/output/content.py new file mode 100644 index 0000000..f880d58 --- /dev/null +++ b/omicron/ssg/output/content.py @@ -0,0 +1,49 @@ +import re +from omicron.ssg.output.output import Output +from pathlib import Path +from datetime import date + + +class ContentError(RuntimeError): + pass + + +class Content(Output): + _SECTION_RE = re.compile(r"^[a-z0-9_-]+(/[a-z0-9_-]+)*$") + + def __init__( + self, + path: Path, + section: str, + slug: str, + created: date, + updated: date | None = None, + tags: set[str] | None = None, + summary: str | None = None, + ): + super().__init__(Content.build_uri(section, slug)) + if tags is None: + tags = set() + self.source = path + self.section: str = section + self.created = created + self.updated = updated + self.tags: set[str] = tags + self.summary: str | None = summary + + @staticmethod + def build_uri(section: str, slug: str) -> str: + if not Content._SECTION_RE.match(section): + raise ContentError(f"Invalid section '{section}'") + uri = f"{section}/{slug}.html" + return uri + + def read_body(self) -> str: + with self.source.open("r", encoding="utf-8") as f: + f.readline() # opening --- + while (line := f.readline()) != "---\n": + if line == "": + raise ContentError( + f"Frontmatter closing delimiter not found for {self.source}" + ) + return f.read() diff --git a/omicron/ssg/output/factory.py b/omicron/ssg/output/factory.py new file mode 100644 index 0000000..98fdd08 --- /dev/null +++ b/omicron/ssg/output/factory.py @@ -0,0 +1,45 @@ +import yaml +from pathlib import Path +from typing import Any, cast +from omicron.ssg.output.content import Content, ContentError +from omicron.ssg.output.article import Article + +FRONTMATTER_CONTENT = {".md"} +CONTENT_EXTENSIONS = FRONTMATTER_CONTENT | {".yml"} + + +def is_content(path: Path) -> bool: + return path.suffix in CONTENT_EXTENSIONS + + +def read_frontmatter(path: Path) -> dict[str, Any]: + lines = [] + with path.open("r", encoding="utf-8") as f: + if f.readline() != "---\n": + raise ContentError(f"Frontmatter opening delimiter not found for {path}") + while (line := f.readline()) != "---\n": + if line == "": + raise ContentError( + f"Frontmatter closing delimiter not found for {path}" + ) + lines.append(line) + data = yaml.safe_load("".join(lines)) + if not isinstance(data, dict) or not all(isinstance(k, str) for k in data): + raise ContentError( + f"Frontmatter must be a YAML mapping with string keys in {path}" + ) + return cast(dict[str, Any], data) + + +def create_content(path: Path) -> Content: + if path.suffix in FRONTMATTER_CONTENT: + meta = read_frontmatter(path) + elif path.suffix == ".yml": + meta = yaml.safe_load(path.read_text()) + else: + raise ContentError(f"Unhandled file type for '{path}'") + + if meta["type"] == "article": + return Article(path, meta) + else: + raise ContentError(f"Invalid type '{meta['type']}' for '{path}'") diff --git a/omicron/ssg/output/output.py b/omicron/ssg/output/output.py new file mode 100644 index 0000000..1256f70 --- /dev/null +++ b/omicron/ssg/output/output.py @@ -0,0 +1,16 @@ +from typing import TYPE_CHECKING +from abc import ABC, abstractmethod + +if TYPE_CHECKING: + from omicron.ssg.site import Site # for static checking with mypy +else: + Site = "omicron.ssg.site.Site" # for runtime checking with beartype + + +class Output(ABC): + def __init__(self, uri: str): + self.uri = uri + + @abstractmethod + def write(self, site: Site) -> None: + pass diff --git a/omicron/ssg/site.py b/omicron/ssg/site.py index 540c1eb..1a0e1fd 100644 --- a/omicron/ssg/site.py +++ b/omicron/ssg/site.py @@ -4,13 +4,9 @@ from urllib.parse import urlparse from itertools import batched from math import ceil -from omicron.ssg.content import ( - Content, - create_content, - CONTENT_EXTENSIONS, - ContentError, - highlight_style, -) +from omicron.ssg.output import Content, ContentError, is_content, create_content +from omicron.ssg.markdown import highlight_style + from pathlib import Path import yaml from jinja2 import Environment, FileSystemLoader @@ -106,7 +102,7 @@ class Site: log.info("Discovering content...") content_path = self.site_path / "content" for file in content_path.rglob("*"): - if file.is_file() and file.suffix in CONTENT_EXTENSIONS: + if file.is_file() and is_content(file): content = create_content(file) self.add_by_uri(content) self.add_by_section(content) @@ -192,5 +188,5 @@ class Site: highlight_style(), encoding="utf-8" ) for content in self.content: - content.render(self) + content.write(self) self.render_indices() diff --git a/omicron/ssg/templates/plain/article.html b/omicron/ssg/templates/plain/article.html index 4ac00e9..3ff1587 100644 --- a/omicron/ssg/templates/plain/article.html +++ b/omicron/ssg/templates/plain/article.html @@ -30,6 +30,6 @@ {% endif %} - {{ article.content }} + {{ content }} {% endblock %}