Add referenced file handling to Content

Before this change it was possible for a file and a directory have the
same path. This change resolves that. Conflicts between files and
directories are now correctly discovered.

omicron/ssg/markdown.py

@@ -1,6 +1,6 @@
 from typing import cast
 from markdown_it import MarkdownIt
-from markdown_it.token import Token
+from markdown_it.token import Token as Token
 from pygments import highlight as pygments_highlight
 from pygments.formatters import HtmlFormatter
 from pygments.lexers import get_lexer_by_name
@@ -32,7 +32,7 @@ def highlight(code: str, lang: str, attrs: str) -> str:
 md = MarkdownIt(options_update={"highlight": highlight})
 
 
-def extract_summary(tokens: list[Token]) -> Markup:
+def render_summary(tokens: list[Token]) -> Markup:
     if not tokens or tokens[0].type != "paragraph_open" or tokens[0].hidden:
         raise MarkdownError(
             "Body does not start with a paragraph; set 'summary' in frontmatter"
@@ -43,11 +43,9 @@ def extract_summary(tokens: list[Token]) -> Markup:
     raise MarkdownError("Content has no paragraph to use as summary")
 
 
-def parse(body: str, include_summary: bool = True) -> tuple[Markup, Markup | None]:
-    tokens = md.parse(body)
-    if include_summary:
-        summary = extract_summary(tokens)
-    else:
-        summary = None
-    content = Markup(md.renderer.render(tokens, md.options, {}))
-    return content, summary
+def render(tokens: list[Token]) -> Markup:
+    return Markup(md.renderer.render(tokens, md.options, {}))
+
+
+def parse(body: str) -> list[Token]:
+    return md.parse(body)

omicron/ssg/output/__init__.py

@@ -1,13 +1,16 @@
-from omicron.ssg.output.output import Output
+from omicron.ssg.output.output import Output, OutputError
 from omicron.ssg.output.content import Content, ContentError
 from omicron.ssg.output.factory import create_content, is_content
 from omicron.ssg.output.file import File
 from omicron.ssg.output.index import Index, discover_index
 from omicron.ssg.output.memory import Memory
+from omicron.ssg.output.directory import Directory
+from omicron.ssg.output.symlink import Symlink
 from omicron.ssg.output.tags import Tags, discover_tags
 
 __all__ = [
     "Output",
+    "OutputError",
     "Content",
     "ContentError",
     "create_content",
@@ -15,7 +18,9 @@ __all__ = [
     "File",
     "Index",
     "discover_index",
+    "Directory",
     "Memory",
+    "Symlink",
     "Tags",
     "discover_tags",
 ]

omicron/ssg/output/article.py

@@ -1,6 +1,8 @@
 import logging
+from urllib.parse import urlparse
 from typing import TYPE_CHECKING, Any
 from omicron.ssg.output.content import Content
+from omicron.ssg.output.file import File
 from pathlib import Path
 import omicron.ssg.markdown as md
 
@@ -28,10 +30,44 @@ class Article(Content):
         )
         self.title: str = meta["title"]
 
+    def handle_reference(self, path: Path) -> Content | File:
+        site = self.site
+        if path in site.by_source:
+            return site.by_source[path]
+        return self.copy_referenced_file(path)
+
+    def resolve_references(self, tokens: list[md.Token]) -> None:
+        # MarkdownIt guarantees links are always in the second level
+        for token in tokens:
+            if token.type != "inline" or token.children is None:
+                continue
+            for child in token.children:
+                if child.type == "link_open":
+                    attr = "href"
+                elif child.type == "image":
+                    attr = "src"
+                else:
+                    continue
+
+                href = child.attrs.get(attr)
+                assert type(href) is str, "unexpected link/image href type"
+                parsed = urlparse(href)
+
+                if parsed.scheme or parsed.netloc:
+                    continue
+                if not parsed.path or parsed.path.startswith("/"):
+                    continue
+                target = self.locate_reference(Path(parsed.path))
+
+                output = self.handle_reference(target)
+                child.attrs[attr] = parsed._replace(path=output.url).geturl()
+
     def write(self) -> None:
-        content, summary = md.parse(self.read_body(), self.summary is None)
-        if summary is not None:
-            self.summary = summary
+        tokens = md.parse(self.read_body())
+        self.resolve_references(tokens)
+        if self.summary is None:
+            self.summary = md.render_summary(tokens)
+        content = md.render(tokens)
 
         template = self.site.jinja_env.get_template("article.html")
         html = template.render(page=self, content=content, site=self.site)

omicron/ssg/output/content.py

@@ -1,6 +1,7 @@
 import re
 from typing import TYPE_CHECKING
-from omicron.ssg.output.output import Output
+from omicron.ssg.output.output import Output, OutputError, is_normalized
+from omicron.ssg.output.file import File
 from pathlib import Path
 from datetime import date
 
@@ -10,7 +11,7 @@ else:
     Site = "omicron.ssg.site.Site"
 
 
-class ContentError(RuntimeError):
+class ContentError(OutputError):
     pass
 
 
@@ -31,6 +32,10 @@ class Content(Output):
         super().__init__(site, Content.build_path(section, slug))
         if tags is None:
             tags = set()
+        if not source.is_absolute():
+            raise ContentError("content source path must be absolute")
+        if not is_normalized(source):
+            raise ContentError("content source path must be normalized")
         self.source = source
         self.section: str = section
         self.created = created
@@ -42,9 +47,32 @@ class Content(Output):
     def build_path(section: str, slug: str) -> Path:
         if not Content._SECTION_RE.match(section):
             raise ContentError(f"Invalid section '{section}'")
-        path = Path(f"{section}/{slug}.html")
+        path = Path(f"{section}/{slug}/index.html")
         return path
 
+    def locate_reference(self, path: Path) -> Path:
+        if path.is_absolute():
+            raise ContentError("Can't resolve absolute references")
+        base = self.site.site_path / "content"
+        file = (self.source.parent / path).resolve()
+        if not file.is_relative_to(base):
+            raise ContentError("Reference escapes content directory")
+        return file
+
+    def copy_referenced_file(self, path: Path) -> File:
+        if not path.is_file():
+            raise ContentError(f"Referenced file '{path}' does not exist")
+
+        dest = self.destination.parent / path.name
+        existing = self.site.by_path.get(dest)
+        if isinstance(existing, File) and existing.source == path:
+            return existing
+
+        file = File(self.site, dest, path)
+        self.site.add_by_path(file)
+        file.write()
+        return file
+
     def read_body(self) -> str:
         with self.source.open("r", encoding="utf-8") as f:
             f.readline()  # opening ---

omicron/ssg/output/directory.py

@@ -0,0 +1,24 @@
+import logging
+from typing import TYPE_CHECKING
+from omicron.ssg.output.output import Output
+from pathlib import Path
+
+log = logging.getLogger(__name__)
+
+if TYPE_CHECKING:
+    from omicron.ssg.site import Site  # for static checking with mypy
+else:
+    Site = "omicron.ssg.site.Site"  # for runtime checking with beartype
+
+
+class Directory(Output):
+    def __init__(self, site: Site, destination: Path):
+        super().__init__(site, destination)
+        url_path = Path("/") / site.base_dir / destination
+        # rebuild url because Output strips /index.html which is a valid dir
+        self.url = url_path.as_posix()
+
+    def write(self) -> None:
+        dest = self.site.output_path / self.destination
+        dest.mkdir(parents=True, exist_ok=True)
+        log.debug("Created directory %s", dest)

omicron/ssg/output/index.py

@@ -44,7 +44,7 @@ class Index(Output):
         if page_num == 1:
             file = Path("index.html")
         else:
-            file = Path(f"page-{page_num}.html")
+            file = Path(f"page-{page_num}/index.html")
 
         if kind == "root":
             path = file

omicron/ssg/output/output.py

@@ -9,13 +9,27 @@ else:
     Site = "omicron.ssg.site.Site"  # for runtime checking with beartype
 
 
+def is_normalized(path: Path) -> bool:
+    return ".." not in path.parts
+
+
+class OutputError(RuntimeError):
+    pass
+
+
 class Output(ABC):
     def __init__(self, site: Site, destination: Path):
         self._site_ref = weakref.ref(site)
+        if destination.is_absolute():
+            raise OutputError("destination path must be relative")
+        if not is_normalized(destination):
+            raise OutputError("destination path must be normalized")
         self.destination = destination
-        url = site.base_dir + "/" + destination.as_posix()
-        url = url.removesuffix("/index.html") or "/"
-        self.url = url
+
+        url = site.base_dir / destination
+        if url.name == "index.html":
+            url = url.parent
+        self.url = url.as_posix()
 
     @property
     def site(self) -> Site:

omicron/ssg/output/symlink.py

@@ -0,0 +1,34 @@
+import logging
+import os
+from typing import TYPE_CHECKING
+from omicron.ssg.output.output import Output, OutputError, is_normalized
+from pathlib import Path
+
+log = logging.getLogger(__name__)
+
+if TYPE_CHECKING:
+    from omicron.ssg.site import Site  # for static checking with mypy
+else:
+    Site = "omicron.ssg.site.Site"  # for runtime checking with beartype
+
+
+class Symlink(Output):
+    def __init__(self, site: Site, link: Path, target: Path):
+        if target.is_absolute():
+            raise OutputError("Symlinks target must be relative")
+        if not is_normalized(target):
+            raise OutputError("Symlinks target must be normalized")
+        super().__init__(site, link)
+        self.target = target
+
+    def write(self) -> None:
+        site = self.site
+        if self.target not in site.by_path:
+            raise OutputError(f"Symlink target '{self.target}' is not a known output")
+        link = site.output_path / self.destination
+        link.parent.mkdir(parents=True, exist_ok=True)
+        if link.exists() or link.is_symlink():
+            link.unlink()
+        rel_target = os.path.relpath(site.output_path / self.target, link.parent)
+        link.symlink_to(rel_target)
+        log.debug("Symlinked %s -> %s", link, rel_target)

omicron/ssg/site.py

@@ -4,8 +4,10 @@ from urllib.parse import urlparse
 
 from omicron.ssg.output import (
     Output,
+    OutputError,
     Content,
     ContentError,
+    Directory,
     is_content,
     create_content,
     File,
@@ -30,11 +32,13 @@ class ConfigError(RuntimeError):
 
 class Site:
     def __init__(self, site: Path):
-        self.site_path = site.absolute()
+        self.site_path = site.resolve()
         self.content: list[Content] = []
+        self.directories: list[Directory] = []
         self.other_outputs: list[Output] = []
         self.by_tag: dict[str, list[Content]] = {}
         self.by_path: dict[Path, Output] = {}
+        self.by_source: dict[Path, Content] = {}
         self.by_section: dict[str, list[Content]] = {}
         self.tags_by_count: list[tuple[str, int]] = []
 
@@ -61,9 +65,9 @@ class Site:
         return cast(dict[str, Any], config)
 
     @staticmethod
-    def parse_base_url(value: str | None) -> tuple[str, str]:
+    def parse_base_url(value: str | None) -> tuple[str, Path]:
         if not value:
-            return ("", "")
+            return ("", Path("/"))
         parsed = urlparse(value)
         if parsed.scheme and parsed.netloc:
             origin = f"{parsed.scheme}://{parsed.netloc}"
@@ -72,7 +76,11 @@ class Site:
         else:
             origin = ""
 
-        path = parsed.path.removesuffix("/")
+        if parsed.path:
+            path = Path(parsed.path)
+        else:
+            path = Path("/")
+
         return (origin, path)
 
     @staticmethod
@@ -98,10 +106,35 @@ class Site:
             raise ContentError(
                 f"path '{output.destination}' is already claimed by another output"
             )
+        new_dirs: list[Directory] = []
+        for parent in output.destination.parents:
+            if parent == Path("."):
+                break
+            if parent in self.by_path:
+                if not isinstance(self.by_path[parent], Directory):
+                    raise OutputError(
+                        f"path '{parent}' is claimed by a file but needed as a directory"
+                    )
+                break
+            new_dirs.append(Directory(self, parent))
         self.by_path[output.destination] = output
+        if isinstance(output, Directory):
+            self.directories.append(output)
+        for d in new_dirs:
+            self.by_path[d.destination] = d
+            self.directories.append(d)
+
+    def add_by_source(self, content: Content) -> None:
+        assert content.source.is_absolute(), "Expecting content to have absolute source"
+        self.by_source[content.source] = content
 
     def home_url(self) -> str:
-        return self.base_dir or "/"
+        return self.base_dir.as_posix()
+
+    def url_for(self, path: str) -> str:
+        if Path(path).is_absolute():
+            raise OutputError("path must be relative")
+        return (self.base_dir / path).as_posix()
 
     def section_url(self, section: str) -> str:
         path = Index.build_path("section", 1, section)
@@ -144,6 +177,7 @@ class Site:
                 self.add_by_path(content)
                 self.add_by_section(content)
                 self.add_by_tag(content)
+                self.add_by_source(content)
                 self.content.append(content)
         self.update_tags_by_count()
         for index in discover_index(self):
@@ -160,11 +194,11 @@ class Site:
                 "base_url config value is missing a domain name, "
                 "can't add canonical url to content"
             )
-        if self.base_dir and not self.base_dir.startswith("/"):
+        if not self.base_dir.is_absolute():
             raise ConfigError("base_url config value must be an absolute path")
         self.discover()
-        self.output_path.mkdir(parents=True, exist_ok=True)
-        for content in self.content:
-            content.write()
+        outputs: list[Output] = [*self.directories, *self.content]
+        for output in sorted(outputs, key=lambda o: o.destination.as_posix()):
+            output.write()
         for output in self.other_outputs:
             output.write()

omicron/ssg/templates/plain/base.html

@@ -4,9 +4,9 @@
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <title>{% block title %}{{ site.name }}{% endblock %}</title>
-    <link rel="stylesheet" href="{{ site.base_dir }}/assets/style.css">
-    <link rel="stylesheet" href="{{ site.base_dir }}/assets/pygments.css">
-    <link rel="canonical" href="{{ site.origin }}{{ page.url }}">
+    <link rel="stylesheet" href="{{ site.url_for('assets/style.css') }}">
+    <link rel="stylesheet" href="{{ site.url_for('assets/pygments.css') }}">
+    {% if site.origin %}<link rel="canonical" href="{{ site.origin }}{{ page.url }}">{% endif %}
     {% block head %}{% endblock %}
 </head>
 <body>