Generate html Files Through Build Scripts
authorGeorgios Atheridis <georgios@atheridis.org>
Sun, 12 Feb 2023 22:20:23 +0000 (22:20 +0000)
committerGeorgios Atheridis <georgios@atheridis.org>
Sun, 12 Feb 2023 22:20:23 +0000 (22:20 +0000)
Users can now create build scripts in python that will allow them to
generate html files dynamically--based on other data.

This is useful for cases where one has a blog with posts being tagged,
it is now possible to create an html file for each tag and group all
posts within that tag's page.

README
TODO
data.toml
pages/final.md [new file with mode: 0644]
pages/index.html [deleted file]
pages/index.md [new file with mode: 0644]
pages/posts/_build.py [new file with mode: 0644]
requirements.txt [new file with mode: 0644]
sigma
templates/scripts/build_nav.py
templates/scripts/build_posts.py

diff --git a/README b/README
index 4df619599fcd24b1552c5d96a28b22bfba70bf5b..fb5d978984d3d3473df4b3deb01a940dd9e8c4c5 100644 (file)
--- a/README
+++ b/README
@@ -4,7 +4,7 @@ sigma is a static site generator written in python
 data.toml:
 ----------
 It is used to define variables for each file. Variables in the root are
-global and are inherited by their children.  If a child has a variable
+global and are inherited by their children. If a child has a variable
 of the same name, then the child's variable is used instead.
 
 -----------------------------------------------------------------------
@@ -15,6 +15,8 @@ Keywords:
 - `_TEMPLATES` specifies the template directory
 - `_OUT` designates the directory where the output files go
 - `_DATE_FORMAT` designates the date format to use when using `_date`
+- `_EXT` is used to find the file extension; it is assigned during
+  execution and should not be changed.
 
 + `self` the file which invoked the template
 + `_extend` the template which the file should invoke
@@ -47,6 +49,22 @@ Functionality:
   │  ├─ _test.py
   ```
   Where the file names inside the directory become the variables of the
-  file.  Python files are not run here, as they will often be used to
-  write code which can then be imported into the _value variable.  Files
-  that are meant to be ran should go into the templates directory.
+  file. Python files are not run here, as they will often be used to
+  write code which can then be imported into the `_value` variable.
+  Files that are meant to be ran should go into the templates directory.
+
+
+- Ability to create html files on the fly using `_build.py` python
+  scripts. The `_value` variable when the script finishes will produce
+  the html files with their values.
+
+  The `_value` variable must be a list containing 2-tuples. Each 2-tuple
+  is an html file, the first element of the 2-tuple is a list containing
+  a split path to the html file, with the last value being the name of
+  the file without the extension; for example, the path `post/one.html`
+  will have the list ["post", "one"]. The second value of the 2-tuple is
+  the contents of the html file.
+
+  This is useful for cases where the amount of files changes dynamically
+  based on content. For example, having an html file for each tag to
+  group posts by.
diff --git a/TODO b/TODO
index d37fbc101c03a99344aaf4f16184503918d263ba..7e47ebb21c00aaedcc5d2d5643755b2e596d7946 100644 (file)
--- a/TODO
+++ b/TODO
@@ -1,19 +1,12 @@
 TODO
 ====
 
-- Allow for a _build variable which will produce html files through
-  key-value pairs.  Directories will also be able to be created by
-  having another key-value pair inside of the value.  This will allow
-  for recursive creation of html files and directories.
+- Do not update files that haven't changed. Include `data.toml` values.
 
-  If the _build variable is a python file inside of the pages directory
-  it will run the python file and the _value variable inside the file
-  will result in the value of the _build variable.
+- Refactor code so a lot of logic isn't duplicated in multiple places.
 
-- Do not update files that haven't changed.  Include data.toml values.
-
-- Allow any executable to be executed as a template.  Its standard
-  output becomes the _value of that template.  Make sure to transfer
+- Allow any executable to be executed as a template. Its standard
+  output becomes the `_value` of that template. Make sure to transfer
   appropriate data to the executable.
 
-- Add logging.
+- Add logging, docstrings, and comments.
index 076926d367ed185b3fbccd69ae0ca195c8066436..8ae54199adff28b9d7d56bc122c87df5398bf3fe 100644 (file)
--- a/data.toml
+++ b/data.toml
@@ -18,10 +18,32 @@ _email = "georgios@atheridis.org"
 [index]
 _name = "Home"
 _title = "My Blog"
+_nav = 0
+
+[final]
+_nav = 5
+
+[textfile]
+_nav = 20
+
+[about]
+_nav = 10
+
+[page_dir.index]
+_name = "Page Dir"
+_nav = 30
 
 [posts]
 _extend = "post.html"
 
+[posts.tags]
+_extend = "base.html"
+
+[posts.tags.index]
+_name = "Tags"
+_nav = 40
+
+
 [posts.example]
 _owner = "John Doe"
 _email = "john_doe@example.com"
@@ -39,8 +61,12 @@ _tags = ["test", "post"]
 [posts.other]
 _title = "Another Post"
 _description = "The description of another post"
-_tags = ["other", "post"]
+_tags = ["other", "post", "sigma"]
 [posts.other._table]
 name = ["Georgios Atheridis", "John Doe", "Jane Doe"]
 email = ["georgios@atheridis.org", "johndoe@example.com", "janedoe@example.com"]
 "phone number" = ["1234567890", "0777777777"]
+
+[posts.python-code]
+_title = "Python Code"
+_tags = ["post", "python", "example"]
diff --git a/pages/final.md b/pages/final.md
new file mode 100644 (file)
index 0000000..facbd28
--- /dev/null
@@ -0,0 +1 @@
+# Is this a markdown file?
diff --git a/pages/index.html b/pages/index.html
deleted file mode 100644 (file)
index 6c835c9..0000000
+++ /dev/null
@@ -1,2 +0,0 @@
-<h1>These are my posts</h1>
-{% file 'scripts/build_posts.py' %}
diff --git a/pages/index.md b/pages/index.md
new file mode 100644 (file)
index 0000000..6c835c9
--- /dev/null
@@ -0,0 +1,2 @@
+<h1>These are my posts</h1>
+{% file 'scripts/build_posts.py' %}
diff --git a/pages/posts/_build.py b/pages/posts/_build.py
new file mode 100644 (file)
index 0000000..b616694
--- /dev/null
@@ -0,0 +1,36 @@
+tag_list = "<ul>"
+tag_page = {}
+
+tags = {}
+
+posts = data["posts"]
+for post in posts:
+    if post.startswith(("_", ".")):
+        continue
+    if not (_tags := get_value(data, namespace + (post, ), "_tags")):
+        continue
+    for tag in _tags:
+        tags[tag] = tags.setdefault(tag, [])
+        tags[tag].append(post)
+        tag_page[tag] = "<ul>"
+
+
+for tag in tags:
+    path = f"/posts/tags/{tag}.html"
+    tag_list += f'<li><span><a href="{path}">{tag}</a></span></li>'
+    tag_page[tag] = "<ul>"
+    for post in tags[tag]:
+        path = "/posts/" + post + get_value(data, namespace + (post, ), "_EXT")
+        tag_page[tag] += (
+            '<li><span><a href="%(path)s">'
+            "{{ posts.%(post)s._title }}<a/></span></li>"
+            % {
+                "path": path,
+                "post": post,
+            }
+        )
+    tag_page[tag] += "</ul>"
+tag_list += "</ul>"
+
+_value = [(["tags", page], tag_page[page]) for page in tag_page]
+_value.append((["tags", "index"], tag_list))
diff --git a/requirements.txt b/requirements.txt
new file mode 100644 (file)
index 0000000..9597cac
--- /dev/null
@@ -0,0 +1 @@
+Markdown==3.4.1
diff --git a/sigma b/sigma
index 1f72c08e2da9855b69f3513cdae7f50cbf4af141..f1b29d4fd53ac0b104b1289218b0754546694455 100755 (executable)
--- a/sigma
+++ b/sigma
@@ -55,6 +55,26 @@ def get_max_mtime(dir: str) -> float:
     return mtime
 
 
+def create_page_index(data: dict):
+    pages: list[tuple] = []
+    for root, _, files in os.walk(data["_PAGE_ROOT"]):
+        root = os.path.relpath(root, data["_PAGE_ROOT"])
+        if root.startswith(("_", ".")) and root != ".":
+            continue
+        if root == ".":
+            root = ""
+        root_name, root_ext = os.path.splitext(root)
+        if root_ext:
+            pages.append(split_path(root_name))
+            continue
+        for file in files:
+            if file.startswith(("_", ".")):
+                continue
+            file_name, _ = os.path.splitext(os.path.join(root, file))
+            pages.append(split_path(file_name))
+    data["_PAGES"] = pages
+
+
 def initialize_values(data: dict):
     for dir in os.walk(data["_PAGE_ROOT"]):
         _, ext = os.path.splitext(dir[0])
@@ -71,12 +91,13 @@ def initialize_values(data: dict):
                 time.strftime(data["_DATE_FORMAT"], time.localtime(mtime)),
                 False,
             )
+            update_value(data, namespace, "_EXT", ext)
             continue
         for file in dir[2]:
             path = os.path.join(dir[0], file)
             rel_path = os.path.relpath(path, data["_PAGE_ROOT"])
             mtime = os.path.getmtime(path)
-            rel_path_no_type, _ = os.path.splitext(rel_path)
+            rel_path_no_type, rel_path_ext = os.path.splitext(rel_path)
             namespace = split_path(rel_path_no_type)
             update_value(data, namespace, "_name", namespace[-1], False)
             update_value(
@@ -86,6 +107,10 @@ def initialize_values(data: dict):
                 time.strftime(data["_DATE_FORMAT"], time.localtime(mtime)),
                 False,
             )
+            if rel_path_ext == ".md":
+                update_value(data, namespace, "_EXT", ".html")
+            else:
+                update_value(data, namespace, "_EXT", rel_path_ext)
 
 
 def get_value_from_file(
@@ -116,10 +141,13 @@ def get_value_from_file(
 
 
 def get_value(data: dict, namespace: tuple, key: str, interpret_ok=True):
-    if key != "_value" and (
-        value := get_value_from_file(data, namespace, key, interpret_ok)
-    ):
-        return value
+    try:
+        if key != "_value" and (
+            value := get_value_from_file(data, namespace, key, interpret_ok)
+        ):
+            return value
+    except FileNotFoundError:
+        pass
     value = data.get(key)
     for namespace_item in namespace:
         data = data.get(namespace_item, data)
@@ -232,6 +260,58 @@ def generate_output(file: str, data: dict, namespace: tuple) -> str:
     return str(get_value(data, namespace, "_value"))
 
 
+def generate_builds(build_file: str, data: dict) -> list[tuple[tuple, str]]:
+    rel_path = os.path.relpath(build_file, data["_PAGE_ROOT"])
+    rel_path_no_type, _ = os.path.splitext(rel_path)
+    namespace = split_path(rel_path_no_type)[:-1]
+    d = {"data": data, "namespace": namespace, "get_value": get_value}
+    with open(build_file, "r") as f:
+        exec(f.read(), d)
+    for path, _ in d["_value"]:
+        rel_path = os.path.relpath(os.path.split(build_file)[0], data["_PAGE_ROOT"])
+        if rel_path != ".":
+            # path = split_path(rel_path) + path
+            path.insert(0, *split_path(rel_path))
+        update_value(data, path, "_EXT", ".html", False)
+        update_value(data, path, "_name", path[-1], False)
+        update_value(
+            data,
+            path,
+            "_date",
+            time.strftime(
+                data["_DATE_FORMAT"], time.localtime(os.path.getmtime(build_file))
+            ),
+            False,
+        )
+
+        data["_PAGES"].append(tuple(path))
+    for i in range(len(d["_value"])):
+        d["_value"][i] = (tuple(d["_value"][i][0]), d["_value"][i][1])
+    return d["_value"]
+
+
+def interpret_builds(build_data: list[tuple[tuple, str]], data: dict):
+    for path, _value in build_data:
+        path_with_ext = path[:-1] + (path[-1] + ".html",)
+        _value = interpret(_value, data, path)
+        update_value(data, path, "_value", _value)
+
+        if _extend := get_value(data, path, "_extend"):
+            update_value(data, path, "_extend", "")
+            generate_output(os.path.join(data["_TEMPLATES"], _extend), data, path)
+        update_value(
+            data,
+            path,
+            "_value",
+            interpret_no_recursion(str(get_value(data, path, "_value")), data, path),
+        )
+
+        os.makedirs(os.path.join(data["_OUT"], *path_with_ext[:-1]), exist_ok=True)
+        with open(os.path.join(data["_OUT"], *path_with_ext), "w") as out_file:
+            out_file.write(get_value(data, path, "_value"))
+
+
+
 def main(args):
     # Load toml file
     data = tomllib.load(args.data)
@@ -260,14 +340,30 @@ def main(args):
 
     initialize_values(data)
 
+    create_page_index(data)
+
+    builds = []
+    for root, _, files in os.walk(data["_PAGE_ROOT"]):
+        if root.startswith(("_", ".")):
+            continue
+        if "_build.py" not in files:
+            continue
+        builds.extend(generate_builds(os.path.join(root, "_build.py"), data))
+
+    interpret_builds(builds, data)
+
     if not args.pages:
         pages = []
         for dir in os.walk(data["_PAGE_ROOT"]):
             _, ext = os.path.splitext(dir[0])
+            if dir[0].startswith(("_", ".")):
+                continue
             if ext:
                 pages.append(dir[0])
                 continue
             for file in dir[2]:
+                if file.startswith(("_", ".")):
+                    continue
                 pages.append(os.path.join(dir[0], file))
     else:
         pages = args.pages
@@ -279,7 +375,7 @@ def main(args):
         os.makedirs(
             os.path.join(data["_OUT"], os.path.split(rel_path)[0]), exist_ok=True
         )
-        if file_type == ".md":
+        if file_type == ".md" and not os.path.isdir(page):
             rel_path = rel_path_no_type + ".html"
         with open(os.path.join(data["_OUT"], rel_path), "w") as out_file:
             out_file.write(generate_output(page, data, namespace))
index 46644f67be53fc24c3c8ac5e9acb32efac9e0c88..40f7c8343b2ef253319222132d70d4440f655b00 100644 (file)
@@ -1,21 +1,25 @@
-import os
-
 _value = "<nav>%s</nav>"
 
 links = ""
 
-for page in os.listdir(data["_PAGE_ROOT"]):
-    if page.startswith(".") or page == "posts":
+page_navs = []
+for page in data["_PAGES"]:
+    if (_nav := get_value(data, page, "_nav")) is None:
         continue
-    if page == "index.html":
-        path = "/"
-    else:
-        path = "/" + page
-    page = os.path.splitext(page)[0]
+    page_navs.append((_nav, page + (get_value(data, page, "_EXT"),)))
+page_navs = sorted(page_navs)
+
+for _, page in page_navs:
+    path = "/"
+    for p in page[:-2]:
+        path += p + "/"
+    if page[-2] != "index":
+        path += page[-2]
+        path += page[-1]
 
     links += '<a href="%(path)s">{{ %(page)s._name }}</a>' % {
         "path": path,
-        "page": page,
+        "page": ".".join(page[:-1]),
     }
 
 _value = _value % links
index 5b372cda81f353bfa674045022b612c72c9bd068..d4419185780f50b3dfb0edefa0d9466b18c54af1 100644 (file)
@@ -6,7 +6,9 @@ _value = "<ul>\n"
 posts = data["posts"]
 post_timestamp = []
 for post in posts:
-    if post.startswith("_"):
+    if post.startswith(("_", ".")):
+        continue
+    if post == "tags":
         continue
     post_timestamp.append(
         (time.strptime(posts[post]["_date"], data["_DATE_FORMAT"]), post)