Lots of new documentation

* Add a tool that automatically generates parts of a graphviz input from
an sql file
* Use that tool to generate bits of graphviz input from all the `create_` sql
files in `src/`
* Copy all those bits of graphviz input togeather to generate diagrams
for each table.

Makefile

@@ -77,6 +77,8 @@ built_parts=$(part_files:src/%=$(app_root)/%)
 built_pages=$(page_files:src/pages/%.etlua=$(app_root)/pages/%.etlua)
 built_sql=$(sql_files:src/sql/%.sql=$(app_root)/sql/%.sql)
 built=$(built_files) $(built_sql) $(built_pages) $(built_tests)
+sql_create_table_files=$(sql_files:src/sql/create_table_%.sql=doc/schema/%.dot)
+sql_docs=$(sql_create_table_files)
 initscript=/lib/systemd/system/smr.service
 config=$(conf_path)/smr.conf
 built_bin=$(smr_bin_path)/smr.so
@@ -170,14 +172,17 @@ test: $(built) ## run the unit tests
 	$(Q)$(CD) $(app_root) && busted -v --no-keep-going --exclude-tags "slow,todo,working"
 
 cov: $(built) ## code coverage (based on unit tests)
-	$(Q)$(RM) $(kore_chroot)/luacov.stats.out
+	$(Q)$(RM) $(app_root)/luacov.stats.out
-	$(Q)$(CD) $(kore_chroot) && busted -v -c --no-keep-going --exclude-tags slow
+	$(Q)$(CD) $(app_root) && busted -v -c --no-keep-going --exclude-tags "slow,todo,working"
-	$(Q)$(CD) $(kore_chroot) && luacov endpoints/
+	$(Q)$(CD) $(app_root) && luacov endpoints/
-	$(Q)$(ECHO) "open kore_chroot/luacov.report.out to view coverage results."
+	$(Q)$(ECHO) "open $(app_root)/luacov.report.out to view coverage results."
 
-doc: 
+$(sql_docs) : doc/schema/%.dot : src/sql/create_table_%.sql
+	cat $< | tools/doc_sql.sh > $@
+
+doc: $(sql_docs)
 	rm -rf .trblcache
-	~/Programs/trbldoc/trbldoc.sh src
+	~/Programs/trbldoc/trbldoc.sh doc src README.md
 	cd .trblcache/built && python3 -m http.server
 
 .PHONY: doc

README.md

@@ -10,7 +10,7 @@ concerns with pastebin.com taking down certain kinds of content. SMR aims to
 be small, fast, and secure. It is built on top of [Kore](https://kore.io), using
 [luajit](https://luajit.org) to expose a Lua programming environment. It uses
 [sqlite3](https://sqlite.org) as it's database. SMR is implemented in about 
-4k SLOC and is expected to never exceed 5k SLOC. Contributions welcome.
+5k SLOC. Contributions welcome.
 
 Language|files|blank|comment|code
 :-------|-------:|-------:|-------:|-------:

doc/hooks.md

@@ -1,3 +1,6 @@
+!{ md
+@file doc/hooks
+
 # Hooks
 
 Various functions that are exposed to the lua environment. These functions may be detoured to effect their behavior.
@@ -17,4 +20,4 @@ authenticate(data :: table) :: number | nil, string
 ```
 Called when a user attempts to log in. Return a number, userid if the login is successful, or nil and an error message if the login is not successful. By default, smr puts "user" and "passfile" fields into the data table.
 
-
+!}

doc/schema/.gitignore

@@ -0,0 +1 @@
+*.dot

src/lua/db.lua

@@ -7,6 +7,13 @@ Does most of the database interaction.
 Creates default empty database during configure()
 Notably, holds a connection to the open sqlite3 database in .conn
 ]]
+
+--[[ sh
+@name sql/table
+echo "digraph schema {" \
+	"$(cat doc/schema/*.dot)" \
+	"}" | dot -Tsvg
+]]
 local sql = require("lsqlite3")
 
 local queries = require("queries")

src/sql/create_table_authors.sql

@@ -1,11 +1,18 @@
 /* md
 @name sql/table/authors
-If/when an author deletes their account, all posts
+If an author deletes their account, all posts
 and comments by that author are also deleted (on
 delete cascade) this is intentional. This also
 means that all comments by other users on a post
 an author makes will also be deleted.
 */
+
+/* sh
+@name sql/table/authors
+echo "digraph authors {" \
+	"$(cat doc/schema/authors.dot)" \
+	"}" | dot -Tsvg
+*/
 CREATE TABLE IF NOT EXISTS authors (
 	id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
 	name TEXT UNIQUE ON CONFLICT FAIL,

src/sql/create_table_comments.sql

@@ -1,9 +1,19 @@
-/*
+/* md
+@name sql/table/comments
 Comments on a post.
 
 When an author deletes their account or the posts this comment
 is posted on is deleted, this comment will also be deleted.
 */
+
+/* sh
+@name sql/table/comments
+echo "digraph comments{" \
+	"$(cat doc/schema/authors.dot)" \
+	"$(cat doc/schema/posts.dot)" \
+	"$(cat doc/schema/comments.dot)" \
+	"}" | dot -Tsvg
+*/
 CREATE TABLE IF NOT EXISTS comments (
 	id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
 	postid REFERENCES posts(id) ON DELETE CASCADE,

src/sql/create_table_images.sql

@@ -1,6 +1,15 @@
-/*
+/* md
+@name sql/table/images
 We may want to store images one day. This is unused for now
 */
+
+/* sh
+@name sql/table/images
+echo "digraph images {" \
+	"$(cat doc/schema/images.dot)" \
+	"$(cat doc/schema/authors.dot)" \
+	"}" | dot -Tsvg
+*/
 CREATE TABLE IF NOT EXISTS images (
 	id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
 	name TEXT,

src/sql/create_table_posts.sql

@@ -1,4 +1,5 @@
-/*
+/* md
+@name sql/table/posts
 If/when an author delets their account, all posts
 and comments by that author are also deleted (on
 delete cascade) this is intentional. This also
@@ -9,6 +10,14 @@ Post text uses zlib compression
 
 Unlisted hashes are SHAv3 521
 */
+
+/* sh
+@name sql/table/posts
+echo "digraph comments{" \
+	"$(cat doc/schema/authors.dot)" \
+	"$(cat doc/schema/posts.dot)" \
+	"}" | dot -Tsvg
+*/
 CREATE TABLE IF NOT EXISTS posts (
 	id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
 	post_text BLOB,

src/sql/create_table_raw_text.sql

@@ -1,7 +1,18 @@
-/*
+/* md
-Store the raw text so people can download it later, maybe
+@name sql/table/raw_text
-we can use it for "download as image" or "download as pdf"
+Store the raw text.
-in the future too. Stil stored zlib compressed
+Used so people can edit their posts and get their original uploaded text.
+Also used so users can download it,
+maybe we can use it for "download as image" or "download as pdf" in the future.
+Stored zlib compressed
+*/
+
+/* sh
+@name sql/table/raw_text
+echo "digraph comments{" \
+	"$(cat doc/schema/raw_text.dot)" \
+	"$(cat doc/schema/posts.dot)" \
+	"}" | dot -Tsvg
 */
 CREATE TABLE IF NOT EXISTS raw_text (
 	id INTEGER PRIMARY KEY REFERENCES posts(id) ON DELETE CASCADE,

src/sql/create_table_session.sql

@@ -1,10 +1,19 @@
-/*
+/* md
+@name sql/table/sessions
 Store a cookie for logged in users. Logged in users can edit
 their own posts, edit their biographies, and post stories and comment under their own name.
-TODO: WE can hash the "key" so that even if the database gets
+TODO: We can hash the "key" so that even if the database gets
 dumped, a hacker can't cookie-steal with only read access
 to the db.
 */
+
+/* sh
+@name sql/table/sessions
+echo "digraph comments{" \
+	"$(cat doc/schema/sessions.dot)" \
+	"$(cat doc/schema/authors.dot)" \
+	"}" | dot -Tsvg
+*/
 CREATE TABLE IF NOT EXISTS sessions (
 	key TEXT PRIMARY KEY,
 	author REFERENCES authors(id) ON DELETE CASCADE,

src/sql/create_table_tags.sql

@@ -2,6 +2,46 @@
 Tags on a post
 A post's tags are deleted if the post is deleted.
 */
+/* dot -Tsvg
+@name db/schema/tags
+digraph tags {
+tags [
+	shape="plaintext"
+	label=<<table>
+		<tr><td colspan="3"><b>tags</b></td></tr>
+		<tr>
+			<td port="id">PK</td>
+			<td>id</td>
+			<td>INT, AUTOINCREMENT, NOT NULL</td>
+		</tr>
+		<tr>
+			<td>FK</td>
+			<td>postid</td>
+			<td port="postid">ON DELETE CASCADE</td>
+		</tr>
+		<tr>
+			<td></td>
+			<td>tag</td>
+			<td>TEXT</td>
+		</tr>
+	</table>>
+
+];
+posts [
+	shape="plaintext"
+	label=<<table>
+		<tr><td colspan="3"><b>posts</b></td></tr>
+		<tr>
+			<td port="id">PK</td>
+			<td>id</td>
+			<td>INT, AUTOINCREMENT, NOT NULL</td>
+		</tr>
+		<tr><td colspan="3">...</td></tr>
+	</table>>
+];
+	tags:postid -> posts:id
+}
+*/
 CREATE TABLE IF NOT EXISTS tags (
 	id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
 	postid REFERENCES posts(id) ON DELETE CASCADE,

src/sql/insert_comment.sql

@@ -1,4 +1,6 @@
-/* Add a new comment to a story */
+/*
+Add a new comment to a story
+*/
 INSERT INTO comments(
 	postid,
 	author,

src/sql/select_author_index.sql

@@ -1,4 +1,6 @@
-/* Get the data we need to display a particular author's latest stories */
+/*
+Get the data we need to display a particular author's latest stories
+*/
 
 SELECT
 	posts.id,

src/sql/select_comments.sql

@@ -1,4 +1,6 @@
-/* Retreive comments on a story */
+/*
+Retreive comments on a story
+*/
 SELECT
 	authors.name,
 	comments.isanon,

src/sql/select_site_index.sql

@@ -1,4 +1,6 @@
-/* Select the data we need to display the on the front page */
+/*
+Select the data we need to display the on the front page
+*/
 SELECT
 	posts.id,
 	posts.post_title,

src/sql/update_views.sql

@@ -1,2 +1,4 @@
-/* Update the view counter when someone reads a story */
+/*
+Update the view counter when someone reads a story
+*/
 UPDATE posts SET views = views + 1 WHERE id = :id;

tools/doc_sql.sh

@@ -0,0 +1,70 @@
+#!/bin/bash -ex
+
+# Tool for documenting sql with graphviz
+# Usage:
+# $ cat sql_file.sql | tools/doc_sql.sh -flags -passed -to -dot > output_file.svg
+# Outputs a subgraph and some edges that can be grouped togeather in a full graph
+
+flags=$@
+input_file=$(cat -)
+# First, find the table name
+table_name=$(echo "${input_file}" "SELECT name FROM sqlite_master WHERE type='table';" | sqlite3 | head -n 1)
+# And get the data we need to display
+table_schema=$(echo "${input_file}" "
+SELECT
+	info.name,
+	info.type,
+	info.pk,
+	info.\"notnull\",
+	fk_from.\"from\" AS fk_from_col,
+	fk_from.\"on_delete\" AS fk_on_delete,
+	fk_from.\"table\" AS fk_from_tbl,
+	fk_from.\"to\" AS fk_from_to
+FROM
+	pragma_table_info('$table_name') info
+	LEFT JOIN pragma_foreign_key_list('$table_name') fk_from
+		ON fk_from.\"from\" = info.\"name\"
+ORDER BY
+	info.\"notnull\" DESC
+;" | sqlite3 )
+table_pk_1=$(echo "$table_schema" | head -n 1)
+table_pk=$(echo "$table_schema" | head -n 1 | awk '
+BEGIN {FS="|"}
+{print $1}
+')
+#foreign_keys=$(echo "${input_file}" "pragma foreign_key_list('$table_name');" | sqlite3)
+
+echo "$table_name ["
+echo "    shape=\"plaintext\""
+echo "    label=<<table>"
+echo "    <tr><td colspan=\"3\"><b>" $table_name "</b></td></tr>"
+echo "$table_schema" | awk '
+BEGIN {FS="|"}
+{
+	if ($4)
+		print "<tr><td port=\"" $1 "\">PK</td>";
+	else if ($5)
+		print "<tr><td port=\"" $1 "\">FK</td>";
+	else
+		print "<tr><td></td>";
+	print "<td>" $1 "</td>";
+	printf "<td";
+	if($4)
+		printf " port=\"" $1 "\">";
+	else
+		printf ">";
+	if($2)
+		print $2;
+	else
+		print $2 "ON DELETE CASCADE";
+	print "</td></tr>";
+}'
+echo "</table>> ];"
+echo "$table_schema" | awk -v "table_name=$table_name" -v "table_pk=$table_pk" '
+BEGIN {FS="|"}
+{
+	if ($5) {
+		print table_name ":" $5 "->" $7 ":" $8 ;
+	}
+}
+'