Store processed inputs in db

Also, use named tuples for row factory.

src/gistillery/db.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import sqlite3
+from collections import namedtuple
 from contextlib import contextmanager
 from typing import Generator
 
@@ -57,20 +58,41 @@ CREATE TABLE jobs
 )
 """
 
+# store the processed inputs
+schema_inputs = """
+CREATE TABLE inputs
+(
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    entry_id TEXT NOT NULL,
+    input TEXT NOT NULL,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY(entry_id) REFERENCES entries(id)
+)
+"""
+
 TABLES = {
-    "entries": schema_entries,
-    "summaries": schema_summary,
-    "tags": schema_tag,
-    "jobs": schema_job,
+    'entries': schema_entries,
+    'summaries': schema_summary,
+    'tags': schema_tag,
+    'jobs': schema_job,
+    'inputs': schema_inputs,
 }
 TABLES_CREATED = False
 
 
+# https://docs.python.org/3/library/sqlite3.html#how-to-create-and-use-row-factories
+def namedtuple_factory(cursor, row):
+    fields = [column[0] for column in cursor.description]
+    cls = namedtuple("Row", fields)
+    return cls._make(row)
+
+
 def _get_db_connection() -> sqlite3.Connection:
     global TABLES_CREATED
 
     # sqlite cannot deal with concurrent access, so we set a big timeout
     conn = sqlite3.connect(db_file, timeout=30)
+    conn.row_factory = namedtuple_factory
     if TABLES_CREATED:
         return conn
 

src/gistillery/webservice.py

@@ -58,7 +58,7 @@ def recent() -> list[EntriesResult]:
         # get the last 10 entries, join summary and tag, where each tag is
         # joined to a comma separated str
         cursor.execute("""
-            SELECT e.id, e.author, s.summary, GROUP_CONCAT(t.tag, ","), e.created_at
+            SELECT e.id, e.author, s.summary, GROUP_CONCAT(t.tag, ",") tags, e.created_at
             FROM entries e
             JOIN summaries s ON e.id = s.entry_id
             JOIN tags t ON e.id = t.entry_id
@@ -86,7 +86,7 @@ def recent_tag(tag: str) -> list[EntriesResult]:
     with get_db_cursor() as cursor:
         cursor.execute(
             """
-            SELECT e.id, e.author, s.summary, GROUP_CONCAT(t.tag, ","), e.created_at
+            SELECT e.id, e.author, s.summary, GROUP_CONCAT(t.tag, ",") tags, e.created_at
             FROM entries e
             JOIN summaries s ON e.id = s.entry_id
             JOIN tags t ON e.id = t.entry_id

src/gistillery/worker.py

@@ -33,6 +33,7 @@ def check_pending_jobs() -> list[JobInput]:
 
 @dataclass
 class JobOutput:
+    processed: str
     summary: str
     tags: list[str]
     processor_name: str
@@ -54,6 +55,7 @@ def _process_job(job: JobInput, registry: MlRegistry) -> JobOutput:
     summary = summarizer(processed)
 
     return JobOutput(
+        processed=processed,
         summary=summary,
         tags=tags,
         processor_name=processor_name,
@@ -64,7 +66,10 @@ def _process_job(job: JobInput, registry: MlRegistry) -> JobOutput:
 
 def store(job: JobInput, output: JobOutput) -> None:
     with get_db_cursor() as cursor:
-        # write to entries, summary, tags tables
+        cursor.execute(
+            "INSERT INTO inputs (entry_id, input) VALUES (?, ?)",
+            (job.id, output.processed),
+        )
         cursor.execute(
             (
                 "INSERT INTO summaries (entry_id, summary, summarizer_name)"

tests/{test_webservice.py → test_app.py}

@@ -19,6 +19,13 @@ class TestWebservice:
         filename = tmp_path / "test-db.sqlite"
         os.environ["DB_FILE_NAME"] = str(filename)
 
+    @pytest.fixture
+    def cursor(self):
+        from gistillery.db import get_db_cursor
+
+        with get_db_cursor() as cursor:
+            yield cursor
+
     @pytest.fixture
     def client(self):
         from gistillery.webservice import app
@@ -117,6 +124,31 @@ class TestWebservice:
         }
         assert last_updated is None
 
+    def test_submitted_job_failed(self, client, mlregistry, monkeypatch):
+        # monkeypatch uuid4 to return a known value
+        job_id = "abc1234"
+        monkeypatch.setattr("uuid.uuid4", lambda: SimpleNamespace(hex=job_id))
+        client.post("/submit", json={"author": "ben", "content": "this is a test"})
+        # patch gistillery.worker._process_job to raise an exception
+
+        def raise_(ex):
+            raise ex
+
+        # make the job processing fail
+        monkeypatch.setattr(
+            "gistillery.worker._process_job",
+            lambda job, registry: raise_(RuntimeError("something went wrong")),
+        )
+        self.process_jobs(mlregistry)
+
+        resp = client.get(f"/check_job_status/{job_id}")
+        output = resp.json()
+        output.pop("last_updated")
+        assert output == {
+            "id": job_id,
+            "status": "failed",
+        }
+
     def test_submitted_job_status_done(self, client, mlregistry, monkeypatch):
         # monkeypatch uuid4 to return a known value
         job_id = "abc1234"
@@ -182,26 +214,17 @@ class TestWebservice:
         assert resp0["summary"] == "this would"
         assert resp0["tags"] == sorted(["#this", "#would", "#be"])
 
-    def test_submitted_job_failed(self, client, mlregistry, monkeypatch):
-        # monkeypatch uuid4 to return a known value
-        job_id = "abc1234"
-        monkeypatch.setattr("uuid.uuid4", lambda: SimpleNamespace(hex=job_id))
+    def test_clear(self, client, cursor, mlregistry):
         client.post("/submit", json={"author": "ben", "content": "this is a test"})
-        # patch gistillery.worker._process_job to raise an exception
+        self.process_jobs(mlregistry)
+        assert cursor.execute("SELECT COUNT(*) c FROM entries").fetchone()[0] == 1
 
-        def raise_(ex):
-            raise ex
+        client.get("/clear")
+        assert cursor.execute("SELECT COUNT(*) c FROM entries").fetchone()[0] == 0
 
-        monkeypatch.setattr(
-            "gistillery.worker._process_job",
-            lambda job, registry: raise_(RuntimeError("something went wrong")),
-        )
+    def test_inputs_stored(self, client, cursor, mlregistry):
+        client.post("/submit", json={"author": "ben", "content": "  this is a test\n"})
         self.process_jobs(mlregistry)
-
-        resp = client.get(f"/check_job_status/{job_id}")
-        output = resp.json()
-        output.pop("last_updated")
-        assert output == {
-            "id": job_id,
-            "status": "failed",
-        }
+        rows = cursor.execute("SELECT * FROM inputs").fetchall()
+        assert len(rows) == 1
+        assert rows[0].input == "this is a test"