Build a persistent text search index

import argparse
from contextlib import closing
from pathlib import Path
import re
import sqlite3
import textwrap


def passages(folder):
    if not folder.is_dir():
        raise ValueError(f"Notes folder not found: {folder}")
    for path in sorted(folder.rglob("*")):
        if path.is_symlink() or not path.is_file() or path.suffix.lower() not in {".md", ".txt"}:
            continue
        source = path.relative_to(folder).as_posix()
        blocks = re.split(r"\n[ \t]*\n", path.read_text(encoding="utf-8").strip())
        chunks = (chunk for block in blocks for chunk in textwrap.wrap(" ".join(block.split()), 600))
        for number, chunk in enumerate(chunks, start=1):
            yield source, number, chunk


def build_index(folder, database):
    rows = list(passages(folder))
    if not rows:
        raise ValueError("Add at least one nonempty .txt or .md note.")
    with closing(sqlite3.connect(database)) as connection, connection:
        connection.execute(
            "CREATE VIRTUAL TABLE IF NOT EXISTS notes "
            "USING fts5(source UNINDEXED, passage UNINDEXED, text)"
        )
        connection.execute("DELETE FROM notes")
        connection.executemany(
            "INSERT INTO notes(source, passage, text) VALUES (?, ?, ?)", rows
        )
    print(f"Indexed {len(rows)} passages in {database}.")


def find_notes(database, query):
    words = re.findall(r"[^\W_]+", query)
    if not words:
        raise ValueError("Use at least one word to search.")
    if not database.is_file():
        raise ValueError("Build the index before searching.")
    terms = " AND ".join(f'"{word}"' for word in words)
    with closing(sqlite3.connect(database.resolve().as_uri() + "?mode=ro", uri=True)) as connection:
        matches = connection.execute(
            "SELECT source, passage, snippet(notes, 2, '[', ']', ' … ', 24) "
            "FROM notes WHERE notes MATCH ? ORDER BY rank LIMIT 5", (terms,)
        ).fetchall()
    for source, passage, snippet in matches:
        print(f"{source}, passage {passage}\n{snippet}\n")
    if not matches:
        print("No matching passages.")


def main():
    parser = argparse.ArgumentParser(description="Build and search a local SQLite index.")
    parser.add_argument("--db", type=Path, default=Path("notes.sqlite3"))
    commands = parser.add_subparsers(dest="command", required=True)
    build = commands.add_parser("build")
    build.add_argument("folder", type=Path)
    find = commands.add_parser("find")
    find.add_argument("query")
    args = parser.parse_args()

    try:
        if args.command == "build":
            build_index(args.folder, args.db)
        else:
            find_notes(args.db, args.query)
    except (OSError, sqlite3.Error, ValueError) as error:
        raise SystemExit(str(error)) from None


if __name__ == "__main__":
    main()

Workshop A begins at 14:30 in Room 204. Register at the front desk.

The media lab has a 3D printer. Reserve a slot at the media desk.

Standard book loans last 21 days. Renewals depend on holds.

The reading room closes at 6 p.m. on Saturdays.