From 74718f7fb5e106a6dd9e9fdd99a6ec96b1cced93 Mon Sep 17 00:00:00 2001 From: "0m.ax" Date: Sat, 22 Aug 2026 15:33:55 +0200 Subject: [PATCH] init --- .gitignore | 2 + PROCESS.md | 567 +++++++++++++++++++++++++++++++++++++++ README.md | 23 ++ flake.lock | 27 ++ flake.nix | 46 ++++ tools/check-links.py | 87 ++++++ tools/extract-figures.py | 424 +++++++++++++++++++++++++++++ 7 files changed, 1176 insertions(+) create mode 100644 .gitignore create mode 100644 PROCESS.md create mode 100644 README.md create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 tools/check-links.py create mode 100644 tools/extract-figures.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..754e510 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.tmp +__pycache__ diff --git a/PROCESS.md b/PROCESS.md new file mode 100644 index 0000000..a221911 --- /dev/null +++ b/PROCESS.md @@ -0,0 +1,567 @@ +# Datasheet → Markdown: process + +A repeatable runbook for turning a large vendor datasheet PDF into per-chapter +Markdown with extracted figures, a table of contents, and an accuracy audit. + +Worked examples in this repo: +- Renesas RAA489400, 80 pages → 9 chapters, ~200 KB of Markdown, 25 extracted + figures. +- USB Power Delivery Specification r3.2 v1.2, 416 pages → 10 chapters + 2 + appendices, ~1.1 MB of Markdown, 178 extracted figures. This one stress-tests + the process at 5x the page count and surfaced the gaps folded into this + document — see the caption-position note in step 4, the seam-defect table in + 3c, and the audit-scaling notes in step 7. + +The approach is: **split the PDF, transcribe chapters in parallel with agents +that read the rendered pages, extract figures mechanically, then audit each +chapter against the PDF with a second, independent set of agents.** + +--- + +## 0. Why it is shaped this way + +Two constraints drive the whole design. + +**Text extraction is not enough.** `pdftotext` loses table structure, and +`pdfimages` finds nothing at all in a FrameMaker-authored datasheet because the +figures are vector line art, not embedded rasters. So transcription agents must +read the *rendered pages* as images, and figures must be rendered-and-cropped +rather than extracted. + +**An agent cannot hold a 45-page chapter.** Context runs out silently — you get +a completed task with no file written. Large chapters must be split into +~8-page chunks processed in parallel, which then introduces seam defects that +have to be cleaned up afterwards. Budget for this. + +--- + +## 1. Prerequisites + +Everything lives in the flake devshell: + +```bash +nix develop +``` + +Provides `qpdf`, `poppler-utils` (`pdftoppm`, `pdftotext`, `pdfinfo`, +`pdfimages`), `mupdf` (`mutool`), `imagemagick` (`magick`), and `python3`. + +> If `nix develop` errors with *"Path 'flake.nix' … is not tracked by Git"*, +> either `git add flake.nix flake.lock`, or use `nix develop "path:$PWD"` to +> bypass the git-tracking requirement. + +--- + +## 2. Split the source PDF into chapters + +Find the chapter boundaries from the PDF bookmarks or the printed contents +page, then cut losslessly with `qpdf`: + +```bash +pdfinfo REN_RAA489400_DST_20240827.pdf # page count +pdftotext -q REN_RAA489400_DST_20240827.pdf - | grep -n "^[0-9]\+\. " + +mkdir -p chapters +qpdf source.pdf --pages . 5-10 -- chapters/01-overview.pdf +qpdf source.pdf --pages . 11-13 -- chapters/02-pin-information.pdf +# ... +``` + +Name files `NN-slug.pdf`. Everything downstream keys off that stem: the +transcription becomes `NN-slug.md` and figures go in `NN-slug/`. + +Record the **printed** page range of each chapter — the number in the page +footer, not the position in the file. You need it in step 5. + +```bash +for f in chapters/*.pdf; do + t=$(pdftotext -q "$f" - | grep -oE "Page [0-9]+" | sed "s/Page //") + echo "$(basename "$f") : $(echo $t | tr ' ' '\n' | head -1)-$(echo $t | tr ' ' '\n' | tail -1)" +done +``` + +--- + +## 3. Transcribe each chapter with a parallel agent + +One agent per chapter PDF, all launched **in a single message** so they run +concurrently. Chapters over ~10 pages need chunking first — see step 3b. + +### The prompt template + +``` +Convert a datasheet chapter PDF into Markdown. + +Source: /abs/path/chapters/NN-slug.pdf +Output: /abs/path/chapters/NN-slug.md + +Steps: +1. Use the native `read` tool with filePath=. The read tool returns + PDFs as attachments so you can see the rendered pages directly. DO NOT use + pdftotext, python, or any bash-based extraction - use the read tool. +2. Transcribe the full contents of every page into GitHub-flavored Markdown, + then write it with the `write` tool to the output path. + +Transcription rules: +- Transcribe all body text VERBATIM. Do NOT summarize, paraphrase, or omit. +- Preserve the heading hierarchy with #/##/###, keeping printed section numbers. +- Reproduce ALL tables as GFM tables. Every row and column, including units and + conditions. Do not truncate. GFM has no rowspan: where the source merges cells + vertically, repeat the value on each row. +- Preserve footnotes as a numbered list, keeping the markers in the cells. +- Transcribe printed errors, typos and inconsistencies EXACTLY as printed. Do + not silently correct them. They are load-bearing for an audit. +- Equations: use LaTeX in $...$, keeping printed equation numbers. +- Use inline code for register names, pin names, bit fields and hex values. +- Add a `` comment at each page boundary, placed BEFORE that + page's content, using the PRINTED page number. This chapter covers pages A-B. + +FIGURES - IMPORTANT: +- For each figure, emit ONLY the printed caption, as bold text, exactly as it + appears (e.g. `**Figure 12. Startup from VBUS**`), plus an image link of the + form `![](NN-slug/figure-.png)`. +- Do NOT write any description of what the figure shows. Do not describe axes, + curve labels, block names, signal paths, colours or annotations. The images + are extracted separately and the description is not in the datasheet. +- If a figure has no printed number or caption, say so plainly and do NOT + invent one. + +Reply with: output path, pages transcribed, number of tables, number of figures. +``` + +### Why the figure rule matters + +The first run of this project asked agents to describe each figure "for +accessibility". Every such description was invented text indistinguishable from +datasheet prose, and three of them were **factually wrong**: + +- a waveform described as settling high when it actually falls back to baseline + (the exact behaviour the plot's measurement arrow was annotating); +- a 32-pin QFN described as "8 pins per side" when it is 11/5/11/5; +- an invented sentence about "RC filter components near the controller" for a + drawing that contains no such components. + +In one chapter the invented prose was ~90% of the file. Removing it later cost +a full audit pass. **Do not ask for figure descriptions.** + +### 3b. Chunking large chapters + +An agent will silently fail on a big chapter — it reports success and writes +nothing. Pre-split anything over ~10 pages: + +```bash +qpdf chapters/07-registers.pdf --pages . 1-8 -- /tmp/reg/07-1-8.pdf +qpdf chapters/07-registers.pdf --pages . 9-16 -- /tmp/reg/07-9-16.pdf +# ... +``` + +Give each chunk agent the same prompt plus: + +``` +IMPORTANT: this is chunk N of M and will be concatenated. Do NOT add a document +title, intro, or closing summary. Start directly with the content of page 1 of +this chunk. Do not write commentary into the file. If the chunk starts or ends +mid-table or mid-register, transcribe the partial content; do not invent the +missing part. This chunk covers printed pages A-B. +``` + +Then concatenate in order: + +```bash +cat /tmp/reg/07-part{1,2,3,4,5,6}.md > chapters/07-registers.md +``` + +### 3c. Clean up the chunk seams — always required + +Parallel chunks drift. Every one of these occurred in this project: + +| Seam defect | Detection | Fix | +|---|---|---| +| Duplicate headings (source heading *and* the agent's own) | `rg "^#{3} " file.md` and look for adjacent pairs naming the same thing | Collapse to one; demote genuine sub-headings to `####` | +| Heading levels differ between chunks | compare `^#+ ` counts per chunk range | Normalize with a scripted pass — see below | +| A chunk emits valid heading text with no leading `#` at all | `rg '^[0-9]+\.[0-9]+(\.[0-9]+){0,4}\. [A-Z]' file.md \| rg -v '^#'` — matches printed-numbered lines that aren't markdown headings | Prepend the correct number of `#` (see the level formula below) | +| A table caption is written as a heading instead of bold text (`##### Table 3.7. ...` instead of `**Table 3.7. ...**`) | `rg -n '^#+ Table ' file.md` | `sed -E 's/^#+ (Table [0-9]+\.[0-9]+\. .+)$/**\1**/'`, then re-check table row/figure counts are unchanged | +| Derived metadata lines in 3 different formats (`\|` vs `·` vs `—`) | `rg -o "^Address:.*" \| sort \| uniq -c` | Pick one canonical form, rewrite | +| Some chunks bold table captions, others don't | `rg -c '^\*\*Table' ; rg -c '^Table '` | Bold them all | +| Address style `0x50` in one chunk, `50h` in another | `rg "0x[0-9A-F]" file.md` | Normalize to the printed style | +| RFC2119 keywords (Shall/Should/May/Shall Not/...) bold in some chunks, plain in others | count `\bShall\b` vs `\*\*Shall\*\*` per file | See "Fixing missing bold emphasis" below — this one has sharp edges | + +### Normalizing heading levels from printed section numbers + +Given a heading whose text starts with a printed number (`9.2.5. Title`), the +correct level is `(number of dots) + 1`: `9.1` → `##`, `9.2.5` → `###`, +`9.2.5.1` → `####`. `Chapter N.` / `Appendix X.` headings are always `#`. + +```python +def desired_level(text): + if re.match(r'^(Chapter \d+|Appendix [A-Z])\.', text): + return 1 + m = re.match(r'^(\d+(?:\.\d+)*)\.?\s', text) + return m and m.group(1).count('.') + 1 +``` + +**Sharp edge:** if you rewrite a heading line with a regex substitution, make +sure the replacement keeps the line's trailing newline. A pattern like +`^(#{1,6})\s+(.*)$` followed by `f"{new_hashes} {text}"` silently drops the +`\n` for every line that actually changes, which merges that heading into the +following paragraph on the same line — invisible in a diff of line *counts*, +very visible once you view the file. Capture the newline explicitly and put it +back: + +```python +heading_re = re.compile(r'^(#{1,6})[ \t]+(.*?)([\r\n]*)$') +# ... hashes, text, eol = heading_re.match(line).groups() +# ... out.append(f"{new_hashes} {text}{eol}") +``` + +Run the script twice — a correct normalizer is idempotent (reports 0 changes +on the second pass). If it isn't, something upstream (the level formula, the +newline handling) is still wrong. + +### Fixing missing bold emphasis + +Before trusting the text layer to tell you whether a word is bold, check a +rendered crop. `mutool draw -F stext`'s per-line `` attribute is not +reliable for detecting an inline bold sub-run in the middle of a line — a +sentence like "...capacitance change **Shall** occur..." can report a single +non-bold font for the whole line even when "Shall" is visibly bold on the +page. Render the page and crop the relevant lines to confirm before writing a +fix: + +```bash +pdftoppm -png -r 200 -f -l -singlefile source.pdf /tmp/page +magick /tmp/page.png -crop x++ /tmp/crop.png # then view it +``` + +If it's genuinely missing, a scripted fix needs two more guards or it +corrupts the file: + +1. **Don't regex-exclude only the exact wrapped form.** Excluding `**Shall**` + but not `**Shall Discard**` means the bare `Shall` inside an + already-bold multi-word phrase gets wrapped again — `**Shall Discard**` → + `****Shall**** Discard**`. Split each line on `**` first and only touch the + even-indexed (outside-any-bold-span) segments: + + ```python + parts = line.split('**') + for i in range(0, len(parts), 2): # even = outside bold, odd = inside + parts[i] = pattern.sub(lambda m: f'**{m.group(0)}**', parts[i]) + line = '**'.join(parts) + ``` + +2. **Watch for bold spans that cross a line break** (`**Shall` at the end of + one line, `Not**` at the start of the next — valid Markdown, renders fine). + The even/odd split above is computed fresh per line, so a line that + *starts* mid-bold-span (because the previous line left it open) gets the + parity backwards. This is rare — find candidates first with + `awk '{n=gsub(/\*\*/,"**"); if (n%2!=0) print FNR}' file.md` (an odd count + on a line is expected *only* for these) — and handle them by hand rather + than generalizing the script further for a handful of cases. + +3. **Don't apply it to keyword/glossary-definition tables.** If a chapter + defines what "Shall" *means* (a keywords/conventions table), the term is + usually printed in plain text even though the same word is bold everywhere + else as normative emphasis — check the specific table's rendering before + assuming the chapter-wide convention applies inside it too. + +### After any scripted rewrite, prove you changed nothing real + +```bash +rg '^\|' before.md > /tmp/a; rg '^\|' after.md > /tmp/b; diff /tmp/a /tmp/b +``` + +An empty diff means no table cell moved. Also compare counts of headings and +`**Figure` captions before/after. This applies to every fix in this section — +heading-level normalization, missing-bold fixes, and any other scripted pass. + +--- + +## 4. Extract the figures + +`pdfimages` will not work on vector art. Use the caption-anchored cropper: + +```bash +python3 tools/extract-figures.py --dry-run chapters/*.pdf # inspect first +python3 tools/extract-figures.py chapters/*.pdf # write PNGs +``` + +**Before running at scale, check the template's caption convention.** Render +one page that has a figure on it and look at where the caption sits relative +to the drawing: + +```bash +pdftoppm -png -r 150 -f -l -singlefile source.pdf /tmp/page +``` + +The tool defaults to the Renesas-style convention — caption **below** the +figure, sequential integer numbers (`Figure 12.`). If the datasheet instead +prints the caption **above** the figure (common in USB-IF/standards-body +documents) and/or numbers figures per-chapter (`Figure 4.1`, `4.2`, ...), pass: + +```bash +python3 tools/extract-figures.py --caption-position above chapters/*.pdf +``` + +`--caption-position above` flips the band direction (bounded above by the +caption, below by the next body text or caption) and the caption regex +accepts `Figure \d+(\.\d+)?\.`, writing `figure-4.1.png` instead of +zero-padded integers. It also requires the caption line's font to be bold — +without that check, a plain prose sentence like "Figure 4.2 illustrates the +application of..." false-matches as a caption and produces a `SKIP: no ink +found in band` entry (or worse, a bogus crop) once you get past the first one. + +It locates `Figure N.` captions in the text layer via `mutool draw -F stext`, +derives a band bounded by the caption and the nearest body-text line, renders +the page, tightens the band to the actual ink, and crops at 300 dpi into +`chapters//figure-NN.png` (or `figure-N.M.png`) using the datasheet's +own figure numbers. + +Always `--dry-run` first and sanity-check the reported band sizes, then eyeball +the output as a contact sheet: + +```bash +magick montage $(find chapters -name 'figure-*.png' | sort -t- -k2 -n) \ + -tile 5x -geometry 300x300+4+4 -background gray70 /tmp/contact.png +``` + +Three tuning constants in the script are page-layout specific and are the first +things to adjust for a different vendor's template: + +- `HEADER_BOTTOM` (62 pt) — must clear the running head **and the horizontal + rule beneath it**, which otherwise gets caught as figure ink. +- `BODY_GAP` (5 pt) — `stext` line boxes clip descenders, so without clearance + the tail of the heading above bleeds into the crop. +- `PAD` (6 pt) — clamped to the band so it cannot reach into the caption. + +Also note captions **wrap**: bound the next figure with the caption's *last* +line, not its first, or the wrapped remainder lands at the top of the following +crop. + +For a figure the datasheet never captioned, crop it explicitly rather than +inventing a number — and record the invocation: + +```bash +python3 tools/extract-figures.py \ + --region '1:195,550,410,640:current-sensing-trace-routing.png' \ + chapters/08-layout.pdf +``` + +### Delegate this whole step to a subagent for big documents + +For a document with 50+ figures, run the dry-run → adjust → contact-sheet → +re-dry-run loop in a subagent rather than the main thread. Each contact sheet +is a sizeable image, and getting the band-detection constants (or the +caption-position convention) right typically takes several rounds — none of +that iteration needs to live in the orchestrating conversation, only the +outcome does. + +Give the subagent: +- the list of chapter PDFs and the expected figure count per chapter (count + `^\*\*Figure` in each transcribed `.md` — the subagent should treat a + mismatch against this count as a bug to chase down, not a fact to report); +- explicit instructions to render one figure page first and report which + caption convention applies before running the tool at scale; +- permission to extend `tools/extract-figures.py` if the template doesn't fit + (new flag, not a fork), **with an explicit instruction to re-run the tool + against the existing worked-example chapter(s) afterwards** to confirm the + change didn't break the default convention; +- a requirement to flag anything that looks visually wrong in a contact sheet + rather than silently accept it, and to use `--region` (recording the exact + invocation) for any figure the automatic detection can't handle. + +Have it report back: final figure counts per chapter (matching the expected +counts above), which figures (if any) needed `--region`, any tool changes +made and the backward-compatibility check result, and one or two contact +sheets for you to spot-check before moving on to the TOC/audit phases. + +--- + +## 5. Normalize the page markers + +Chunk agents tend to number pages from 1 locally instead of using the printed +number. Check, then offset each file by the difference: + +```bash +for f in chapters/*.md; do + echo "$f $(rg -o '' -r '$1' "$f" | head -1)-$(rg -o '' -r '$1' "$f" | tail -1)" +done + +# e.g. chapter 1 printed pages start at 5 but markers start at 1 -> add 4 +perl -i -pe "s//''/ge" chapters/01-overview.md +``` + +Markers must sit **before** the content of the page they name. Anything that +later slices the file by page depends on this — see the pitfall in step 7. + +--- + +## 6. Build the table of contents + +Generate anchors mechanically; do not hand-write them. GitHub's slug rule is: +lowercase, strip punctuation except hyphens/underscores, spaces to hyphens. + +```python +def anchor(h): + s = h.strip().lower().replace("`", "") + s = re.sub(r"[^\w\s-]", "", s, flags=re.UNICODE) + return re.sub(r"\s+", "-", s.strip()) +``` + +A useful TOC has: a chapter table (file, printed page range, source PDF link, +figure count), the numbered section tree, a figure index, and a section listing +**known defects in the source datasheet** so nobody later "fixes" a faithful +transcription. Cap the depth — a 63-register chapter should link to its section, +not enumerate every register. + +Verify: + +```bash +python3 tools/check-links.py chapters +``` + +This checks every relative link, every anchor against real headings, and flags +orphaned PNGs. Exit code is non-zero on failure, so it can gate a commit. + +--- + +## 7. Audit: transcription vs PDF + +**Do not skip this, and use fresh agents.** The agent that wrote a file is not +a reliable auditor of it. + +One agent per chapter, launched in parallel, large chapters split the same way +as in step 3b — but the chunk size doesn't have to match. Transcription chunks +are sized for *writing* a large amount of new content (~8 pages); audit +chunks only need to *read and compare*, so they tolerate roughly 2x the page +count before running into the same silent-context-loss failure mode. On a +400+ page document, auditing in ~15-page chunks instead of ~8-page chunks +roughly halves the number of audit agents needed with no loss of quality. + +``` +You are auditing a Markdown transcription of a datasheet chapter against the +original PDF. This is a READ-ONLY audit. Do NOT edit, write, or create files. + +PDF (ground truth): .pdf +Markdown to audit: .md + +Method: +1. Use the native `read` tool on the PDF. It returns rendered pages as an + attachment. DO NOT use pdftotext/python/bash extraction - compare against + what is actually printed. +2. Use the `read` tool on the .md. +3. Compare page by page. + +Check: +- Missing or extra sections; wrong heading text or numbers. +- Body text summarized or paraphrased rather than transcribed verbatim. +- Every table: all rows and columns present, every value correct digit by + digit including sign, decimal point and unit. +- Figure captions: correct number, exact caption text. +- Any hallucinated content in the MD that is not in the PDF. THIS IS THE MOST + IMPORTANT THING TO CATCH. + +For each finding give: severity (CRITICAL = wrong/invented, MAJOR = missing, +MINOR = formatting), printed page number, what the PDF says, what the MD says. +If a category is clean, say so in one line. End with a one-line verdict: +ACCURATE / MINOR ISSUES / SIGNIFICANT ISSUES. Do not fix anything. +``` + +Add per-chapter emphasis: numeric accuracy for a specifications chapter, every +pin row for a pinout chapter, bit-level detail for a register chapter, field +order and ACK/NACK bit values for protocol frame diagrams. + +Feed forward any suspected source defects and ask the auditor to confirm what +the PDF *actually* prints. This is how you separate a transcription bug from a +vendor typo. In this project every such flag turned out to be a genuine defect +in the datasheet: a threshold printed in mV with volt-magnitude values, a pin +listed as both `NC` and `GPIO2`, `TYPE_C_PRAMETER` vs `TYPE_C_PARAMETER` +between a heading and its own table caption. + +### Prefer letting the audit agent navigate by marker, not pre-slicing + +The simplest way to avoid the slicing pitfall below is to not slice at all: +point the audit agent at the *full* chapter `.md` and tell it which +`` markers bound its assigned range ("only inspect the portion +between `` and ``"). The agent can `read` +the whole file and navigate to the right markers itself. This trades a larger +`read` for eliminating an entire class of off-by-one bugs in a slicing script, +and for the sizes involved (a chapter `.md` is at most a few thousand lines) +the read is cheap. + +Reach for scripted pre-slicing only when a downstream tool (not an agent) +needs an exact byte range — and if you do, remember markers precede their +page: + +```python +start = marks[a] # NOT marks[a-1] + 1 +end = marks[b + 1] +``` + +Getting this wrong shifts every slice one page and produces six confident, +identical, **false** "MAJOR: last page is missing" reports. Before believing any +missing-content finding, grep the real file: + +```bash +rg -o "Table ([0-9]+)\." -r '$1' chapters/07-registers.md | sort -nu | tr '\n' ' ' +rg -o "^### 7\.2\.([0-9]+) " -r '$1' chapters/07-registers.md | sort -n | tr '\n' ' ' +``` + +Contiguous sequences mean nothing is missing. + +--- + +## 8. Act on the findings + +Triage before editing. In this project, **all datasheet data — every table, +value, register and pin — was accurate**; every real defect was in prose the +agents added around the figures. Expect the same shape. + +Distinguish: + +- **Invented content** → delete. +- **Derived content** (e.g. an address/reset summary computed from the register + map) → keep if useful, but normalize it and add a note saying it is derived, + not printed. Do not let it masquerade as transcription. +- **Faithful transcription of a source defect** → leave alone, record in the + TOC's known-issues section. + +When stripping generated text, anchor the pattern tightly. Figure descriptions +were italic paragraphs *directly following an image line*; that shape can be +matched without touching legitimate printed `*Note:*` paragraphs elsewhere. +Verify afterwards that the printed ones survived: + +```bash +echo "before: $(rg -c '^\*Note' backup.md) after: $(rg -c '^\*Note' file.md)" +``` + +--- + +## Checklist + +``` +[ ] Split PDF into chapters/NN-slug.pdf; record printed page ranges +[ ] Transcribe in parallel, one agent per chapter (NO figure descriptions) +[ ] Chunk chapters over ~10 pages; concatenate; clean the seams +[ ] If normalizing heading levels with a script, verify it's idempotent + (0 changes on a second run) and preserves trailing newlines +[ ] Verify table-row diff is empty after every scripted rewrite +[ ] Check the figure caption convention (above/below the figure) on one + rendered page before extracting at scale; consider a subagent for the + extract/tune/contact-sheet loop on documents with 50+ figures +[ ] Extract figures; --dry-run, then contact-sheet review +[ ] Normalize to printed numbers +[ ] Build TOC with generated anchors; record known source defects +[ ] tools/check-links.py passes +[ ] Audit every chapter with fresh agents (chunks can be ~2x transcription + chunk size, since audits only read+compare) +[ ] Verify "missing content" findings against the real file before acting +[ ] Delete invented content; normalize derived content; keep source defects +``` + +## Tools in this repo + +| Tool | Purpose | +|---|---| +| [`tools/extract-figures.py`](tools/extract-figures.py) | Caption-anchored figure cropping from vector PDFs. `--caption-position {below,above}` (default `below`) and decimal figure numbers (`4.1`) support both the Renesas-style and USB-IF-style caption conventions | +| [`tools/check-links.py`](tools/check-links.py) | Link, anchor and orphan-image validation. Anchor generation replicates GitHub's `-1`/`-2` disambiguation for duplicate headings (e.g. two companies with near-identical names in a contributors list) | +| [`chapters/TOC.md`](chapters/TOC.md) | Worked example of the generated TOC | diff --git a/README.md b/README.md new file mode 100644 index 0000000..752a201 --- /dev/null +++ b/README.md @@ -0,0 +1,23 @@ +# Markdowned Datasheets + +Vendor datasheets and technical specifications, converted from PDF into +per-chapter Markdown with extracted figures and a generated table of contents. + +Each PDF is transcribed **verbatim**, full body text, complete tables, and +figures cropped out as PNGs and linked back in with their printed captions. + +## What's here + +Each converted document starts with `TOC.md`, it links every chapter, every figure, and lists the +source document's own known defects so you don't mistake a vendor typo for a +transcription error. + +## Converting another datasheet + +[`PROCESS.md`](PROCESS.md) is the runbook a coding agent follows to do the +conversion — prompt templates, known failure modes, and a checklist. + +To convert a new one, drop the PDF in the repo root and ask your agent to +follow `PROCESS.md` for it. Point it at whichever existing conversion is +closest in scale for reference if the source document is unusual (a single +short datasheet vs. a 400+ page multi-vendor specification). diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..5f9d40f --- /dev/null +++ b/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1783148766, + "narHash": "sha256-uslt2pqShTIXDdAHRHv2QkYLsVdY8Oqwz0EA48/RSM8=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a50de1b7d8a586adc18d2395c19de7d6058e6030", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a50de1b7d8a586adc18d2395c19de7d6058e6030", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..fa651f2 --- /dev/null +++ b/flake.nix @@ -0,0 +1,46 @@ +{ + description = "Tooling for turning datasheet PDFs into text"; + + inputs.nixpkgs.url = + "github:NixOS/nixpkgs/a50de1b7d8a586adc18d2395c19de7d6058e6030"; + + outputs = { self, nixpkgs }: + let + systems = [ "x86_64-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ]; + forAllSystems = f: + nixpkgs.lib.genAttrs systems (system: f nixpkgs.legacyPackages.${system}); + in + { + devShells = forAllSystems (pkgs: { + default = pkgs.mkShell { + name = "datasheet-to-text"; + + packages = with pkgs; [ + # Split / merge / inspect PDFs losslessly. + # qpdf --split-pages=1 in.pdf out-%d.pdf + # qpdf in.pdf --pages . 12-34 -- out.pdf + qpdf + + # pdfseparate, pdfunite, pdftotext, pdfinfo, pdfimages, pdftoppm + poppler-utils + + # mutool: draw/extract/convert, good structured text output + # mutool draw -F text -o out.txt in.pdf 12-34 + mupdf + + # Crop/trim rendered pages down to individual figures. + # magick in.png -crop WxH+X+Y +repage out.png + imagemagick + + # Runs tools/extract-figures.py (stdlib only). + python3 + ]; + + shellHook = '' + echo "datasheet-to-text shell" + echo " qpdf $(qpdf --version | head -1 | grep -o '[0-9.]*$') pdftotext mutool magick python3" + ''; + }; + }); + }; +} diff --git a/tools/check-links.py b/tools/check-links.py new file mode 100644 index 0000000..490ec65 --- /dev/null +++ b/tools/check-links.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Verify every relative link, anchor and image path in a folder of Markdown. + +Catches the two things that break most often after a transcription pass: +image paths that point at figures which were never extracted, and TOC anchors +that no longer match a heading because the heading was reworded. + +Anchor slugs follow GitHub's rules: lowercase, strip punctuation except +hyphens/underscores, spaces to hyphens. + +Usage: + check-links.py [DIR] # defaults to ./chapters + +Exits non-zero if anything is broken, so it can gate a commit. +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +LINK = re.compile(r"\[([^\]]*)\]\(([^)]+)\)") +HEADING = re.compile(r"^#{1,6} +(.*)$") + + +def anchor(heading: str) -> str: + s = heading.strip().lower().replace("`", "") + s = re.sub(r"[^\w\s-]", "", s, flags=re.UNICODE) + return re.sub(r"\s+", "-", s.strip()) + + +def main() -> int: + root = Path(sys.argv[1] if len(sys.argv) > 1 else "chapters") + if not root.is_dir(): + print(f"not a directory: {root}") + return 2 + + md_files = sorted(root.glob("*.md")) + anchors: dict[str, set[str]] = {} + for f in md_files: + seen: dict[str, int] = {} + slugs: set[str] = set() + for line in f.read_text().splitlines(): + m = HEADING.match(line) + if not m: + continue + a = anchor(m.group(1)) + n = seen.get(a, 0) + seen[a] = n + 1 + # GitHub disambiguates repeated headings by appending -1, -2, ... + slugs.add(a if n == 0 else f"{a}-{n}") + anchors[f.name] = slugs + + bad = 0 + linked_images: set[Path] = set() + + for f in md_files: + for _text, target in LINK.findall(f.read_text()): + if target.startswith(("http://", "https://", "mailto:", "#")): + continue + rel, _, frag = target.partition("#") + if not rel: + continue + resolved = (f.parent / rel).resolve() + if not resolved.exists(): + print(f"{f.name}: MISSING FILE {target}") + bad += 1 + continue + if resolved.suffix.lower() == ".png": + linked_images.add(resolved) + if frag and resolved.suffix == ".md": + if frag not in anchors.get(resolved.name, set()): + print(f"{f.name}: MISSING ANCHOR #{frag} in {rel}") + bad += 1 + + for img in sorted(root.rglob("*.png")): + if img.resolve() not in linked_images: + print(f"ORPHAN IMAGE: {img.relative_to(root)}") + bad += 1 + + print(f"\n{len(md_files)} file(s) checked, {bad} problem(s)") + return 1 if bad else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/extract-figures.py b/tools/extract-figures.py new file mode 100644 index 0000000..6390556 --- /dev/null +++ b/tools/extract-figures.py @@ -0,0 +1,424 @@ +#!/usr/bin/env python3 +"""Extract figures from a (mostly vector) datasheet PDF into per-PDF folders. + +Most figures in these Renesas datasheets are vector line art, so `pdfimages` +finds nothing. Instead we: + + 1. Read the structured text layer (`mutool draw -F stext`) to locate the + "Figure N. " captions and the surrounding body text. + 2. Derive a coarse band for each figure: bounded below by its caption and + above by whichever comes last -- the previous row of captions on the page, + the last body-text/heading line, or the top of the content area. + 3. Where a page places figures side by side (performance-graph pages usually + do), split the band horizontally at the midpoints between the captions of + that row, so each figure gets its own column. + 4. Render the page and tighten that band to the actual ink bounding box, so + the crop hugs the drawing instead of the column. + 4. Write `figure-NN.png` named by the datasheet's own global figure number. + +Usage: + extract-figures.py [--dry-run] [--dpi N] [--outdir DIR] file.pdf [file.pdf ...] +""" + +from __future__ import annotations + +import argparse +import re +import shutil +import subprocess +import sys +import tempfile +import xml.etree.ElementTree as ET +from dataclasses import dataclass +from pathlib import Path + +# --- page geometry, in PDF points (origin top-left, y increasing downward) --- +HEADER_BOTTOM = 62.0 # running head + its horizontal rule (ends ~60pt) +FOOTER_TOP = 740.0 # doc id / revision / page number sit below this +BODY_MIN_SIZE = 9.5 # body text and section headings are >= 10pt +BODY_MAX_X0 = 60.0 # ...and start at the left margin (~47.5pt) +CAPTION_MAX_SIZE = 13.0 # captions are bold, 9-12pt depending on template +CAPTION_LINE_GAP = 5.0 # max vertical gap between wrapped caption lines +ROW_TOLERANCE = 6.0 # captions starting within this many points of each + # other sit side by side in the same row +BODY_GAP = 5.0 # clearance below body text; stext line bboxes clip + # descenders, which would otherwise leak into a crop +PAD = 6.0 # padding around the ink bbox, in points + +CAPTION_RE = re.compile(r"^\s*Figure\s+(\d+(?:\.\d+)?)\s*\.\s*(.*)$") + +INK_THRESHOLD = 200 # 0-255 grey; anything darker counts as ink +ANALYSIS_DPI = 100 # cheap render used only for ink detection + + +@dataclass +class Line: + x0: float + y0: float + x1: float + y1: float + size: float + text: str + bold: bool = False + + +@dataclass +class Caption: + number: str + title: str + top: float # y0 of the first caption line + bottom: float # y1 of the last wrapped caption line + x0: float # left edge across all wrapped lines + x1: float # right edge across all wrapped lines + + +@dataclass +class Figure: + number: str + title: str + page: int + top: float + bottom: float + left: float + right: float + + +def run(cmd: list[str]) -> str: + proc = subprocess.run(cmd, capture_output=True, text=True) + if proc.returncode != 0: + raise RuntimeError(f"{cmd[0]} failed: {proc.stderr.strip()}") + return proc.stdout + + +def page_lines(pdf: Path, page: int) -> tuple[float, float, list[Line]]: + xml = run(["mutool", "draw", "-F", "stext", "-o", "-", str(pdf), str(page)]) + root = ET.fromstring(xml) + pg = root.find("page") + if pg is None: + return 0.0, 0.0, [] + lines: list[Line] = [] + for el in pg.iter("line"): + bbox = el.get("bbox") + if not bbox: + continue + x0, y0, x1, y1 = (float(v) for v in bbox.split()) + font = el.find("font") + size = float(font.get("size")) if font is not None else 0.0 + bold = bool(font is not None and "bold" in (font.get("name") or "").lower()) + lines.append(Line(x0, y0, x1, y1, size, el.get("text") or "", bold)) + lines.sort(key=lambda l: (l.y0, l.x0)) + return float(pg.get("width")), float(pg.get("height")), lines + + +def find_captions(lines: list[Line]) -> list[Caption]: + """Locate the "Figure N. <title>" captions on a page. + + Captions wrap onto several lines; the bottom of the *last* line is what + bounds the next figure, otherwise the wrapped remainder of this caption + leaks into the top of the following crop. The horizontal extent spans all + the wrapped lines, and is what separates side-by-side figures. + """ + out: list[Caption] = [] + for i, line in enumerate(lines): + m = CAPTION_RE.match(line.text) + if not m or line.size > CAPTION_MAX_SIZE or not line.bold: + continue + title = m.group(2).strip() + # Absorb continuation lines: same size, directly below, horizontally + # overlapping (a side-by-side caption is not a continuation), and not + # itself a new caption. + prev = line + x0, x1 = line.x0, line.x1 + for nxt in lines[i + 1:]: + if abs(nxt.size - line.size) > 0.5: + continue + # A caption in the neighbouring column is not a continuation, and + # must not stop the search either. + if nxt.x1 < x0 or nxt.x0 > x1: + continue + if nxt.y0 - prev.y1 > CAPTION_LINE_GAP or nxt.y0 < prev.y1 - 1: + break + if CAPTION_RE.match(nxt.text): + break + title = f"{title} {nxt.text.strip()}".strip() + x0, x1 = min(x0, nxt.x0), max(x1, nxt.x1) + prev = nxt + out.append(Caption(m.group(1), title, line.y0, prev.y1, x0, x1)) + out.sort(key=lambda c: (c.top, c.x0)) + return out + + +def caption_rows(captions: list[Caption]) -> list[list[Caption]]: + """Group captions that sit side by side into rows, ordered top to bottom.""" + rows: list[list[Caption]] = [] + for cap in captions: + if rows and abs(cap.top - rows[-1][0].top) <= ROW_TOLERANCE: + rows[-1].append(cap) + else: + rows.append([cap]) + for row in rows: + row.sort(key=lambda c: c.x0) + return rows + + +def figures_on_page(pdf: Path, page: int) -> list[Figure]: + width, _, lines = page_lines(pdf, page) + captions = find_captions(lines) + if not captions: + return [] + + rows = caption_rows(captions) + figures: list[Figure] = [] + for r, row in enumerate(rows): + row_top = min(c.top for c in row) + upper = HEADER_BOTTOM + # The previous row of captions bounds this one from above -- use the + # last wrapped line of that row, not its first. + if r > 0: + upper = max(upper, max(c.bottom for c in rows[r - 1]) + BODY_GAP) + # So does the last body-text / heading line above the row. + for line in lines: + if line.y1 >= row_top or line.y1 <= HEADER_BOTTOM: + continue + if line.y0 >= FOOTER_TOP: + continue + is_body = line.size >= BODY_MIN_SIZE and line.x0 <= BODY_MAX_X0 + if is_body and line.y1 + BODY_GAP > upper: + # Do not step over an earlier row's figures. + if any(upper < c.top < line.y1 for pr in rows[:r] for c in pr): + continue + upper = line.y1 + BODY_GAP + # Split the row horizontally at the midpoints between its captions. + for i, cap in enumerate(row): + left = 0.0 if i == 0 else (row[i - 1].x1 + cap.x0) / 2.0 + right = width if i == len(row) - 1 else (cap.x1 + row[i + 1].x0) / 2.0 + figures.append( + Figure(cap.number, cap.title, page, upper, cap.top, left, right) + ) + return figures + + +def figures_on_page_above(pdf: Path, page: int) -> list[Figure]: + """Same as figures_on_page, but for templates where the caption sits + ABOVE the figure (e.g. USB PD spec) instead of below it (Renesas-style). + """ + width, _, lines = page_lines(pdf, page) + captions = find_captions(lines) + if not captions: + return [] + + rows = caption_rows(captions) + figures: list[Figure] = [] + for r, row in enumerate(rows): + row_bottom = max(c.bottom for c in row) + upper = row_bottom + BODY_GAP + lower = FOOTER_TOP + # The next row of captions bounds this one from below -- use its + # topmost caption line. + if r + 1 < len(rows): + lower = min(lower, min(c.top for c in rows[r + 1]) - BODY_GAP) + # So does the first body-text / heading line below the row. + for line in lines: + if line.y0 <= row_bottom or line.y0 >= lower: + continue + if line.y1 <= HEADER_BOTTOM: + continue + if line.bold and CAPTION_RE.match(line.text): + continue # a real next caption; already bounded via rows[r+1] + is_body = line.size >= BODY_MIN_SIZE and line.x0 <= BODY_MAX_X0 + if is_body and line.y0 - BODY_GAP < lower: + lower = line.y0 - BODY_GAP + # Split the row horizontally at the midpoints between its captions. + for i, cap in enumerate(row): + left = 0.0 if i == 0 else (row[i - 1].x1 + cap.x0) / 2.0 + right = width if i == len(row) - 1 else (cap.x1 + row[i + 1].x0) / 2.0 + figures.append( + Figure(cap.number, cap.title, page, upper, lower, left, right) + ) + return figures + + +def read_pgm(path: Path) -> tuple[int, int, bytes]: + data = path.read_bytes() + if not data.startswith(b"P5"): + raise ValueError(f"{path} is not a binary PGM") + fields, pos = [], 2 + while len(fields) < 3: + while pos < len(data) and data[pos : pos + 1].isspace(): + pos += 1 + if data[pos : pos + 1] == b"#": + while data[pos : pos + 1] not in (b"\n", b""): + pos += 1 + continue + start = pos + while pos < len(data) and not data[pos : pos + 1].isspace(): + pos += 1 + fields.append(int(data[start:pos])) + return fields[0], fields[1], data[pos + 1 :] + + +def ink_bbox( + pgm: tuple[int, int, bytes], + top_pt: float, + bottom_pt: float, + scale: float, + left_pt: float = 0.0, + right_pt: float | None = None, +) -> tuple[float, float, float, float] | None: + """Tightest ink bbox (in points) within the given band.""" + w, h, px = pgm + r0 = max(0, int(top_pt * scale)) + r1 = min(h, int(bottom_pt * scale) + 1) + c0 = max(0, int(left_pt * scale)) + c1 = min(w, int(right_pt * scale) + 1) if right_pt is not None else w + if r1 <= r0 or c1 <= c0: + return None + + rows = [r for r in range(r0, r1) + if min(px[r * w + c0 : r * w + c1]) < INK_THRESHOLD] + if not rows: + return None + top_px, bottom_px = rows[0], rows[-1] + 1 + + span = c1 - c0 + col_min = bytearray(b"\xff" * span) + for r in rows: + row = px[r * w + c0 : r * w + c1] + for c in range(span): + if row[c] < col_min[c]: + col_min[c] = row[c] + cols = [c for c in range(span) if col_min[c] < INK_THRESHOLD] + if not cols: + return None + left_px, right_px = c0 + cols[0], c0 + cols[-1] + 1 + + return (left_px / scale, top_px / scale, right_px / scale, bottom_px / scale) + + +def crop_region(pdf: Path, outdir: Path, dpi: int, spec: str, dry_run: bool) -> str: + """Crop an explicitly given region: PAGE:x0,y0,x1,y1:NAME (points). + + For figures the datasheet never captioned, so caption detection cannot + find them. The given box is still tightened to the ink inside it. + """ + page_s, box_s, name = spec.split(":", 2) + page = int(page_s) + bx0, by0, bx1, by1 = (float(v) for v in box_s.split(",")) + tmp = Path(tempfile.mkdtemp(prefix="figregion-")) + try: + analysis = tmp / "a" + run(["pdftoppm", "-gray", "-r", str(ANALYSIS_DPI), "-f", str(page), + "-l", str(page), "-singlefile", str(pdf), str(analysis)]) + pgm = read_pgm(analysis.with_suffix(".pgm")) + box = ink_bbox(pgm, by0, by1, ANALYSIS_DPI / 72.0, bx0, bx1) + if box is None: + return f"SKIP: no ink in region {box_s} on p{page}" + x0, y0, x1, y1 = box + x0, y0 = max(bx0, x0 - PAD), max(by0, y0 - PAD) + x1, y1 = min(bx1, x1 + PAD), min(by1, y1 + PAD) + if not dry_run: + full = tmp / "f" + run(["pdftoppm", "-png", "-r", str(dpi), "-f", str(page), + "-l", str(page), "-singlefile", str(pdf), str(full)]) + s = dpi / 72.0 + geom = (f"{round((x1 - x0) * s)}x{round((y1 - y0) * s)}" + f"+{round(x0 * s)}+{round(y0 * s)}") + outdir.mkdir(parents=True, exist_ok=True) + run(["magick", str(full.with_suffix(".png")), "-crop", geom, + "+repage", str(outdir / name)]) + return (f"p{page} region -> ink {x0:.0f},{y0:.0f} " + f"{x1 - x0:.0f}x{y1 - y0:.0f}pt") + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +def extract(pdf: Path, outdir: Path, dpi: int, dry_run: bool, + caption_position: str = "below") -> list[tuple[Figure, str]]: + finder = figures_on_page_above if caption_position == "above" else figures_on_page + pages = int( + re.search(r"Pages:\s+(\d+)", run(["pdfinfo", str(pdf)])).group(1) + ) + results: list[tuple[Figure, str]] = [] + tmp = Path(tempfile.mkdtemp(prefix="figextract-")) + try: + for page in range(1, pages + 1): + figs = finder(pdf, page) + if not figs: + continue + + analysis = tmp / f"a{page}" + run(["pdftoppm", "-gray", "-r", str(ANALYSIS_DPI), "-f", str(page), + "-l", str(page), "-singlefile", str(pdf), str(analysis)]) + pgm = read_pgm(analysis.with_suffix(".pgm")) + scale = ANALYSIS_DPI / 72.0 + + full = tmp / f"f{page}" + if not dry_run: + run(["pdftoppm", "-png", "-r", str(dpi), "-f", str(page), + "-l", str(page), "-singlefile", str(pdf), str(full)]) + + for fig in figs: + box = ink_bbox(pgm, fig.top, fig.bottom, scale, + fig.left, fig.right) + if box is None: + results.append((fig, "SKIP: no ink found in band")) + continue + x0, y0, x1, y1 = box + # Pad, but never past the band edges: the caption sits just + # below `fig.bottom` and body text just above `fig.top`. + x0, x1 = max(fig.left, x0 - PAD), min(fig.right, x1 + PAD) + y0 = max(y0 - PAD, fig.top) + y1 = min(y1 + PAD, fig.bottom - 2.0) + name = f"figure-{fig.number}.png" + note = (f"p{fig.page} band {fig.top:.0f}-{fig.bottom:.0f}pt " + f"x {fig.left:.0f}-{fig.right:.0f}pt " + f"-> ink {x0:.0f},{y0:.0f} {x1 - x0:.0f}x{y1 - y0:.0f}pt") + if not dry_run: + s = dpi / 72.0 + geom = (f"{round((x1 - x0) * s)}x{round((y1 - y0) * s)}" + f"+{round(x0 * s)}+{round(y0 * s)}") + outdir.mkdir(parents=True, exist_ok=True) + run(["magick", str(full.with_suffix(".png")), "-crop", geom, + "+repage", str(outdir / name)]) + results.append((fig, note)) + finally: + shutil.rmtree(tmp, ignore_errors=True) + return results + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("pdfs", nargs="+", type=Path) + ap.add_argument("--dpi", type=int, default=300, help="output render DPI") + ap.add_argument("--outdir", type=Path, default=None, + help="output root (default: alongside each PDF)") + ap.add_argument("--dry-run", action="store_true", + help="report detected figures without writing images") + ap.add_argument("--region", action="append", default=[], + metavar="PAGE:x0,y0,x1,y1:NAME", + help="also crop an explicit region (points) from the single " + "given PDF; for figures with no 'Figure N.' caption") + ap.add_argument("--caption-position", choices=["below", "above"], default="below", + help="whether 'Figure N. Title' sits below the figure " + "(Renesas-style, default) or above it (e.g. USB PD spec)") + args = ap.parse_args() + + if args.region and len(args.pdfs) != 1: + ap.error("--region applies to exactly one PDF") + + for pdf in args.pdfs: + root = args.outdir if args.outdir else pdf.parent + outdir = root / pdf.stem + print(f"\n=== {pdf.name} -> {outdir}/") + for fig, note in extract(pdf, outdir, args.dpi, args.dry_run, args.caption_position): + print(f" figure-{fig.number} {note}") + print(f" {fig.title[:88]}") + for spec in args.region: + note = crop_region(pdf, outdir, args.dpi, spec, args.dry_run) + print(f" {spec.split(':', 2)[2]} {note}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())