work_unit: dict,

draft_dir: Path,

output_dir: Path,

label_index_file: Path,

verbose: bool = False,

) -> dict:

"""

from .converter import Converter

draft_dir = Path(draft_dir)

output_dir = Path(output_dir)

stable_name = work_unit["stable_name"]

try:

# Create converter instance for this worker

converter = Converter()

# Determine input file(s)

temp_files = []

if work_unit["type"] == "collision":

# Merge multiple files

chapters = work_unit["chapters"]

chapter_files = []

for ch in chapters:

ch_file = draft_dir / f"{ch}.tex"

if ch_file.exists():

chapter_files.append(ch_file)

# Merge files into temporary file

merged_content = []

for tex_file in chapter_files:

content = tex_file.read_text(encoding="utf-8")

merged_content.append(content)

# Create temporary merged file

file_to_convert = create_temp_tex_file("\n\n".join(merged_content))

temp_files.append(file_to_convert)

else:

# Single file conversion

chapter = work_unit["chapter"]

chapter_file = draft_dir / f"{chapter}.tex"

if not chapter_file.exists():

return {

"success": False,

"stable_name": stable_name,

"error": f"{chapter}.tex not found",

}

file_to_convert = chapter_file

# Expand \input{} commands for front/back

if chapter in ["front", "back"]:

content = chapter_file.read_text(encoding="utf-8")

base_dir = chapter_file.parent

expanded = expand_latex_inputs(content, base_dir)

file_to_convert = create_temp_tex_file(expanded)

temp_files.append(file_to_convert)

# Convert to markdown

output_file = output_dir / f"{stable_name}.md"

converter.convert_file(

file_to_convert,

output_file=output_file,

standalone=True,

verbose=False,

current_file_stem=stable_name,

label_index_file=label_index_file,

source_dir=draft_dir,

)

# Cleanup temporary files

cleanup_temp_files(temp_files)

return {"success": True, "output_file": output_file, "stable_name": stable_name}

except Exception as e:

# Cleanup temporary files on error

cleanup_temp_files(temp_files)

"""Builds complete standard document from std.tex driver file"""

def __init__(self, draft_dir: Path):

"""

self.draft_dir = Path(draft_dir)

self.std_tex = self.draft_dir / "std.tex"

def convert_diagrams_to_svg(self, output_dir: Path, verbose: bool = False) -> list[Path]:

"""

# Find all .dot files in the draft source directory

dot_files = list(self.draft_dir.glob("*.dot"))

if not dot_files:

if verbose:

print("No .dot diagram files found")

return []

# Create images subdirectory

images_dir = output_dir / "images"

ensure_dir(images_dir)

if verbose:

print(f"Converting {len(dot_files)} diagrams to SVG...")

svg_files = []

for dot_file in dot_files:

svg_file = images_dir / (dot_file.stem + ".svg")

try:

# Use Graphviz dot command to convert to SVG

subprocess.run(

["dot", "-Tsvg", str(dot_file), "-o", str(svg_file)],

capture_output=True,

text=True,

check=True,

)

svg_files.append(svg_file)

if verbose:

print(f" {dot_file.name} → {svg_file.name}")

except subprocess.CalledProcessError as e:

print(

f"Warning: Failed to convert {dot_file.name}: {e.stderr}",

file=sys.stderr,

)

except FileNotFoundError:

print(

"Error: Graphviz (dot) not found. Install with: apt install graphviz",

file=sys.stderr,

)

break

if verbose and svg_files:

print(f"Generated {len(svg_files)} SVG diagrams in {images_dir}")

return svg_files

def extract_chapter_order(

self, include_frontmatter: bool = True, include_backmatter: bool = True

) -> list[str]:

r"""

if not self.std_tex.exists():

raise FileNotFoundError(f"std.tex not found at {self.std_tex}")

content = self.std_tex.read_text(encoding="utf-8")

# Use pylatexenc to parse the LaTeX

walker = LatexWalker(content)

nodelist, _, _ = walker.get_latex_nodes()

chapters = []

in_frontmatter = False

in_mainmatter = False

in_backmatter = False

# Walk through nodes looking for \include{} commands

def visit_node(node):

nonlocal in_frontmatter, in_mainmatter, in_backmatter

if isinstance(node, LatexMacroNode):

# Track document sections

if node.macroname == "frontmatter":

in_frontmatter = True

elif node.macroname == "mainmatter":

in_frontmatter = False

in_mainmatter = True

elif node.macroname == "backmatter":

in_mainmatter = False

in_backmatter = True

elif node.macroname == "include":

# Decide whether to include based on section

should_include = False

if (

(in_frontmatter and include_frontmatter)

or in_mainmatter

or (in_backmatter and include_backmatter)

):

should_include = True

if should_include and node.nodeargd and node.nodeargd.argnlist:

# Extract the filename from the argument

arg = node.nodeargd.argnlist[0]

if arg and hasattr(arg, "nodelist"):

# Get the text content

filename = "".join(

n.chars if hasattr(n, "chars") else "" for n in arg.nodelist

)

if filename:

chapters.append(filename.strip())

# Recursively visit child nodes

if hasattr(node, "nodelist") and node.nodelist:

for child in node.nodelist:

visit_node(child)

for node in nodelist:

visit_node(node)

return chapters

def _expand_input_commands(self, tex_file: Path) -> Path:

"""

content = tex_file.read_text(encoding="utf-8")

base_dir = tex_file.parent

expanded = expand_latex_inputs(content, base_dir)

# Create temporary file with expanded content

return create_temp_tex_file(expanded)

def extract_stable_name_from_tex(self, tex_file: Path, converter) -> str:

"""

try:

# Find the first \rSec0[label] line in the file

# This is the top-level section that defines the stable name

rsec0_line = None

with tex_file.open("r", encoding="utf-8") as f:

for line in f:

# Look for \rSec0[label]{Title} pattern

if r"\rSec0[" in line:

rsec0_line = line.strip()

break

if not rsec0_line:

# No \rSec0 found, fall back to filename

return tex_file.stem

# Create temporary file with just the \rSec0 line

# This avoids issues with unclosed LaTeX environments when truncating

tmp_path = create_temp_tex_file(rsec0_line + "\n")

try:

# Convert with Pandoc (cpp-sections.lua extracts the label)

markdown = converter.convert_file(

tmp_path, output_file=None, standalone=False, verbose=False

)

# Extract first <a id="..."> anchor created by cpp-sections.lua

# Format: <a id="stable.name">[stable.name]</a>

match = re.search(r'<a id="([^"]+)">', markdown)

if match:

label = match.group(1)

# Extract prefix before first dot

# "expr.prim" → "expr"

# "stmt.stmt" → "stmt"

# "intro.scope" → "intro"

stable_name = label.split(".")[0] if "." in label else label

return stable_name

finally:

# Clean up temp file

tmp_path.unlink(missing_ok=True)

except (OSError, UnicodeDecodeError, re.error):

# If extraction fails, fall back to filename

# This handles files that don't have \rSec tags or conversion errors

pass

# Fallback: use filename stem

return tex_file.stem

def detect_stable_name_collisions(

self, chapters: list[str], chapter_to_stable: dict[str, str], verbose: bool = False

) -> dict[str, list[str]]:

"""

# Group chapters by stable name

stable_to_chapters = {}

for chapter in chapters:

stable_name = chapter_to_stable.get(chapter)

if not stable_name:

continue

if stable_name not in stable_to_chapters:

stable_to_chapters[stable_name] = []

stable_to_chapters[stable_name].append(chapter)

# Find collision groups (stable names with 2+ chapters)

collision_groups = {

stable: chapters_list

for stable, chapters_list in stable_to_chapters.items()

if len(chapters_list) > 1

}

if verbose and collision_groups:

print(f"\nDetected {len(collision_groups)} stable name collision(s):")

for stable_name, chapters_list in collision_groups.items():

print(f" {stable_name}: {len(chapters_list)} files will be merged")

for chapter in chapters_list:

print(f" - {chapter}.tex")

return collision_groups

def _merge_tex_files(self, chapter_files: list[Path], verbose: bool = False) -> Path:

r"""

if verbose:

print(f" Merging {len(chapter_files)} files:")

for f in chapter_files:

print(f" + {f.name}")

# Concatenate file contents with separators

merged_content = []

for tex_file in chapter_files:

content = tex_file.read_text(encoding="utf-8")

merged_content.append(content)

# Create temporary file with merged content

# Join with double newline to ensure proper spacing

return create_temp_tex_file("\n\n".join(merged_content))

def build_full_standard(

self, converter, output_file: Path, verbose: bool = False, toc_depth: int = 1

) -> tuple[str, list[str]]:

"""

chapters = self.extract_chapter_order()

if verbose:

print(f"Found {len(chapters)} chapters in std.tex")

# Convert .dot diagrams to SVG before processing chapters

# Images go in the same directory as the output file

output_dir = Path(output_file).parent

ensure_dir(output_dir)

self.convert_diagrams_to_svg(output_dir, verbose=verbose)

converted_chapters = []

all_content_parts = []

all_references = {} # Collect all references from all chapters

temp_files = [] # Track temporary files for cleanup

for i, chapter in enumerate(chapters, 1):

chapter_file = self.draft_dir / f"{chapter}.tex"

if not chapter_file.exists():

if verbose:

print(f"Warning: {chapter}.tex not found, skipping")

continue

if verbose:

print(f"[{i}/{len(chapters)}] Converting {chapter}.tex...")

# Expand \input{} commands for chapters that use them

# (front.tex and back.tex contain \input{} references)

file_to_convert = chapter_file

if chapter in ["front", "back"]:

try:

file_to_convert = self._expand_input_commands(chapter_file)

temp_files.append(file_to_convert)

except Exception as e:

if verbose:

print(f"Warning: Could not expand inputs in {chapter}.tex: {e}")

# Convert chapter to markdown

try:

markdown = converter.convert_file(

file_to_convert,

output_file=None, # Return as string

standalone=False, # Don't include standalone wrappers

verbose=False,

source_dir=self.draft_dir,

)

# Extract link definitions and content separately

content, references = self._split_content_and_references(markdown)

all_content_parts.append(content)

all_references.update(references)

converted_chapters.append(chapter)

except Exception as e:

# Always print conversion errors to prevent silent failures

print(f"ERROR: Failed to convert {chapter}.tex: {e}", file=sys.stderr)

if verbose:

import traceback

traceback.print_exc()

continue

# Build final document

full_content = "\n\n---\n\n".join(all_content_parts)

# Generate and prepend Table of Contents

toc = self._generate_toc(full_content, max_depth=toc_depth)

full_content = toc + "\n\n---\n\n" + full_content

# Add all link definitions at the end

if all_references:

link_defs = "\n<!-- Link reference definitions -->\n"

for ref in sorted(all_references.keys()):

link_defs += f"[{ref}]: #{ref}\n"

full_content += "\n\n" + link_defs

# Write to output file

output_file.write_text(full_content, encoding="utf-8")

# Cleanup temporary files

cleanup_temp_files(temp_files)

if verbose:

print(f"\nWrote complete standard to {output_file}")

print(f"Converted {len(converted_chapters)}/{len(chapters)} chapters")

print(f"Total cross-references: {len(all_references)}")

return full_content, converted_chapters

def _generate_toc(self, markdown_content: str, max_depth: int = 1) -> str:

"""

toc_lines = []

toc_lines.append("# Table of Contents\n")

# Pattern to match headings with embedded anchors

# Matches: ## Title <a id="label">[[label]]</a>

heading_pattern = re.compile(

r'^(#{1,6})\s+(.+?)\s+<a id="([^"]+)">\[\[([^\]]+)\]\]</a>\s*$', re.MULTILINE

)

# Track section numbers at each level

section_numbers = [0, 0, 0, 0, 0, 0] # Support up to H6

for match in heading_pattern.finditer(markdown_content):

hashes = match.group(1)

title = match.group(2).strip()

stable_name = match.group(4)

level = len(hashes) - 1 # H1=0, H2=1, etc.

# Update section numbers (always update for correct numbering)

section_numbers[level] += 1

# Reset deeper levels

for i in range(level + 1, 6):

section_numbers[i] = 0

# Skip headings deeper than max_depth

if level >= max_depth:

continue

# Build section number string (e.g., "1.2.3")

section_num = ".".join(str(section_numbers[i]) for i in range(level + 1))

# Create TOC entry with indentation

indent = " " * level

list_marker = "- "

toc_entry = (

f"{indent}{list_marker}{section_num} {title} [[{stable_name}]](#{stable_name})"

)

toc_lines.append(toc_entry)

return "\n".join(toc_lines)

def _split_content_and_references(self, markdown: str) -> tuple[str, dict]:

"""

lines = markdown.split("\n")

content_lines = []

references = {}

in_references = False

for line in lines:

if "<!-- Link reference definitions -->" in line:

in_references = True

continue

if in_references:

# Match [ref]: #ref pattern

match = re.match(r"\[([^\]]+)\]:\s*#\1", line)

if match:

references[match.group(1)] = True

continue

content_lines.append(line)

return "\n".join(content_lines).strip(), references

def _generate_toc_for_separate_files(self, output_files: list[Path], max_depth: int = 1) -> str:

"""

toc_lines = []

toc_lines.append("# Table of Contents\n")

# Pattern to match headings with embedded anchors

# Matches: ## Title <a id="label">[[label]]</a>

# Also matches annexes: ## Title (informative) <a id="label" data-annex="true" data-annex-type="informative">[[label]]</a>

heading_pattern = re.compile(

r'^(#{1,6})\s+(.+?)\s+<a id="([^"]+)"(?:\s+[^>]+)?>\[\[([^\]]+)\]\]</a>\s*$',

re.MULTILINE,

)

# Separate pattern to detect if heading is an annex and extract type

annex_pattern = re.compile(r'data-annex="true"(?:\s+data-annex-type="([^"]+)")?')

# Track section numbers at each level

section_numbers = [0, 0, 0, 0, 0, 0] # Support up to H6

annex_counter = 0 # Track annex letter (A, B, C, etc.)

current_annex_letter = None # Track current annex for subsections

# Process files in order

for md_file in output_files:

content = md_file.read_text(encoding="utf-8")

filename = md_file.stem

for match in heading_pattern.finditer(content):

hashes = match.group(1)

title = match.group(2).strip()

anchor_id = match.group(3)

stable_name = match.group(4)

# Check if this heading is an annex (has data-annex="true")

full_match = match.group(0)

is_annex = annex_pattern.search(full_match) is not None

level = len(hashes) - 1 # H1=0, H2=1, etc.

# For annex headings at H1 level, use letter numbering

if is_annex and level == 0:

annex_counter += 1

current_annex_letter = chr(ord("A") + annex_counter - 1)

# Reset annex subsection numbering

for i in range(1, 6):

section_numbers[i] = 0

else:

# Update section numbers (always update for correct numbering)

section_numbers[level] += 1

# Reset deeper levels

for i in range(level + 1, 6):

section_numbers[i] = 0

# Skip headings deeper than max_depth

if level >= max_depth:

continue

# Build section number string

if is_annex and level == 0:

# Annex A, B, C, etc.

section_num = f"Annex {current_annex_letter}"

elif current_annex_letter and level > 0:

# Annex subsections: A.1, A.2, etc.

subsection = ".".join(str(section_numbers[i]) for i in range(1, level + 1))

section_num = f"{current_annex_letter}.{subsection}"

else:

# Regular numbered sections: 1.2.3, etc.

section_num = ".".join(str(section_numbers[i]) for i in range(level + 1))

# Create TOC entry with cross-file link

indent = " " * level

list_marker = "- "

# Format: - section_num title [[stable_name]](filename.md#anchor)

toc_entry = f"{indent}{list_marker}{section_num} {title} [[{stable_name}]]({filename}.md#{anchor_id})"

toc_lines.append(toc_entry)

return "\n".join(toc_lines)

@staticmethod

def _is_grammar_definition(block: str) -> bool:

"""

# Pattern: lowercase word (with optional hyphens/digits), followed by colon

# at the start of a line. Examples: "declaration:", "namespace-alias-definition:"

return bool(re.search(r"^[a-z][a-z0-9-]*:", block, re.MULTILINE))

def _generate_grammar_appendix(

self, output_files: list[Path], output_dir: Path, verbose: bool = False

) -> None:

"""

# Mapping of stable names to grammar section info (from LaTeX \gramSec commands)

# Order matches the sequence in which \gramSec appears in the LaTeX sources

grammar_sections = [

("lex", "Lexical conventions"),

("basic", "Basic concepts"),

("expr", "Expressions"),

("stmt", "Statements"),

("dcl", "Declarations"),

("class", "Classes"),

("over", "Overloading"),

("temp", "Templates"),

("cpp", "Preprocessing directives"),

("except", "Exception handling"),

("depr", "Deprecated features"),

]

# Pattern to extract BNF code blocks: ``` bnf\n...\n```

bnf_pattern = re.compile(r"^``` bnf\n(.*?)\n```$", re.MULTILINE | re.DOTALL)

# Collect BNF blocks by stable name

bnf_by_file = {}

file_by_stem = {f.stem: f for f in output_files}

for stable_name, section_title in grammar_sections:

if stable_name in file_by_stem:

md_file = file_by_stem[stable_name]

content = md_file.read_text(encoding="utf-8")

# Extract BNF blocks that contain grammar definitions

# Filter out example/explanation blocks (like "behaves as if")

all_blocks = bnf_pattern.findall(content)

blocks = [b for b in all_blocks if self._is_grammar_definition(b)]

if blocks:

bnf_by_file[stable_name] = {

"title": section_title,

"blocks": blocks,

"count": len(blocks),

}

if verbose:

print(f" Extracted {len(blocks)} BNF blocks from {stable_name}.md")

# Find grammar.md in output files

grammar_file = output_dir / "grammar.md"

if not grammar_file.exists():

if verbose:

print("Warning: grammar.md not found, skipping grammar aggregation")

return

# Read current grammar.md to preserve intro and keywords sections

current_content = grammar_file.read_text(encoding="utf-8")

# Extract everything up to and including the Keywords section

# Grammar.md has: ## General, ## Keywords with 6 BNF blocks, then other sections

# We want to preserve General + Keywords only, then replace everything else with aggregated sections

# Find "## Keywords" and then the next "##" heading after it

keywords_match = re.search(r"^## Keywords", current_content, re.MULTILINE)

if keywords_match:

# Find the next ## heading after Keywords

next_heading = re.search(

r"^##\s", current_content[keywords_match.end() :], re.MULTILINE

)

if next_heading:

# Extract up to (but not including) the next heading

intro_and_keywords = current_content[: keywords_match.end() + next_heading.start()]

else:

# No next heading found, use entire content (shouldn't happen)

intro_and_keywords = current_content

else:

# Fallback: if we can't find Keywords, just use first two headings

header_parts = current_content.split("\n## ", 3)

if len(header_parts) >= 3:

intro_and_keywords = "\n## ".join(header_parts[:3])

else:

intro_and_keywords = current_content

# Build new grammar.md content

new_content_parts = [intro_and_keywords.rstrip()]

# Add aggregated grammar sections

total_blocks = 0

aggregated_anchors = [] # Track anchors for link definitions

for stable_name, section_title in grammar_sections:

if stable_name in bnf_by_file:

section_info = bnf_by_file[stable_name]

blocks = section_info["blocks"]

total_blocks += len(blocks)

# Add section heading with anchor

anchor_id = f"gram.{stable_name}"

aggregated_anchors.append(anchor_id)

new_content_parts.append(

f'\n\n## {section_title} <a id="{anchor_id}">[[{anchor_id}]]</a>\n'

)

# Add all BNF blocks from this section with spacing between them

for block in blocks:

new_content_parts.append(f"\n``` bnf\n{block}\n```\n")

# Add link definitions for aggregated sections

if aggregated_anchors:

new_content_parts.append("\n<!-- Link reference definitions -->\n")

for anchor_id in aggregated_anchors:

new_content_parts.append(f"[{anchor_id}]: #{anchor_id}\n")

# Write updated grammar.md

new_content = "".join(new_content_parts)

grammar_file.write_text(new_content, encoding="utf-8")

if verbose:

print(

f"\nRegenerated grammar.md with {total_blocks} BNF blocks from {len(bnf_by_file)} sections"

)

def build_separate_chapters(

self,

converter,

output_dir: Path,

verbose: bool = False,

toc_depth: int = 1,

) -> list[Path]:

"""

chapters = self.extract_chapter_order()

if verbose:

print(f"Found {len(chapters)} chapters in std.tex")

output_dir = Path(output_dir)

ensure_dir(output_dir)

# Convert .dot diagrams to SVG before processing chapters

self.convert_diagrams_to_svg(output_dir, verbose=verbose)

output_files = []

temp_files = [] # Track temporary files for cleanup

chapter_to_stable = {} # Mapping from source filename to stable name

# Pre-extract stable names for all chapters

# This is done upfront to avoid repeated conversions

if verbose:

print("Extracting stable names...")

for chapter in chapters:

chapter_file = self.draft_dir / f"{chapter}.tex"

if chapter_file.exists():

stable_name = self.extract_stable_name_from_tex(chapter_file, converter)

chapter_to_stable[chapter] = stable_name

if verbose and stable_name != chapter:

print(f" {chapter}.tex → {stable_name}.md")

# Detect stable name collisions

collision_groups = self.detect_stable_name_collisions(chapters, chapter_to_stable, verbose)

# Build label index for cross-file references

if verbose:

print("\nBuilding label index for cross-file references...")

from .label_indexer import LabelIndexer

indexer = LabelIndexer(self.draft_dir)

indexer.build_index(use_stable_names=True, stable_name_map=chapter_to_stable)

# Write Lua table file

label_index_file = output_dir / "cpp_std_labels.lua"

indexer.write_lua_table(label_index_file)

if verbose:

stats = indexer.get_statistics()

print(f"Indexed {stats['labels']} labels across {stats['files']} files")

if stats["duplicates"] > 0:

print(f"Warning: Found {stats['duplicates']} duplicate labels")

# Create work units for parallel processing

# Each work unit is either a single chapter or a collision group

work_units = []

processed_chapters = set()

for chapter in chapters:

# Skip if already processed as part of a collision group

if chapter in processed_chapters:

continue

chapter_file = self.draft_dir / f"{chapter}.tex"

if not chapter_file.exists():

if verbose:

print(f"Warning: {chapter}.tex not found, skipping")

continue

# Get stable name for this chapter