diff --git a/src/app/components/blocks/CodeBlock.tsx b/src/app/components/blocks/CodeBlock.tsx index 0c0f518..73d0094 100644 --- a/src/app/components/blocks/CodeBlock.tsx +++ b/src/app/components/blocks/CodeBlock.tsx @@ -33,7 +33,9 @@ export function CodeBlock({ node, id }: { node: Extract; const rawLang = node.lang && node.lang !== 'text' ? node.lang : undefined // Unrendered mermaid keeps its 'mermaid' title but skips tree-sitter (no // grammar for it) — render the raw source as plain highlighted text. - const filetype = rawLang && rawLang !== 'mermaid' ? infoStringToFiletype(rawLang) : undefined + const highlightLang = rawLang ?? node.inferredLang + const filetype = + highlightLang && highlightLang !== 'mermaid' ? infoStringToFiletype(highlightLang) : undefined const title = rawLang ? ` ${rawLang} ` : undefined const lines = node.value.split('\n') diff --git a/src/app/lib/ast.test.ts b/src/app/lib/ast.test.ts index 4235efa..77ae294 100644 --- a/src/app/lib/ast.test.ts +++ b/src/app/lib/ast.test.ts @@ -94,6 +94,16 @@ describe('buildTree', () => { const { nodes } = buildTree('```js\nx\n```') expect(nodes[0]).toMatchObject({ kind: 'code', lang: 'js', value: 'x' }) }) + + test('untagged code block carries an inferred lang', () => { + const { nodes } = buildTree('```\n$ bun test\n```') + expect(nodes[0]).toMatchObject({ kind: 'code', lang: undefined, inferredLang: 'bash' }) + }) + + test('tagged code block skips inference', () => { + const { nodes } = buildTree('```text\n$ bun test\n```') + expect(nodes[0]).toMatchObject({ kind: 'code', lang: 'text', inferredLang: undefined }) + }) }) describe('buildTree — details wrapping', () => { diff --git a/src/app/lib/ast.ts b/src/app/lib/ast.ts index 4af06be..987dfd8 100644 --- a/src/app/lib/ast.ts +++ b/src/app/lib/ast.ts @@ -1,5 +1,6 @@ import { marked } from 'marked' import type { Tokens } from 'marked' +import { detectLang } from './detect-lang' import { htmlContainsBlockMarkdown, htmlToMarkdown, stripHtml } from './html' marked.use({ @@ -37,7 +38,13 @@ export type InlineNode = export type Node = | { kind: 'heading'; level: 1 | 2 | 3 | 4 | 5 | 6; id: string; text: InlineNode[] } | { kind: 'paragraph'; inline: InlineNode[] } - | { kind: 'code'; lang?: string; value: string } + | { + kind: 'code' + lang?: string + value: string + /** Guessed by `detectLang` when the fence has no info string; drives highlighting only, never the title. */ + inferredLang?: string + } | { kind: 'list'; ordered: boolean; start?: number; items: ListItem[] } | { kind: 'blockquote'; children: Node[] } | { kind: 'table'; header: InlineNode[][]; rows: InlineNode[][][] } @@ -261,7 +268,13 @@ function blockToNode(t: Tokens.Generic, ctx: ParseContext): Node | null { } case 'code': { const c = t as Tokens.Code - return { kind: 'code', lang: c.lang || undefined, value: c.text } + const lang = c.lang || undefined + return { + kind: 'code', + lang, + value: c.text, + inferredLang: lang ? undefined : detectLang(c.text), + } } case 'list': { const l = t as Tokens.List diff --git a/src/app/lib/detect-lang.test.ts b/src/app/lib/detect-lang.test.ts new file mode 100644 index 0000000..2fde6f3 --- /dev/null +++ b/src/app/lib/detect-lang.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, test } from 'bun:test' +import { detectLang } from './detect-lang' + +describe('detectLang', () => { + describe('shebang', () => { + test.each([ + ['#!/bin/bash\necho hi', 'bash'], + ['#!/bin/sh\necho hi', 'bash'], + ['#!/usr/bin/env zsh\necho hi', 'bash'], + ['#!/usr/bin/env python3\nprint(1)', 'python'], + ['#!/usr/bin/python3.11\nprint(1)', 'python'], + ['#!/usr/bin/env -S node --no-warnings\nconsole.log(1)', 'javascript'], + ['#!/usr/bin/env bun\nconsole.log(1)', 'typescript'], + ['#!/usr/bin/env FOO=1 deno\nconsole.log(1)', 'typescript'], + ])('%p -> %p', (src, lang) => { + expect(detectLang(src)).toBe(lang) + }) + + test('unknown interpreter yields undefined', () => { + expect(detectLang('#!/usr/bin/env perl\nprint 1;')).toBeUndefined() + }) + }) + + test('shell prompt on first line', () => { + expect(detectLang('$ bun install\n$ bun test')).toBe('bash') + expect(detectLang('\n$ ls\nfile.txt')).toBe('bash') + }) + + test('JSON object or array', () => { + expect(detectLang('{"a": 1, "b": [true, null]}')).toBe('json') + expect(detectLang('[\n 1,\n 2\n]')).toBe('json') + }) + + test('JSON-like but invalid stays undetected', () => { + expect(detectLang('{ a: 1 }')).toBeUndefined() + }) + + test.each([ + '\n', + '\n', + '
\n

Hi

\n
', + 'Logo', + '

\n \n

', + '\n\n \n
1
', + '
', + '\nx', + ])('HTML: %p', src => { + expect(detectLang(src)).toBe('html') + }) + + test.each([ + ['JSX expression attribute', '
\n {children}\n
'], + ['component tag', ''], + ['component named like a tag', ''], + ['unknown tag', 'bar'], + ['tag-name prefix', ''], + ['HTML after prose', 'Use this:\n
'], + ])('not HTML: %s', (_, src) => { + expect(detectLang(src)).toBeUndefined() + }) + + test('Go package clause', () => { + expect(detectLang('// Package main\npackage main\n\nfunc main() {}')).toBe('go') + }) + + test('Java package clause is not Go', () => { + expect(detectLang('package com.example;')).toBeUndefined() + }) + + test.each([ + 'def foo(x):\n return x + 1', + 'async def foo() -> None:\n pass', + 'from os import path', + 'class Foo(Base):\n pass', + ])('Python: %p', src => { + expect(detectLang(src)).toBe('python') + }) + + test.each([ + 'fn main() {\n println!("hi");\n}', + 'pub fn add(a: i32, b: i32) -> i32 {\n a + b\n}', + 'use std::collections::HashMap;', + ])('Rust: %p', src => { + expect(detectLang(src)).toBe('rust') + }) + + test('Zig fn signature is not Rust', () => { + expect(detectLang('pub fn main() void {\n}')).toBeUndefined() + }) + + test('TOML with a table header', () => { + expect(detectLang('[package]\nname = "x"\nversion = "0.1.0"')).toBe('toml') + }) + + test('bare assignment without a table header stays undetected', () => { + expect(detectLang('x = 1')).toBeUndefined() + }) + + test.each([ + ['file tree', 'src/\n├── index.ts\n└── app/'], + ['log output', 'ERROR 2024-01-01 failed to connect\nretrying in 5s...'], + ['prose', 'This is just some plain text.'], + ['empty', ''], + ])('plain text stays undetected: %s', (_, src) => { + expect(detectLang(src)).toBeUndefined() + }) +}) diff --git a/src/app/lib/detect-lang.ts b/src/app/lib/detect-lang.ts new file mode 100644 index 0000000..45f9b33 --- /dev/null +++ b/src/app/lib/detect-lang.ts @@ -0,0 +1,100 @@ +/** + * Guess the language of an untagged fenced code block. Precision over recall: + * a rule fires only on syntax that is near-unique to one language, and + * anything else returns undefined so plain text (trees, logs, output) is not + * miscolored. Returned names are info strings `infoStringToFiletype` resolves + * to a bundled grammar. + */ +export function detectLang(value: string): string | undefined { + if (value.startsWith('#!')) return shebangLang(firstLine(value)) + for (const rule of RULES) { + if (rule.test(value)) return rule.lang + } + return undefined +} + +type Rule = { lang: string; test: (value: string) => boolean } + +const RULES: Rule[] = [ + { lang: 'bash', test: v => /^\$ \S/.test(v.trimStart()) }, + { lang: 'json', test: isJson }, + { lang: 'html', test: isHtml }, + // `package x;` (Java) has a semicolon and dotted path; Go's clause has neither. + { lang: 'go', test: v => /^package [a-z_]\w*\s*$/m.test(v) }, + { + lang: 'python', + test: v => + /^(async )?def \w+\(.*\)(\s*->.*)?:\s*$/m.test(v) || + /^from [\w.]+ import \w/m.test(v) || + /^class \w+(\(.*\))?:\s*$/m.test(v), + }, + { + lang: 'rust', + // Zig's `fn f() void {` has a bare return type, which this signature rejects. + test: v => + /^(pub )?fn \w+(<.*>)?\(.*\)\s*(->.*)?\{?\s*$/m.test(v) || + /^use \w+::[\w:{}, *]+;\s*$/m.test(v), + }, + { + lang: 'toml', + test: v => /^\[[\w.-]+\]\s*$/m.test(v) && /^[\w.-]+\s*=\s*\S/m.test(v), + }, +] + +const INTERPRETER_LANGS: Record = { + sh: 'bash', + bash: 'bash', + zsh: 'bash', + dash: 'bash', + ksh: 'bash', + python: 'python', + node: 'javascript', + bun: 'typescript', + deno: 'typescript', + tsx: 'typescript', + 'ts-node': 'typescript', +} + +/** Interpreter parsing follows GitHub Linguist: unwrap `env` flags and `K=V` pairs, drop version suffixes. */ +function shebangLang(line: string): string | undefined { + const [path = '', ...args] = line.slice(2).trim().split(/\s+/) + let interpreter = basename(path) + if (interpreter === 'env') { + interpreter = args.find(a => !a.startsWith('-') && !a.includes('=')) ?? '' + } + const name = interpreter.replace(/\.\d+$/, '').replace(/\d+$/, '') + return INTERPRETER_LANGS[name] +} + +const HTML_TAG_START = + /^( + +``` + +### Go + +``` +// Package main is the entry point. +package main + +import "fmt" + +func main() { + fmt.Println("hi") +} +``` + +### Python: def + +``` +def greet(name: str) -> str: + return f"hello {name}" +``` + +### Python: from-import + +``` +from pathlib import Path + +print(Path.cwd()) +``` + +### Rust: fn + +``` +pub fn add(a: i32, b: i32) -> i32 { + a + b +} +``` + +### Rust: use + +``` +use std::collections::HashMap; + +let mut m = HashMap::new(); +``` + +### TOML + +``` +[package] +name = "viewmd" +version = "0.1.0" + +[dependencies] +serde = "1" +``` + +## Plain (not detected) + +### File tree + +``` +src/ +├── index.tsx +└── app/ + ├── App.tsx + └── lib/ +``` + +### Log output + +``` +ERROR 2024-01-01T10:00:00Z failed to connect to db +WARN retrying in 5s... +``` + +### Prose + +``` +This is just some plain text the author put in a fence. +Note: it should stay uncolored. +``` + +### Command output without prompt + +``` +added 42 packages in 1.2s +``` + +### Unknown shebang (perl) + +``` +#!/usr/bin/env perl +print "hi\n"; +``` + +### Invalid JSON (JS object) + +``` +{ name: 'viewmd', private: true } +``` + +### JSX (expression attribute) + +``` +
+ {children} +
+``` + +### JSX component + +``` + +``` + +### HTML tag outside the list + +``` +
+

Hi

+
+``` + +### Java package (not Go) + +``` +package com.example; + +public class Main {} +``` + +### Zig fn (not Rust) + +``` +pub fn main() void { + std.debug.print("hi\n", .{}); +} +``` + +### Bare assignment (not TOML) + +``` +x = 1 +y = 2 +``` + +## Tagged (unchanged) + +### Explicit `text` stays plain + +```text +$ bun install +``` + +### Explicit `json` keeps its title + +```json +{ "a": 1 } +```