Publish only es modules

.oxlintrc.json

@@ -0,0 +1,15 @@
+{
+  "$schema": "./node_modules/oxlint/configuration_schema.json",
+  "plugins": [
+    "typescript",
+    "unicorn",
+    "oxc"
+  ],
+  "categories": {
+    "correctness": "error"
+  },
+  "rules": {},
+  "env": {
+    "builtin": true
+  }
+}

README.md

@@ -31,12 +31,14 @@ tokenizer errors in debug mode.
 
 # Usage
 
-You can tokenize HTML:
+This package is published as ES modules only. You can tokenize HTML:
 
 ```js
-var tokens = HTML5Tokenizer.tokenize("<div id='foo' href=bar class=\"bat\">");
+import { tokenize } from 'simple-html-tokenizer';
 
-var token = tokens[0];
+let tokens = tokenize("<div id='foo' href=bar class=\"bat\">");
+
+let token = tokens[0];
 token.tagName     //=> "div"
 token.attributes  //=> [["id", "foo"], ["href", "bar"], ["class", "bat"]]
 token.selfClosing //=> false
@@ -45,6 +47,13 @@ token.selfClosing //=> false
 ## Building and running the tests
 
 ```bash
-npm install
-npm test
+pnpm install
+pnpm test
+```
+
+To regenerate the files in `src/generated` from the upstream W3C HTML spec:
+
+```bash
+pnpm codegen:char-refs
+pnpm codegen:tokenizer-states
 ```

commands/build-char-refs.js

@@ -0,0 +1,11 @@
+import codegen from './codegen.js';
+import entities from './codegen/entities.js';
+
+await codegen(
+  'w3c',
+  'html',
+  'master',
+  'entities.json',
+  new URL('../src/generated/html5-named-char-refs.ts', import.meta.url),
+  entities
+);

commands/build-tokenizer-states.js

@@ -1,24 +1,11 @@
-'use strict';
-const tokenizerStates = require('./codegen/tokenizer-states');
-const codegen = require('./codegen');
-const path = require('path');
+import codegen from './codegen.js';
+import tokenizerStates from './codegen/tokenizer-states.js';
 
-module.exports = {
-  name: 'build-tokenizer-states',
-  works: 'insideProject',
-  description: `Rebuild "tokenizer-states"`,
-
-  availableOptions: [],
-
-  run() {
-    return codegen(
-      this.ui,
-      'w3c',
-      'html',
-      'master',
-      'sections/syntax.include',
-      path.join(this.project.root, 'src', 'generated', 'tokenizer-states.ts'),
-      tokenizerStates
-    );
-  }
-};
+await codegen(
+  'w3c',
+  'html',
+  'master',
+  'sections/syntax.include',
+  new URL('../src/generated/tokenizer-states.ts', import.meta.url),
+  tokenizerStates
+);

commands/codegen.js

@@ -1,28 +1,20 @@
-'use strict';
-const fs = require('fs');
-const { getGithubRefSHA, githubContentURL, getText } = require('./utils');
+import { writeFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { getGithubRefSHA, githubContentURL, getText } from './utils.js';
 
-module.exports = async function codegen(
-  ui,
-  owner,
-  repo,
-  ref,
-  src,
-  dest,
-  generator
-) {
-  ui.writeInfoLine(`getting sha for ${owner}/${repo} ${ref}`);
+export default async function codegen(owner, repo, ref, src, dest, generator) {
+  console.log(`getting sha for ${owner}/${repo} ${ref}`);
   const sha = await getGithubRefSHA(owner, repo, ref);
   const url = githubContentURL(owner, repo, sha, src);
 
-  ui.writeInfoLine(`downloading ${url}`);
+  console.log(`downloading ${url}`);
   const body = await getText(url);
   let code = `/**
  * generated from ${url}
  * do not edit
  */
 `;
   code += generator(body);
-  fs.writeFileSync(dest, code);
-  ui.writeInfoLine(`generated ${dest}`);
-};
+  writeFileSync(dest, code);
+  console.log(`generated ${fileURLToPath(dest)}`);
+}

commands/codegen/entities.js

@@ -1,8 +1,8 @@
-'use strict';
 // lifted from jshint
+// oxlint-disable-next-line no-control-regex -- matching control chars is the point
 const UNSAFE = /[\u0000-\u001f\u007f-\u009f\u00ad\u0600-\u0604\u070f\u17b4\u17b5\u200c-\u200f\u2028-\u202f\u2060-\u206f\ufeff\ufff0-\uffff]/;
 
-module.exports = function buildEntitiesModule(body) {
+export default function buildEntitiesModule(body) {
   let data = JSON.parse(body);
   let dest = `export default {\n`;
   let seen = Object.create(null);
@@ -33,14 +33,10 @@ module.exports = function buildEntitiesModule(body) {
   }
   dest += '\n};\n';
   return dest;
-};
+}
 
 function codepointLiteral(codepoint) {
-  let n = codepoint.toString(16);
-  if (n.length < 4) {
-    n = new Array(4 - n.length + 1).join('0') + n;
-  }
-  return '\\u' + n;
+  return '\\u' + codepoint.toString(16).padStart(4, '0');
 }
 
 function codepointsLiteral(codepoints) {

commands/codegen/tokenizer-states.js

@@ -1,8 +1,7 @@
-'use strict';
 const STATE_REGEX = /tokenizer-([\w-]+?)-state/g;
 const KEBOB_REGEX = /-([a-z])/g;
 
-module.exports = function buildTokenizerStates(syntax) {
+export default function buildTokenizerStates(syntax) {
   const states = ['beforeData'];
   let m;
   while ((m = STATE_REGEX.exec(syntax))) {
@@ -11,8 +10,8 @@ module.exports = function buildTokenizerStates(syntax) {
     });
     states.push(state);
   }
-  return `export const enum TokenizerState {
+  return `export enum TokenizerState {
 ${states.map(s => `  ${s} = '${s}'`).join(',\n')}
 }
 `;
-};
+}

commands/utils.js

@@ -1,49 +1,26 @@
-'use strict';
-const os = require('os');
-const got = require('got');
-const fs = require('fs');
-
-const CACHE = getCache();
-
-process.on('exit', writeCache.bind(null, CACHE));
-
-exports.getGithubRefSHA = getGithubRefSHA;
-exports.githubContentURL = githubContentURL;
-exports.getText = getText;
-exports.getJSON = getJSON;
+async function get(url) {
+  const res = await fetch(url);
+  if (!res.ok) {
+    throw new Error(`GET ${url} failed: ${res.status} ${res.statusText}`);
+  }
+  return res;
+}
 
-function getText(url) {
-  return got(url, { cache: CACHE }).then(res => res.body);
+export async function getText(url) {
+  return (await get(url)).text();
 }
 
-function getJSON(url) {
-  return got(url, { cache: CACHE, json: true }).then(res => res.body);
+export async function getJSON(url) {
+  return (await get(url)).json();
 }
 
-async function getGithubRefSHA(owner, repo, ref) {
+export async function getGithubRefSHA(owner, repo, ref) {
   let body = await getJSON(
     `https://api.github.com/repos/${owner}/${repo}/git/refs/heads/${ref}`
   );
   return body.object.sha;
 }
 
-function githubContentURL(owner, repo, sha, path) {
+export function githubContentURL(owner, repo, sha, path) {
   return `https://raw.githubusercontent.com/${owner}/${repo}/${sha}/${path}`;
 }
-
-function getCache() {
-  try {
-    return new Map(
-      JSON.parse(fs.readFileSync(`${os.tmpdir()}/___got_cache__.json`, 'utf8'))
-    );
-  } catch (e) {
-    return new Map();
-  }
-}
-
-function writeCache(cache) {
-  fs.writeFileSync(
-    `${os.tmpdir()}/___got_cache__.json`,
-    JSON.stringify(Array.from(cache))
-  );
-}