Token[Atom] = ref object
  case t*: TokenType
  of DOCTYPE:
    quirks*: bool
    name*: Option[string]
    pubid*: Option[string]
    sysid*: Option[string]
  of START_TAG, END_TAG:
    selfclosing*: bool
    tagname*: Atom
    attrs*: Table[Atom, string]
  of CHARACTER, CHARACTER_WHITESPACE:
    s*: string
  of COMMENT:
    data*: string
  of EOF, CHARACTER_NULL:
    nil

Tokenizer[Handle; Atom] = object
  state*: TokenizerState
  laststart*: Token[Atom]
  hasnonhtml*: bool
  tokqueue*: seq[Token[Atom]]
  inputBufIdx*: int

TokenizeResult = enum
  trDone, trEmit

TokenType = enum
  DOCTYPE, START_TAG, END_TAG, COMMENT, CHARACTER, CHARACTER_WHITESPACE,
  CHARACTER_NULL, EOF