Token[Atom] = ref object
  flags*: set[TokenFlag]
  case t*: TokenType
  of ttDoctype:
    name*: string
    pubid*: string
    sysid*: string
  of ttStartTag, ttEndTag:
    tagname*: Atom
    attrs*: Table[Atom, string]
  of ttCharacter, ttWhitespace, ttComment:
    s*: string
  of ttNull:
    nil

TokenFlag = enum
  tfQuirks, tfPubid, tfSysid, tfSelfClosing

Tokenizer[Handle; Atom] = object
  laststart*: Token[Atom]
  state*: TokenizerState
  hasnonhtml*: bool
  tokqueue*: seq[Token[Atom]]
  inputBufIdx*: int

TokenizeResult = enum
  trDone, trEmit

TokenType = enum
  ttDoctype, ttStartTag, ttEndTag, ttComment, ttCharacter, ttWhitespace, ttNull