    {-# LANGUAGE OverloadedStrings #-}
    {-# LANGUAGE DeriveDataTypeable #-}
    {-# LANGUAGE BangPatterns #-}

    module Commonmark.Tokens
      ( Tok(..)
      , TokType(..)
      , SourcePos
      , tokenize
      , untokenize
      ) where

    import           Unicode.Char    (isAlphaNum)
    import           Unicode.Char.General.Compat  (isSpace)
    import           Data.Text       (Text)
    import qualified Data.Text       as T
    import           Data.Data       (Data, Typeable)
    import           Text.Parsec.Pos
    import           Data.Text.Normalize (normalize, NormalizationMode(NFC))

    data Tok = Tok { Tok -> TokType
    tokType     :: !TokType
                   , Tok -> SourcePos
    tokPos      :: !SourcePos
                   , Tok -> Text
    tokContents :: {-# UNPACK #-} !Text
                   }
                   deriving (Int -> Tok -> ShowS
    [Tok] -> ShowS
    Tok -> String
    (Int -> Tok -> ShowS)
    -> (Tok -> String) -> ([Tok] -> ShowS) -> Show Tok
    forall a.
    (Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
    $cshowsPrec :: Int -> Tok -> ShowS
    showsPrec :: Int -> Tok -> ShowS
    $cshow :: Tok -> String
    show :: Tok -> String
    $cshowList :: [Tok] -> ShowS
    showList :: [Tok] -> ShowS
    Show, Tok -> Tok -> Bool
    (Tok -> Tok -> Bool) -> (Tok -> Tok -> Bool) -> Eq Tok
    forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
    $c== :: Tok -> Tok -> Bool
    == :: Tok -> Tok -> Bool
    $c/= :: Tok -> Tok -> Bool
    /= :: Tok -> Tok -> Bool
    Eq, Typeable Tok
    Typeable Tok
    -> (forall (c :: * -> *).
        (forall d b. Data d => c (d -> b) -> d -> c b)
        -> (forall g. g -> c g) -> Tok -> c Tok)
    -> (forall (c :: * -> *).
        (forall b r. Data b => c (b -> r) -> c r)
        -> (forall r. r -> c r) -> Constr -> c Tok)
    -> (Tok -> Constr)
    -> (Tok -> DataType)
    -> (forall (t :: * -> *) (c :: * -> *).
        Typeable t =>
        (forall d. Data d => c (t d)) -> Maybe (c Tok))
    -> (forall (t :: * -> * -> *) (c :: * -> *).
        Typeable t =>
        (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c Tok))
    -> ((forall b. Data b => b -> b) -> Tok -> Tok)
    -> (forall r r'.
        (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> Tok -> r)
    -> (forall r r'.
        (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> Tok -> r)
    -> (forall u. (forall d. Data d => d -> u) -> Tok -> [u])
    -> (forall u. Int -> (forall d. Data d => d -> u) -> Tok -> u)
    -> (forall (m :: * -> *).
        Monad m =>
        (forall d. Data d => d -> m d) -> Tok -> m Tok)
    -> (forall (m :: * -> *).
        MonadPlus m =>
        (forall d. Data d => d -> m d) -> Tok -> m Tok)
    -> (forall (m :: * -> *).
        MonadPlus m =>
        (forall d. Data d => d -> m d) -> Tok -> m Tok)
    -> Data Tok
    Tok -> Constr
    Tok -> DataType
    (forall b. Data b => b -> b) -> Tok -> Tok
    forall a.
    Typeable a
    -> (forall (c :: * -> *).
        (forall d b. Data d => c (d -> b) -> d -> c b)
        -> (forall g. g -> c g) -> a -> c a)
    -> (forall (c :: * -> *).
        (forall b r. Data b => c (b -> r) -> c r)
        -> (forall r. r -> c r) -> Constr -> c a)
    -> (a -> Constr)
    -> (a -> DataType)
    -> (forall (t :: * -> *) (c :: * -> *).
        Typeable t =>
        (forall d. Data d => c (t d)) -> Maybe (c a))
    -> (forall (t :: * -> * -> *) (c :: * -> *).
        Typeable t =>
        (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c a))
    -> ((forall b. Data b => b -> b) -> a -> a)
    -> (forall r r'.
        (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> a -> r)
    -> (forall r r'.
        (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> a -> r)
    -> (forall u. (forall d. Data d => d -> u) -> a -> [u])
    -> (forall u. Int -> (forall d. Data d => d -> u) -> a -> u)
    -> (forall (m :: * -> *).
        Monad m =>
        (forall d. Data d => d -> m d) -> a -> m a)
    -> (forall (m :: * -> *).
        MonadPlus m =>
        (forall d. Data d => d -> m d) -> a -> m a)
    -> (forall (m :: * -> *).
        MonadPlus m =>
        (forall d. Data d => d -> m d) -> a -> m a)
    -> Data a
    forall u. Int -> (forall d. Data d => d -> u) -> Tok -> u
    forall u. (forall d. Data d => d -> u) -> Tok -> [u]
    forall r r'.
    (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> Tok -> r
    forall r r'.
    (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> Tok -> r
    forall (m :: * -> *).
    Monad m =>
    (forall d. Data d => d -> m d) -> Tok -> m Tok
    forall (m :: * -> *).
    MonadPlus m =>
    (forall d. Data d => d -> m d) -> Tok -> m Tok
    forall (c :: * -> *).
    (forall b r. Data b => c (b -> r) -> c r)
    -> (forall r. r -> c r) -> Constr -> c Tok
    forall (c :: * -> *).
    (forall d b. Data d => c (d -> b) -> d -> c b)
    -> (forall g. g -> c g) -> Tok -> c Tok
    forall (t :: * -> *) (c :: * -> *).
    Typeable t =>
    (forall d. Data d => c (t d)) -> Maybe (c Tok)
    forall (t :: * -> * -> *) (c :: * -> *).
    Typeable t =>
    (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c Tok)
    $cgfoldl :: forall (c :: * -> *).
    (forall d b. Data d => c (d -> b) -> d -> c b)
    -> (forall g. g -> c g) -> Tok -> c Tok
    gfoldl :: forall (c :: * -> *).
    (forall d b. Data d => c (d -> b) -> d -> c b)
    -> (forall g. g -> c g) -> Tok -> c Tok
    $cgunfold :: forall (c :: * -> *).
    (forall b r. Data b => c (b -> r) -> c r)
    -> (forall r. r -> c r) -> Constr -> c Tok
    gunfold :: forall (c :: * -> *).
    (forall b r. Data b => c (b -> r) -> c r)
    -> (forall r. r -> c r) -> Constr -> c Tok
    $ctoConstr :: Tok -> Constr
    toConstr :: Tok -> Constr
    $cdataTypeOf :: Tok -> DataType
    dataTypeOf :: Tok -> DataType
    $cdataCast1 :: forall (t :: * -> *) (c :: * -> *).
    Typeable t =>
    (forall d. Data d => c (t d)) -> Maybe (c Tok)
    dataCast1 :: forall (t :: * -> *) (c :: * -> *).
    Typeable t =>
    (forall d. Data d => c (t d)) -> Maybe (c Tok)
    $cdataCast2 :: forall (t :: * -> * -> *) (c :: * -> *).
    Typeable t =>
    (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c Tok)
    dataCast2 :: forall (t :: * -> * -> *) (c :: * -> *).
    Typeable t =>
    (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c Tok)
    $cgmapT :: (forall b. Data b => b -> b) -> Tok -> Tok
    gmapT :: (forall b. Data b => b -> b) -> Tok -> Tok
    $cgmapQl :: forall r r'.
    (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> Tok -> r
    gmapQl :: forall r r'.
    (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> Tok -> r
    $cgmapQr :: forall r r'.
    (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> Tok -> r
    gmapQr :: forall r r'.
    (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> Tok -> r
    $cgmapQ :: forall u. (forall d. Data d => d -> u) -> Tok -> [u]
    gmapQ :: forall u. (forall d. Data d => d -> u) -> Tok -> [u]
    $cgmapQi :: forall u. Int -> (forall d. Data d => d -> u) -> Tok -> u
    gmapQi :: forall u. Int -> (forall d. Data d => d -> u) -> Tok -> u
    $cgmapM :: forall (m :: * -> *).
    Monad m =>
    (forall d. Data d => d -> m d) -> Tok -> m Tok
    gmapM :: forall (m :: * -> *).
    Monad m =>
    (forall d. Data d => d -> m d) -> Tok -> m Tok
    $cgmapMp :: forall (m :: * -> *).
    MonadPlus m =>
    (forall d. Data d => d -> m d) -> Tok -> m Tok
    gmapMp :: forall (m :: * -> *).
    MonadPlus m =>
    (forall d. Data d => d -> m d) -> Tok -> m Tok
    $cgmapMo :: forall (m :: * -> *).
    MonadPlus m =>
    (forall d. Data d => d -> m d) -> Tok -> m Tok
    gmapMo :: forall (m :: * -> *).
    MonadPlus m =>
    (forall d. Data d => d -> m d) -> Tok -> m Tok
    Data, Typeable)

    data TokType =
           Spaces
         | UnicodeSpace
         | LineEnd
         | WordChars
         | Symbol {-# UNPACK #-} !Char
         deriving (Int -> TokType -> ShowS
    [TokType] -> ShowS
    TokType -> String
    (Int -> TokType -> ShowS)
    -> (TokType -> String) -> ([TokType] -> ShowS) -> Show TokType
    forall a.
    (Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
    $cshowsPrec :: Int -> TokType -> ShowS
    showsPrec :: Int -> TokType -> ShowS
    $cshow :: TokType -> String
    show :: TokType -> String
    $cshowList :: [TokType] -> ShowS
    showList :: [TokType] -> ShowS
    Show, TokType -> TokType -> Bool
    (TokType -> TokType -> Bool)
    -> (TokType -> TokType -> Bool) -> Eq TokType
    forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
    $c== :: TokType -> TokType -> Bool
    == :: TokType -> TokType -> Bool
    $c/= :: TokType -> TokType -> Bool
    /= :: TokType -> TokType -> Bool
    Eq, Eq TokType
    Eq TokType
    -> (TokType -> TokType -> Ordering)
    -> (TokType -> TokType -> Bool)
    -> (TokType -> TokType -> Bool)
    -> (TokType -> TokType -> Bool)
    -> (TokType -> TokType -> Bool)
    -> (TokType -> TokType -> TokType)
    -> (TokType -> TokType -> TokType)
    -> Ord TokType
    TokType -> TokType -> Bool
    TokType -> TokType -> Ordering
    TokType -> TokType -> TokType
    forall a.
    Eq a
    -> (a -> a -> Ordering)
    -> (a -> a -> Bool)
    -> (a -> a -> Bool)
    -> (a -> a -> Bool)
    -> (a -> a -> Bool)
    -> (a -> a -> a)
    -> (a -> a -> a)
    -> Ord a
    $ccompare :: TokType -> TokType -> Ordering
    compare :: TokType -> TokType -> Ordering
    $c< :: TokType -> TokType -> Bool
    < :: TokType -> TokType -> Bool
    $c<= :: TokType -> TokType -> Bool
    <= :: TokType -> TokType -> Bool
    $c> :: TokType -> TokType -> Bool
    > :: TokType -> TokType -> Bool
    $c>= :: TokType -> TokType -> Bool
    >= :: TokType -> TokType -> Bool
    $cmax :: TokType -> TokType -> TokType
    max :: TokType -> TokType -> TokType
    $cmin :: TokType -> TokType -> TokType
    min :: TokType -> TokType -> TokType
    Ord, Typeable TokType
    Typeable TokType
    -> (forall (c :: * -> *).
        (forall d b. Data d => c (d -> b) -> d -> c b)
        -> (forall g. g -> c g) -> TokType -> c TokType)
    -> (forall (c :: * -> *).
        (forall b r. Data b => c (b -> r) -> c r)
        -> (forall r. r -> c r) -> Constr -> c TokType)
    -> (TokType -> Constr)
    -> (TokType -> DataType)
    -> (forall (t :: * -> *) (c :: * -> *).
        Typeable t =>
        (forall d. Data d => c (t d)) -> Maybe (c TokType))
    -> (forall (t :: * -> * -> *) (c :: * -> *).
        Typeable t =>
        (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c TokType))
    -> ((forall b. Data b => b -> b) -> TokType -> TokType)
    -> (forall r r'.
        (r -> r' -> r)
        -> r -> (forall d. Data d => d -> r') -> TokType -> r)
    -> (forall r r'.
        (r' -> r -> r)
        -> r -> (forall d. Data d => d -> r') -> TokType -> r)
    -> (forall u. (forall d. Data d => d -> u) -> TokType -> [u])
    -> (forall u. Int -> (forall d. Data d => d -> u) -> TokType -> u)
    -> (forall (m :: * -> *).
        Monad m =>
        (forall d. Data d => d -> m d) -> TokType -> m TokType)
    -> (forall (m :: * -> *).
        MonadPlus m =>
        (forall d. Data d => d -> m d) -> TokType -> m TokType)
    -> (forall (m :: * -> *).
        MonadPlus m =>
        (forall d. Data d => d -> m d) -> TokType -> m TokType)
    -> Data TokType
    TokType -> Constr
    TokType -> DataType
    (forall b. Data b => b -> b) -> TokType -> TokType
    forall a.
    Typeable a
    -> (forall (c :: * -> *).
        (forall d b. Data d => c (d -> b) -> d -> c b)
        -> (forall g. g -> c g) -> a -> c a)
    -> (forall (c :: * -> *).
        (forall b r. Data b => c (b -> r) -> c r)
        -> (forall r. r -> c r) -> Constr -> c a)
    -> (a -> Constr)
    -> (a -> DataType)
    -> (forall (t :: * -> *) (c :: * -> *).
        Typeable t =>
        (forall d. Data d => c (t d)) -> Maybe (c a))
    -> (forall (t :: * -> * -> *) (c :: * -> *).
        Typeable t =>
        (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c a))
    -> ((forall b. Data b => b -> b) -> a -> a)
    -> (forall r r'.
        (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> a -> r)
    -> (forall r r'.
        (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> a -> r)
    -> (forall u. (forall d. Data d => d -> u) -> a -> [u])
    -> (forall u. Int -> (forall d. Data d => d -> u) -> a -> u)
    -> (forall (m :: * -> *).
        Monad m =>
        (forall d. Data d => d -> m d) -> a -> m a)
    -> (forall (m :: * -> *).
        MonadPlus m =>
        (forall d. Data d => d -> m d) -> a -> m a)
    -> (forall (m :: * -> *).
        MonadPlus m =>
        (forall d. Data d => d -> m d) -> a -> m a)
    -> Data a
    forall u. Int -> (forall d. Data d => d -> u) -> TokType -> u
    forall u. (forall d. Data d => d -> u) -> TokType -> [u]
    forall r r'.
    (r -> r' -> r)
    -> r -> (forall d. Data d => d -> r') -> TokType -> r
    forall r r'.
    (r' -> r -> r)
    -> r -> (forall d. Data d => d -> r') -> TokType -> r
    forall (m :: * -> *).
    Monad m =>
    (forall d. Data d => d -> m d) -> TokType -> m TokType
    forall (m :: * -> *).
    MonadPlus m =>
    (forall d. Data d => d -> m d) -> TokType -> m TokType
    forall (c :: * -> *).
    (forall b r. Data b => c (b -> r) -> c r)
    -> (forall r. r -> c r) -> Constr -> c TokType
    forall (c :: * -> *).
    (forall d b. Data d => c (d -> b) -> d -> c b)
    -> (forall g. g -> c g) -> TokType -> c TokType
    forall (t :: * -> *) (c :: * -> *).
    Typeable t =>
    (forall d. Data d => c (t d)) -> Maybe (c TokType)
    forall (t :: * -> * -> *) (c :: * -> *).
    Typeable t =>
    (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c TokType)
    $cgfoldl :: forall (c :: * -> *).
    (forall d b. Data d => c (d -> b) -> d -> c b)
    -> (forall g. g -> c g) -> TokType -> c TokType
    gfoldl :: forall (c :: * -> *).
    (forall d b. Data d => c (d -> b) -> d -> c b)
    -> (forall g. g -> c g) -> TokType -> c TokType
    $cgunfold :: forall (c :: * -> *).
    (forall b r. Data b => c (b -> r) -> c r)
    -> (forall r. r -> c r) -> Constr -> c TokType
    gunfold :: forall (c :: * -> *).
    (forall b r. Data b => c (b -> r) -> c r)
    -> (forall r. r -> c r) -> Constr -> c TokType
    $ctoConstr :: TokType -> Constr
    toConstr :: TokType -> Constr
    $cdataTypeOf :: TokType -> DataType
    dataTypeOf :: TokType -> DataType
    $cdataCast1 :: forall (t :: * -> *) (c :: * -> *).
    Typeable t =>
    (forall d. Data d => c (t d)) -> Maybe (c TokType)
    dataCast1 :: forall (t :: * -> *) (c :: * -> *).
    Typeable t =>
    (forall d. Data d => c (t d)) -> Maybe (c TokType)
    $cdataCast2 :: forall (t :: * -> * -> *) (c :: * -> *).
    Typeable t =>
    (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c TokType)
    dataCast2 :: forall (t :: * -> * -> *) (c :: * -> *).
    Typeable t =>
    (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c TokType)
    $cgmapT :: (forall b. Data b => b -> b) -> TokType -> TokType
    gmapT :: (forall b. Data b => b -> b) -> TokType -> TokType
    $cgmapQl :: forall r r'.
    (r -> r' -> r)
    -> r -> (forall d. Data d => d -> r') -> TokType -> r
    gmapQl :: forall r r'.
    (r -> r' -> r)
    -> r -> (forall d. Data d => d -> r') -> TokType -> r
    $cgmapQr :: forall r r'.
    (r' -> r -> r)
    -> r -> (forall d. Data d => d -> r') -> TokType -> r
    gmapQr :: forall r r'.
    (r' -> r -> r)
    -> r -> (forall d. Data d => d -> r') -> TokType -> r
    $cgmapQ :: forall u. (forall d. Data d => d -> u) -> TokType -> [u]
    gmapQ :: forall u. (forall d. Data d => d -> u) -> TokType -> [u]
    $cgmapQi :: forall u. Int -> (forall d. Data d => d -> u) -> TokType -> u
    gmapQi :: forall u. Int -> (forall d. Data d => d -> u) -> TokType -> u
    $cgmapM :: forall (m :: * -> *).
    Monad m =>
    (forall d. Data d => d -> m d) -> TokType -> m TokType
    gmapM :: forall (m :: * -> *).
    Monad m =>
    (forall d. Data d => d -> m d) -> TokType -> m TokType
    $cgmapMp :: forall (m :: * -> *).
    MonadPlus m =>
    (forall d. Data d => d -> m d) -> TokType -> m TokType
    gmapMp :: forall (m :: * -> *).
    MonadPlus m =>
    (forall d. Data d => d -> m d) -> TokType -> m TokType
    $cgmapMo :: forall (m :: * -> *).
    MonadPlus m =>
    (forall d. Data d => d -> m d) -> TokType -> m TokType
    gmapMo :: forall (m :: * -> *).
    MonadPlus m =>
    (forall d. Data d => d -> m d) -> TokType -> m TokType
    Data, Typeable)

    -- | Convert a 'Text' into a list of 'Tok'. The first parameter
    -- species the source name.
    tokenize :: String -> Text -> [Tok]
    tokenize :: String -> Text -> [Tok]
    tokenize String
    name =
      {-# SCC tokenize #-} SourcePos -> [Text] -> [Tok]
    go (String -> SourcePos
    initialPos String
    name) ([Text] -> [Tok]) -> (Text -> [Text]) -> Text -> [Tok]
    forall b c a. (b -> c) -> (a -> b) -> a -> c
    . (Char -> Char -> Bool) -> Text -> [Text]
    T.groupBy Char -> Char -> Bool
    f (Text -> [Text]) -> (Text -> Text) -> Text -> [Text]
    forall b c a. (b -> c) -> (a -> b) -> a -> c
    . NormalizationMode -> Text -> Text
    normalize NormalizationMode
    NFC
      where
        -- We group \r\n, consecutive spaces, and consecutive alphanums;
        -- everything else gets in a token by itself.
        f :: Char -> Char -> Bool
    f Char
    '\r' Char
    '\n' = Bool
    True
        f Char
    ' ' Char
    ' '   = Bool
    True
        f Char
    x   Char
    y     = Char -> Bool
    isAlphaNum Char
    x Bool -> Bool -> Bool
    && Char -> Bool
    isAlphaNum Char
    y

        go :: SourcePos -> [Text] -> [Tok]
    go !SourcePos
    _pos [] = []
        go !SourcePos
    pos (!Text
    t:[Text]
    ts) = -- note that t:ts are guaranteed to be nonempty
          case HasCallStack => Text -> Char
    Text -> Char
    T.head Text
    t of
             Char
    ' ' ->  TokType -> SourcePos -> Text -> Tok
    Tok TokType
    Spaces SourcePos
    pos Text
    t Tok -> [Tok] -> [Tok]
    forall a. a -> [a] -> [a]
    :
                     SourcePos -> [Text] -> [Tok]
    go (SourcePos -> Int -> SourcePos
    incSourceColumn SourcePos
    pos (Text -> Int
    T.length Text
    t)) [Text]
    ts
             Char
    '\t' -> TokType -> SourcePos -> Text -> Tok
    Tok TokType
    Spaces SourcePos
    pos Text
    t Tok -> [Tok] -> [Tok]
    forall a. a -> [a] -> [a]
    :
                     SourcePos -> [Text] -> [Tok]
    go (SourcePos -> Int -> SourcePos
    incSourceColumn SourcePos
    pos
                           (Int
    4 Int -> Int -> Int
    forall a. Num a => a -> a -> a
    - (SourcePos -> Int
    sourceColumn SourcePos
    pos Int -> Int -> Int
    forall a. Num a => a -> a -> a
    - Int
    1) Int -> Int -> Int
    forall a. Integral a => a -> a -> a
    `mod` Int
    4)) [Text]
    ts
             Char
    '\r' -> TokType -> SourcePos -> Text -> Tok
    Tok TokType
    LineEnd SourcePos
    pos Text
    t Tok -> [Tok] -> [Tok]
    forall a. a -> [a] -> [a]
    :
                     SourcePos -> [Text] -> [Tok]
    go (SourcePos -> Int -> SourcePos
    incSourceLine (SourcePos -> Int -> SourcePos
    setSourceColumn SourcePos
    pos Int
    1) Int
    1) [Text]
    ts
             Char
    '\n' -> TokType -> SourcePos -> Text -> Tok
    Tok TokType
    LineEnd SourcePos
    pos Text
    t Tok -> [Tok] -> [Tok]
    forall a. a -> [a] -> [a]
    :
                     SourcePos -> [Text] -> [Tok]
    go (SourcePos -> Int -> SourcePos
    incSourceLine (SourcePos -> Int -> SourcePos
    setSourceColumn SourcePos
    pos Int
    1) Int
    1) [Text]
    ts
             Char
    thead
               | Char -> Bool
    isAlphaNum Char
    thead ->
                     TokType -> SourcePos -> Text -> Tok
    Tok TokType
    WordChars SourcePos
    pos Text
    t Tok -> [Tok] -> [Tok]
    forall a. a -> [a] -> [a]
    :
                     SourcePos -> [Text] -> [Tok]
    go (SourcePos -> Int -> SourcePos
    incSourceColumn SourcePos
    pos (Text -> Int
    T.length Text
    t)) [Text]
    ts
               | Char -> Bool
    isSpace Char
    thead ->
                     TokType -> SourcePos -> Text -> Tok
    Tok TokType
    UnicodeSpace SourcePos
    pos Text
    t Tok -> [Tok] -> [Tok]
    forall a. a -> [a] -> [a]
    :
                     SourcePos -> [Text] -> [Tok]
    go (SourcePos -> Int -> SourcePos
    incSourceColumn SourcePos
    pos Int
    1) [Text]
    ts
               | Bool
    otherwise ->
                     TokType -> SourcePos -> Text -> Tok
    Tok (Char -> TokType
    Symbol Char
    thead) SourcePos
    pos Text
    t Tok -> [Tok] -> [Tok]
    forall a. a -> [a] -> [a]
    :
                     SourcePos -> [Text] -> [Tok]
    go (SourcePos -> Int -> SourcePos
    incSourceColumn SourcePos
    pos Int
    1) [Text]
    ts

    -- | Reverses 'tokenize'.  @untokenize . tokenize@ should be
    -- the identity.
    untokenize :: [Tok] -> Text
    untokenize :: [Tok] -> Text
    untokenize = {-# SCC untokenize #-} [Text] -> Text
    forall a. Monoid a => [a] -> a
    mconcat ([Text] -> Text) -> ([Tok] -> [Text]) -> [Tok] -> Text
    forall b c a. (b -> c) -> (a -> b) -> a -> c
    . (Tok -> Text) -> [Tok] -> [Text]
    forall a b. (a -> b) -> [a] -> [b]
    map Tok -> Text
    tokContents