% Copyright 2012-2020, Alexander Shibakov
% This file is part of SPLinT
%
% SPLinT is free software: you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation, either version 3 of the License, or
% (at your option) any later version.
%
% SPLinT is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
% GNU General Public License for more details.
%
% You should have received a copy of the GNU General Public License
% along with SPLinT.  If not, see <http://www.gnu.org/licenses/>.

% prototypes of all the macros produced by the parser
% we will follow the convention: 12string is a string of category 12 characters and spaces, tex_string: a string
% of TeX tokens; sptr is a pointer to the stash stream, fptr is a pointer to the format stream

\def\yyuniontag{\yyunion}
\def\parserstrictnamespace{parser-strict}
\def\parserprototypesnamespace{parser-strict:headers}
\def\parserdebugnamespace{parser-debug}

\def\yyunion{\currentyyunionnamespace}
\def\currentyyunionnamespace{generic}

% types inserted by the input routines: these should not be tampered with randomly
% therefore they are not included in the yyunion list and only defined once below and redefined later
% permanently

\def\cwebstreamchars{% characters seen by the input routine
    \|\`\\\{\}\~\ \_\&\^\$\#\n% the last one is inserted by the preprocessing scripts
}


% insidemost

\long\def\stashed#1{} % stashed material (usually \Cee\ code) :: \stashed{tex_string}
\long\def\sflush#1#2{} % stash flushing marker :: \sflush{tex_string}
\def\format#1{} % format material triggered by braces and the start of production :: \format{tex_string}
\def\formatlocal#1{} % format material triggered by every term :: \formatlocal{tex_string}

% level 1

\long\def\strm[#1]#2.[#3]{} % a stream element :: \strm[digits]{\stashed|\format|\formatlocal}.[digits]

% types returned by the lexer; the integer type (hex vs. decimal) is determined at the lexer level so
% these are tagged by the lexer as well

\defp\anint#1#2#3{} % INT :: \anint{digits}{fptr}{sptr}
\defp\hexint#1#2#3{} % INT :: \hexint{0[xX][0-9A-F]+}{fptr}{sptr}

% types inserted by the parser (* marks the types that are essentially returned by the lexer
% and `tagged' by the parser)

\defp\idit#1#2#3#4{} % *ID :: \idit{12string}{tex_string}{fptr}{sptr}
\defp\charit#1#2#3#4{} % *CHAR :: \charit{12string}{tex_string}{fptr}{sptr}
\defp\stringify#1#2#3#4{} % *STRING :: \stringify{12string}{tex_string}{fptr}{sptr}
\defp\tagit#1#2#3#4{} % *TAG :: \tagit{12string}{tex_string}{fptr}{sptr}
\defp\braceit#1#2#3{} % *BRACED_CODE :: \braceit{tex_string}{fptr}{sptr}
\defp\midf#1#2{} % *rule separator :: \midf{fptr}{sptr}
\defp\preckind#1#2#3{} % *precedence operator :: \preckind{12string}{fptr}{sptr}
%
\defp\flexoptionpair#1#2{} % a flex option :: \flexoptoinpair{\idit}{\idit|\stringify}
\defp\termname#1#2{} % a production term :: \termname{\idit|\stringify|\charit}{\idit}
\defp\mergeop#1#2#3{} % merge directive :: \mergeop{\tagit}{fptr}{sptr}
\defp\dprecop#1#2#3{} % dprec directive :: \dprecop{\anint|\hexint}{fptr}{sptr}
\defp\sprecop#1#2#3{} % prec directive :: \sprecop{\idit|\stringify|\charit}{fptr}{sptr}
\defp\rrhssep#1#2{} %  :: \rrhssep{fptr}{sptr}
\defp\arhssep#1#2{} %  :: \arhssep{fptr}{sptr}
\defp\rarhssep#1#2{} %  :: \rarhssep{fptr}{sptr}
\defp\hspace#1#2{} % a space has to take two parameters just like any connector :: \hspace{fptr}{sptr}
\defp\emptyterm{}
\defp\onesymbol#1#2#3{} % a symbol definition :: \onesymbol{\idit}{\anint|\hexint}{\stringify}
\defp\bracedvalue#1#2#3{} % a braced variable value :: \bracedvalue{tex_string}{fptr}{sptr}

%level 1

\defp\rhs#1#2#3{} % a `right-hand side' :: \rhs{[\termname\hspace\rarhssep\arhssep\actbraces]+}{\arhssep|<nothing>}{\ifrhsfull}
\defp\actbraces#1#2#3#4#5\bdend{} % action braces :: \actbraces{tex_string}{\idit|<nothing>}{fptr}{sptr}directive*\bdend
\defp\bpredicate#1#2#3#4#5\bdend{} % predicate :: \bpredicate{tex_string}{<nothing>}{fptr}{sptr}directive*\bdend

% level 2

\defp\prodheader#1#2#3#4{} % a `left-hand side' of a production :: \prodheader{\idit}{\idit}{fptr}{sptr}
\defp\rules#1#2#3{} % a complete right-hand side :: \rules{[\rhs|\rrhssep]}{max fptr}{max sptr}
\defp\symbolprec#1#2{} % symbol precedence :: \symbolprec{\idit|\stringify|\charit}{\anint|\hexint}
%\def\pcluster#1#2{} % a production cluster :: \pcluster{\prodheader}{\termname|...} should never appear in the final output

% level 3

\defp\oneproduction#1#2#3{} % a complete production :: \oneproduction{\prodheader\rules}{fptr}{sptr}
\defp\prodprodsepsmall{} % production separator
\defp\prodprodsep{} % production separator
\defp\prodprodseplarge{} % production separator
\defp\optoptsepsmall{} % separator between options
\defp\optoptsepnone{} % (trivial) separator between options
\defp\posmark#1#2{} % the position marker :: \posmark{fptr}{sptr}
\defp\tokendecls#1#2#3{} % token declarations :: \tokendecls{[\onesymbol]+}{fptr}{sptr}
\defp\ntermdecls#1#2#3{} % nterm declarations :: \ntermdecls{[\onesymbol]+}{fptr}{sptr}
\defp\typedecls#1#2#3#4{} % type declarations :: \typedecls{\tagit}{[\idit\stringify\charit]+}{fptr}{sptr}
\defp\precdecls#1#2#3#4#5{} % precedence declarations :: \precdecls{left|right|nonassoc|precedence}{\tagit}{[\idit\stringify\charit]+}{fptr}{sptr}
\defp\flexsstatedecls#1#2#3{} % flex state declarations (nonexlusive) :: \flexsstatedecls{[\idit\stringify]+}{fptr}{sptr}
\defp\flexxstatedecls#1#2#3{} % flex state declarations (exclusive) :: \flexxstatedecls{[\idit\stringify]+}{fptr}{sptr}
\defp\flexoptiondecls#1#2#3{} % flex options :: \flexoptiondecls{[\flexoptionpair]+}{fptr}{sptr}
\defp\oneparametricoption#1#2#3#4{} % option :: \oneparametricoption{option-name}{\stringify|\idit}{fptr}{sptr}
\defp\optionflag#1#2#3#4{} % option flag :: \optionflag{flag name}{abbreviated name}{fptr}{sptr}
\defp\paramdef#1#2#3#4{} % parameters :: \paramdef{\braceit}{parameter domain}{fptr}{sptr}
\defp\codepropstype#1#2#3#4#5#6#7{} % term method :: \codepropstype{printer|destructor}{tex_string}{\stringit|\idit}{fptr}{sptr}{fptr}{sptr}
\defp\initaction#1#2#3{} % initial action :: \initaction{tex_string}{fptr}{sptr}
\defp\codeassoc#1#2#3#4#5#6#7{} % code association :: \codeassoc{code kind}{|\idit}{tex_string}{fptr}{sptr}{fptr}{sptr}
\defp\vardef#1#2#3#4{} % variable definition :: \vardef{\idit|\stringify}{|\idit|\stringify|\bracedvalue}{fptr}{sptr}
\defp\prologuecode#1#2#3#4#5{} % prologue code :: \prologuecode{tex_string}{fptr}{sptr}{fptr}{sptr}

% level 4

%\def\grammar#1#2{} % a grammar: should never appear in the final output

\savecslist{parser-prototypes}\yyunion

% table construction: the macros run to build the contents of the table of productions to be
% typeset.

\newtoks\table
\newtoks\tokdectoks
\newtoks\opttable
\newtoks\typestable
\newtoks\prectable

\newtoks\tokstoks
\newtoks\typesset

\let\rhscont\getsecond
\let\rhscnct\getthird
\let\rhsbool\getfourth

\newif\ifrhsfull

\newtoks\yystashlocal
\newtoks\yyformatlocal

% index implementation
% note that the rank and domain parameters are handed down by the
% `outer' macro while the `qualifier' (\term...string) is decided
% at the point of invocation; some auxilary sequences first

% a macro that always produces the current value of the register

\def\thewrap#1#2{%
    \noexpand\thewrap{\noexpand#1}{\the#1}%
}

% a macro that always produces the current page number.

\def\selfpageno#1{%
   \noexpand\selfpageno{\the\pageno}%
}

% a macro that keeps the (single) argument intact

\def\stickycs#1{\noexpand\stickycs\noexpand#1}

% for finer control over indexing, there are two levels on which
% the programmer can disable the production of a whatsit node (\write):
% (1) by setting the \iftermindex conditional to \iffalse, and (2) by defining
% the \termindex macro to ignore its argument; both of these are expanded
% at the point where the index entry is made; the intention is that
% the conditional controls indexing at large (say, the index entries
% inside the index should not be indexed again), while redefining the macro
% provides a finer control over the indexing style

\newif\iftermindex

\let\termindex\eatone

% the index is split into different domains: say, the \bison\ domain contains
% the entries for the terms of a grammar;

\def\bisonidxdomain{B} % marker for the domain (section of the index)

\def\defidxrank{0} % index rank of definitions is lowest so they are listed first
\def\headeridxrank{1} % index rank of lhs
\def\termidxrank{2} % ordinary term rank

% the expansion of the various components of an index entry must be carefully
% controlled: it is not possible to predict how many times the entry will be expanded;
% there are four types of expandable entries in general:
% o the entries that must be expanded at the point of invocation (say, \secno)
% o the entries that must be expanded only when the \write is inside a \box that is
%   being shipped out (say, the \pageno)
% o the `lazy' entries that can be expanded any time (although at least one expansion
%   must take place)
% o finally, the entries that must be expanded at a specific point between the point
%   of invocation and the \shipout
% the indexing macro below treats the first three cases for the entries that the macro
% below produce; the last case can be implemented by redefining the \stickycs macro
% at the appropriate point

\def\indxe#1#2{% index entry:
               % #1 is the output stream
               % #2 is the entry (see below for the proper form)
    \expandafter\ind@@\expandafter#2\end{#1}%          
}

\def\ind@@#1\end#2{%
    \ind@e{#2}{#1}%
}

\def\ind@e#1#2{% #1 is the output stream
               % #2 is a 8-parameter sequence, see below
    \write#1{\in@@e#2\in@@ecomment#2}%
}

\def\in@@e#1#2#3#4#5#6#7#8{% #1: early expandable (like \secno)
                           % #2: auxilary info (expandable, invariant)
                           % #3: domain
                           % #4: rank
                           % #5: type1
                           % #6: type2
                           % #7: key1
                           % #8: key2
    \i@@@e{#1}{\errmessage{\noexpand\\in@@e error: expanded page number placeholder}}%
        {#2}{#3}{#4}{#5}{#6}{\charstonumberse#7\end}{\charstonumberse#8\end}%
}

\def\in@@ecomment#1#2#3#4#5#6#7#8{% #1: early expandable (like \secno)
                                  % #2: auxilary info (expandable, invariant)
                                  % #3: domain
                                  % #4: rank
                                  % #5: type1
                                  % #6: type2
                                  % #7: key1
                                  % #8: key2
    \harmlesscomment\space\charstocharsx#7\end :: \charstocharsx#8\end%
}

\def\i@@@e#1#2#3#4#5#6#7#8#9{% #1: early expandable (like \secno)
                             % #2: page number placeholder
                             % #3: auxilary info
                             % #4: domain
                             % #5: rank
                             % #6: type1
                             % #7: type2
                             % #8: key1
                             % #9: key2
    \noexpand\i@@@e{#1}{\the\pageno}{#3}{#4}{#5}{\noexpand#6}{\noexpand#7}{#8}{#9}%
}

\def\appendfitotoksa{\toksa\expandafter{\the\toksa\else<>\fi}} % hide \fi inside the loop

% defining expandable macros for string transformations

\tempca\@ne
\toksa{\ifcase#1<NUL>}
\tempcb=\uccode`.

\bloop
\uccode`.=\tempca
\uppercase{\toksb\expandafter{.}}%
\toksa\expandafter\expandafter\expandafter{\expandafter\the\expandafter\toksa\expandafter\or\the\toksb}%
\advance\tempca by\@ne
\ifnum\tempca>"FF
    \appendfitotoksa
\else
\repeat

\def\charstocharsx#1\end{% expandable version of a `safety' macro, ascii characters only, no tricky space characters
    \yystartsinspace{#1.}{%
        \space\ch@rstocharsx#1\end
    }{%
        \yystringempty{#1}{}{%
            \ch@rst@charsx#1\end
        }%
    }%
}

\def\ch@rstocharsx#1 #2\end{\charstocharsx#2\end}

\def\ch@rst@charsx#1#2\end{\asciiprint{`#1}\charstocharsx#2\end}

\edef\next{\toksa{\def\noexpand\asciiprint##1{\the\toksa}}}\next
\the\toksa

\def\writeidxhentry#1{%
      \indxe\gindex{{\secno}{{\currentrulecontext}{\hostparsernamespace}}{\bisonidxdomain}{\headeridxrank}#1}%
}

\def\writeidxentry#1{%
      \indxe\gindex{{\secno}{{\currentrulecontext}{\hostparsernamespace}}{\bisonidxdomain}{\termidxrank}#1}%
}

\def\writetokenidxentry#1{%
      \indxe\gindex{{\secno}{{\currentrulecontext}{\hostparsernamespace}}{\bisonidxdomain}{\defidxrank}#1}%
}

\def\gidxentry#1#2#3{% a generic index entry:
                     % #1 is the type processor,
                     % #2 is the key
                     % #3 is the visual key
    \iftermindex
        \termindex{{}{#1}{#2}{#3}}%
    \fi
}

% a convenient shortcut, as many visual keys come in a token register

\def\gidxentryxv#1#2#3{% a generic index entry, e(x)pand (v)isual key:
                       % #1 is the type processor,
                       % #2 is the key
                       % #3 is the visual key
    \expandafter\gidx@ntryxv\expandafter{#3}{#1}{#2}%
}

\def\gidxentryxb#1#2#3{% a generic index entry, e(x)pand (b)oth keys:
                       % #1 is the type processor,
                       % #2 is the key
                       % #3 is the visual key
    \expandafter\gidx@ntryxb\expandafter{#2}{#3}{#1}%
}

\def\gidx@ntryxb#1#2#3{%
    \expandafter\gidx@ntryxv\expandafter{#2}{#3}{#1}%
}

\def\gidx@ntryxv#1#2#3{%
    \gidxentry{#2}{#3}{#1}%
}

\def\tidxentry#1#2{% a text index entry, TODO: expand \currentrulecontext 
    \iftermindex
        \indxe\gindex{{\secno}{{\currentrulecontext}{\indexpseudonamespace}}{\bisonidxdomain}{\termidxrank}{}{#1}{#2}{}}%
    \fi
}

\def\idxinline#1{% index the term in text
    \edef\next{\tidxentry{\termidstring}{#1}}\next
}

\def\gatherstash#1#2\strm[#3]#4.[#5]{% #1 contains the stash gathered
    \yystringempty{#3}{ #1}%
    {%
        \expandafter\g@therstash\expandafter{\romannumeral0\unstash#4}{#1}%
    }%
}

% stash processing wrapper macros

\def\g@therstash#1#2{\gatherstash{#2#1}}

\def\unstash#1#2{\expandafter\space#2}% #1 is \stashed or \sflush

\def\yyreadstash#1\to#2\in#3\with#4{%
    \yyreadfifo#1\to#2\in#3%
    #4{#3}%
    \setbox\z@=\vbox{
        \hsize\maxdimen\emergencystretch\maxdimen
        \rightskip\z@ plus 1fill
        \insertstash{#3}%
        \ifmmode
            {}$\aftergroup\yyr@adstash % $o that the box can be finished
        \else
            \aftergroup\eatthree
        \fi
    }{#1}{#2}{#3}%
}

\def\yyr@adstash#1#2#3{%
    #3\expandafter{\the#3{}$}% $
    \appendlnx#1{\strm[NaN]{\stashed{{}${}}}.[NaN]}% {}, in case \yystash ends with $
}

% the sequence below may be redefined so that the #2 parameter to \yyreadstash may be a list of
% token registers

\def\insertstash#1{\the#1\relax}

\def\readstash#1{\yyreadstash\yystash\to#1\in\yystashlocal\with\unpackstash}

\def\readstashwredirect#1#2#3{% helper macro to read the stash when the stash and the format pointers appear together
                              % #1(the format pointer) is ignored
                              % #2 is the stash marker
                              % #3 is the local stash register
    \yyreadstash\yystash\to#2\in#3\with\unpackstash
}

\def\unpackstash#1{#1\expandafter{\romannumeral0\expandafter\unp@ckstash\the#1\strm[].[]}}

\def\unp@ckstash{\gatherstash{}}

\def\unwrapstash#1{% to use where only the removal of \strm and \stashed is needed
    #1\expandafter{\romannumeral0\expandafter\unwr@pstash\the#1\strm[].[]}%
}

\def\unwr@pstash{\gatherstash{}}

% parser invocation

\def\newparserstateextra{}

\def\d@parse#1{\yyparse#1\yyeof\yyeof\endparseinput\endparse}

\def\astarraylastcs{\expandafter\noexpand\romannumeral\gettopofstackcsx\astarray}

\long\def\strm[#1]#2.[#3]{#2} % this is used with both the format and the stash streams; TODO: deprecate

\long\def\inmath#1{$#1$}

%\long\def\stashed#1{\toksa\expandafter{\the\toksa#1}} % to be used with unpackaged stash
%\long\def\stashed#1{\toksa\expandafter\expandafter\expandafter{\expandafter\the\expandafter\toksa#1}} % to use with AST array
\long\def\stashed#1{\errmessage{making \noexpand\stashed\space expandable is deprecated}}

\long\def\sflush#1#2{}

\def\ntt{\noexpand\ntt@} % a convenient shortcut
\def\ntt@{\tt\let\_\UL}

\def\prodheader#1#2#3#4{%
    \yyreadfifo\yyformat\to#3\in\yyformatlocal
    \restorecs{table-render}{\strm}\the\yyformatlocal
    \let\termindex\writeidxhentry
    \prodhead@r{#1}{#2}%
    \let\termindex\eatone
}

\def\prodhead@r#1#2{%
    \let\idit\prodhdridit
    #1\relax\concat\table\toksa\hrhss@p
}

\def\hrhss@p{\toksa{&\omit\hfil&\omit\hfil&\omit\hfil\cr\noalign{\penalty100}\omit\hfil&\termspostformat}\concat\table\toksa}

\def\rrhssep#1#2{%
    \yyreadfifot\yyformat\to#1\in\yyformatlocal\with\formatsort\relax
    \the\yyformatlocal % this will have a chance to change the definition of \rrhss@p
    \rrhss@p
}

\def\rrhss@p{\toksa{\omit\hfil&}\concat\table\toksa}

\def\arhssep#1#2{%
    \yyreadfifo\yyformat\to#1\in\yyformatlocal
    \the\yyformatlocal % this will have a chance to change the definition of \arhss@p
    \arhss@p
}

%% between the inline action and next term:

\def\arhss@p{\toksa{\omit\hfil&\hbox to 2em{\hfil}}\concat\table\toksa}

\def\rarhssep#1#2{%
    \yyreadfifo\yyformat\to#1\in\yyformatlocal
    \the\yyformatlocal % this will have a chance to change the definition of \rarhss@p
    \rarhss@p
}

%% between a term and the following action

\def\rarhss@p{\appendr\table&}

\newtoks\hstoks

\def\hspace#1#2{%
    \hstoks\yyformat
    \yyreadfifot\yyformat\to#1\in\yyformatlocal\with\formatsort\relax
    \the\yyformatlocal % may change \hspac@
    \hspac@
}

\def\hspac@{%
    \appendr\table{ }%
}

\def\rhs#1#2#3{#1}

\def\supplybdirective#1#2{%
    \edef\next{\nx\s@pplybdirective#1{\the#2}\the#1\nx\bdend}\next
}

\def\s@pplybdirective#1#2#3\bdend\bdend{%
    #1{#3#2\bdend}%
}

\def\termname#1#2{%
    \let\idit\termnameidit
    \let\stringify\termnamestringify
    \let\optstrextra\optstrextraesc
    \let\termindex\writeidxentry
    \termnam@{#1}{#2}%
    \let\termindex\eatone
}

\def\termnam@#1#2{\toksa{}\toksb{}\toksc{}#1\expandafter\termn@m@\expandafter{\the\toksa}{#2}}

\def\termn@m@#1#2{\toksa{}\toksb{}\toksc{}#2\expandafter\t@rmn@m@\expandafter{\the\toksa}{#1}}

% several versions of term typesetting

\def\t@rmn@m@#1#2{% the termname is #2, the bracketed part is in #1
    \yystringempty{#1}{\toksa{#2}}{\toksa{#2\hbox{$\langle${}#1{}$\rangle$}}}%
    \concat\table\toksa
}

\newif\ifbracketedvisible % decide whether to show the bracketed name in productions

\def\t@rmn@m@#1#2{% the termname is #2, the bracketed part is in #1
    \yystringempty{#1}{\toksa{#2}}{%
        \ifbracketedvisible
            \toksa{#2\hbox{$\,\cdot$#1}}%
        \else
            \toksa{#2}%
        \fi
    }%
    \concat\table\toksa
}

%

\newtoks\bdirects

\def\actbraces#1#2#3#4#5\bdend{%
    \yyreadfifo\yyformat\to#3\in\yyformatlocal
    \restorecs{table-render}{\strm}%
    #5% accumulate all the directives
    \yytoksempty\bdirects{}{%
        \edef\next{\bdirects{%
             $\vtop{\halign{\nx\quad####\nx\hfil&####\nx\hfil\cr\the\bdirects}}$}}\next
    }%
    \the\yyformatlocal
    \actbrac@s{#1}{#2}{#4}%
}

\def\actbrac@s#1#2#3{%
    \readstash{#3}%
    \appendr\table{\the\bdirects&\toksa{\the\yystashlocal}\cr}%
    \bdirects{}%
}

\def\bpredicate#1#2#3#4#5\bdend{%
    \yyreadfifo\yyformat\to#3\in\yyformatlocal
    \restorecs{table-render}{\strm}%
    #5% accumulate all the directives
    \yytoksempty\bdirects{%
        \bdirects{\quad$\dashv\ \nx\vdash$}%
    }{%
        \edef\next{\bdirects{%
             $\vtop{\halign{\nx\quad####\nx\hfil&####\nx\hfil\cr$\dashv\ $&\omit\nx\hfil$\nx\vdash$\cr
                                                                \the\bdirects}}$}}\next
    }%
    \the\yyformatlocal
    \bpredicat@{#1}{#2}{#4}%
}

\def\bpredicat@#1#2#3{% 
    \readstash{#3}%
    \appendr\table{\the\bdirects&\toksa{\the\yystashlocal}\cr}%
    \bdirects{}%
}

\def\bdirective#1#2#3#4{%
    \savecslist{local-namespace}\yyunion
    \restorecslist{table-render:directives}\yyunion
    \toksa{}#2% make a directive list
    \appendr\bdirects{$\nx\langle${\ntt #1} &\the\toksa$\nx\rangle$\nx\cr}%
    \restorecslist{local-namespace}\yyunion
}

\def\mergeop#1#2#3{%
    \bdirective{merge}{#1}{#2}{#3}%
}

\def\dprecop#1#2#3{%
    \bdirective{dprec}{#1}{#2}{#3}%
}

\def\sprecop#1#2#3{%
    \bdirective{prec}{#1}{#2}{#3}%
}

\def\rules#1#2#3{#1}

\def\oneproduction#1#2#3{#1\setrulecontext{}\stopproduction}

% to make it possible to typeset tokens depending on their context
% the context name is passed to the token typeseting macros; the most
% common context name is the left hand side of a production; special
% context names (such as *left, *right, etc) are defined for terms that are
% not part of a rule; all such context names start with `*' since this
% character cannot be part of a term name

\def\setrulecontext#1{\def\currentrulecontext{#1}}

\let\currentrulecontext\empty

% an empty left hand side is indexed in the index pseudonamespace and not in the main
% parser namespace, since the name parser never encounters this term (as the \emptyterm
% control sequence is inserted by the main parser) while typesetting productions;
% because of this it would be unnatural to define a \prettywordpair sequence for it in the main
% parser namespace; using a special non-\bison\ name should avoid any confusion with
% the regular grammar terms
\def\emptyterm{\appendr\table{$\nx\circ$\tidxentry{\termexception}{emptyrhs&}}}

\def\termnameidit#1#2#3#4{%
    \yyreadfifot\yyformat\to#3\in\yyformatlocal\with\formatsort\relax
    \the\yyformatlocal
    \nameproc{#2}\with\parsebin
    \peekstash{#4}% does not clobber \toksa and \toksb
    \expandafter\settermstyle\expandafter{\the\toksb}{#1}%
 }

\def\settermstyle#1#2{% put the \whatsit node (\write) at the end so that the
                      % term macros can remove the whitespace preceeding it
    \edef\next{%
        \toksa{%
            \termmetastyle{%
                \let\nx\idxfont\nx\empty\nx\it\the\toksa
                \gidxentry{\termidstring}{#2}{#1}\nx\/%
            }%
        }%
    }\next
}

\def\termnamestringify#1#2#3#4{%
    \nameproc{#2}\with\parsebin
    \ifyyparsefail
        \toksb{"#1"}\toksc{\termstring}%
    \else
        \toksc{\termidstring}%
    \fi
    \edef\next{\toksa{{%
            \expandafter\gidxentryxv\expandafter{\the\toksc}{#1}{\the\toksb}%
            \ntt\the\toksa
        }}}\next
}

\def\termmetastyle#1{{#1}}

\let\peekstash\eatone

\newif\ifinheader

\def\prodhdridit#1#2#3#4{%
    \setrulecontext{}% TODO: ?
    \nameproc{#2}\with\parsebin
    \setrulecontext{#1}%
    \edef\next{\toksa{{%
            \gidxentryxv{\termidstring}{#1}{\the\toksb}%
            \nx\inheadertrue\let\nx\idxfont\nx\bf\nx\itbold\the\toksa\nx\/%
        }}}\next
}

\def\charit#1#2#3#4{%
    \expandafter\ifx\csname\prettynamecs\hostparsernamespace{#1}\endcsname\relax
        \toksa{#1}\sansfirst\toksa
        \edef\next{\toksa{{\ntt\the\toksa}\gidxentryxb{\termvstring}{\the\toksa}{}}}% remove the possible \\
    \else
        \expandafter\ifx\csname\viskeyref\hostparsernamespace{#1}\endcsname\relax
            \tokse{#1}% we are stripping the quotes so the visible key is adjusted
        \else
            \edef\next{\tokse{\expandafter\expandafter\csname\viskeyref
                \hostparsernamespace{#1}\endcsname\expandafter{\currentrulecontext}}}\next
        \fi
        \edef\next{\expandafter\toksa\expandafter{\expandafter\noexpand
            %\csname\prettynamecs\hostparsernamespace{#1}\endcsname{\currentrulecontext}\gidxentryxv{\termvstring}{'#1'}{\the\tokse}}}%
            \csname\prettynamecs\hostparsernamespace{#1}\endcsname{\currentrulecontext}\gidxentryxv{\termidstring}{'#1'}{\the\tokse}}}%
    \fi
    \next
}

\def\anint#1#2#3{%
    \appendrnx\table{#1}%
}

\def\hexint#1#2#3{%
     \expandafter\appendrnx\expandafter\table\expandafter{\eattwo#1$_{16}$}%
}

\def\optoptsepnone{%
    \concat\opttable\postoks
}

\def\optoptsepsmall{% keep the alignment going but insert a small space
    \concat\opttable\postoks
    \appendrnx\opttable{\noalign{\smallskip}}%
}

\def\prodprodsepsmall{%
    \pushothertables
    \appendrnx\table{\noalign{\penalty-150 \smallskip}}%
}

\def\prodprodsep{%
    \pushothertables
    \appendrnx\table{\noalign{\medskip}}%
}

\def\prodprodseplarge{%
    \pushothertables
    \appendrnx\table{\noalign{\bigskip}}%
}

\def\separatorswitcheq{%
    \oneproduction {%
        \toksd{\prodprodsepsmall}%
    }
    \codepropstype\codeassoc {%
        \toksd{\optoptsepsmall}%
    }
}

\def\separatorswitchdefaulteq{%
    \toksd{\optoptsepnone}%
}

% a switch for inserting position information into the parsed table
% clobbers \toksd

\newtoks\postoks

\def\positionswitch{
    \empty
    {%
        \postoks{}% semicolons should be invisible
    }
    \paramdef\oneparametricoption\optionflag\typedecls\vardef
    {%
        \getfourth{\yy(2)}\to\postoks % the format marker
        \getfifth{\yy(2)}\to\toksd % the stash marker
        \edef\next{\postoks{\nx\posmark{\the\postoks}{\the\toksd}}}\next
    }
    \prologuecode\ntermdecls\tokendecls
        \flexsstatedecls\flexxstatedecls\flexoptiondecls
    {%
        \getthird{\yy(2)}\to\postoks % the format marker
        \getfourth{\yy(2)}\to\toksd % the stash marker
        \edef\next{\postoks{\nx\posmark{\the\postoks}{\the\toksd}}}\next
    }
    \precdecls
    {%
        \getfifth{\yy(2)}\to\postoks % the format marker
        \getsixth{\yy(2)}\to\toksd % the stash marker
        \edef\next{\postoks{\nx\posmark{\the\postoks}{\the\toksd}}}\next
    }
    \oneproduction
    {%
        \getsecond{\yy(2)}\to\postoks % the \prodheader
        \getfourth{\postoks}\to\toksd % the format marker
        \getfifth{\postoks}\to\postoks % the stash marker
        \edef\next{\postoks{\nx\posmark{\the\toksd}{\the\postoks}}}\next
    }
    \codeassoc\codepropstype
    {%
        \getseventh{\yy(2)}\to\postoks % the format marker
        \geteightth{\yy(2)}\to\toksd % the stash marker
        \edef\next{\postoks{\nx\posmark{\the\postoks}{\the\toksd}}}\next
    }
}

\def\positionswitchdefault{%
    \errmessage{unknown chunk type in: \the\yy(2)}%
}

\def\posmark#1#2{%
%    \yyreadfifo\yyformat\to#1\in\yyformatlocal
%    no formatting for locations currently
%    \the\yyformatlocal
    \posm@rk{#2}%
}

\def\posm@rk#1{%
    \readstash{#1}%
    \yytoksempty\yystashlocal{\postoks{}}{%
        \toksa\expandafter{\the\yystashlocal}\cleanstash\stripstash
        \setbox\z@=\vbox{\setlazyc\the\toksa}%
        \ifnum\ht\z@=\z@
            \postoks{}%
        \else
            \edef\next{\postoks{\noalign{\indent\toksa{\the\yystashlocal}\nx\makestashbox}}}\next
        \fi
    }%
}

\def\codeoptionlist{%
    \codepropstype\codeassoc\prologuecode
}

\def\simpleoptionsplit{%
    \oneparametricoption\vardef\paramdef\optionflag
}

\def\noncodeoptionlist{%
    \oneparametricoption\vardef\paramdef\optionflag
    \codepropstype\codeassoc\prologuecode
}

\def\alloptionlist{%
    \oneparametricoption\vardef\paramdef\optionflag
    \codepropstype\codeassoc\prologuecode
    \typedecls\tokendecls\precdecls\oneproduction
    \flexsstatedecls\flexxstatedecls\flexoptiondecls
}

\def\emptyoption{\empty}

\diagprod\codeoptionlist\noncodeoptionlist\in\codenoncodeseplist
\diagprod\simpleoptionsplit\simpleoptionsplit\in\noncodenoncodeseplist
\diagprod\alloptionlist\emptyoption\in\emptyotherseplist

% this is a switch for inserting separators between dissimilar items

\def\separatorswitchneq{% 
    \classexpand\codenoncodeseplist
    {% separate options with code (\codepropstype, \codeassoc) from other options by a small space
        \toksd{\optoptsepsmall}%
    }
    \classexpand\noncodenoncodeseplist
    {%
        \toksd{\optoptsepnone}%
    }
    % a separator after the first rule or before or after a semicolon
    \classexpand\emptyotherseplist
    {%
        \toksd{\optoptsepnone}%
    }
}

\setspecialcharsfrom\separatorswitchneq

\def\separatorswitchdefaultneq{%
    \toksd{\prodprodsep}%
}

\def\pushothertables{%
    %token's
    \yytoksempty\tokdectoks{}{\attachtokentable}%
    %type's
    \yytoksempty\typestable{}{\attachtypestable}%
    %precedence's
    \yytoksempty\prectable{}{\attachprectable}%
    % options
    \yytoksempty\opttable{}{\attachoptionstable}%
    \concat\table\postoks
} %

% the various \attach... macros above are defined elsewhere and are the macros that 
% determine the actual typesetting of the corresponding tables.

\let\stopproduction\relax

\def\tokendecls#1#2#3{%
    \savecslist{local-namespace}{\yyunion\termindex}%
    \restorecslist{table-render:token-declarations}\yyunion
    \let\termindex\writetokenidxentry
    \let\currenttokentype\empty#1%
    \restorecslist{local-namespace}{\yyunion\termindex}%
}

\def\typedecls#1#2#3#4{%
    \savecslist{local-namespace}{\yyunion\termindex}%
    \restorecslist{table-render:type-declarations}\yyunion
    \let\termindex\writetokenidxentry
    \typesset{}#2\relax#1%
    \appendr\typestable{\nx\onetype{\the\toksa}{\the\typesset}}%
    \restorecslist{local-namespace}{\yyunion\termindex}%
}

% \onetype is a typeseting macro defined elsewhere
% TODO: move these to the \flex\ section

\def\flexsstatedecls#1#2#3{% 
    \savecslist{local-namespace}{\yyunion\termindex}%
    \restorecslist{table-render:flex-state-declarations}\yyunion
    \let\termindex\writeidxfsdentry%\writetokenidxentry
    \typesset{}#1\relax
    \appendr\typestable{\nx\flexsstatelist{\the\typesset}}%
    \restorecslist{local-namespace}{\yyunion\termindex}%
}

\def\flexxstatedecls#1#2#3{% 
    \savecslist{local-namespace}{\yyunion\termindex}%
    \restorecslist{table-render:flex-state-declarations}\yyunion
    \let\termindex\writeidxfsdentry%\writetokenidxentry
    \typesset{}#1\relax
    \appendr\typestable{\nx\flexxstatelist{\the\typesset}}%
    \restorecslist{local-namespace}{\yyunion\termindex}%
}

% \flexsstatelist and \flexxstatelist are typesetting macros defined elsewhere

\def\flexoptiondecls#1#2#3{% 
    \savecslist{local-namespace}{\yyunion\termindex}%
    \restorecslist{table-render:flex-option-definitions}\yyunion
    \let\termindex\writeidxfsrnentry%\writetokenidxentry
    \toksa{}#1\relax
    \restorecslist{local-namespace}{\yyunion\termindex}%
}

\def\precdecls#1#2#3#4#5{% #1 type
                         % #2 tag
                         % #3 list of symbols
                         % #4 format pointer
                         % #5 stash pointer
    \savecslist{local-namespace}{\yyunion\termindex}%
    \restorecslist{table-render:prec-declarations}\yyunion
    \setrulecontext{*#1}%
    \let\termindex\writetokenidxentry
    \typesset{}#3\toksa{}#2%
    \appendr\prectable{\nx\oneprec{#1}{\the\toksa}{\the\typesset}}%
    \restorecslist{local-namespace}{\yyunion\termindex}%
    \setrulecontext{}%
}

% \oneprec is a typesetting macro defined elsewhere

% options typesetting

\def\oneparametricoption#1#2#3#4{%
    \restorecs{table-render:token-declarations}{\braceit\stringify\idit\anint\hexint}%
    \toksa{}\toksb{}\toksc{}#2%
    \edef\next{\nx\oneparametricopti@n{#1}{\the\toksa}{\the\toksb}{\the\toksc}}\next
}

\def\oneparametricopti@n#1#2#3#4{%
    \yystringempty{#2}{% there is no identifier
        \yystringempty{#4}{% there is no digit
            \edef\next{\toksa{{\ntt"\the\toksb"}}}\next
        }{%
            \edef\next{\toksa{{\nx\rm\the\toksc}}}\next
        }%
    }{%
        \nameproc{#2}\with\parsebin
    }% now \toksa contains the value of the parameter
    \toksb{#1}%
    \let\termindex\writetokenidxentry
    \appendr\opttable{\gidxentry{\termostring}{#1}{}$\nx\langle${\nx\bf\the\toksb}$\nx\rangle$&\the\toksa\nx\/\nx\cr}%
    \let\termindex\eatone
}

\def\paramdef#1#2#3#4{%
    \restorecs{table-render:token-declarations}\braceit
    \let\termindex\writetokenidxentry
    \appendr\opttable{\gidxentry{\termostring}{#2}{}}%
    \let\termindex\eatone
    \appendrnx\opttable{$\langle${\bf #2}$\rangle$}%
    \toksa{}#1%
    \concat\opttable\toksa
}

\def\optionflag#1#2#3#4{%
    \let\termindex\writetokenidxentry
    \appendr\opttable{\gidxentry{\termostring}{#1}{}}%
    \let\termindex\eatone
    \yystringempty{#2}{%
        \appendrnx\opttable{$\langle${\bf#1}$\rangle$\rlap{$\,\star$}&\cr}%
    }{%
        \appendrnx\opttable{$\langle${\bf#1}$\rangle$\rlap{$\,\star$}& {\rm(set as $\langle${\bf#2}$\rangle$)}\cr}%
    }%
}

\def\codepropstype#1#2#3#4#5#6#7{%
    \yyreadfifo\yyformat\to#4\in\yyformatlocal
    \restorecs{table-render}{\strm}%
    \the\yyformatlocal % this can change \codepropstyp@
    \codepropstyp@{#2}{#3}{#1}{#5}%
}

\def\codepropstyp@#1#2#3#4{%
    \readstash{#4}%
    \restorecs{table-render:type-declarations}{\idit\stringify\hspace}%
    \let\termindex\writetokenidxentry
    \typesset{}#2% collect all the symbols in \typesset
    \appendr\opttable{\gidxentry{\termostring}{#3}{}$\nx\langle${\nx\bf #3}$\nx\rangle$&%
         \toksa{\the\yystashlocal}\nx\makestashbox\nx\hfil\cr
         \noalign{\nx\smallskip}%
         \noalign{\indent$\nx\rlap{\hbox to2em{\nx\hfil$\nx\star$\nx\hfil}}\vcenter{
             \advance\hsize by-\parindent
             \emergencystretch10pt\nx\raggedright\noindent\hangafter\z@\hangindent2em\nx\strut\the\typesset\nx\strut}$}%
    }%
    \let\termindex\eatone
}

\def\codeassoc#1#2#3#4#5#6#7{%
    \yyreadfifo\yyformat\to#4\in\yyformatlocal
    \restorecs{table-render}{\strm}%
    \the\yyformatlocal % this can change \codeass@c
    \codeass@c{#1}{#2}{#3}{#5}%
}

\def\codeass@c#1#2#3#4{%
    \readstash{#4}%
    \let\idit\termnameidit
    \let\termindex\writetokenidxentry
    \toksa{}#2% put the typeset form of the identifier in \toksa
    \appendr\opttable{%
        \yystringempty{#1}{}{\gidxentry{\termostring}{#1}{}}%
        $\nx\langle${\nx\bf #1}$\nx\rangle$\nx\quad{\the\toksa}&%
        \toksa{\the\yystashlocal}\nx\makestashbox\nx\hfil\cr
    }%
    \let\termindex\eatone
}

\def\vardef#1#2#3#4{%
    \yyreadfifo\yyformat\to#3\in\yyformatlocal
    \restorecs{table-render}{\strm}%
    \the\yyformatlocal % this can change \vard@f
    \vard@f{#1}{#2}{#4}%
}

\def\vard@f#1#2#3{%
    \let\idit\termnameidit
    \let\stringify\termnamestringify
    \let\termindex\writetokenidxentry
    #1% put the typeset form of the identifier or string in \toksa
    \toksb\toksa
    \savecslist{local-namespace}\yyunion
    \restorecslist{table-render:variable-definitions}\yyunion
    \toksa{}#2% put the value in toksa
    \toksc\toksa
    \restorecslist{local-namespace}\yyunion
    \let\termindex\eatone
    \appendr\opttable{$\nx\langle${\nx\bf define}$\nx\rangle$\nx\quad{\the\toksb}&\the\toksc\cr}%
}

\def\prologuecode#1#2#3#4#5{% similar to \actbrac@s
    \readstash{#5}%
    \appendr\opttable{%
        \omit\span\omit\nx\cdotfill\cr
        \omit\span\omit\toksa{\the\yystashlocal}\nx\makestashbox\nx\cr
        \omit\span\omit\nx\cdotfill\cr
    }%
}

\def\yyunion@tablerender{%
    \strm\stashed\rarhss@p\rrhss@p\prodhead@r\hrhss@p\arhss@p\termnam@\termn@m@\t@rmn@m@
    \actbrac@s\hspac@\codepropstyp@\vard@f\termmetastyle\peekstash\stopproduction
}

\savecslist{table-render}\yyunion
\savecslist{table-render}\yyunion@tablerender

\def\setprodtable{%
    \restorecslist{table-render}\yyunion
    \restorecslist{table-render}\yyunion@tablerender
}

% formatting macros

\def\midf#1#2{}

\def\format#1{%
    #1%
}
\let\formatlocal\format
\let\formatbegin\format
\let\formatp\format

\newtoks\yystashlastcontent % contents of the last action
\newif\ifshowlastaction % if true, show the last action of a flattened set of rules

% flatten: omit the actions, just list all the rules in a single line;
% if \showlastactiontrue show just the last action, otherwise display \actionfiller
% instead

\def\flatten{%
    \let\stopproduction\stopproduction@flatten
    \let\actbrac@s\actbrac@s@flatten
    \let\rrhss@p\rrhss@p@flatten
    \let\arhss@p\relax
    \let\rarhss@p\relax
}

\def\actbrac@s@flatten#1#2#3{% do everything but add the stash contents to the table
    \readstash{#3}%
    \yystashlastcontent\yystashlocal % keep the contents of the braces
                                     % in case we want to show the last action
    \futurelet\actsep\actbrac@s@fl@tten
}

% see the remarks before \emptyterm above for the reasons for choosing the index
% pseudonamespace for this term
\def\actbrac@s@fl@tten{%
    \ifx\actsep\arhssep
        \appendr\table{$\nx\ \nx\diamond\nx\ $\tidxentry{\termexception}{inline_action&}}%
    \fi
}

\def\rrhss@p@flatten{%
    \appendrnx\table{$\ \vert\ $}%
}%

\def\stopproduction@flatten{%
    \ifshowlastaction
        \yystashlastcontent\expandafter{\expandafter\toksa\expandafter{\the\yystashlastcontent}}%
    \else
        \yystashlastcontent{\actionfiller}%
    \fi
    \appendr\table{&&\the\yystashlastcontent\nx\cr}%
}

% fold: resume normal display

\def\fold{%
    \restorecs{table-render}{\actbrac@s\rrhss@p\arhss@p\rarhss@p\stopproduction}%
    \stopproduction@flatten
}

\def\actionfiller{\omit\quad$\ldots$\hfil}

% inline: put the left hand side of the production on the same line as the rule

\def\inline{%
    \let\prodhead@r\prodhead@r@inline
    \let\hrhss@p\hrhss@p@inline
}

\def\prodhead@r@inline#1#2{%
    \let\idit\prodhdridit
    #1\relax
    \concat\table\toksa
    \setbox\z@=\hbox{\strut\the\toksa\/$\,$\rm:\quad}%
    \tempda=\wd\z@\relax
    \advance\tempda-2em
    \hrhss@p
}

\newif\ifsquashterms

\def\termspostformat{\ifsquashterms\aftergroup\noexpand\squashtermstrue\fi}

\def\hrhss@p@inline{%
    \appendr\table{&\termspostformat\hbox to \the\tempda{\hss}}%
}

% breakline: break the production line

\def\breakline{%
    \let\hspac@\hspac@@breakline
}

\def\hspac@@breakline{%
    \appendrnx\table{{$\,\hookleftarrow$}&\omit\hfil&\omit\hfil\cr\omit\hfil&\hbox to 2em{\hfil}}%
    \restorecs{table-render}\hspac@
}

% breakahead: break the production line after #1 terms
% use: \breakahead{[0-9]+}

\newcount\breakaheadcount

\def\breakahead#1{%
    \let\bahspac@\hspac@
    \let\hspac@\hspac@@breakahead
    \breakaheadcount#1\relax
}

\def\hspac@@breakahead{%
    \ifnum\breakaheadcount=\@ne
        \yybreak{%  
            \appendrnx\table{{$\,\hookleftarrow$}&\omit\hfil&\omit\hfil\cr\omit\hfil&\hbox to 2em{\hfil}}%
            \let\hspac@\bahspac@
        }%
    \else
        \advance\breakaheadcount by \m@ne
        \yybreak{\bahspac@}%
    \yycontinue        
}

% skipheader: do not show the left hand side

\def\skipheader{%
    \let\prodhead@r\prodhead@r@skipheader
    \let\hrhss@p\hrhss@p@skipheader
}

\def\prodhead@r@skipheader#1#2{%
    \hrhss@p
}

\def\hrhss@p@skipheader{\appendrnx\table{\omit\hbox to 2em{\hfil}&\termspostformat}\restorecs{table-render}{\hrhss@p}}

\def\stashtoterm#1{% in case one needs to insert previous stash as a term
                   % usage: \let\peekstash\stashtoterm
    \readstash{#1}%
    {\edef\next{\toksc{{\toksa{\the\yystashlocal}\nx\makestashbox}}}\expandafter}\next
    \let\termmetastyle\xtoksc
    \restorecs{table-render}\peekstash
}

\def\xtoksc#1{\the\toksc}%

% resetf: resume normal formatting

\def\resetf{%
    \restorecs{table-render}{\prodheader\prodhead@r\hrhss@p\rrhss@p\arhss@p\rhs\termname\actbraces\actbrac@s\rarhss@p\hspac@\rules
        \oneproduction\emptyterm\stringify\idit\charit\prodprodsep\onesymbol\strm\stopproduction}%
    \squashtermsfalse
}

\def\prodstyle#1{% to typeset token names in text;
                 % CAUTION: when many parsers are at play at the same time,
                 % one should use the version of this macro below that takes a
                 % namespace as a parameter, to make sure the term is typeset properly
                 % when another namespace might be in effect (such as when section names are output)  
    {%  
        \let\optstrextra\optstrextraesc
        \def\termidxrank{5}%
        \let\termindex\writeidxentry
        \nameproc{#1}\with\parsebin
        \edef\next{%
            \toksa{%
                \termmetastyle{%
                    \let\nx\idxfont\nx\empty\nx\it\the\toksa
                    \expandafter\gidxentryxv\expandafter{\romannumeral
                        \ifyyparsefail
                            \yybreak{0 \termidstring}%
                        \else
                            \yybreak{\expandafter\tssextract\the\toksa\end}%
                        \yycontinue}{#1}{\the\toksb}\nx\/%
                }%
            }%
        }\next
        \let\_\uscore
        \the\toksa\nobreak
    }%
}

\def\tssextract#1#2#3\end{%
    \yystringempty{#2}{0 \termidstring}{0 #2}%
}

\def\prodtstyle#1{% to typeset terminal string names in text;
    {%  
        \let\optstrextra\optstrextraesc
        \def\termidxrank{5}%
        \let\termindex\writeidxentry
        \nameproc{#1}\with\parsebin
        \yytoksempty\toksb{\toksb{"#1"}}{}%
        \edef\next{%
            \toksa{%
                \termmetastyle{%
                    \let\nx\idxfont\nx\empty\ntt\the\toksa
                    \gidxentryxv{\termstring}{#1}{\the\toksb}%
                }%
            }%
        }\next
        \let\_\uscore
        \the\toksa\nobreak
    }%
}

\def\prodstylens#1#2{% to typeset token names in text, in a predefined namespace
    {%
        \def\hostparsernamespace{#2}%
        \prodstyle{#1}%
    }%
}

% macros used in typesetting the result of the prologue parsing

%type declarations

\restorecslist{parser-strict}\yyunion


\def\idit#1#2#3#4{%
    \termnameidit{#1}{#2}{#3}{#4}%
    \concat\typesset\toksa
}

\def\stringify#1#2#3#4{%
    \termnamestringify{#1}{#2}{#3}{#4}%
    \concat\typesset\toksa
}

\def\hspace#1#2{%
    \appendrnx\typesset{ }%
}

\def\charit#1#2#3#4{%
    \toksa{\.{#2}}%
    \concat\typesset\toksa
}

\def\tagit#1#2#3#4{%
    \toksa{\\{#2}}%
}

\toyyunion{table-render:type-declarations}

%token declarations

\newif\iftracetokennames

\restorecslist{parser-strict}\yyunion

\def\hspace#1#2{}

\def\idit#1#2#3#4{\toksa{#2}}

\def\charit#1#2#3#4{\toksa{'#1'}}

\def\stringify#1#2#3#4{\toksb{#2}}

\def\anint#1#2#3{\toksc{#1}}

\def\hexint#1#2#3{\toksc\expandafter{\eattwo#1$_{16}$}}

\def\tagit#1#2#3#4{%
    \def\currenttokentype{#2}%
}

\def\braceit#1#2#3{% similar to \actbrac@s
    \readstash{#3}%
    \edef\next{\toksa{&\toksa{\the\yystashlocal}\nx\makestashbox\nx\cr}}\next
}

\def\onesymbol#1#2#3{% #1 is the macro name used internally by bison (\idit{}{}{}{})
                     % #2 is the integer value (\anint{}{}{})
                     % #3 is the string value (\stringify{}{}{}{})
    \toksa{}\toksb{}\toksc{}%
    #1#2#3\relax
    \edef\next{\nx\onesymb@l{\the\toksa}{\the\toksc}{\the\toksb}}\next
}

\def\onesymb@l#1#2#3{%         #1 is the macro name used internally by bison 
                             % #2 is the integer value 
                             % #3 is the string value 
    \let\optstrextra\optstrextraesc
    \tokse{}\toksf{}\toksg{}% these registers will contain the appropriate visual keys
                            % and index entry processing control sequences
    \yystringempty{#3}{}{%
        \nameproc{#3}\with\parsebin
        \ifyyparsefail % if the parsing failed, create a visible key, process the 
                       % index entry as a \.{\\tt} string
            \tokse{"#3"}\toksf{\termstring}%
        \else % otherwise use the value of the string as the visual key,
              % process the string as a \bison\ identifier
            \tokse\toksb\toksf{\termidstring}%
        \fi
        \tokstoks\toksa
    }%
    \yystringempty{#1}{}{\nameproc{#1}\with\parsebin\toksg\toksb}%
    \iftracetokennames\ferrmessage{token: \the\toksa, string: \the\tokstoks}\fi
    \toksb\expandafter{\currenttokentype}%
    \appendr\tokdectoks{\nx\toksdefline
                            {\the\toksa}%
                            {% 
                                \yystringempty{#1}{}{\gidxentryxv{\termidstring}{#1}{\the\toksg}}%
                                \yystringempty{#3}{}{\expandafter\gidxentryxv\expandafter{\the\toksf}{#3}{\the\tokse}}%
                            }% 
                            {\yytoksempty\toksb{auto}{\the\toksb}}%
                            {#2}%
                            {\the\tokstoks}%
    }%
    \restorecs{nameparser}{\optstrextra}%
}

\toyyunion{table-render:token-declarations}

%precedence declarations

\restorecslist{parser-strict}\yyunion

\def\idit#1#2#3#4{%
    \termnameidit{#1}{#2}{#3}{#4}%
    \concat\typesset\toksa
}

\def\stringify#1#2#3#4{%
    \termnamestringify{#1}{#2}{#3}{#4}%
    \concat\typesset\toksa
}

\def\charit#1#2#3#4{%
    \expandafter\ifx\csname\prettynamecs\hostparsernamespace{#1}\endcsname\relax
        \toksa{#1}\sansfirst\toksa
        \edef\next{\toksa{{\ntt\the\toksa}\gidxentryxb{\termvstring}{\the\toksa}{}}}% remove the possible \\
    \else
        \expandafter\ifx\csname\viskeyref\hostparsernamespace{#1}\endcsname\relax
            \tokse{#1}%
        \else
            \edef\next{\tokse{\expandafter\expandafter\csname\viskeyref
                \hostparsernamespace{#1}\endcsname\expandafter{\currentrulecontext}}}\next
        \fi
        \edef\next{\expandafter\toksa\expandafter{\expandafter\noexpand
            %\csname\prettynamecs\hostparsernamespace{#1}\endcsname{\currentrulecontext}\gidxentryxv{\termvstring}{'#1'}{\the\tokse}}}%
            \csname\prettynamecs\hostparsernamespace{#1}\endcsname{\currentrulecontext}\gidxentryxv{\termidstring}{'#1'}{\the\tokse}}}%
    \fi
    \next
    \concat\typesset\toksa
}

\def\anint#1#2#3{%
    \appendrnx\typesset{${}=\hbox{\ntt#1}$}%
}

\def\hexint#1#2#3{%
    \toksa\expandafter{\eattwo#1}%
    \appendr\typesset{${}=\hbox{\ntt\the\toksa}_{16}$}%
}

\def\hspace#1#2{%
    \appendrnx\typesset{ }%
}

\def\tagit#1#2#3#4{%
    \toksa{\\{#2}}%
}

\def\symbolprec#1#2{%
    #1#2%
}

\toyyunion{table-render:prec-declarations}

% variable definitions

\restorecslist{parser-strict}\yyunion

\def\idit#1#2#3#4{\toksa{#2}}

\def\stringify#1#2#3#4{\toksa{#2}}

\def\bracedvalue#1#2#3{% similar to \actbrac@s
    \readstash{#3}%
    \edef\next{\toksa{\toksa{\the\yystashlocal}\nx\makestashbox}}\next
}

\toyyunion{table-render:variable-definitions}

% directives

\restorecslist{parser-strict}\yyunion

\def\idit#1#2#3#4{%
    \termnameidit{#1}{#2}{#3}{#4}%
}

\def\stringify#1#2#3#4{%
    \termnamestringify{#1}{#2}{#3}{#4}%
}

\def\tagit#1#2#3#4{%
    \toksa{\\{#2}}%
}

\def\anint#1#2#3{%
    \toksa{#1}%
}

\def\hexint#1#2#3{%
     \toksa\expandafter{\eattwo#1$_{16}$}%
}

\toyyunion{table-render:directives}

% flex state declarations

\restorecslist{parser-strict}\yyunion

\def\idit#1#2#3#4{%
    \nameproc{#2}\with\parsebin
    \appendr\typesset{{%
            \gidxentryxv{\termttstring}{#1}{\the\toksb}%
            \ntt\the\toksa
        }}%
}

\def\hspace#1#2{%
    \appendr\typesset{ }%
}

\toyyunion{table-render:flex-state-declarations}

% flex option definitions

\restorecslist{parser-strict}\yyunion

\def\flexoptionpair#1#2{%
    #1%
    \yystringempty{#2}{% there is no right hand side
        \appendr\opttable{$\nx\langle${\ntt\the\toksa}$\nx\rangle_{\rm f}\,\star$&\cr}%
    }{%
        \toksb\toksa\toksa{}#2%
        \appendr\opttable{$\nx\langle${\ntt\the\toksb}$\nx\rangle_{\rm f}$&\nx\it\the\toksa\nx\/\nx\cr}%
    }% 
}

\def\idit#1#2#3#4{%
    \toksa{#2}%
    \edef\next{\toksa{{%
            \gidxentry{\termfsopstring}{#2}{}%
            \ntt\the\toksa
        }}}\next
}

\def\stringify#1#2#3#4{\toksa{{\ntt@"#2"}}}

\def\hspace#1#2{}

\toyyunion{table-render:flex-option-definitions}

% rule listing macros

\restorecslist{parser-prototypes}\yyunion % we just want to ignore the non-rule related code

\def\prodheader#1#2#3#4{%
    #1\relax
    \lastlastname\lastname
    \setnumname\numname
    \lastname{}%
    #2\relax
    \edef\next{\toksc{\numname:}\toksd{\noexpand\lhs{\the\lastlastname}{\the\lastname}}}\next
}

% the macros below are defined in `parser-prototypes'
%\def\rrhssep#1#2{}
%\def\arhssep#1#2{}
%\def\rarhssep#1#2{}
%\def\hspace#1#2{}
%\def\prodprodsep{}
%\def\onesymbol#1#2#3{}

\def\rhs#1#2#3{%
    \concat\toksa\toksc
    \concat\toksb\toksd
    #1\relax
    \edef\next{\toksb{\noexpand\onerule{\the\toksb}}}\next
    \appendr\toksa\def
    \possibleimplicitfalse
    \yyreplacestring\toksa\in\newsymswitch\with\toksb
    \toksa{}\toksb{}%
}

\newtoks\lastname
\newtoks\lastlastname

\def\termname#1#2{%
    \flushact
    #1\relax
    \lastlastname\lastname
    \setnumname\numname
    \lastname{}%
    #2\relax
    \appendr\toksa{ \numname}\appendr\toksb{\noexpand\term{\the\lastlastname}{\the\lastname}}%
}

\def\setnumname#1{%
    \expandafter\let\expandafter\next\csname term\parsernamespace\the\lastname \endcsname
    \ifx\next\relax % this is a terminal that was renamed
        \expandafter\ifx\csname token\parsernamespace\the\lastname \endcsname\relax % this is probably a symbol from another grammar
            \def#1{-1}%
            \errmessage{token \csname token\parsernamespace\the\lastname \endcsname\space is undefined}%
        \else
            \expandafter\tempca\csname token\parsernamespace\the\lastname \endcsname
            \edef#1{\fgetelemof{yytranslate}\at\tempca}%
        \fi
    \else
        \let#1\next
    \fi
}

\def\actbraces#1#2#3#4#5\bdend{%
    \flushact
    \lastname{}%
    #2\relax
    \edef\next{\lastactsym{\the\lastname}}\next
    \possibleimplicittrue
}

\def\rules#1#2#3{#1}

\def\oneproduction#1#2#3{#1}

\def\emptyterm{%
    \flushact
    \lastname{}%
}

\def\stringify#1#2#3#4{%
    \lastname{"#1"}%
}

\def\idit#1#2#3#4{%
    \lastname{#1}%
}

\def\charit#1#2#3#4{%
    \lastname{'#1'}%
}

\def\anint#1#2#3{%
    \lastname{#1}%
}

\def\hexint#1#2#3{%
     \lastname\expandafter{\expandafter\number\eattwo#1$_{16}$}%
}

\newtoks\lastactsym
\newif\ifpossibleimplicit

\def\flushact{%
    \ifpossibleimplicit
        \appendr\toksa{ @implicit@}\appendr\toksb{\noexpand\implicitterm{@implicit@}{\the\lastactsym}}%
        \lastactsym{}%
        \possibleimplicitfalse
    \fi
}

\savecslist{symbols}\yyunion

%% token typesetting macros

\def\prettynamecs#1#2{'parser'#1#2} % naming convention for token typesetting alternatives

\def\setsafe#1{\def\saferword{#1}\setspecialcharsfrom\saferword}
\def\prettyword#1{\setsafe{#1}\expandafter\edef\csname\prettynamecs\parsernamespace\saferword\endcsname##1{{\ntt\lowercase{\saferword}}}}
\def\prettytoken#1{\setsafe{#1}\expandafter\edef\csname\prettynamecs\parsernamespace\saferword\endcsname##1{{\ntt\saferword}}}
\def\prettytoken@#1{\toksa{}\numberstocharsandspaces#1\end\expandafter\edef
    \csname\prettynamecs\parsernamespace{\the\toksa}\endcsname##1{{\ntt\the\toksa}}%
}

\def\prettywordpair@@#1#2{\setsafe{#1}\expandafter\def\csname\prettynamecs\parsernamespace\saferword\endcsname##1{#2}}

\def\prettywordpair#1#2{\prettywordpair@@{#1}{{\ntt@#2}}}

\def\prettywordpair@#1#2{% a `weak symbol' version of the above macro, so that the new typesetting can be overriden 
    \setsafe{#1}\toksa{}\numberstocharsandspaces#2\end\expandafter\nameproc\expandafter{\the\toksa}\with\parsebin
    \expandafter\edef\csname\prettynamecs\parsernamespace\saferword\endcsname##1{%
        \noexpand\ifdisplaytokenraw
            {\ntt\saferword}% \ntt expands to \noexpand\ntt@
        \noexpand\else
            {\ntt\def\noexpand\_{\char`\noexpand\_}\the\toksa}%
        \noexpand\fi
    }%
}

\def\prettywordpairwvis#1#2#3{%
    \prettywordpair{#1}{#2}%
    \expandafter\def\csname\viskeyref\parsernamespace{\saferword}\endcsname##1{#3}%
}

\def\prettywordpair@wvis#1#2#3{%
    \prettywordpair@{#1}{#2}%
    \expandafter\def\csname\viskeyref\parsernamespace{\saferword}\endcsname##1{#3}%
}

\def\prettywordpair@@wvis#1#2#3{%
    \prettywordpair@@{#1}{#2}%
    \expandafter\def\csname\viskeyref\parsernamespace{\saferword}\endcsname##1{#3}%
}

% a control sequence to reuse the token definition file for typesetting

\newif\iftraceprettytokens % display prettyfying control sequences

\def\tokeneqpretty#1#2{%
    \setsafe{#1}\toksa{}\numberstocharsandspaces#2\end
    \expandafter\nameproc\expandafter{\the\toksa}\with\parsebin
    \ifyyparsefail
        \toksc{{}$_{\rm m}${}}\toksd\toksa
    \else
        \toksc{}\toksd\toksb % quotes should trigger the creation of a visible key
    \fi
    \expandafter\edef\csname\prettynamecs\parsernamespace\saferword\endcsname##1{%
        \noexpand\ifdisplaytokenraw
            {\ntt\saferword}%
        \noexpand\else
            {\ntt\def\noexpand\_{\char`\noexpand\_}\the\toksa\the\toksc}%
        \noexpand\fi
    }%
    \expandafter\edef\csname\viskeyref\parsernamespace{\saferword}\endcsname##1{\the\toksd}%
    \iftraceprettytokens
        \message{token: \expandafter\meaning\csname\prettynamecs\parsernamespace\saferword\endcsname}%
        \message{visual key: \expandafter\meaning\csname\viskeyref\parsernamespace{\saferword}\endcsname}%
    \fi
}

% the bootstrap macros

\def\initbootstrap{%
    \restorecslist{bootstrap}\yyunion
}

\restorecslist{parser-prototypes}\yyunion

\def\tokendecls#1#2#3{#1}

\def\stringify#1#2#3#4{\toksa{"#1"}}

\def\idit#1#2#3#4{\toksb{#1}}

%\def\anint#1#2#3{\toksc{#1}}% this definition is not needed as all
                             % the information is extracted from yytname

% note that the definitions below do not handle cases such as 
%token '{' "left brace" since the lexer knows how to return '{'

\def\onesymbol#1#2#3{%
    \toksa{}\toksb{}%\toksc{}%
    #1#2#3%
    \yytoksempty\toksa{\expandafter\charstonumbers\the\toksb\end}%
                      {\expandafter\toksa\expandafter{\expandafter}\expandafter\charstonumbers\the\toksa\end}%
    \yytoksempty\toksb{%
        \yytoksempty\toksa{}{%
            \immediate\write\tokendefs{\noexpand\tokeneq{\the\toksb}{\the\toksa}}%
        }%
    }{%
        \immediate\write\tokendefs{\noexpand\tokeneq{\the\toksb}{\the\toksa}}%
    }%
}

\def\flexsstatedecls#1#2#3{% bootstrap parser does not produce state declarations so it is fine to share states
    \savecslist{local-namespace}\yyunion
    \restorecslist{bootstrap:flex-state-declarations}\yyunion
        #1\relax
    \restorecslist{local-namespace}\yyunion
}

\let\flexxstatedecls\flexsstatedecls

\savecslist{bootstrap}\yyunion

\restorecslist{parser-strict}\yyunion

\def\idit#1#2#3#4{{\immediate\write\stlist{_register_name(#1);}}}

\def\hspace#1#2{}

\savecslist{bootstrap:flex-state-declarations}\yyunion

% correct a few macros for debugging

\restorecslist{parser-debug}\yyunion

\defc\actbraces{% action braces :: \actbraces{tex_string}{\idit|<nothing>}{fptr}{sptr}directive*\bdend
    \noexpand\actbraces{#1}{#2}{#3}{#4}#5\noexpand\bdend
}

\defc\bpredicate{% predicate :: \bpredicate{tex_string}{<nothing>}{fptr}{sptr}directive*\bdend
    \noexpand\bpredicate{#1}{}{#3}{#4}#5\noexpand\bdend
}

\defc\rhs{% a `right-hand side' :: \rhs{[\termname\hspace\rarhssep\arhssep\actbraces]+}{\arhssep|<nothing>}{\ifrhsfull}
    \noexpand\rhs{#1}{#2}{}% omit setting the conditional
}

\toyyunion{parser-debug}

% \romannumeral-expandable macros for character translation

\def\charstonumbersre#1\end{\charstonumbersr@0\end#1\end}

\def\charstonumbersr@#1\end#2\end{%
    \yystringempty{#2}{#1}%
        {\yystartsinspace{#2}{\expandafter\charstonumbers@@swap\expandafter{\number`\ }{#1}#2\end}{\charstonumb@rsr@#1\end#2\end}}%
}

\def\charstonumb@rsr@#1\end#2{%
    \expandafter\charstonumbersr@swap\expandafter{\number`#2}{#1}%
}

\def\charstonumbersr@swap#1#2{\charstonumbersr@#2{#1}\end}

\def\charstonumbers@@swap#1#2 {\charstonumbersr@#2{#1}\end}

% variations: set \toksa to the expansion, \edef-expandable version

\def\charstonumbers#1\end{\toksa\expandafter{\romannumeral\charstonumbersre#1\end}}
\def\charstonumberse#1\end{\romannumeral\charstonumbersre#1\end}

% name parser macros: currently only one suffix is handled, the very last one

\def\yyuniontag{\yynpunion}
\def\parserstrictnamespace{nameparser-strict}
\def\parserprototypesnamespace{nameparser-strict:headers}
\def\parserdebugnamespace{nameparser-debug}

\def\yynpunion{\currentyyunionnamespace}

\defp\idstr#1#2{} % idenifier ::
\defp\bidstr#1#2{} % bison variable ::
\defp\chstr#1#2{} % character ::
\defp\sfxi#1#2{} % literal suffix ::
\defp\sfxn#1#2{} % integer suffix ::
\defp\dotsp{} % suffix separator ::
\defp\optstr#1#2{} % option ::
\defp\qual#1#2{} % qualifier ::
\defp\visflag#1#2{} % visible key change flag ::

\toyyunion{nameparser-prototypes}

\defc\idstr{%
   \appendrnx\toksa{#1}%
   \appendrnx\toksc{#2}%
   \toksb{}%
}

\defc\bidstr{%
   \appendrnx\toksa{{}$\Upsilon${}}%
   \appendrnx\toksc{#2}%
   \toksb{}%
}

\defc\chstr{%
   \appendrnx\toksa{{\ntt@#1}}%
   \appendrnx\toksc{#2}%
   \toksb{}%
}

% the \toksb, set in the next two macros is used by \optstr later

\defc\sfxi{\toksb{\/${}_{\idxfont#1}$}\appendrnx\toksa{\/${}_{\idxfont#1}$}}

\defc\sfxn{\toksb{#1}\appendrnx\toksa{#1}}

\defc\dotsp{\futurelet\next\d@tsp}

\def\d@tsp{%
    \ifx\next\sfxi
    \else
        \ifx\next\qual
        \else
            \appendrnx\toksa{.}%
        \fi
    \fi
}

\defc\optstr{%
   \toksb\expandafter{\eatone#1}%
   \optstrextra
   \concat\toksa\toksb
   \appendrnx\toksc{#2}%
   \toksb{}%
}

\def\optstrextra{%
    \edef\next{\toksb{{$\nx\ulcorner$\ntt\the\toksb$\nx\urcorner$}}}\next
}

\def\optstrextraesc{%
    \edef\next{\toksb{{$\nx\langle$\ntt\the\toksb$\nx\rangle$}}}\next
}

\defc\qual{\appendrnx\toksd{\/${}_{\rm#2}$}\toksb{}}

\let\idxfont\relax

\defc\visflag{\toksf{#1}\appendrnx\tokse{#1}}% flag visible key change
                                   % \toksf contains the typesetting hint sequence

\toyyunion{nameparser}

\newtoks\namechars

\newif\iftracebadnames
\newif\iftracenames

\def\viskeyref#1#2{% getting the possible visual key from the index key
                   % #1 is the namespace of the original key
                   % #2 is the key
      \restorecsname{[nameparser:visual]}{.\prettynamecs{#1}{#2}}%
}

% ignore explicit (\ ) spaces while parsing names
% the macro will be called whenever \tosmallparser is executed (see yyinit.sty);
% this is neccessary due to a shortcut taken by the macros that display
% \bison\ productions in text (see \beginprod macros in yyinit.sty).

\def\ignoreexplicitspace{\yyinput}

% the main name processing routine
% does not have any side effects;
% the parsed term is returned as #1 of the sequence in #2; #2 of #2 will hold a visual key if
% one exists, otherwise it will be empty; #3 is \yyparsefailtrue or empty
% #4 is the prettyfied version of the term if one exists; #5 is the typesetting hint from \visflag sequences;
% the version below prettifies tokens only if the parsing was successful; if
% needed, prettyfication may be applied after the parsing is complete

\def\nameproc#1\with#2{%
    {% keeping all changes local
        \iftracenames
            \toksa{#1}\ferrmessage{original name: \the\toksa}%
        \fi
        \tosmallparser\basicparserinit\d@parse{#1}%
        \ifyyparsefail
            \iftracebadnames
                \toksa{#1}\ferrmessage{bad name: \the\toksa}%
            \fi
            \toksb{#2{#1}{}{\yyparsefailtrue}{}{}}%
        \else
            \npbuildnames{#2}%   
        \fi
        \expandafter
    }\the\toksb
}

\def\npbuildnames#1{%
    \iftracenames
        \ferrmessage{parsed name (namespace: \parsernamespace): \the\namechars}%
    \fi
    \toksa{}\toksb{}\toksc{}\toksd{}\tokse{}\toksf{}%
    \restorecslist{nameparser}\yynpunion    
    \the\namechars\relax
    \iftracenames
        \ferrmessage{processed name: \the\toksa<identifier>, \the\toksb<suffices>, %
            \the\toksc<hash name>, \the\toksd<options>, \the\tokse<visible key>}%
    \fi
    \expandafter\npsetkeys\expandafter{\the\toksc}%
    \iftracenames
        \ferrmessage{final name: \the\toksa\yytoksempty\toksc{}{, prettyfied name: \the\toksc}%
                                           \yytoksempty\toksf{}{, hint: \the\toksf}%
                                           \yytoksempty\tokse{}{, visual key: \the\tokse}, %
                                           parsing \ifyyparsefail failed\else successful\fi
        }%
    \fi
    \toksb{#1}%
    \edef\next{%
        \toksb{%
            \the\toksb
            {\the\toksa}% the parsed name
            {\the\tokse}% the visual key
            {}% parseflag
            {\the\toksc}% the prettified name
            {\the\toksf}% the typesetting hint
        }%
    }\next
}

\def\npsetkeys#1{%
    \expandafter\ifx\csname\prettynamecs\hostparsernamespace{#1}\endcsname\relax
        \toksc{}%
    \else
        \toksc\expandafter{\csname\prettynamecs\hostparsernamespace{#1}\endcsname}%
        \appendr\toksc{{\currentrulecontext}\the\toksb\the\toksd}%
        \toksf{\termidstring}%
    \fi
    \concat\toksa\toksd % changes \toksa and \toksc, no need for \toksb or \toksd after this
    \expandafter\ifx\csname\viskeyref\hostparsernamespace{#1}\endcsname\relax
        % no special visual key for this token
        \yybreak{%
            \yytoksempty\tokse{}{\makeviskeydirect}% if the visual key changed, output the key,
                                                   % attempt to compute it form the parsed output
        }%
    \else
        \yybreak{\makeviskeyfrompretty{#1}}% modify the production of the visible key to use the
                                       % prettified version
    \yycontinue % changes \toksb, \toksd, and \tokse
}

% standard postprocessing function

\def\parsebin#1#2#3#4#5{%
    \yystringempty{#4}{%
        \yystringempty{#3}{\toksa{\eatone{#5}#1}}{\toksa{#1}}%
    }{%
        \yystringempty{#3}{\toksa{\eatone{#5}#4}}{\toksa{#4}}%
    }%
    \toksb{#2}%
    \yystringempty{#3}{\yyparsefailfalse}{\yyparsefailtrue}%
}

% creating a visual key from the stored visual key and the sufixes

\restorecslist{nameparser-prototypes}\yynpunion % make most sequences do nothing

\defc\qual{\toksb{#2}\concat\toksd\toksb}

\toyyunion{nameparser:vispretty}

\def\makeviskeyfrompretty#1{%
    \edef\next{\tokse{\expandafter\expandafter\csname\viskeyref
        \hostparsernamespace{#1}\endcsname\expandafter{\currentrulecontext}}}\next
    \restorecslist{nameparser:vispretty}\yynpunion%
    \toksd{}%
    \the\namechars
    \yytoksempty\toksd{}{\appendr\tokse.\concat\tokse\toksd}%
}

% creating a visual key directly from parsed output

\restorecslist{nameparser-strict}\yynpunion

\defc\idstr{\toksb{#2}\concat\tokse\toksb}

\defc\bidstr{\toksb{#2}\concat\tokse\toksb}

\defc\chstr{\toksb{#2}\concat\tokse\toksb}

\defc\sfxi{\toksb{#2}\concat\tokse\toksb}

\defc\sfxn{\toksb{#2}\concat\tokse\toksb}

\defc\dotsp{\appendr\tokse.}

\defc\optstr{\toksb{#2}\concat\tokse\toksb}

\defc\qual{\toksb{#2}\concat\tokse\toksb}

\defc\visflag{}

\toyyunion{nameparser:visdirect}

\def\makeviskeydirect{%
    \tokse{}%
    \restorecslist{nameparser:visdirect}\yynpunion
    \the\namechars
}

\restorecslist{nameparser-strict}\yynpunion

% \flex\ regex typesetting routine and \flex\ parser value stack types

\def\frexproc#1\with#2{%
    {% keeping all changes local
        \iftracenames
            \toksa{#1}\ferrmessage{original regex: \the\toksa}%
        \fi
        \toflexreparser
        \basicparserinit
        \flexreparserinit
        \flexreparserdatainit
        \d@parse{\ #1}% the parsed regex is put in the \table register
        \ifyyparsefail
            \iftracebadnames
                \toksa{#1}\ferrmessage{bad regex: \the\toksa}%
            \fi
            \toksb{#2{#1}{}{\yyparsefailtrue}}%
        \else
            \toksd{#1}%
            \frebuildx{#2}%   
        \fi
        \expandafter
    }\the\toksb
}

\def\frebuildx#1{%
    \toksb{#1}\toksc{}%
    \restorecslist{flexparser-re}\yyflunion
    \the\table % typeset the regex (appears in \toksc)
    \edef\next{%
        \toksb{%
            \the\toksb % the name of the processing routine
            {\the\toksd}% the original regex
            {\the\toksc}% processed regex (or empty)
            {}% parse flag
        }%
    }\next
}

\def\ppregex#1#2#3{%
    \toksc{#2}%
}

\def\setregextable{%
    \restorecslist{flexparser-sect2}\yyflunion
}

\def\setregexdeftable{%
    \restorecslist{flexparser-sect1}\yyflunion
}

% \flex\ parser indexing macros

\def\flexidxdomain{F}
\def\fstatedefidxrank{0}
\def\fstateidxrank{1}
\def\fregexidxrank{2}
\def\ftextrefrank{5}

\def\writeidxfsentry#1{% \flex\ state reference entry
      \indxe\gindex{{\secno}{{}{\flexpseudonamespace}}{\flexidxdomain}{\fstateidxrank}#1}%
}

\def\writeidxfstextentry#1{% \flex\ state reference in text
      \indxe\gindex{{\secno}{{}{\flexpseudonamespace}}{\flexidxdomain}{\ftextrefrank}#1}%
}

\def\writeidxfsdentry#1{% \flex\ state definition entry
      \indxe\gindex{{\secno}{{}{\flexpseudonamespace}}{\flexidxdomain}{\fstatedefidxrank}#1}%
}

\def\writeidxfsrdentry#1{% \flex\ regex name definition entry
      \indxe\gindex{{\secno}{{}{\flexpseudorenamespace}}{\flexidxdomain}{\fstatedefidxrank}#1}%
}

\def\writeidxfsrnentry#1{% \flex\ regex name entry
      \indxe\gindex{{\secno}{{}{\flexpseudorenamespace}}{\flexidxdomain}{\fregexidxrank}#1}%
}

\def\writeidxfscclentry#1{% \flex\ chracter class name entry
      \indxe\gindex{{\secno}{{}{\flexpseudonamespace}}{\flexidxdomain}{\fregexidxrank}#1}%
}

% \flex\ parser typesetting macros and stack types

\def\yyuniontag{\yyflunion}
\def\parserstrictnamespace{flexparser-strict}
\def\parserprototypesnamespace{flexparser-strict:headers}
\def\parserdebugnamespace{flexparser-debug}

\def\yyflunion{\currentyyunionnamespace}

\defp\flnametok#1#2{} % named definition :: \flnametok{matched text}{formatting command}, the text is \{text\}\ *
%
\defp\flcclexpr#1#2#3#4{} % character class expression :: \flcclexpr{matched text}{ccl token}{fptr}{sptr}
\defp\flchar#1#2#3#4{} % character :: \flchar{char}{char12}{fptr}{sptr}
\defp\flstring#1#2#3{} % string :: \flstring{{fptr}{sptr}}{\flchar{...}*}{{fptr}{sptr}}
\defp\flcclrnge#1#2{} % character range :: \flcclrnge{/flchar{...}}{/flchar{...}}
\defp\flbraceccl#1#2#3{} % character class :: \flbraceccl{{fptr}{sptr}}{ccl}{{fptr}{sptr}}
\defp\flbracecclneg#1#2#3{} % negative character class :: \flbracecclneg{{fptr}{sptr}}{ccl}{{fptr}{sptr}}
\defp\flcclunion#1#2{} % union of character classes :: \flcclunion{ccl}{ccl}
\defp\flccldiff#1#2{} % difference of character classes :: \flccldiff{ccl}{ccl}
\defp\flrepeat#1{} % asterisk operation :: \flrepeat{re}
\defp\flrepeatstrict#1{} % plus operation :: \flrepeatstrict{re}
\defp\flrepeatonce#1{} % ? operation :: \flrepeatonce{re}
\defp\flrepeatnm#1#2#3{} % {n, m} operation :: \flrepeatnm{re}{num}{num}
\defp\flrepeatgen#1#2{} % {n,} operation :: \flrepeatgen{re}{num}
\defp\flrepeatn#1#2{} % {n} operation :: \flrepeatn{re}{num}
\defp\flparens#1#2#3#4{} % parenthesized regular expression :: \flparens{{fptr}{sptr}}{re}{{fptr}{sptr}}{formatting command}
\defp\fldot#1#2{} % dot expression :: \fldot{fptr}{sptr}
\defp\flor#1#2{} % `|' expression :: \flor{fptr}{sptr}
\defp\fltrail#1#2{} % regular expression sans trailing suffix :: \fltrail{re}{{fptr}{sptr}}
\defp\flretrail#1#2#3{} % regular expression with a trailing context :: \flretrail{re}{{fptr}{sptr}}{re}
\defp\flreateol#1#2#3{} % regular expression with the end of line as a trailing context :: \flreateol{re}{fptr}{sptr}
\defp\flrule#1#2{} % full regular expression :: \flrule{re}{formatting command}
\defp\flbolrule#1#2{} % full regular expression at the beginning of the line :: \flbolrule{re}{formatting command}
\defp\fleof#1#2{} % end of file rule :: \fleof{fptr}{sptr}
\defp\flsconlist#1#2#3{} % start condition list :: \flsconlist{{fptr}{sptr}}{\flname...}{{fptr}{sptr}}
\defp\flsconuniv#1#2{} % universal start condition :: \flsconuniv{fptr}{sptr}
\defp\flnamesep#1#2{} % separator between names :: \flnamesep{fptr}{sptr}
\defp\flname#1#2#3#4{} % name :: \flname{name}{name12}{fptr}{sptr}
\defp\flopt#1#2#3#4#5{} % \flex option :: \flopt{option}{text}{text12}{fptr}{sptr}
\defp\floptions#1{} % \flex options :: \floptions{\flopt ...}
\defp\flscondecl#1#2#3#4{} % start condition declarations :: \flscondecl{x|s}{fptr}{sptr}{\flname ...}
\defp\flaction#1#2#3#4#5#6#7{} % \flex action :: \flaction{\flsconlist...}{\fl..rule ...}{fptr}{sptr}{fptr}{sptr}{formatting command}
\defp\flactionc#1#2#3#4#5#6#7{} % \flex continued action :: \flaction{\flsconlist...}{\fl..rule ...}{fptr}{sptr}{fptr}{sptr}{formatting command}
\defp\flactiongroup#1#2#3#4#5#6{} % a group of \flex actions :: \flactiongroup{\flsconlist...}{fptr}{sptr}{\flaction...}{fptr}{sptr}
\defp\flbareaction#1#2#3#4#5#6{} % bare action :: \flbareaction{fptr}{sptr}
\defp\flptropt#1#2{} % %ponter directive :: \flptropt{fptr}{sptr}
\defp\flarrayopt#1#2{} % %array directive :: \flarrayopt{fptr}{sptr}
\defp\fltopopt#1#2#3#4{} % %top directive :: \fltopopt{fptr}{sptr}{fptr}{sptr}
\defp\flredef#1#2#3#4#5#6#7#8{} % regular expression definition :: \flredef{name}{name12}{fptr}{sptr}{text}{text12}{fptr}{sptr}

\toyyunion{flexparser-prototypes}

\restorecslist{flexparser-strict}\yyflunion

\defc\flrule{%#1#2% full regular expression :: \flrule{re}{formatting command}
    {\toksc{}#1#2\expandafter}\expandafter\toksd\expandafter{\the\toksc}%
    \concat\toksc\toksd
}

\newif\iftextre % check if this is a free standing regular expression or appears in the definitions table

\defc\flbolrule{%#1#2% full regular expression at the beginning of the line :: \flbolrule{re}{formatting command}
    {\toksc{}#1#2\appendlnx\toksc{\raise0.5pt%
        \iftextre\yybreak{\hbox}\else\yybreak{\llap}\yycontinue{\sscmd$\dashv$\ }}\expandafter}\expandafter%
        \toksd\expandafter{\the\toksc}\concat\toksc\toksd
}

\let\astformat@flrule\empty

\def\inscomment#1{% insert a comment before the rule
    \def\astformat@flrule{%
        \appendlnx\toksc{\noalign{#1}}%
    }%
}

\def\insrulealign#1#2{%
    \def\astformat@flrule{%
        \tokse{#1}\toksf{#2}%
        \concatl\tokse\toksc
        \concat\toksc\toksf
    }%
}

\def\rulealigntemplate{% to hide the hash symbols from \\edef operators
    \vbox\bgroup\halign\bgroup##\hfil&##\hfil\cr
}

% regular expression symbols

\defc\flstring{%#1#2#3% string :: \flstring{{fptr}{sptr}}{\flchar{...}*}{{fptr}{sptr}}
%    \toksc\expandafter{\the\toksc\hbox{#2}}%
    #2%
}
  
\defc\flnametok{%#1#2% named definition :: \flnametok{matched text}{formatting command}, the text is \{text\}\ *
    \let\termindex\writeidxfsrnentry
    \fln@metok#1\end{#2}%
    \let\termindex\eatone
}

\def\fln@metok\{#1\}#2\end#3{%
    {%
        \let\hostparsernamespace\flexpseudorenamespace
        \nameproc{#1}\with\parsebin
        \edef\next{%
            \toksd{\toksd{\the\toksa}\tokse{\the\toksb}}%
        }\next % a trick to `reshuffle' the output of \nameproc:
        % the parsed name goes to \toksd and the visual key is put in \tokse
        \expandafter
    }\the\toksd
    \toksf{}\toksg{}%
    #3%
    \edef\next{%
        \toksc{\the\toksc\the\toksf{}\nx\flexrendisplay{\the\toksd}%
            \gidxentryxv{\termfsrestring}{#1}{\the\tokse}\the\toksg
        }%
    }\next
}

\def\insnamealign#1#2{%
    \def\astformat@flnametok{%
        \toksf{#1}%
        \toksg{#2}%
    }%
}

\let\astformat@flnametok\empty

\defc\flchar{%#1#2#3#4% character :: \flchar{char}{char12}{fptr}{sptr}
    \yystringempty{#2}{#1}{\flch@r#2\end}%
}

\def\flch@r#1#2\end{%
    \ifnum`#1=`\\
        \fl@h@r#2\end
    \else
        \toksc\expandafter{\the\toksc{\ntt#1#2}}%
    \fi
}

\def\fl@h@r#1#2\end{% process the sequence after an escape
    {%
        \let\default\flescdefault
        \tokse{#1}\toksd{#2}%
        \switchon{\the\tokse}\in\flesccharswitch
        \edef\next{\toksc{\toksc{\the\toksc\the\toksd}}}\next
        \expandafter
    }\the\toksc
}

\def\flesccharswitch{% recognizing different types of escape sequences
01234567
    {% octal constant
        \edef\next{\toksd{\nx\.{\the\tokse\the\toksd}$_8${}}}\next
    }
nfrtabv
    {% \Cee\ escape sequence
        \edef\next{\toksd{{\nx\sscmd\nx\sscmd$\nx\langle$\nx\.{\the\tokse\the\toksd}$\nx\rangle$}}}\next
    }
x
    {% hexadecimal constant
        \edef\next{\toksd{\nx\.{\the\toksd}$_{16}${}}}\next
    }
}

\def\flescdefault{%
    \edef\next{\toksd{\nx\.{\the\tokse\the\toksd}}}\next
}

\setspecialcharsfrom\flesccharswitch

\defc\fldot{%#1#2% dot expression :: \fldot{fptr}{sptr}
    \toksc\expandafter{\the\toksc.}%
}

\defc\flparens{%#1#2#3#4% parenthesized regular expression :: \flparens{{fptr}{sptr}}{re}{{fptr}{sptr}}{format command}
    {\toksc{}#2\expandafter}\expandafter\toksd\expandafter
        {\the\toksc{\rm)}}%
    \toksf{}\toksg{}#4%
    \tokse\expandafter{\the\toksf{\rm(}}%
    \concatl\tokse\toksd
    \concat\toksc\toksd
    \concat\toksc\toksg
}

\def\insparensalign#1#2{%
    \def\astformat@flparens{%
        \toksf{#1}%
        \toksg{#2}%
    }%
}

\let\astformat@flparens\empty

\defc\flrepeat{%#1% asterisk operation :: \flrepeat{re}
    {\toksc{}#1\expandafter}\expandafter\toksd\expandafter
        {\the\toksc{{}${}_{*}$}}%
    \flselectiveseparator
    \concat\toksc\toksd%
}

\defc\flrepeatn{%#1#2% {n} operation :: \flrepeatn{re}{num}
    {\toksc{}#1\expandafter}\expandafter\toksd\expandafter
        {\the\toksc{{}${}_{\{\yyfirstoftwo#2\}}$}}%
    \flselectiveseparator
    \concat\toksc\toksd%
}

\defc\flrepeatnm{%#1#2#3% {n, m} operation :: \flrepeatnm{re}{num}{num}
    {\toksc{}#1\expandafter}\expandafter\toksd\expandafter
        {\the\toksc{{}${}_{\{\yyfirstoftwo#2,\yyfirstoftwo#3\}}$}}%
    \flselectiveseparator
    \concat\toksc\toksd%
}

\defc\flrepeatonce{%#1% ? operation :: \flrepeatonce{re}
    {\toksc{}#1\expandafter}\expandafter\toksd\expandafter
        {\the\toksc{{}${}_{?}$}}%
    \flselectiveseparator
    \concat\toksc\toksd%
}

\defc\flrepeatstrict{%#1% plus operation :: \flrepeatstrict{re}
    {\toksc{}#1\expandafter}\expandafter\toksd\expandafter{\the\toksc{{}${}_{+}$}}%
    \flselectiveseparator
    \concat\toksc\toksd%
}

\def\flselectiveseparator{%
    \yytoksempty\toksc{}{%
        \tokse{{{}$\,$}}%
        \concatl\tokse\toksd
    }%
}

\defc\flcclrnge{%#1#2% character range :: \flcclrnge{\flchar{...}}{\flchar{...}}
%    \toksd{#1}\getfirst\toksd\to\toksd
%    \tokse{#2}\getfirst\tokse\to\tokse
    #1\appendr\toksc{{\nx\rm--}}#2%
%    \edef\next{\toksc{\the\toksc\the\toksd{\nx\rm--}\the\tokse}}\next
}

\defc\flcclexpr{%#1#2#3#4% character class expression :: \flcclexpr{matched text}{ccl token}{fptr}{sptr}
    {%
        \let\hostparsernamespace\flexpseudonamespace
        \nameproc{#2}\with\parsebin
        \edef\next{%
            \toksd{\toksd{\the\toksa}\tokse{\the\toksb}}%
        }\next % a trick to `reshuffle' the output of \nameproc:
        % the parsed name goes to \toksd and the visual key is put in \tokse
        \expandafter
    }\the\toksd
    \let\termindex\writeidxfscclentry
    \edef\next{\toksc{\the\toksc{\the\toksd}%
        \gidxentryxv{\termidstring}{#2}{\the\tokse}%
    }}\next
    \let\termindex\eatone
}

\defc\flbraceccl{%#1#2#3% character class :: \flbraceccl{{fptr}{sptr}}{ccl}{{fptr}{sptr}}
    \appendrnx\toksc{{}$[${}}%
    #2%
    \appendrnx\toksc{{}$]${}}%
}

\def\flbraceccldemo#1#2#3{% character class :: \flbraceccl{{fptr}{sptr}}{ccl}{{fptr}{sptr}}
    % a demo version without the beginning bracket
    \appendrnx\toksc{{}$\ldots${}}%
    #2%
    \appendrnx\toksc{{}$]${}}%
}

\defc\flccldiff{%#1#2% difference of character classes :: \flccldiff{ccl}{ccl}
    #1%
    \appendr\toksc{${}\nx\setminus{}$}%
    #2%
}

\defc\flbracecclneg{%#1#2#3% negative character class :: \flbracecclneg{{fptr}{sptr}}{ccl}{{fptr}{sptr}}
    \appendr\toksc{{}$[${}}%
    #2%
    \appendr\toksc{{}$]^c${}}%
}

\defc\fleof{%#1#2% end of file rule :: \fleof{fptr}{sptr}
    \let\termindex\writeidxfsrnentry
    \edef\next{\toksc{\the\toksc{}\nx\flexrendisplay{EOF}%
        \gidxentry{\termfsrestring}{EOF}{EOF}}}\next
    \let\termindex\eatone
}

\defc\flor{%#1#2% `|' expression :: \flor{fptr}{sptr}
    \appendr\toksc{{}${}\nx\mid{}${}}%
}

\defc\flsconuniv{%#1#2% universal start condition :: \flsconuniv{fptr}{sptr}
    \let\termindex\writeidxfsentry
    \edef\next{\toksb{\the\toksb{}$\langle*\rangle${}%
         \gidxentryxv{\termvstring}{*}{*}}}\next % TODO: create a separate name space for state names
    \let\termindex\eatone
}

\defc\fltrail{%#1#2% regular expression sans trailing suffix :: \fltrail{re}{{fptr}{sptr}}
    \errmessage{\nx\fltrail\space macro has not been implemented (see yyunion.sty).}
}

\defc\flretrail{%#1#2#3% regular expression with a trailing context :: \flretrail{re}{{fptr}{sptr}}{re}
    \errmessage{\nx\flretrail\space macro has not been implemented (see yyunion.sty).}
}

\defc\flreateol{%#1#2#3% regular expression with the end of line as a trailing context :: \flreateol{re}{fptr}{sptr}
    {\toksc{}#1\expandafter}\expandafter\toksd\expandafter{\the\toksc{\raise0.5pt\rlap{\sscmd\ $\vdash$}}}%
    \concat\toksc\toksd%
}

\toyyunion{flexparser-re}

\def\flexsndisplay#1{% state name typesetting style
    \hbox{\tt#1}%
}

\def\flexrendisplay#1{% regular expression name typesetting style
    \penalty-100 \hbox{$\langle$\tt#1$\rangle$}%
}

\def\flexsnstyle#1{% typesetting \flex\ state names in text
    {%
        \let\parsernamespace\flexpseudonamespace
        \nameproc{#1}\with\parsebin
        \edef\next{%
            \toksd{\the\toksa}\tokse{\the\toksb}%
        }\next % a trick to `reshuffle' the output of \nameproc:
        % the parsed name goes to \toksd and the visual key is put in \tokse
        \let\termindex\writeidxfstextentry
        \edef\next{\toksd{\nx\flexsndisplay{\the\toksd}%
            \gidxentryxv{\termvstring}{#1}{\the\tokse}}}\next
        \expandafter
    }\the\toksd
}

\def\flexrenstyle#1{% typesetting \flex\ regular expression names in text
    {%
        \let\parsernamespace\flexpseudorenamespace
        \nameproc{#1}\with\parsebin
        \edef\next{%
            \toksd{\the\toksa}\tokse{\the\toksb}%
        }\next % a trick to `reshuffle' the output of \nameproc:
        % the parsed name goes to \toksd and the visual key is put in \tokse
        \let\termindex\writeidxfstextentry
        \edef\next{\toksd{\nx\flexrendisplay{\the\toksd}%
            \gidxentryxv{\termfsrestring}{#1}{\the\tokse}}}\next
        \expandafter
    }\the\toksd
}

\def\flexrestyle#1{% typesetting \flex\ regular expressions in text
    {%
        \frexproc{#1}\with\ppregex
        \edef\next{\toksc{\toksc{\the\toksc}}}\next
        \expandafter
    }\the\toksc
    \hbox{\textretrue\tt\the\toksc}%
}

\defc\flactiongroup{% #1#2#3#4#5#6% a group of \flex actions :: \flactiongroup{\flsconlist...}{fptr}{sptr}{\flaction...}{fptr}{sptr}
    \toksb{}#1% collect all the state name information in \toksb
    % gather the preceeding stash
    \readstash{#3}% read stash late, after \flsconslist (ignore it)
    \edef\next{%
        \toksb{\noalign{\hbox to\nx\hsize{\hskip\the\rgindent
        \nx\flexsndisplay{\the\toksb}:\toksa{\relax\the\toksg}\nx\makestashbox\nx\hfil}}}%
    }\next
    \yytoksempty\table{}{\appendrnx\table{\noalign{\medskip}}}%
    \concat\table\toksb
    \advance\rgindent by\parindent
    #4%
    \advance\rgindent by-\parindent
}

\defc\flsconlist{% #1#2#3% start condition list :: \flsconlist{{fptr}{sptr}}{\flname...}{{fptr}{sptr}}
    \readstashwredirect#1\toksg % save the stash preceeding the conditions list to reinsert later
    #2% collect the state names (and the stash in between)
    \readstashwredirect#3\toksf
    \appendr\toksb{\nx\rm\toksa{\the\toksf}\nx\makestashbox}%
}

\defc\flname{% #1#2#3#4% name :: \flname{name}{name12}{fptr}{sptr}
    {%
        \let\parsernamespace\flexpseudonamespace
        \nameproc{#2}\with\parsebin
        \edef\next{%
            \toksd{\toksd{\the\toksa}\tokse{\the\toksb}}%
        }\next % a trick to `reshuffle' the output of \nameproc:
        % the parsed name goes to \toksd and the visual key is put in \tokse
        \expandafter
    }\the\toksd
    \let\termindex\writeidxfsentry
    \readstashwredirect{}{#4}\toksf % do not touch \yystashlocal
    \appendr\toksb{{\nx\rm\toksa{\the\toksf}\nx\makestashbox}%
        \the\toksd\gidxentryxv{\termvstring}{#1}{\the\tokse}}%
    \let\termindex\eatone
}

\defc\flnamesep{%#1#2% separator between names :: \flnamesep{fptr}{sptr}
    \toksb\expandafter{\the\toksb\ }%
}

\newskip\rgindent

\rgindent=\parindent

\defc\flaction{%#1#2#3#4#5#6#7% \flex action :: \flaction{\flsconlist...}{\fl..rule ...}{fptr}{sptr}{fptr}{sptr}{formatting command}
    \toksb{}#1%
    \toksc{}#2%
    #7% apply formatting
    \readstash{#6}% read stash late, after \flsconlist and \fl...rule
    \yytoksempty\toksb{%
        \appendr\table{\hskip\the\rgindent\hbox{\nx\tt\nx\strut\the\toksc}}%
    }{%
        \appendr\table{%
            \hskip\the\rgindent
            \vbox{%
                {\nx\sscmd\nx\expandafter}\nx\expandafter\baselineskip\nx\the\baselineskip
                \halign{&####\hfil\cr
                    \hbox to\rgindent{\nx\sscmd\nx\tt\nx\strut\the\toksb\hss}\cr
                    \nx\tt\nx\strut\the\toksc\cr
                }%
            }%
        }%
    }%
    \appendr\table{&\toksa{\relax\the\yystashlocal}\nx\cr}%
}

\defc\flactionc{%#1#2#3#4#5#6#7% \flex continued action :: \flaction{\flsconlist...}{\fl..rule ...}{fptr}{sptr}{fptr}{sptr}{formatting command}
    \toksb{}#1%
    \toksc{}#2%
    \readstash{#6}% read stash late, after \flsconlist and \fl...rule
    #7% apply formatting
    \yytoksempty\toksb{%
        \appendr\table{\hskip\the\rgindent\hbox{\nx\tt\nx\strut\the\toksc}}%
    }{%
        \appendr\table{%
            \hskip\the\rgindent
            \vbox{%
                {\nx\sscmd\nx\expandafter}\nx\expandafter\baselineskip\nx\the\baselineskip
                \halign{&####\hfil\cr
                    \hbox to\rgindent{\nx\sscmd\nx\tt\nx\strut\the\toksb\hss}\cr
                    \nx\tt\nx\strut\the\toksc\cr
                }%
            }%
        }%
    }%
    \appendr\table{&\toksa{{}$\nx\hookleftarrow$\relax\the\yystashlocal}\nx\cr}% in the event \cr has been redefined
}

\def\inscomment#1{%
    \def\astformat@flaction{%
        \toksd{#1}%
        \appendr\table{\noalign{\nx\smallskip\noindent\hskip\the\rgindent
            $\nx\triangleright\;$\the\toksd\nx\smallskip}}%
    }%
}

\let\astformat@flaction\empty

\toyyunion{flexparser-sect2}

% \flex\ setion~1 typesetting
% currently, all \flex\state declarations are handled by an extended \bison\ parser
% to collect the state data; we thus only implement regular expression definitions;

\restorecslist{flexparser-strict}\yyflunion

\defc\flredef{%#1#2#3#4#5#6#7#8% regular expression definition :: \flredef{name}{name12}{fptr}{sptr}{text}{text12}{fptr}{sptr}
    \let\termindex\writeidxfsrdentry
    {%
        \let\hostparsernamespace\flexpseudorenamespace
        \nameproc{#1}\with\parsebin
        \edef\next{%
            \toksd{\toksd{\the\toksa}\tokse{\the\toksb}}%
        }\next % a trick to `reshuffle' the output of \nameproc:
        % the parsed name goes to \toksd and the visual key is put in \tokse
        \expandafter
    }\the\toksd
    \frexproc{#5}\with\ppregex
    \appendr\table{{}$\nx\langle\hbox{\ntt\the\toksd}\nx\rangle${}%
        \gidxentryxv{\termfsrestring}{#1}{\the\tokse}&\the\toksc\cr% TODO: make a special type processor for regex names
    }%
    \let\termindex\eatone
}

\defc\flopt{%#1#2#3#4#5% \flex option :: \flopt{option}{text}{text12}{fptr}{sptr}
    \csname flex@ption_#1\endcsname{#2}{#3}%
}

\expandafter\def\csname flex@ption_deprecated\endcsname#1#2{%
    \flex@opt@deprecated#2%
}

\def\flex@opt@deprecated#1#2#3^^J{%
    \let\termindex\writeidxfsrnentry
    \appendr\table{$\langle${\nx\tt #2}$\rangle_{\rm f}$\gidxentryxv{\termfsopstring}{#2}{#2}%
        \the\toksg&\number\flexgetnumber#3.\nx\cr}%
    \let\termindex\empty
}

\def\flexgetnumber#1{%
    \ifnum`#1<"3A
        \yybreak{%
            \ifnum`#1>"2F
                \yybreak{#1\flexgetnumber}%
            \else
                \yybreak{%
                    \ifnum`#1="20
                        \yybreak{\flexgetnumber}%
                    \else
                        \yybreak{}%
                    \yycontinue
                }
            \yycontinue
        }%
    \else
        \yybreak{}%
    \yycontinue
}

\toyyunion{flexparser-sect1}

% correct a few sequences for debugging purposes

\restorecslist{flexparser-debug}\yyflunion

\defc\flnametok{%#1#2% named definition :: \flnametok{matched text}{formatting command}, the text is \{text\}\ *
    \noexpand\flnametok{#1}{}%
}

\defc\flparens{%#1#2#3#4% parenthesized regular expression :: \flparens{{fptr}{sptr}}{re}{{fptr}{sptr}}{formatting command}
    \noexpand\flparens{#1}{#2}{#3}{}%
}

\defc\flrule{%#1#2% full regular expression :: \flrule{re}{formatting command}
    \noexpand\flrule{#1}{}%
}

\defc\flbolrule{%#1#2% full regular expression at the beginning of the line :: \flbolrule{re}{formatting command}
    \noexpand\flbolrule{#1}{}%
}

\defc\flaction{%#1#2#3#4#5#6#7% \flex action :: \flaction{\flsconlist...}{\fl..rule ...}{fptr}{sptr}{fptr}{sptr}{formatting command}
    \noexpand\flaction{#1}{#2}{#3}{#4}{#5}{#6}{}%
}

\defc\flactionc{%#1#2#3#4#5#6#7% \flex continued action :: \flaction{\flsconlist...}{\fl..rule ...}{fptr}{sptr}{fptr}{sptr}{formatting command}
    \noexpand\flactionc{#1}{#2}{#3}{#4}{#5}{#6}{}%
}

\toyyunion{flexparser-debug}