% Copyright 2012-2020, Alexander Shibakov
% This file is part of SPLinT
%
% SPLinT is free software: you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation, either version 3 of the License, or
% (at your option) any later version.
%
% SPLinT is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
% GNU General Public License for more details.
%
% You should have received a copy of the GNU General Public License
% along with SPLinT.  If not, see <http://www.gnu.org/licenses/>.

% finish parser initialization

\let\yylexreturn\yylexreturnregular

\genericparser 
    name: main, 
    ptables: cweb/gyytab.tex, 
    ltables: cweb/ltab.tex, 
    tokens: bo.tok, 
    asetup: {}, 
    dsetup: \newlexerstateextra\newparserstateextra, 
    rsetup: {},
    optimization: \optimizeall;%

% prologue parser

\genericparser 
    name: prologue, 
    ptables: cweb/dyytab.tex, 
    ltables: cweb/ltab.tex, 
    tokens: bo.tok, 
    asetup: {}, 
    dsetup: \newlexerstateextra\newparserstateextra, 
    rsetup: {},
    optimization: \optimizeall;%

% flex parser

\genericparser 
    name: flex, 
    ptables: cweb/fiptab.tex, 
    ltables: cweb/filtab.tex, 
    tokens: fo.tok, 
    asetup: {}, 
    dsetup: {},
    rsetup: {},
    optimization: {};% optimized by the driver (--optimize-tables, --optimize-actions)

% flex regex parser

\genericparser 
    name: flexre, 
    ptables: cweb/reptab.tex, 
    ltables: cweb/filtab.tex, 
    tokens: fo.tok, 
    asetup: {}, 
    dsetup: {},
    rsetup: {},
    optimization: {};% optimized by the driver (--optimize-tables, --optimize-actions)

% flex section 2 parser

\genericparser 
    name: flextwo, 
    ptables: cweb/raptab.tex, 
    ltables: cweb/filtab.tex, 
    tokens: fo.tok, 
    asetup: {}, 
    dsetup: \newparserstateextra,
    rsetup: {},
    optimization: {};% optimized by the driver (--optimize-tables, --optimize-actions)

% flex section 1 parser

\genericparser 
    name: flexone, 
    ptables: cweb/ddptab.tex, 
    ltables: cweb/filtab.tex, 
    tokens: fo.tok, 
    asetup: {}, 
    dsetup: {},
    rsetup: {},
    optimization: {};% optimized by the driver (--optimize-tables, --optimize-actions)

\let\flexpseudonamespace\flexnamespace
\let\flexpseudorenamespace\flexrenamespace

% parser for term names: this is not really a great idea in itself but rather an
% illustration of what is possible

\genericparser 
    name: small, 
    ptables: cweb/small_tab.tex, 
    ltables: cweb/small_dfa.tex, 
    tokens: {}, 
    asetup: {}, 
    dsetup: {}, 
    rsetup: \let\returnexplicitspace\ignoreexplicitspace, % ignore spaces in names
    optimization: \optimizeall;%

\tomainparser

% stage two macros: parsing

% \flex parser stack

% section 1 parser

\def\flexoneparserinit{%
    \yylessusedtrue
    \floption@sensetrue
}

\def\flexoneparserdatainit{%
    \table{}%
}

% regular expression parser

\def\flexreparserinit{%
    \yyBEGIN{SECT2}%
    \flin@ruletrue
    \yylessusedtrue
}

\def\flexreparserdatainit{%
    \table{}%
}

% section 2 parser

\def\flexparserinit{%
    \yyBEGIN{SECT2}%
    \def\flbracelevel{0}%
    \yylessusedtrue
}

\def\flexparserdatainit{%
    \table{}%
}

% output parsed tables

\long\def\displayoutputcode#1#2#3{% #1 is the output code (will be expanded by \write)
                                  % #2 is the output stream
                                  % #3 is the preamble (will be expanded by \write)
    \immediate\write#2{#3#1}%
}

\def\displayflextable#1{{%
    \hidecslist\cwebstreamchars
    \restorecslist{flexparser-debug}\yyflunion
    \newlinechar=`^^J%
    \expandafter\displayoutputcode\expandafter{\the\table}\exampletable
        {^^J\harmlesscomment \parsernamespace::parsed table #1:^^J}%
}}

\def\displaybisontable#1{{%
    \hidecslist\cwebstreamchars
    \restorecslist{parser-debug}\yyunion
    \newlinechar=`^^J%
    \expandafter\displayoutputcode\expandafter{\the\table}\exampletable
        {^^J\harmlesscomment \parsernamespace::parsed table #1:^^J}%
}}

% stage two parsing macros

\def\preparsebisongrammar{%
    \let\postparse\postparsebisongrammar
    \tomainparser
    \displayrawtable % do this after the parse namespaces are setup
    \basicparserinit
    \bisonparserinit
    \bisonparserdatainit
    \yyparse
}

\def\preparsebisonprologue{%
    \let\postparse\postparsebisonprologue
    \toprologueparser
    \displayrawtable % do this after the namespaces are setup
    \basicparserinit
    \bisonparserinit
    \bisonparserdatainit
    \yyparse
}

\def\preparseflexone{%
    \let\postparse\postparseflexone
    \toflexoneparser
    \displayrawtable % do this after the namespaces are setup
    \basicparserinit
    \flexoneparserinit
    \flexoneparserdatainit
    \yyparse
}

\def\preparseflextwo{%
    \let\postparse\postparseflextwo
    \toflextwoparser
    \displayrawtable % do this after the namespaces are setup
    \basicparserinit
    \flexparserinit
    \flexparserdatainit
    \yyparse
}

\def\preparsefallback#1{%
    \let\postparse\relax
    \message{#1}%
}

% stage three, postprocessing and typesetting

\newif\ifparseverbose

\def\postparsegeneric#1{%
    \ifyyparsefail
        \yybreak{%
            \ifparseverbose\ferrmessage{#1 parsing failed.}\fi
            \parserreset
        }%
    \else
        \yybreak{%
            \ifparseverbose\ferrmessage{#1 parsing successful.}\fi
            \ifsaveparseoutput\displayparsedtable{#1}\fi
            \typesetparsedtables
        }%
    \yycontinue
}

\def\parserreset#1\par{%
    \yyparsefailfalse % in case the next pass is a \relax
    \let\postparse\empty % ...
    \expandafter\skiptolsection\the\Binputtoks\par% start the next parsing pass, skip \6\hbox{} trash
}

\def\postparsebisongrammar{%
    \let\displayparsedtable\displaybisontable
    \let\typesetparsedtables\typesetalltables
    \postparsegeneric{(grammar)}%
}

\def\postparsebisonprologue{%
    \let\displayparsedtable\displaybisontable
    \let\typesetparsedtables\typesetalltables
    \postparsegeneric{(prologue)}%
}

\def\postparseflexone{%
    \let\displayparsedtable\displayflextable
    \let\typesetparsedtables\typesetfsonetables
    \postparsegeneric{(section 1)}%
}

\def\postparseflextwo{%
    \let\displayparsedtable\displayflextable
    \let\typesetparsedtables\typesetfstwotables
    \postparsegeneric{(section 2)}%
}

\def\displayrawtable{%
    \ifsaveparseoutput
        {\newlinechar=`^^J\immediate\write\exampletable{^^J\harmlesscomment 
         table before parsing:^^J\the\Binputtoks}}%
    \fi
    \ifchecktable 
        \ferrmessage{table before parsing: \the\Binputtoks}%
    \fi
}

\fillpstack{}{%
    \preparsebisongrammar
    \preparsebisonprologue
    {\preparsefallback{**}}%
    \relax % this \relax is necessary so that the braces above 
           % are not stripped by \poppstack
}

\fillpstack{b}{%
    \preparsebisongrammar
    \preparsebisonprologue
    {\preparsefallback{**}}%
    \relax % this \relax is necessary so that the braces above 
           % are not stripped by \poppstack
}

\fillpstack{fs1}{%
    \preparseflexone
    \preparseflextwo
    {\preparsefallback{==}}%
    \relax % this \relax is necessary so that the braces above 
           % are not stripped by \poppstack
}

\fillpstack{fs2}{%
    \preparseflextwo
    {\preparsefallback{--}}%
    \relax % this \relax is necessary so that the braces above 
           % are not stripped by \poppstack
}

\fillpstack{t}{%
    \relax
}

% stage 3.5 macros: typesetting

\newtoks\symstream

\def\tlskip{\z@}
\def\tfskip{\parindent}

\newif\ifchecktrailingstash

\def\typesetalltables{%
    \begingroup
        \ifsaveparseoutput
            {\newlinechar=`^^J\immediate\write\exampletable{^^J\harmlesscomment
             stashed stream:^^J\the\yystash^^J^^J\harmlesscomment
             format stream: ^^J\the\yyformat}}%
        \fi
        \ifchecktable 
            \ferrmessage{parsed table: \the\table^^J^^J%
                         stashed stream: \the\yystash^^J^^J%
                         format stream: \the\yyformat}%
        \fi
        \extractprodtableinfo    
        \symstream\table
        \table{}%
        \setprodtable
        \the\symstream\relax
        \postoks{}\pushothertables
        \ifchecktable 
            \ferrmessage{table after processing: \the\table}%
        \fi
        \ifsaveparseoutput
            {\newlinechar=`^^J\immediate\write\exampletable{^^J\harmlesscomment
             processed table:^^J\the\table}}%
        \fi
        \parindent1em
        \checkforpropertable\table
        \tabskip\tfskip
        \ruletableset
        \ifchecktrailingstash
            \ferrmessage{remaining stash: \the\yystash}%
        \fi
        \unwrapstash\yystash
        \toksa\expandafter{\the\yystash}%
        \cleanstash\stripstash\checkforccode
        \ifchecktrailingstash
            \ferrmessage{stash after cleaning: \the\toksa}%
        \fi
        \ifnum\wd0>\z@
        %\ifchecktable
        %    \showboxdepth=1000
        %    \showboxbreadth=1000
        %    \showbox0
        %\fi
        % currently testing for nontrivial leftover stash involves packaging the stash material
        % into a \vbox; as a result, the stash containing ${}{}$\hbox{} will have a nonzero length
        % which is why the test below is necessary
            \ifnum\ht0>\z@
                \indent\boxstash
            \fi
        \fi
    \expandafter % export the value of the alignment
    \endgroup
    \expandafter\gaglue\the\gaglue\relax
}

\def\typesetfstwotables{%
    \begingroup
        \ifchecktable 
            \ifsaveparseoutput
                {\newlinechar=`^^J\immediate\write\exampletable{^^J\harmlesscomment
                 stashed stream:^^J\the\yystash^^J^^J\harmlesscomment
                 format stream: ^^J\the\yyformat}}%
            \else
                \errmessage{parsed table: \the\table^^J^^J%
                            stashed stream: \the\yystash^^J^^J%
                            format stream: \the\yyformat}%
            \fi
        \fi
        \extractregextableinfo    
        \symstream\table
        \table{}%
        \setregextable
        \the\symstream\relax
        \regextableset
        \ifchecktable 
            \ifsaveparseoutput
            \else
                \errmessage{table after processing: \the\table}%
            \fi
        \fi
        \ifsaveparseoutput
            {\newlinechar=`^^J\immediate\write\exampletable{^^J\harmlesscomment
             processed table:^^J\the\table}}%
        \fi
        \ifchecktable
            \ifchecktrim
                \ferrmessage{remaining stash: \the\yystash}%
            \fi
        \fi
        \unwrapstash\yystash
        \toksa\expandafter{\the\yystash}%
        \cleanstash\stripstash\checkforccode
        \ifchecktable
            \ifchecktrim
                \ferrmessage{stash after cleaning: \the\toksa}%
            \fi
        \fi
        \ifnum\wd0>\z@
        %\ifchecktable
        %    \showboxdepth=1000
        %    \showboxbreadth=1000
        %    \showbox0
        %\fi
        % currently testing for nontrivial leftover stash involves packaging the stash material
        % into a \vbox; as a result, the stash containing ${}{}$\hbox{} will have a nonzero length
        % which is why the test below is necessary
            \ifnum\ht0>\z@
                \indent\boxstash
            \fi
        \fi
    \endgroup
}

\def\typesetfsonetables{%
    \begingroup
        \ifchecktable 
            \ifsaveparseoutput
                {\newlinechar=`^^J\immediate\write\exampletable{^^J\harmlesscomment
                 stashed stream:^^J\the\yystash^^J^^J\harmlesscomment
                 format stream: ^^J\the\yyformat}}%
            \else
                \errmessage{parsed table: \the\table^^J^^J%
                            stashed stream: \the\yystash^^J^^J%
                            format stream: \the\yyformat}%
            \fi
        \fi
%        \extractregextableinfo    
        \symstream\table
        \table{}%
        \setregexdeftable
        \the\symstream\relax
        \regexdeftableset
        \ifchecktable 
            \ifsaveparseoutput
            \else
                \errmessage{table after processing: \the\table}%
            \fi
        \fi
        \ifsaveparseoutput
            {\newlinechar=`^^J\immediate\write\exampletable{^^J\harmlesscomment
             processed table:^^J\the\table}}%
        \fi
        \ifchecktable
            \ifchecktrim
                \ferrmessage{remaining stash: \the\yystash}%
            \fi
        \fi
        \unwrapstash\yystash
        \toksa\expandafter{\the\yystash}%
        \cleanstash\stripstash\checkforccode
        \ifchecktable
            \ifchecktrim
                \ferrmessage{stash after cleaning: \the\toksa}%
            \fi
        \fi
        \ifnum\wd0>\z@
        %\ifchecktable
        %    \showboxdepth=1000
        %    \showboxbreadth=1000
        %    \showbox0
        %\fi
        % currently testing for nontrivial leftover stash involves packaging the stash material
        % into a \vbox; as a result, the stash containing ${}{}$\hbox{} will have a nonzero length
        % which is why the test below is necessary
            \ifnum\ht0>\z@
                \indent\boxstash
            \fi
        \fi
    \endgroup
}

\let\extractprodtableinfo\empty % we do not preprocess the productions table

\let\extractregextableinfo\empty % we do not preprocess the regex table

% setting the rule table: cross-section alignment and other effects are applied here;
% in order to produce the proper line skips before and after \unvbox, the rules followed by
% \TeX\ while adding an \halign to a vertical list have to be reproduced explicitly

\newdimen\gaglue % the width of the action box of the last alignment

\def\ruletableset{%
    \par
    \vskip-\baselineskip
    \ifx\gatoks\relax
    \else
        \table\expandafter{\the\table\gatoks}%
    \fi
    \setbox0\vbox\expandafter{\expandafter
        \null\expandafter\prevdepth\the\prevdepth
        \halign \ifpropertable to \hsize \fi
            {\hbox to 2em{##\/$\,${\rm:}\hss}\hfil\tabskip\z@&\setallterms{##}&##\hfil\tabskip0pt plus1fil&%
                \toksa{}##\makestashbox\hfil\tabskip\tlskip\cr
                \the\table
            }
            \expandafter
    }\expandafter
    \unvcopy\expandafter0\expandafter
    \prevdepth\the\prevdepth\relax
    \setbox\z@=\vbox{\unvbox\z@ \setbox\z@=\lastbox % set the alignment dimension 
        \hbox{\unhbox\z@ \unskip\setbox\z@=\lastbox\expandafter}\expandafter}\expandafter\gaglue\the\wd\z@
}

\def\setallterms#1{\setbox\z@=\hbox{\it#1}\ifsquashterms\hbox to0pt{\unhbox\z@\hss}\else\unhbox\z@\fi\hfil}

% typesetting the scanner automaton rules

\def\regextableset{%
    \par
    \vskip-\baselineskip
    \setbox0 \vbox\expandafter{\expandafter
        \null\expandafter\prevdepth\the\prevdepth
        \halign to\hsize
            {##\hfil\tabskip0 pt plus1fil\ &\relax\tabskip\tlskip\toksa{}##\makestashbox\hfil\cr
                \the\table
            }%
            \expandafter
    }\expandafter
    \unvbox\expandafter0\expandafter
    \prevdepth\the\prevdepth\relax
}

% typesetting named regular expression definitions

\def\regexdeftableset{%
    \par
    \vskip-\baselineskip
    \setbox0\vbox\expandafter{\expandafter
        \null\expandafter\prevdepth\the\prevdepth
        \halign to\hsize
            {\hskip\parindent##\hfil\tabskip0 pt plus1fil\ &\relax\tabskip\tlskip\tt##\hfil\cr
                \the\table
            }%
            \expandafter
    }\expandafter
    \unvbox\expandafter0\expandafter
    \prevdepth\the\prevdepth\relax
}

% quick and dirty global alignment: the size of the last box (and those in between)
% can be chosen automatically after one pass and read in for the final pass;
% in the future this will be the default implementation; for now, the inelegant
% solution below works as well.

\def\setglobalalignrules#1{%
    \def\gatoks{%
        \omit\hfil&\omit\hfil&\omit\hfil&\omit\hfil\hbox to #1{\hfil}\cr
        \noalign{\vskip-\baselineskip}%
        %\noalign{\centerline{$\diamond$}}
    }%
}

\setglobalalignrules{3 in}%

% to align all the actions across the document, comment out the next line

\let\gatoks\relax

\def\checkforccode{%
    \setbox0=\vbox{\setlazyc\the\toksa}%
}

\def\setlazyc{%
    \hidecs{\1\4\5\6\8}%
}

\newif\ifdisplaytokenraw

\def\toksdefline#1#2#3#4#5{%   #1 is the parsed version of the internal bison name
                             % #2 is the index entry
                             % #3 is the type
                             % #4 is the explicit value
                             % #5 is the parsed version of the string value
    \ifnum\tempca=\z@
        \tokdectoks\expandafter{\the\tokdectoks&{#2}#1\hfil\cr}%
        \tempca\tempcb
    \else
        \ifnum\tempcb=\tempca
            \tokdectoks\expandafter{\the\tokdectoks{#2}#1\hfil}%
        \else
            \tokdectoks\expandafter{\the\tokdectoks&{#2}#1\hfil}%
        \fi
        \advance\tempca\m@ne
    \fi
}

\def\attachtokentable{% typesetting the token declarations as part of the grammar
    \tempcb3 \tempca\tempcb
    \expandafter\tokdectoks\expandafter{\expandafter}\the\tokdectoks
    \ifnum\tempca=\tempcb
    \else
        \bloop
            \tokdectoks\expandafter{\the\tokdectoks&\omit\hfil}%
        \ifnum\tempca=\z@
        \else
            \advance\tempca\m@ne
        \repeat
        \tokdectoks\expandafter{\the\tokdectoks\cr}%
    \fi
    \toksa{\tt\strut##\hfil\tabskip=0pt plus 1fil}%
    \bloop
        \toksa\expandafter{\the\toksa&\tt\strut##\hfil}%
    \ifnum\tempcb=\tw@
    \else
        \advance\tempcb\m@ne
    \repeat
    \toksa\expandafter{\the\toksa&\tt\strut##\hfil\tabskip\z@\cr}%
    \concat\toksa\tokdectoks
    \tokdectoks\toksa
    \edef\next{\table{\the\table\noalign{%
                       \tabskip\parindent
                       %\nx\displaytokenrawtrue % this controls how tokens are displayed
                                                % in the declarations: if true, the macro names will be shown
                       \halign to\hsize{\the\tokdectoks}%
               }%
    }}\next
    \tokdectoks{}%
}

% typesetting token lists, precedence declaratins, etc
% these belong to yyunion.sty but they use `global' settings that are
% too specific to put there 

\def\attachtypestable{% typesetting the type declarations as part of the grammar
    \toksa{}\the\typestable
    \edef\next{\table{\the\table\noalign{%
                   {%
                       \the\toksa
                   }%
              }%
    }}\next\typestable{}%
}

\long\def\onetype#1#2{%
    \toksb{{%
        \hbadness\@M
        \rightskip=\z@
        \tempda\hsize \advance\tempda-1in 
        \indent{$\langle$\tt union$\rangle.#1$:}\ \hfill\nobreak\hfill\nobreak\null\penalty1\relax\null\nobreak\hfill
        \hbox to\tempda{$\vtop{\noindent\hsize\tempda\raggedright #2\strut}$}\par
    }}%
    \concat\toksa\toksb
}

\long\def\flexsstatelist#1{%
    \toksb{{%
        \hbadness\@M
        \rightskip=\z@
        \tempda\hsize \advance\tempda-1in 
        \indent{$\langle$\tt states-s$\rangle_{\rm f}$:}\ \hfill\nobreak\hfill\nobreak\null\penalty1\relax\null\nobreak\hfill
        \hbox to\tempda{$\vtop{\noindent\hsize\tempda\raggedright #1\strut}$}\par
    }}%
    \concat\toksa\toksb
}

\long\def\flexxstatelist#1{%
    \toksb{{%
        \hbadness\@M
        \rightskip=\z@
        \tempda\hsize \advance\tempda-1in 
        \indent{$\langle$\tt states-x$\rangle_{\rm f}$:}\ \hfill\nobreak\hfill\nobreak\null\penalty1\relax\null\nobreak\hfill
        \hbox to\tempda{$\vtop{\noindent\hsize\tempda\raggedright #1\strut}$}\par
    }}%
    \concat\toksa\toksb
}

\def\attachprectable{% typesetting the precedence declarations as part of the grammar
    \toksa{}\the\prectable
    \edef\next{\table{\the\table\noalign{%
                   {%
                       \the\toksa
                   }%
              }%
    }}\next\prectable{}%
}

\long\def\oneprec#1#2#3{%
    \toksc{%
        \hbadness\@M
        \rightskip=\z@
        \tempda\hsize \advance\tempda-1in 
        \indent
    }%
    \toksd{%
        \ \hfill\nobreak\hfill\nobreak\null\penalty1\relax\null\nobreak\hfill
        \hbox to\tempda{$\vtop{\noindent\hsize\tempda\raggedright #3\strut}$}\par
    }
    \def\next{#2}%
    \ifx\next\empty
        \toksb{}%
    \else
        \toksb{${}:{}$#2}%
    \fi
    \edef\next{\toksb{{$\nx\langle$\nx\bf #1\the\toksb$\nx\rangle$}}}\next
    \appendr\toksa{{\the\toksc\the\toksb\the\toksd}}%
}

\def\attachoptionstable{% typesetting the options as part of the grammar
    \edef\next{\table{\the\table\noalign{%
                   \tabskip\parindent
                   \halign{####\nx\hfil\tabskip\z@&\nx\qquad\nx\it####\nx\hfil\cr
                       \the\opttable
                   }%
              }%
    }}\next\opttable{}%
}

\newif\ifpropertable

\def\checkforpropertable#1{{% checking if there is any table material
% this is a hack to avoid underfull boxes when there is no actual alignment material in 
% \halign to \hsize
     \hbadness\@M
     \let\noalign\trivialnoalign
     \let\omit\relax
     \setbox\z@=\vbox{
         \halign
             {\eatone{##}&\eatone{##}.&\eatone{##}&\eatone{##}\cr
                 \the#1%
             }%
     }%
     \ifnum\wd\z@>\z@
         \aftergroup\propertabletrue
     \else
         \aftergroup\propertablefalse
     \fi
}}

\long\def\trivialnoalign#1{}

% macros for processing \Cee\ mode material

\newif\ifchecktrim

%\long\def\buildstash#1{\toksa\expandafter{\the\toksa#1}} % = stashed

\def\cleanstash{%
  \ifchecktrim\ferrmessage{collected stash: \the\toksa}\fi
  \expandafter\cleanst@sh\the\toksa\packagebox}

\def\cleanst@sh{\let\6\testsbox\setbox0=\vbox\bgroup}

\def\testsbox{%
  \ifmmode
      \let\next\relax
  \else
      \egroup
      \ifnum\wd\z@>\z@
          \ifnum\ht\z@>\z@
              \let\next\scoopupstash
          \else
              \let\next\rebuildstash
          \fi
      \else
          \let\next\rebuildstash
      \fi
  \fi
  \next
}

\long\def\rebuildstash#1\packagebox{\toksa{#1}\cleanstash}

\long\def\scoopupstash#1\packagebox{}

\def\packagebox{\egroup\ifnum\wd0>\z@\else\toksa{}\fi}

\def\stripstash{%
  \ifchecktrim\ferrmessage{before trimming: \the\toksa}\fi
  \def\6{}\expandafter\stripst@sh\expandafter\ignorespaces\the\toksa\6\str@pst@sh}

\def\stripst@sh{\toksa{}\stripst@shi}

\long\def\stripst@shi#1\6{%
  \toksb{#1}\futurelet\next\str@pst@sh
}

\def\str@pst@sh{%
  \ifx\next\str@pst@sh
      \iftrailingreturn
          \striptrim
      \fi
      \concat\toksa\toksb
      \iftrailingreturn % not done yet
          \let\next\stripagain
      \else
          \let\next\eatone
      \fi
      \trailingreturnfalse
  \else
      \edef\next{\toksa{\the\toksa\the\toksb\noexpand\6}}\next
      \def\next{\stripst@shi\relax\ignorespaces}%
      \trailingreturntrue
  \fi
  \next
}

\def\stripagain#1{\stripstash}

\newif\iftrailingreturn

\def\striptrim{%
  \ifchecktrim\ferrmessage{trimming: \the\toksb}\fi
  \edef\next{\the\toksb}%
  \expandafter\striptr@m\the\toksb\relax\end
}

\def\striptr@m
        #1% \relax (from last \stripst@shi)
        #2% \ignorespaces (from last \stripst@shi)
        % intervening spaces
        #3% \relax
        #4% ?
        \end{%
%  \toksc{#3#4}\showthe\toksc
%
  \setbox\z@\vbox{
      #3#4}%
  \ifnum\wd\z@=\z@
      \expandafter\trimreturn\the\toksa\end
      \toksb{}%
  \else
      \ifnum\ht\z@=\z@
          \expandafter\trimreturn\the\toksa\end
          \toksb{}%
      \else
          \trailingreturnfalse
      \fi
  \fi
}

\def\trimreturn#1\6\end{%
  \toksa{#1}%
}

\def\boxstash{%
  \ifchecktrim\ferrmessage{stash contents: \the\toksa}\fi
  $\vtop{\activateinlinec\tabskip\z@\halign{\strut\ignorespaces##\hfil\cr\relax\the\toksa\crcr}}$}

\def\makestashbox{\cleanstash\stripstash\boxstash}

\newbox\indentbox

\def\activateinlinec{%
  \setbox\indentbox=\hbox{}%
  \def\1{\setbox\indentbox\hbox{\box\indentbox\hbox to 3em{\hfil}}}%
  \def\2{\setbox\indentbox\hbox{\box\indentbox\hbox to -3em{}}}%
  \def\4{\hbox to -3em{}}%
  \def\5{}%
%  \let\6\cr
  \def\6{%
      \edef\setindentbox{%
          \setbox\indentbox\hbox to\the\wd\indentbox{\noexpand\hfil}%
          \copy\indentbox\ignorespaces
      }%
      \expandafter\crcontainer\setindentbox
  }%
  \def\7{%
      \edef\setindentbox{%
          \setbox\indentbox\hbox to\the\wd\indentbox{\noexpand\hfil}%
          \copy\indentbox\ignorespaces
      }%
      \expandafter\crcontainerspread\setindentbox
  }%
  \def\8{\hskip-1em}%
  \let\$\prodterm
}

\def\crcontainer{\cr}
\def\crcontainerspread{\cr\noalign{\yskip}}

% rough version of the term typesetting

\def\prodterm{%
    \futurelet\next\analyzepterm
}

\def\analyzepterm{%
    \ifx\next\$%
        \let\next\pr@dterm
    \else
        \ifcat\noexpand\next0%
            \let\next\pr@dterm
        \else
            \if\noexpand\next[%
                \let\next\pr@dterm
            \else
                \let\next\oldmathS
            \fi
        \fi
    \fi
    \next
}
            
\def\pr@dterm#1{%
  \ifx#1\$%
      \def\next{\hbox{$\Upsilon$}}%
  \else
      \if\noexpand#1[%
          \let\next\seeksym
      \else
          \ifnum`#1<"3A\relax
              \ifnum`#1>`0\relax
                  \def\next{\seekno#1}%
              \else
                  \def\next{\hbox{$\Upsilon$}#1}% TODO: look for an identifier
              \fi
          \else
              \def\next{\hbox{$\Upsilon$}#1}% TODO: look for an identifier
          \fi
      \fi
  \fi
  \next
}%

\let\$\prodterm
\defreserved\${\prodterm}

\def\seekno{\afterassignment\printterm\tempca}%

\def\seeksym#1]{%
  \hbox{$\Upsilon\kern-1pt{}_{\def\\##1{\hbox{\sscmd\prodstyle{##1}}}\rm#1}$}}

\def\seeksym#1]{% a better version of the above
  \hbox{$\ulcorner\def\\##1{##1}\let\.\\\let\|\\\let\ous\_\let\_\relax
    \edef\next{#1}\let\_\ous
    \hbox{\expandafter\prodstyle\expandafter{\next}}\urcorner$}}

\def\printterm{\hbox{$\Upsilon\kern-1pt{}_{\number\tempca}$}}%

% typesetting examples of \bison\ productions and \flex\ input in text

\long\def\setproduction#1{%
    \def\termidxrank{5}%
    \def\headeridxrank{4}%
    \def\defidxrank{3}%
    \def\texcsidxrank{5}%
    \textproductionsetup
    \hbox{\strut}%
    \Binputtoks{\lsectionbegin{b}#1\yyeof\yyeof\endparseinput\endparse\postparse}%
    \the\Binputtoks\par% Stage two, start the parsing
}

\def\textproductionsetup{%
    \ninepoint
    \let\returnexplicitspace\splitexplicitspace
    \let\acharswitch\texcharadjust
    \let\onecharswitch\texcsadjust
    \let\extractprodtableinfo\empty % we do not preprocess the table
    \showlastactionfalse
    \let\actionfiller\empty
    \let\postparsetext\postparsebproduction
    \fillpstack{b}{%
        \preparsebisongrammar
        \preparsebisonprologue
        {\preparsefallback{**}}%
        \relax % this \relax is necessary so that the braces above 
               % are not stripped by \poppstack
    }%
}

\def\splitexplicitspace{%
    \yyinput\{\}% 
}

\def\texcharadjust{
    ` {%
          \yybyte{|}%
          \expandafter\yycp@\expandafter`\the\yybyte\relax
          \mkpurebyte
          \yyreturn
      }
    _ {%
          \yybyte{\_}%
          \expandafter\yycp@\expandafter`\the\yybyte\relax
          \mkpurebyte
          \yyreturn
      }
}

\def\texcsadjust{
      \n {%
          \yycp@=\n
          \mkpurebyte
          \yyreturn
      }
      \^^M {%
          \yyinput\{\}% 
      }
      \' {%
          \expandafter\yyinput\benignescape'%
      }
      \\ {%
          \expandafter\expandafter\expandafter\yyinput\expandafter\benignescape\benignescape%
      }
}%

% production typesetting

\def\beginprod{%
    \par
    \begingroup
        \b@ginprod
}

\long\def\b@ginprod#1\endprod{%
        \setproduction{#1}%
    \expandafter
    \endgroup
    \expandafter\gaglue\the\gaglue\relax % export the alignment width
}

\def\beginmprod{%
    $$
    \vbox\bgroup
        \def\checkforpropertable##1{\propertablefalse} % so that the table is set to its natural width
        \b@ginmprod
}

\long\def\b@ginmprod#1\endmprod{%
        \setproduction{#1}%
    \egroup
    $$%
}

% centering productions: works for rule listing only (no token declarations, etc.)

\def\begincprod#1\endcprod{{\def\tlskip{0 pt plus1fill}\let\tfskip\tlskip\beginprod#1\endprod}}

% flex examples typesetting

\long\def\setflex#1{%
    \def\fstatedefidxrank{3}%
    \def\fstateidxrank{4}%
    \def\fregexidxrank{5}%
    \textflexsetup
    \hbox{\strut}%    
    \Binputtoks{\lsectionbegin{fs1}#1\yyeof\yyeof\endparseinput\endparse\postparse}%
    \the\Binputtoks\par% Stage two, start the parsing, the \par is expected by the \parserreset
}

\def\textflexsetup{%
    \ninepoint
    %\let\returnexplicitspace\splitexplicitspace
    \let\acharswitch\texcharadjust
    %\let\onecharswitch\texcsadjust
    \let\extractprodtableinfo\empty % we do not preprocess the table
    \let\postparsetext\postparsefsection
    \fillpstack{fs1}{%
        \preparseflexone % TODO
        \preparseflextwo
        {\preparsefallback{**}}%
        \relax % this \relax is necessary so that the braces above 
               % are not stripped by \poppstack
    }%
}

\def\beginflex{%
    \par
    \begingroup
    \catcode`\^^M=12 %
    \catcode`\#=12 %
        \b@ginflex%
}

\long\def\b@ginflex#1\endflex{%
        \setflex{#1}%
    \expandafter
    \endgroup
    \expandafter\gaglue\the\gaglue\relax % export the alignment width, TODO: set \gaglue in \flex
}



% the following macros assume that \gindex or \xrefstream having been defined implies 
% that all the bookkeeping required for maintaining the custom index and local cross referencing 
% has been taken care of; this way during the bootstrapping mode, index entries or cross references
% are not generated.

\ifx\gindex\UNDEFINED
\else
    \termindextrue
    \immediate\openout\gindex=\jobname.gdx
\fi

\let\endcprod\endgroup
\let\endmprod\endgroup
\let\endprod\endgroup

% stringing all the manuals together (disabled for now)

%\newwrite\lastpageinfo
%\newread\testeof

%\toksa\expandafter{\fin}
%\edef\fin{\the\toksa\noexpand\savelastpagenumber}
%\def\savelastpagenumber{%
%  \openout\lastpageinfo=\jobname.lpg%
%  \write\lastpageinfo{\def\noexpand\contentspagenumber{\number\pageno}\pageno=\noexpand\contentspagenumber \advance\pageno by 1}%
%  \write\lastpageinfo{\def\noexpand\secno{\number\secno}}%
%  \closeout\lastpageinfo
%}