ctbuilder.rs

//! Build grammars at compile-time so that they can be statically included into a binary.

use std::{
    any::type_name,
    collections::{HashMap, HashSet},
    env::{current_dir, var},
    error::Error,
    fmt::{self, Debug, Write as fmtWrite},
    fs::{self, File, create_dir_all, read_to_string},
    hash::Hash,
    io::Write,
    marker::PhantomData,
    path::{Path, PathBuf},
    sync::{LazyLock, Mutex},
};

use crate::{
    LexerTypes, RTParserBuilder, RecoveryKind,
    diagnostics::{DiagnosticFormatter, SpannedDiagnosticFormatter},
};

#[cfg(feature = "_unstable_api")]
use crate::unstable_api::UnstableApi;

use bincode::{Decode, Encode, decode_from_slice, encode_to_vec};
use cfgrammar::{
    Location, RIdx, Span, Symbol,
    header::{GrmtoolsSectionParser, Header, HeaderValue, Value},
    markmap::{Entry, MergeBehavior},
    yacc::{YaccGrammar, YaccKind, YaccOriginalActionKind, ast::ASTWithValidityInfo},
};
use filetime::FileTime;
use lrtable::{Minimiser, StateGraph, StateTable, from_yacc, statetable::Conflicts};
use num_traits::{AsPrimitive, PrimInt, Unsigned};
use proc_macro2::{Literal, TokenStream};
use quote::{ToTokens, TokenStreamExt, format_ident, quote};
use syn::{Generics, parse_quote};

const ACTION_PREFIX: &str = "__gt_";
const GLOBAL_PREFIX: &str = "__GT_";
const ACTIONS_KIND: &str = "__GtActionsKind";
const ACTIONS_KIND_PREFIX: &str = "Ak";
const ACTIONS_KIND_HIDDEN: &str = "__GtActionsKindHidden";

const RUST_FILE_EXT: &str = "rs";

const WARNING: &str = "[Warning]";
const ERROR: &str = "[Error]";

static GENERATED_PATHS: LazyLock<Mutex<HashSet<PathBuf>>> =
    LazyLock::new(|| Mutex::new(HashSet::new()));

struct CTConflictsError<StorageT: Eq + Hash> {
    conflicts_diagnostic: String,
    #[cfg(test)]
    #[cfg_attr(test, allow(dead_code))]
    stable: StateTable<StorageT>,
    phantom: PhantomData<StorageT>,
}

/// The quote impl of `ToTokens` for `Option` prints an empty string for `None`
/// and the inner value for `Some(inner_value)`.
///
/// This wrapper instead emits both `Some` and `None` variants.
/// See: [quote #20](https://github.com/dtolnay/quote/issues/20)
struct QuoteOption<T>(Option<T>);

impl<T: ToTokens> ToTokens for QuoteOption<T> {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        tokens.append_all(match self.0 {
            Some(ref t) => quote! { ::std::option::Option::Some(#t) },
            None => quote! { ::std::option::Option::None },
        });
    }
}

/// The quote impl of `ToTokens` for `usize` prints literal values
/// including a type suffix for example `0usize`.
///
/// This wrapper omits the type suffix emitting `0` instead.
struct UnsuffixedUsize(usize);

impl ToTokens for UnsuffixedUsize {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        tokens.append(Literal::usize_unsuffixed(self.0))
    }
}

/// This wrapper adds a missing impl of `ToTokens` for tuples.
/// For a tuple `(a, b)` emits `(a.to_tokens(), b.to_tokens())`
struct QuoteTuple<T>(T);

impl<A: ToTokens, B: ToTokens> ToTokens for QuoteTuple<(A, B)> {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        let (a, b) = &self.0;
        tokens.append_all(quote!((#a, #b)));
    }
}

/// The wrapped `&str` value will be emitted with a call to `to_string()`
struct QuoteToString<'a>(&'a str);

impl ToTokens for QuoteToString<'_> {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        let x = &self.0;
        tokens.append_all(quote! { #x.to_string() });
    }
}

impl<StorageT> fmt::Display for CTConflictsError<StorageT>
where
    StorageT: 'static + Debug + Hash + PrimInt + Unsigned,
    usize: AsPrimitive<StorageT>,
{
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{}", self.conflicts_diagnostic)
    }
}

impl<StorageT> fmt::Debug for CTConflictsError<StorageT>
where
    StorageT: 'static + Debug + Hash + PrimInt + Unsigned,
    usize: AsPrimitive<StorageT>,
{
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{}", self.conflicts_diagnostic)
    }
}

impl<StorageT> Error for CTConflictsError<StorageT>
where
    StorageT: 'static + Debug + Hash + PrimInt + Unsigned,
    usize: AsPrimitive<StorageT>,
{
}

/// A string which uses `Display` for it's `Debug` impl.
struct ErrorString(String);
impl fmt::Display for ErrorString {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let ErrorString(s) = self;
        write!(f, "{}", s)
    }
}
impl fmt::Debug for ErrorString {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let ErrorString(s) = self;
        write!(f, "{}", s)
    }
}
impl Error for ErrorString {}

/// Specify the visibility of the module generated by `CTBuilder`.
#[derive(Clone, PartialEq, Eq, Debug)]
#[non_exhaustive]
pub enum Visibility {
    /// Module-level visibility only.
    Private,
    /// `pub`
    Public,
    /// `pub(super)`
    PublicSuper,
    /// `pub(self)`
    PublicSelf,
    /// `pub(crate)`
    PublicCrate,
    /// `pub(in {arg})`
    PublicIn(String),
}

/// Specifies the [Rust Edition] that will be emitted during code generation.
///
/// [Rust Edition]: https://doc.rust-lang.org/edition-guide/rust-2021/index.html
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
#[non_exhaustive]
pub enum RustEdition {
    Rust2015,
    Rust2018,
    Rust2021,
}

impl RustEdition {
    fn to_variant_tokens(self) -> TokenStream {
        match self {
            RustEdition::Rust2015 => quote!(::lrpar::RustEdition::Rust2015),
            RustEdition::Rust2018 => quote!(::lrpar::RustEdition::Rust2018),
            RustEdition::Rust2021 => quote!(::lrpar::RustEdition::Rust2021),
        }
    }
}

impl ToTokens for Visibility {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        tokens.extend(match self {
            Visibility::Private => quote!(),
            Visibility::Public => quote! {pub},
            Visibility::PublicSuper => quote! {pub(super)},
            Visibility::PublicSelf => quote! {pub(self)},
            Visibility::PublicCrate => quote! {pub(crate)},
            Visibility::PublicIn(data) => {
                let other = str::parse::<TokenStream>(data).unwrap();
                quote! {pub(in #other)}
            }
        })
    }
}

impl Visibility {
    fn to_variant_tokens(&self) -> TokenStream {
        match self {
            Visibility::Private => quote!(::lrpar::Visibility::Private),
            Visibility::Public => quote!(::lrpar::Visibility::Public),
            Visibility::PublicSuper => quote!(::lrpar::Visibility::PublicSuper),
            Visibility::PublicSelf => quote!(::lrpar::Visibility::PublicSelf),
            Visibility::PublicCrate => quote!(::lrpar::Visibility::PublicCrate),
            Visibility::PublicIn(data) => {
                let data = QuoteToString(data);
                quote!(::lrpar::Visibility::PublicIn(#data))
            }
        }
    }
}

/// A `CTParserBuilder` allows one to specify the criteria for building a statically generated
/// parser.
pub struct CTParserBuilder<'a, LexerTypesT: LexerTypes>
where
    LexerTypesT::StorageT: Eq + Hash,
    usize: AsPrimitive<LexerTypesT::StorageT>,
{
    // Anything stored in here (except `output_path`, `conflicts`, and `error_on_conflict`) almost
    // certainly needs to be included as part of the rebuild_cache function below so that, if it's
    // changed, the grammar is rebuilt.
    grammar_path: Option<PathBuf>,
    // If specified rather than reading source from `grammar_path`, use this string directly
    grammar_src: Option<String>,
    // If specified along with `grammar_src`, use this rather than building an ast from `grammar_src`.
    from_ast: Option<ASTWithValidityInfo>,
    output_path: Option<PathBuf>,
    mod_name: Option<&'a str>,
    recoverer: Option<RecoveryKind>,
    yacckind: Option<YaccKind>,
    error_on_conflicts: bool,
    warnings_are_errors: bool,
    show_warnings: bool,
    visibility: Visibility,
    rust_edition: RustEdition,
    inspect_rt: Option<
        Box<
            dyn for<'b> FnMut(
                &'b mut Header<Location>,
                RTParserBuilder<LexerTypesT::StorageT, LexerTypesT>,
                &'b HashMap<String, LexerTypesT::StorageT>,
                &PathBuf,
            ) -> Result<(), Box<dyn Error>>,
        >,
    >,
    // test function for inspecting private state
    #[cfg(test)]
    inspect_callback: Option<Box<dyn Fn(RecoveryKind) -> Result<(), Box<dyn Error>>>>,
    phantom: PhantomData<LexerTypesT>,
}

impl<
    'a,
    StorageT: 'static + Debug + Hash + PrimInt + Encode + Unsigned,
    LexerTypesT: LexerTypes<StorageT = StorageT>,
> CTParserBuilder<'a, LexerTypesT>
where
    usize: AsPrimitive<StorageT>,
{
    /// Create a new `CTParserBuilder`.
    ///
    /// `StorageT` must be an unsigned integer type (e.g. `u8`, `u16`) which is:
    ///   * big enough to index (separately) all the tokens, rules, productions in the grammar,
    ///   * big enough to index the state table created from the grammar,
    ///   * less than or equal in size to `u32`.
    ///
    /// In other words, if you have a grammar with 256 tokens, 256 rules, and 256 productions,
    /// which creates a state table of 256 states you can safely specify `u8` here; but if any of
    /// those counts becomes 257 or greater you will need to specify `u16`. If you are parsing
    /// large files, the additional storage requirements of larger integer types can be noticeable,
    /// and in such cases it can be worth specifying a smaller type. `StorageT` defaults to `u32`
    /// if unspecified.
    ///
    /// # Examples
    ///
    /// ```text
    /// CTParserBuilder::<DefaultLexerTypes<u8>>::new()
    ///     .grammar_in_src_dir("grm.y")?
    ///     .build()?;
    /// ```
    pub fn new() -> Self {
        CTParserBuilder {
            grammar_path: None,
            grammar_src: None,
            from_ast: None,
            output_path: None,
            mod_name: None,
            recoverer: None,
            yacckind: None,
            error_on_conflicts: true,
            warnings_are_errors: true,
            show_warnings: true,
            visibility: Visibility::Private,
            rust_edition: RustEdition::Rust2021,
            inspect_rt: None,
            #[cfg(test)]
            inspect_callback: None,
            phantom: PhantomData,
        }
    }

    /// Set the input grammar path to a file relative to this project's `src` directory. This will
    /// also set the output path (i.e. you do not need to call [CTParserBuilder::output_path]).
    ///
    /// For example if `a/b.y` is passed as `inp` then [CTParserBuilder::build] will:
    ///   * use `src/a/b.y` as the input file.
    ///   * write output to a file which can then be imported by calling `lrpar_mod!("a/b.y")`.
    ///   * create a module in that output file named `b_y`.
    ///
    /// You can override the output path and/or module name by calling [CTParserBuilder::output_path]
    /// and/or [CTParserBuilder::mod_name], respectively, after calling this function.
    ///
    /// This is a convenience function that makes it easier to compile grammar files stored in a
    /// project's `src/` directory: please see [CTParserBuilder::build] for additional constraints
    /// and information about the generated files. Note also that each `.y` file can only be
    /// processed once using this function: if you want to generate multiple grammars from a single
    /// `.y` file, you will need to use [CTParserBuilder::output_path].
    pub fn grammar_in_src_dir<P>(mut self, srcp: P) -> Result<Self, Box<dyn Error>>
    where
        P: AsRef<Path>,
    {
        if !srcp.as_ref().is_relative() {
            return Err(format!(
                "Grammar path '{}' must be a relative path.",
                srcp.as_ref().to_str().unwrap_or("<invalid UTF-8>")
            )
            .into());
        }

        let mut grmp = current_dir()?;
        grmp.push("src");
        grmp.push(srcp.as_ref());
        self.grammar_path = Some(grmp);

        let mut outp = PathBuf::new();
        outp.push(var("OUT_DIR").unwrap());
        outp.push(srcp.as_ref().parent().unwrap().to_str().unwrap());
        create_dir_all(&outp)?;
        let mut leaf = srcp
            .as_ref()
            .file_name()
            .unwrap()
            .to_str()
            .unwrap()
            .to_owned();
        write!(leaf, ".{}", RUST_FILE_EXT).ok();
        outp.push(leaf);
        Ok(self.output_path(outp))
    }

    /// If set, specifies that this grammar should be built from a pre-validated AST
    /// instead of a `.y`` file. When this is specified, `grammar_path` will not be read.
    #[cfg(feature = "_unstable_api")]
    pub fn grammar_ast(mut self, valid_ast: ASTWithValidityInfo, _api_key: UnstableApi) -> Self {
        self.from_ast = Some(valid_ast);
        self
    }

    /// Set the input grammar path to `inp`. If specified, you must also call
    /// [CTParserBuilder::output_path]. In general it is easier to use
    /// [CTParserBuilder::grammar_in_src_dir].
    pub fn grammar_path<P>(mut self, inp: P) -> Self
    where
        P: AsRef<Path>,
    {
        self.grammar_path = Some(inp.as_ref().to_owned());
        self
    }

    #[cfg(feature = "_unstable_api")]
    pub fn with_grammar_src(mut self, src: String, _api_key: UnstableApi) -> Self {
        self.grammar_src = Some(src);
        self
    }

    /// Set the output grammar path to `outp`. Note that there are no requirements on `outp`: the
    /// file can exist anywhere you can create a valid [Path] to. However, if you wish to use
    /// [crate::lrpar_mod!] you will need to make sure that `outp` is in
    /// [std::env::var]`("OUT_DIR")` or one of its subdirectories.
    pub fn output_path<P>(mut self, outp: P) -> Self
    where
        P: AsRef<Path>,
    {
        self.output_path = Some(outp.as_ref().to_owned());
        self
    }

    /// Set the generated module name to `mod_name`. If no module name is specified,
    /// [CTParserBuilder::build] will attempt to create a sensible default based on the grammar
    /// filename.
    pub fn mod_name(mut self, mod_name: &'a str) -> Self {
        self.mod_name = Some(mod_name);
        self
    }

    /// Set the visibility of the generated module to `vis`. Defaults to `Visibility::Private`.
    pub fn visibility(mut self, vis: Visibility) -> Self {
        self.visibility = vis;
        self
    }

    /// Set the recoverer for this parser to `rk`. Defaults to `RecoveryKind::CPCTPlus`.
    pub fn recoverer(mut self, rk: RecoveryKind) -> Self {
        self.recoverer = Some(rk);
        self
    }

    /// Set the `YaccKind` for this parser to `ak`.
    pub fn yacckind(mut self, yk: YaccKind) -> Self {
        self.yacckind = Some(yk);
        self
    }

    /// If set to true, [CTParserBuilder::build] will return an error if the given grammar contains
    /// any Shift/Reduce or Reduce/Reduce conflicts. Defaults to `true`.
    pub fn error_on_conflicts(mut self, b: bool) -> Self {
        self.error_on_conflicts = b;
        self
    }

    /// If set to true, [CTParserBuilder::build] will return an error if the given grammar contains
    /// any warnings. Defaults to `true`.
    pub fn warnings_are_errors(mut self, b: bool) -> Self {
        self.warnings_are_errors = b;
        self
    }

    /// If set to true, [CTParserBuilder::build] will print warnings to stderr, or via cargo when
    /// running under cargo. Defaults to `true`.
    pub fn show_warnings(mut self, b: bool) -> Self {
        self.show_warnings = b;
        self
    }

    /// Sets the rust edition to be used for generated code. Defaults to the latest edition of
    /// rust supported by grmtools.
    pub fn rust_edition(mut self, edition: RustEdition) -> Self {
        self.rust_edition = edition;
        self
    }

    #[cfg(test)]
    pub fn inspect_recoverer(
        mut self,
        cb: Box<dyn for<'h, 'y> Fn(RecoveryKind) -> Result<(), Box<dyn Error>>>,
    ) -> Self {
        self.inspect_callback = Some(cb);
        self
    }

    #[doc(hidden)]
    pub fn inspect_rt(
        mut self,
        cb: Box<
            dyn for<'b, 'y> FnMut(
                &'b mut Header<Location>,
                RTParserBuilder<'y, StorageT, LexerTypesT>,
                &'b HashMap<String, StorageT>,
                &PathBuf,
            ) -> Result<(), Box<dyn Error>>,
        >,
    ) -> Self {
        self.inspect_rt = Some(cb);
        self
    }

    /// Statically compile the Yacc file specified by [CTParserBuilder::grammar_path()] into Rust,
    /// placing the output into the file spec [CTParserBuilder::output_path()]. Note that three
    /// additional files will be created with the same name as specified in [self.output_path] but
    /// with the extensions `grm`, and `stable`, overwriting any existing files with those names.
    ///
    /// If `%parse-param` is not specified, the generated module follows the form:
    ///
    /// ```text
    ///   mod <modname> {
    ///     pub fn parse<'lexer, 'input: 'lexer>(lexer: &'lexer dyn NonStreamingLexer<...>)
    ///       -> (Option<ActionT>, Vec<LexParseError<...>> { ... }
    ///
    ///     pub fn token_epp<'a>(tidx: ::cfgrammar::TIdx<StorageT>) -> ::std::option::Option<&'a str> {
    ///       ...
    ///     }
    ///
    ///     ...
    ///   }
    /// ```
    ///
    /// If `%parse-param x: t` is specified, the generated module follows the form:
    ///
    /// ```text
    ///   mod <modname> {
    ///     pub fn parse<'lexer, 'input: 'lexer>(lexer: &'lexer dyn NonStreamingLexer<...>, x: t)
    ///       -> (Option<ActionT>, Vec<LexParseError<...>> { ... }
    ///
    ///     pub fn token_epp<'a>(tidx: ::cfgrammar::TIdx<StorageT>) -> ::std::option::Option<&'a str> {
    ///       ...
    ///     }
    ///
    ///     ...
    ///   }
    /// ```
    ///
    /// where:
    ///  * `modname` is either:
    ///    * the module name specified by [CTParserBuilder::mod_name()];
    ///    * or, if no module name was explicitly specified, then for the file `/a/b/c.y` the
    ///      module name is `c_y` (i.e. the file's leaf name, minus its extension, with a prefix of
    ///      `_y`).
    ///  * `ActionT` is either:
    ///    * if the `yacckind` was set to `YaccKind::GrmTools` or
    ///      `YaccKind::Original(YaccOriginalActionKind::UserAction)`, it is
    ///      the return type of the `%start` rule;
    ///    * or, if the `yacckind` was set to
    ///      `YaccKind::Original(YaccOriginalActionKind::GenericParseTree)`, it
    ///      is `Node<StorageT>` where the `Node` type is defined within your `lrpar_mod!`.
    ///
    /// # Panics
    ///
    /// If `StorageT` is not big enough to index the grammar's tokens, rules, or productions.
    pub fn build(mut self) -> Result<CTParser<StorageT>, Box<dyn Error>> {
        let grmp = self
            .grammar_path
            .as_ref()
            .expect("grammar_path must be specified before processing.");
        let outp = self
            .output_path
            .as_ref()
            .expect("output_path must be specified before processing.");
        let mut header = Header::new();

        match header.entry("yacckind".to_string()) {
            Entry::Occupied(_) => unreachable!(),
            Entry::Vacant(mut v) => match self.yacckind {
                Some(YaccKind::Eco) => panic!("Eco compile-time grammar generation not supported."),
                Some(yk) => {
                    let yk_value = Value::try_from(yk)?;
                    let mut o = v.insert_entry(HeaderValue(
                        Location::Other("CTParserBuilder".to_string()),
                        yk_value,
                    ));
                    o.set_merge_behavior(MergeBehavior::Ours);
                }
                None => {
                    v.mark_required();
                }
            },
        }
        if let Some(recoverer) = self.recoverer {
            match header.entry("recoverer".to_string()) {
                Entry::Occupied(_) => unreachable!(),
                Entry::Vacant(v) => {
                    let rk_value: Value<Location> = Value::try_from(recoverer)?;
                    let mut o = v.insert_entry(HeaderValue(
                        Location::Other("CTParserBuilder".to_string()),
                        rk_value,
                    ));
                    o.set_merge_behavior(MergeBehavior::Ours);
                }
            }
        }

        {
            let mut lk = GENERATED_PATHS.lock().unwrap();
            if lk.contains(outp.as_path()) {
                return Err(format!("Generating two parsers to the same path ('{}') is not allowed: use CTParserBuilder::output_path (and, optionally, CTParserBuilder::mod_name) to differentiate them.", &outp.to_str().unwrap()).into());
            }
            lk.insert(outp.clone());
        }

        let inc = if let Some(grammar_src) = &self.grammar_src {
            grammar_src.clone()
        } else {
            read_to_string(grmp).map_err(|e| format!("When reading '{}': {e}", grmp.display()))?
        };

        let yacc_diag = SpannedDiagnosticFormatter::new(&inc, grmp);
        let parsed_header = GrmtoolsSectionParser::new(&inc, false).parse();
        if let Err(errs) = parsed_header {
            let mut out = String::new();
            out.push_str(&format!(
                "\n{ERROR}{}\n",
                yacc_diag.file_location_msg(" parsing the `%grmtools` section", None)
            ));
            for e in errs {
                out.push_str(&indent("     ", &yacc_diag.format_error(e).to_string()));
            }
            return Err(ErrorString(out))?;
        }
        let (parsed_header, _) = parsed_header.unwrap();
        header.merge_from(parsed_header)?;
        self.yacckind = header
            .get("yacckind")
            .map(|HeaderValue(_, val)| val)
            .map(YaccKind::try_from)
            .transpose()?;
        header.mark_used(&"yacckind".to_string());
        let ast_validation = if let Some(ast) = &self.from_ast {
            ast.clone()
        } else if let Some(yk) = self.yacckind {
            ASTWithValidityInfo::new(yk, &inc)
        } else {
            Err("Missing 'yacckind'".to_string())?
        };

        header.mark_used(&"recoverer".to_string());
        let rk_val = header.get("recoverer").map(|HeaderValue(_, rk_val)| rk_val);

        if let Some(rk_val) = rk_val {
            self.recoverer = Some(RecoveryKind::try_from(rk_val)?);
        } else {
            // Fallback to the default recoverykind.
            self.recoverer = Some(RecoveryKind::CPCTPlus);
        }
        self.yacckind = Some(ast_validation.yacc_kind());
        let warnings = ast_validation.ast().warnings();
        let res = YaccGrammar::<StorageT>::new_from_ast_with_validity_info(&ast_validation);
        let grm = match res {
            Ok(_) if self.warnings_are_errors && !warnings.is_empty() => {
                let mut out = String::new();
                out.push_str(&format!(
                    "\n{ERROR}{}\n",
                    yacc_diag.file_location_msg("", None)
                ));
                for e in warnings {
                    out.push_str(&format!(
                        "{}\n",
                        indent("     ", &yacc_diag.format_warning(e).to_string())
                    ));
                }
                return Err(ErrorString(out))?;
            }
            Ok(grm) => {
                if !warnings.is_empty() {
                    for w in warnings {
                        let ws_loc = yacc_diag.file_location_msg("", None);
                        let ws = indent("     ", &yacc_diag.format_warning(w).to_string());
                        // Assume if this variable is set we are running under cargo.
                        if std::env::var("OUT_DIR").is_ok() && self.show_warnings {
                            for line in ws_loc.lines().chain(ws.lines()) {
                                println!("cargo:warning={}", line);
                            }
                        } else if self.show_warnings {
                            eprintln!("{}", ws_loc);
                            eprintln!("{WARNING} {}", ws);
                        }
                    }
                }
                grm
            }
            Err(errs) => {
                let mut out = String::new();
                out.push_str(&format!(
                    "\n{ERROR}{}\n",
                    yacc_diag.file_location_msg("", None)
                ));
                for e in errs {
                    out.push_str(&indent("     ", &yacc_diag.format_error(e).to_string()));
                    out.push('\n');
                }

                return Err(ErrorString(out))?;
            }
        };

        #[cfg(test)]
        if let Some(cb) = &self.inspect_callback {
            cb(self.recoverer.expect("has a default value"))?;
        }

        let rule_ids = grm
            .tokens_map()
            .iter()
            .map(|(&n, &i)| (n.to_owned(), i.as_storaget()))
            .collect::<HashMap<_, _>>();

        let derived_mod_name = match self.mod_name {
            Some(s) => s.to_owned(),
            None => {
                // The user hasn't specified a module name, so we create one automatically: what we
                // do is strip off all the filename extensions (note that it's likely that inp ends
                // with `y.rs`, so we potentially have to strip off more than one extension) and
                // then add `_y` to the end.
                let mut stem = grmp.to_str().unwrap();
                loop {
                    let new_stem = Path::new(stem).file_stem().unwrap().to_str().unwrap();
                    if stem == new_stem {
                        break;
                    }
                    stem = new_stem;
                }
                format!("{}_y", stem)
            }
        };

        let cache = self.rebuild_cache(&derived_mod_name, &grm);

        // We don't need to go through the full rigmarole of generating an output file if all of
        // the following are true: the output file exists; it is newer than the input file; and the
        // cache hasn't changed. The last of these might be surprising, but it's vital: we don't
        // know, for example, what the IDs map might be from one run to the next, and it might
        // change for reasons beyond lrpar's control. If it does change, that means that the lexer
        // and lrpar would get out of sync, so we have to play it safe and regenerate in such
        // cases.
        if let Ok(ref inmd) = fs::metadata(grmp) {
            if let Ok(ref out_rs_md) = fs::metadata(outp) {
                if FileTime::from_last_modification_time(out_rs_md)
                    > FileTime::from_last_modification_time(inmd)
                {
                    if let Ok(outc) = read_to_string(outp) {
                        if outc.contains(&cache.to_string()) {
                            return Ok(CTParser {
                                regenerated: false,
                                rule_ids,
                                yacc_grammar: grm,
                                grammar_src: inc,
                                grammar_path: self.grammar_path.unwrap(),
                                conflicts: None,
                            });
                        } else {
                            #[cfg(grmtools_extra_checks)]
                            if std::env::var("CACHE_EXPECTED").is_ok() {
                                eprintln!("outc: {}", outc);
                                eprintln!("using cache: {}", cache,);
                                // Primarily for use in the testsuite.
                                panic!("The cache regenerated however, it was expected to match");
                            }
                        }
                    }
                }
            }
        }

        // At this point, we know we're going to generate fresh output; however, if something goes
        // wrong in the process between now and us writing /out/blah.rs, rustc thinks that
        // everything's gone swimmingly (even if build.rs errored!), and tries to carry on
        // compilation, leading to weird errors. We therefore delete /out/blah.rs at this point,
        // which means, at worse, the user gets a "file not found" error from rustc (which is less
        // confusing than the alternatives).
        fs::remove_file(outp).ok();

        let (sgraph, stable) = from_yacc(&grm, Minimiser::Pager)?;
        if self.error_on_conflicts {
            if let Some(c) = stable.conflicts() {
                match (grm.expect(), grm.expectrr()) {
                    (Some(i), Some(j)) if i == c.sr_len() && j == c.rr_len() => (),
                    (Some(i), None) if i == c.sr_len() && 0 == c.rr_len() => (),
                    (None, Some(j)) if 0 == c.sr_len() && j == c.rr_len() => (),
                    (None, None) if 0 == c.rr_len() && 0 == c.sr_len() => (),
                    _ => {
                        let conflicts_diagnostic = yacc_diag.format_conflicts::<LexerTypesT>(
                            &grm,
                            ast_validation.ast(),
                            c,
                            &sgraph,
                            &stable,
                        );
                        return Err(Box::new(CTConflictsError {
                            conflicts_diagnostic,
                            phantom: PhantomData,
                            #[cfg(test)]
                            stable,
                        }));
                    }
                }
            }
        }

        if let Some(ref mut inspector_rt) = self.inspect_rt {
            let rt: RTParserBuilder<'_, StorageT, LexerTypesT> =
                RTParserBuilder::new(&grm, &stable);
            let rt = if let Some(rk) = self.recoverer {
                rt.recoverer(rk)
            } else {
                rt
            };
            inspector_rt(&mut header, rt, &rule_ids, grmp)?
        }

        let unused_keys = header.unused();
        if !unused_keys.is_empty() {
            return Err(format!("Unused keys in header: {}", unused_keys.join(", ")).into());
        }
        let missing_keys = header
            .missing()
            .iter()
            .map(|s| s.as_str())
            .collect::<Vec<_>>();
        if !missing_keys.is_empty() {
            return Err(format!(
                "Required values were missing from the header: {}",
                missing_keys.join(", ")
            )
            .into());
        }

        self.output_file(
            &grm,
            &stable,
            &derived_mod_name,
            outp,
            &format!("/* CACHE INFORMATION {} */\n", cache),
            &yacc_diag,
        )?;
        let conflicts = if stable.conflicts().is_some() {
            Some((sgraph, stable))
        } else {
            None
        };
        Ok(CTParser {
            regenerated: true,
            rule_ids,
            yacc_grammar: grm,
            grammar_src: inc,
            grammar_path: self.grammar_path.unwrap(),
            conflicts,
        })
    }

    /// Given the filename `a/b.y` as input, statically compile the grammar `src/a/b.y` into a Rust
    /// module which can then be imported using `lrpar_mod!("a/b.y")`. This is a convenience
    /// function around [`process_file`](#method.process_file) which makes it easier to compile
    /// grammar files stored in a project's `src/` directory: please see
    /// [`process_file`](#method.process_file) for additional constraints and information about the
    /// generated files.
    #[deprecated(
        since = "0.11.0",
        note = "Please use grammar_in_src_dir(), build(), and token_map() instead"
    )]
    #[allow(deprecated)]
    pub fn process_file_in_src(
        &mut self,
        srcp: &str,
    ) -> Result<HashMap<String, StorageT>, Box<dyn Error>> {
        let mut inp = current_dir()?;
        inp.push("src");
        inp.push(srcp);
        let mut outp = PathBuf::new();
        outp.push(var("OUT_DIR").unwrap());
        outp.push(Path::new(srcp).parent().unwrap().to_str().unwrap());
        create_dir_all(&outp)?;
        let mut leaf = Path::new(srcp)
            .file_name()
            .unwrap()
            .to_str()
            .unwrap()
            .to_owned();
        write!(leaf, ".{}", RUST_FILE_EXT).ok();
        outp.push(leaf);
        self.process_file(inp, outp)
    }

    /// Statically compile the Yacc file `inp` into Rust, placing the output into the file `outp`.
    /// Note that three additional files will be created with the same name as `outp` but with the
    /// extensions `grm`, and `stable`, overwriting any existing files with those names.
    ///
    /// `outp` defines a module as follows:
    ///
    /// ```text
    ///   mod modname {
    ///     pub fn parse(lexemes: &::std::vec::Vec<::lrpar::Lexeme<StorageT>>) { ... }
    ///         -> (::std::option::Option<ActionT>,
    ///             ::std::vec::Vec<::lrpar::LexParseError<StorageT>>)> { ...}
    ///
    ///     pub fn token_epp<'a>(tidx: ::cfgrammar::TIdx<StorageT>) -> ::std::option::Option<&'a str> {
    ///       ...
    ///     }
    ///
    ///     ...
    ///   }
    /// ```
    ///
    /// where:
    ///  * `modname` is either:
    ///    * the module name specified [`mod_name`](#method.mod_name)
    ///    * or, if no module name was explicitly specified, then for the file `/a/b/c.y` the
    ///      module name is `c_y` (i.e. the file's leaf name, minus its extension, with a prefix of
    ///      `_y`).
    ///  * `ActionT` is either:
    ///    * the `%actiontype` value given to the grammar
    ///    * or, if the `yacckind` was set YaccKind::Original(YaccOriginalActionKind::UserAction),
    ///      it is [`Node<StorageT>`](../parser/enum.Node.html)
    ///
    /// # Panics
    ///
    /// If `StorageT` is not big enough to index the grammar's tokens, rules, or
    /// productions.
    #[deprecated(
        since = "0.11.0",
        note = "Please use grammar_path(), output_path(), build(), and token_map() instead"
    )]
    pub fn process_file<P, Q>(
        &mut self,
        inp: P,
        outp: Q,
    ) -> Result<HashMap<String, StorageT>, Box<dyn Error>>
    where
        P: AsRef<Path>,
        Q: AsRef<Path>,
    {
        self.grammar_path = Some(inp.as_ref().to_owned());
        self.output_path = Some(outp.as_ref().to_owned());
        let cl: CTParserBuilder<LexerTypesT> = CTParserBuilder {
            grammar_path: self.grammar_path.clone(),
            grammar_src: None,
            from_ast: None,
            output_path: self.output_path.clone(),
            mod_name: self.mod_name,
            recoverer: self.recoverer,
            yacckind: self.yacckind,
            error_on_conflicts: self.error_on_conflicts,
            warnings_are_errors: self.warnings_are_errors,
            show_warnings: self.show_warnings,
            visibility: self.visibility.clone(),
            rust_edition: self.rust_edition,
            inspect_rt: None,
            #[cfg(test)]
            inspect_callback: None,
            phantom: PhantomData,
        };
        Ok(cl.build()?.rule_ids)
    }

    fn output_file<P: AsRef<Path>>(
        &self,
        grm: &YaccGrammar<StorageT>,
        stable: &StateTable<StorageT>,
        mod_name: &str,
        outp_rs: P,
        cache: &str,
        diag: &SpannedDiagnosticFormatter,
    ) -> Result<(), Box<dyn Error>> {
        let visibility = self.visibility.clone();
        let user_actions = if let Some(
            YaccKind::Original(YaccOriginalActionKind::UserAction) | YaccKind::Grmtools,
        ) = self.yacckind
        {
            Some(self.gen_user_actions(grm, diag)?)
        } else {
            None
        };
        let rule_consts = self.gen_rule_consts(grm)?;
        let token_epp = self.gen_token_epp(grm)?;
        let parse_function = self.gen_parse_function(grm, stable)?;
        let action_wrappers = match self.yacckind.unwrap() {
            YaccKind::Original(YaccOriginalActionKind::UserAction) | YaccKind::Grmtools => {
                Some(self.gen_wrappers(grm)?)
            }
            YaccKind::Original(YaccOriginalActionKind::NoAction)
            | YaccKind::Original(YaccOriginalActionKind::GenericParseTree) => None,
            _ => unreachable!(),
        };

        let additional_decls =
            if let Some(YaccKind::Original(YaccOriginalActionKind::GenericParseTree)) =
                self.yacckind
            {
                // `lrpar::Node`` is deprecated within the lrpar crate, but not from within this module,
                // Once it is removed from `lrpar`, we should move the declaration here entirely.
                Some(quote! {
                            #[allow(unused_imports)]
                            pub use ::lrpar::parser::_deprecated_moved_::Node;
                })
            } else {
                None
            };

        let mod_name =
            match syn::parse_str::<proc_macro2::Ident>(mod_name) {
                Ok(s) => s,
                Err(e) => return Err(format!(
                    "CTParserBuilder::mod_name(\"{}\") is not a valid rust identifier due to '{}'",
                    mod_name, e
                )
                .into()),
            };
        let out_tokens = quote! {
            #visibility mod #mod_name {
                // At the top so that `user_actions` may contain #![inner_attribute]
                #user_actions
                mod _parser_ {
                    #![allow(clippy::type_complexity)]
                    #![allow(clippy::unnecessary_wraps)]
                    #![deny(unsafe_code)]
                    #[allow(unused_imports)]
                    use super::*;
                    #additional_decls
                    #parse_function
                    #rule_consts
                    #token_epp
                    #action_wrappers
                } // End of `mod _parser_`
                #[allow(unused_imports)]
                pub use _parser_::*;
                #[allow(unused_imports)]
                use ::lrpar::Lexeme;
            } // End of `mod #mod_name`
        };
        // Try and run a code formatter on the generated code.
        let unformatted = out_tokens.to_string();
        let outs = syn::parse_str(&unformatted)
            .map(|syntax_tree| prettyplease::unparse(&syntax_tree))
            .unwrap_or(unformatted);
        let mut f = File::create(outp_rs)?;
        f.write_all(outs.as_bytes())?;
        f.write_all(cache.as_bytes())?;
        Ok(())
    }

    /// Generate the cache, which determines if anything's changed enough that we need to
    /// regenerate outputs and force rustc to recompile.
    fn rebuild_cache(&self, derived_mod_name: &'_ str, grm: &YaccGrammar<StorageT>) -> TokenStream {
        // We don't need to be particularly clever here: we just need to record the various things
        // that could change between builds.
        //
        // Record the time that this version of lrpar was built. If the source code changes and
        // rustc forces a recompile, this will change this value, causing anything which depends on
        // this build of lrpar to be recompiled too.
        let Self {
            // All variables except for `output_path`, `inspect_callback` and `phantom` should
            // be written into the cache.
            grammar_path,
            // I struggle to imagine the correct thing for `grammar_src`.
            grammar_src: _,
            // I struggle to imagine the correct thing for `from_ast`.
            from_ast: _,
            mod_name,
            recoverer,
            yacckind,
            output_path: _,
            error_on_conflicts,
            warnings_are_errors,
            show_warnings,
            visibility,
            rust_edition,
            inspect_rt: _,
            #[cfg(test)]
                inspect_callback: _,
            phantom: _,
        } = self;
        let build_time = env!("VERGEN_BUILD_TIMESTAMP");
        let grammar_path = grammar_path.as_ref().unwrap().to_string_lossy();
        let mod_name = QuoteOption(mod_name.as_deref());
        let visibility = visibility.to_variant_tokens();
        let rust_edition = rust_edition.to_variant_tokens();
        let yacckind = yacckind.expect("is_some() by this point");
        let rule_map = grm
            .iter_tidxs()
            .map(|tidx| {
                QuoteTuple((
                    usize::from(tidx),
                    grm.token_name(tidx).unwrap_or("<unknown>"),
                ))
            })
            .collect::<Vec<_>>();
        let cache_info = quote! {
            BUILD_TIME = #build_time
            DERIVED_MOD_NAME = #derived_mod_name
            GRAMMAR_PATH = #grammar_path
            MOD_NAME = #mod_name
            RECOVERER = #recoverer
            YACC_KIND = #yacckind
            ERROR_ON_CONFLICTS = #error_on_conflicts
            SHOW_WARNINGS = #show_warnings
            WARNINGS_ARE_ERRORS = #warnings_are_errors
            RUST_EDITION = #rust_edition
            RULE_IDS_MAP = [#(#rule_map,)*]
            VISIBILITY = #visibility
        };
        let cache_info_str = cache_info.to_string();
        quote!(#cache_info_str)
    }

    /// Generate the main parse() function for the output file.
    fn gen_parse_function(
        &self,
        grm: &YaccGrammar<StorageT>,
        stable: &StateTable<StorageT>,
    ) -> Result<TokenStream, Box<dyn Error>> {
        let storaget = str::parse::<TokenStream>(type_name::<StorageT>())?;
        let lexertypest = str::parse::<TokenStream>(type_name::<LexerTypesT>())?;
        let recoverer = self.recoverer;
        let run_parser = match self.yacckind.unwrap() {
            YaccKind::Original(YaccOriginalActionKind::GenericParseTree) => {
                quote! {
                    ::lrpar::RTParserBuilder::new(&grm, &stable)
                        .recoverer(#recoverer)
                        .parse_map(
                            lexer,
                            &|lexeme| Node::Term{lexeme},
                            &|ridx, nodes| Node::Nonterm{ridx, nodes}
                        )
                }
            }
            YaccKind::Original(YaccOriginalActionKind::NoAction) => {
                quote! {
                    ::lrpar::RTParserBuilder::new(&grm, &stable)
                        .recoverer(#recoverer)
                        .parse_map(lexer, &|_| (), &|_, _| ()).1
                }
            }
            YaccKind::Original(YaccOriginalActionKind::UserAction) | YaccKind::Grmtools => {
                let actionskind = str::parse::<TokenStream>(ACTIONS_KIND)?;
                let parsed_parse_generics = make_generics(grm.parse_generics().as_deref())?;
                let (_, type_generics, _) = parsed_parse_generics.split_for_impl();
                // actions always have a parse_param argument, and when the `parse` function lacks one
                // that parameter will be unit.
                let (action_fn_parse_param, action_fn_parse_param_ty) = match grm.parse_param() {
                    Some((name, ty)) => {
                        let name = str::parse::<TokenStream>(name)?;
                        let ty = str::parse::<TokenStream>(ty)?;
                        (quote!(#name), quote!(#ty))
                    }
                    None => (quote!(()), quote!(())),
                };
                let wrappers = grm.iter_pidxs().map(|pidx| {
                    let pidx = usize::from(pidx);
                    format_ident!("{}wrapper_{}", ACTION_PREFIX, pidx)
                });
                let edition_lifetime = if self.rust_edition != RustEdition::Rust2015 {
                    quote!('_,)
                } else {
                    quote!()
                };
                let ridx = usize::from(self.user_start_ridx(grm));
                let action_ident = format_ident!("{}{}", ACTIONS_KIND_PREFIX, ridx);

                quote! {
                    let actions: ::std::vec::Vec<
                            &dyn Fn(
                                    ::cfgrammar::RIdx<#storaget>,
                                    &'lexer dyn ::lrpar::NonStreamingLexer<'input, #lexertypest>,
                                    ::cfgrammar::Span,
                                    ::std::vec::Drain<#edition_lifetime ::lrpar::parser::AStackType<<#lexertypest as ::lrpar::LexerTypes>::LexemeT, #actionskind #type_generics>>,
                                    #action_fn_parse_param_ty
                            ) -> #actionskind #type_generics
                        > = ::std::vec![#(&#wrappers,)*];
                    match ::lrpar::RTParserBuilder::new(&grm, &stable)
                        .recoverer(#recoverer)
                        .parse_actions(lexer, &actions, #action_fn_parse_param) {
                            (Some(#actionskind::#action_ident(x)), y) => (Some(x), y),
                            (None, y) => (None, y),
                            _ => unreachable!()
                    }
                }
            }
            kind => panic!("YaccKind {:?} not supported", kind),
        };

        let parsed_parse_generics: Generics = match self.yacckind.unwrap() {
            YaccKind::Original(YaccOriginalActionKind::UserAction) | YaccKind::Grmtools => {
                make_generics(grm.parse_generics().as_deref())?
            }
            _ => make_generics(None)?,
        };
        let (generics, _, where_clause) = parsed_parse_generics.split_for_impl();

        // `parse()` may or may not have an argument for `%parseparam`.
        let parse_fn_parse_param = match self.yacckind.unwrap() {
            YaccKind::Original(YaccOriginalActionKind::UserAction) | YaccKind::Grmtools => {
                if let Some((name, tyname)) = grm.parse_param() {
                    let name = str::parse::<TokenStream>(name)?;
                    let tyname = str::parse::<TokenStream>(tyname)?;
                    Some(quote! {#name: #tyname})
                } else {
                    None
                }
            }
            _ => None,
        };
        let parse_fn_return_ty = match self.yacckind.unwrap() {
            YaccKind::Original(YaccOriginalActionKind::UserAction) | YaccKind::Grmtools => {
                let actiont = grm
                    .actiontype(self.user_start_ridx(grm))
                    .as_ref()
                    .map(|at| str::parse::<TokenStream>(at))
                    .transpose()?;
                quote! {
                    (::std::option::Option<#actiont>, ::std::vec::Vec<::lrpar::LexParseError<#storaget, #lexertypest>>)
                }
            }
            YaccKind::Original(YaccOriginalActionKind::GenericParseTree) => quote! {
                (::std::option::Option<Node<<#lexertypest as ::lrpar::LexerTypes>::LexemeT, #storaget>>,
                    ::std::vec::Vec<::lrpar::LexParseError<#storaget, #lexertypest>>)
            },
            YaccKind::Original(YaccOriginalActionKind::NoAction) => quote! {
                ::std::vec::Vec<::lrpar::LexParseError<#storaget, #lexertypest>>
            },
            _ => unreachable!(),
        };

        let grm_data = encode_to_vec(grm, bincode::config::standard())?;
        let stable_data = encode_to_vec(stable, bincode::config::standard())?;
        Ok(quote! {
            const __GRM_DATA: &[u8] = &[#(#grm_data,)*];
            const __STABLE_DATA: &[u8] = &[#(#stable_data,)*];

            #[allow(dead_code)]
            pub fn parse #generics (
                 lexer: &'lexer dyn ::lrpar::NonStreamingLexer<'input, #lexertypest>,
                 #parse_fn_parse_param
            ) -> #parse_fn_return_ty
            #where_clause
            {
                let (grm, stable) = ::lrpar::ctbuilder::_reconstitute(__GRM_DATA, __STABLE_DATA);
                #run_parser
            }
        })
    }

    fn gen_rule_consts(
        &self,
        grm: &YaccGrammar<StorageT>,
    ) -> Result<TokenStream, proc_macro2::LexError> {
        let mut toks = TokenStream::new();
        for ridx in grm.iter_rules() {
            if !grm.rule_to_prods(ridx).contains(&grm.start_prod()) {
                let r_const = format_ident!("R_{}", grm.rule_name_str(ridx).to_ascii_uppercase());
                let storage_ty = str::parse::<TokenStream>(type_name::<StorageT>())?;
                let ridx = UnsuffixedUsize(usize::from(ridx));
                toks.extend(quote! {
                    #[allow(dead_code)]
                    pub const #r_const: #storage_ty = #ridx;
                });
            }
        }
        Ok(toks)
    }

    fn gen_token_epp(
        &self,
        grm: &YaccGrammar<StorageT>,
    ) -> Result<TokenStream, proc_macro2::LexError> {
        let mut tidxs = Vec::new();
        for tidx in grm.iter_tidxs() {
            tidxs.push(QuoteOption(grm.token_epp(tidx)));
        }
        let const_epp_ident = format_ident!("{}EPP", GLOBAL_PREFIX);
        let storage_ty = str::parse::<TokenStream>(type_name::<StorageT>())?;
        Ok(quote! {
            const #const_epp_ident: &[::std::option::Option<&str>] = &[
                #(#tidxs,)*
            ];

            /// Return the %epp entry for token `tidx` (where `None` indicates \"the token has no
            /// pretty-printed value\"). Panics if `tidx` doesn't exist.
            #[allow(dead_code)]
            pub fn token_epp<'a>(tidx: ::cfgrammar::TIdx<#storage_ty>) -> ::std::option::Option<&'a str> {
                #const_epp_ident[usize::from(tidx)]
            }
        })
    }

    /// Generate the wrappers that call user actions
    fn gen_wrappers(&self, grm: &YaccGrammar<StorageT>) -> Result<TokenStream, Box<dyn Error>> {
        let parsed_parse_generics = make_generics(grm.parse_generics().as_deref())?;
        let (generics, type_generics, where_clause) = parsed_parse_generics.split_for_impl();

        let (parse_paramname, parse_paramdef);
        match grm.parse_param() {
            Some((name, tyname)) => {
                parse_paramname = str::parse::<TokenStream>(name)?;
                let ty = str::parse::<TokenStream>(tyname)?;
                parse_paramdef = quote!(#parse_paramname: #ty);
            }
            None => {
                parse_paramname = quote!(());
                parse_paramdef = quote! {_: ()};
            }
        };

        let mut wrappers = TokenStream::new();
        for pidx in grm.iter_pidxs() {
            let ridx = grm.prod_to_rule(pidx);

            // Iterate over all $-arguments and replace them with their respective
            // element from the argument vector (e.g. $1 is replaced by args[0]). At
            // the same time extract &str from tokens and actiontype from nonterminals.
            let wrapper_fn = format_ident!("{}wrapper_{}", ACTION_PREFIX, usize::from(pidx));
            let ridx_var = format_ident!("{}ridx", ACTION_PREFIX);
            let lexer_var = format_ident!("{}lexer", ACTION_PREFIX);
            let span_var = format_ident!("{}span", ACTION_PREFIX);
            let args_var = format_ident!("{}args", ACTION_PREFIX);
            let storaget = str::parse::<TokenStream>(type_name::<StorageT>())?;
            let lexertypest = str::parse::<TokenStream>(type_name::<LexerTypesT>())?;
            let actionskind = str::parse::<TokenStream>(ACTIONS_KIND)?;
            let edition_lifetime = if self.rust_edition != RustEdition::Rust2015 {
                Some(quote!('_,))
            } else {
                None
            };
            let mut wrapper_fn_body = TokenStream::new();
            if grm.action(pidx).is_some() {
                // Unpack the arguments passed to us by the drain
                for i in 0..grm.prod(pidx).len() {
                    let arg = format_ident!("{}arg_{}", ACTION_PREFIX, i + 1);
                    wrapper_fn_body.extend(match grm.prod(pidx)[i] {
                        Symbol::Rule(ref_ridx) => {
                            let ref_ridx = usize::from(ref_ridx);
                            let actionvariant = format_ident!("{}{}", ACTIONS_KIND_PREFIX, ref_ridx);
                            quote! {
                                #[allow(clippy::let_unit_value)]
                                let #arg = match #args_var.next().unwrap() {
                                    ::lrpar::parser::AStackType::ActionType(#actionskind::#type_generics::#actionvariant(x)) => x,
                                    _ => unreachable!()
                                };
                            }
                        }
                        Symbol::Token(_) => {
                            quote! {
                                let #arg = match #args_var.next().unwrap() {
                                    ::lrpar::parser::AStackType::Lexeme(l) => {
                                        if l.faulty() {
                                            Err(l)
                                        } else {
                                            Ok(l)
                                        }
                                    },
                                    ::lrpar::parser::AStackType::ActionType(_) => unreachable!()
                                };
                            }
                        }
                    })
                }

                // Call the user code
                let args = (0..grm.prod(pidx).len())
                    .map(|i| format_ident!("{}arg_{}", ACTION_PREFIX, i + 1))
                    .collect::<Vec<_>>();
                let action_fn = format_ident!("{}action_{}", ACTION_PREFIX, usize::from(pidx));
                let actionsvariant = format_ident!("{}{}", ACTIONS_KIND_PREFIX, usize::from(ridx));

                wrapper_fn_body.extend(match grm.actiontype(ridx) {
                    Some(s) if s == "()" => {
                        // If the rule `r` that we're calling has the unit type then Clippy will warn that
                        // `enum::A(wrapper_r())` is pointless. We thus have to split it into two:
                        // `wrapper_r(); enum::A(())`.
                        quote! {
                            #action_fn(#ridx_var, #lexer_var, #span_var, #parse_paramname, #(#args,)*);
                            #actionskind::#type_generics::#actionsvariant(())
                        }
                    }
                    _ => {
                        quote! {
                            #actionskind::#type_generics::#actionsvariant(#action_fn(#ridx_var, #lexer_var, #span_var, #parse_paramname, #(#args,)*))
                        }
                    }
                })
            } else if pidx == grm.start_prod() {
                wrapper_fn_body.extend(quote!(unreachable!()));
            } else {
                unreachable!(
                    "Production in rule '{}' must have an action body, which should have been handled by gen_user_actions.",
                    grm.rule_name_str(grm.prod_to_rule(pidx))
                );
            };

            let attrib = if pidx == grm.start_prod() {
                // The start prod has an unreachable body so it doesn't use it's variables.
                Some(quote!(#[allow(unused_variables)]))
            } else {
                None
            };
            wrappers.extend(quote! {
                #attrib
                fn #wrapper_fn #generics (
                    #ridx_var: ::cfgrammar::RIdx<#storaget>,
                    #lexer_var: &'lexer dyn ::lrpar::NonStreamingLexer<'input, #lexertypest>,
                    #span_var: ::cfgrammar::Span,
                    mut #args_var: ::std::vec::Drain<#edition_lifetime ::lrpar::parser::AStackType<<#lexertypest as ::lrpar::LexerTypes>::LexemeT, #actionskind #type_generics>>,
                    #parse_paramdef
                ) -> #actionskind #type_generics
                #where_clause
                {
                    #wrapper_fn_body
                }
             })
        }
        let mut actionskindvariants = Vec::new();
        let actionskindhidden = format_ident!("_{}", ACTIONS_KIND_HIDDEN);
        let actionskind = str::parse::<TokenStream>(ACTIONS_KIND).unwrap();
        let mut phantom_data_type = Vec::new();
        for ridx in grm.iter_rules() {
            if let Some(actiont) = grm.actiontype(ridx) {
                let actionskindvariant =
                    format_ident!("{}{}", ACTIONS_KIND_PREFIX, usize::from(ridx));
                let actiont = str::parse::<TokenStream>(actiont).unwrap();
                actionskindvariants.push(quote! {
                    #actionskindvariant(#actiont)
                })
            }
        }
        for lifetime in parsed_parse_generics.lifetimes() {
            let lifetime = &lifetime.lifetime;
            phantom_data_type.push(quote! { &#lifetime () });
        }
        for type_param in parsed_parse_generics.type_params() {
            let ident = &type_param.ident;
            phantom_data_type.push(quote! { #ident });
        }
        actionskindvariants.push(quote! {
            #actionskindhidden(::std::marker::PhantomData<(#(#phantom_data_type,)*)>)
        });
        wrappers.extend(quote! {
            #[allow(dead_code)]
            enum #actionskind #generics #where_clause {
                #(#actionskindvariants,)*
            }
        });
        Ok(wrappers)
    }

    /// Generate the user action functions (if any).
    fn gen_user_actions(
        &self,
        grm: &YaccGrammar<StorageT>,
        diag: &SpannedDiagnosticFormatter,
    ) -> Result<TokenStream, Box<dyn Error>> {
        let programs = grm
            .programs()
            .as_ref()
            .map(|s| str::parse::<TokenStream>(s))
            .transpose()?;
        let mut action_fns = TokenStream::new();
        // Convert actions to functions
        let parsed_parse_generics = make_generics(grm.parse_generics().as_deref())?;
        let (generics, _, where_clause) = parsed_parse_generics.split_for_impl();
        let (parse_paramname, parse_paramdef, parse_param_unit);
        match grm.parse_param() {
            Some((name, tyname)) => {
                parse_param_unit = tyname.trim() == "()";
                parse_paramname = str::parse::<TokenStream>(name)?;
                let ty = str::parse::<TokenStream>(tyname)?;
                parse_paramdef = quote!(#parse_paramname: #ty);
            }
            None => {
                parse_param_unit = true;
                parse_paramname = quote!(());
                parse_paramdef = quote! {_: ()};
            }
        };
        for pidx in grm.iter_pidxs() {
            if pidx == grm.start_prod() {
                continue;
            }

            // Work out the right type for each argument
            let mut args = Vec::with_capacity(grm.prod(pidx).len());
            for i in 0..grm.prod(pidx).len() {
                let argt = match grm.prod(pidx)[i] {
                    Symbol::Rule(ref_ridx) => {
                        if let Some(action_type) = grm.actiontype(ref_ridx).as_ref() {
                            str::parse::<TokenStream>(action_type)?
                        } else {
                            let mut s = String::from("\n");
                            let rule_span = grm.rule_name_span(ref_ridx);
                            s.push_str(&diag.file_location_msg("Error", Some(rule_span)));
                            s.push_str("\n");
                            s.push_str(&diag.underline_span_with_text(
                                rule_span,
                                "Rule missing action type".to_string(),
                                '^',
                            ));
                            return Err(ErrorString(s).into());
                        }
                    }
                    Symbol::Token(_) => {
                        let lexemet =
                            str::parse::<TokenStream>(type_name::<LexerTypesT::LexemeT>())?;
                        quote!(::std::result::Result<#lexemet, #lexemet>)
                    }
                };
                let arg = format_ident!("{}arg_{}", ACTION_PREFIX, i + 1);
                args.push(quote!(mut #arg: #argt));
            }

            // If this rule's `actiont` is `()` then Clippy will warn that the return type `-> ()`
            // is pointless (which is true). We therefore avoid outputting a return type if actiont
            // is the unit type.
            let returnt = {
                let actiont = grm.actiontype(grm.prod_to_rule(pidx)).as_ref().unwrap();
                if actiont == "()" {
                    None
                } else {
                    let actiont = str::parse::<TokenStream>(actiont)?;
                    Some(quote!( -> #actiont))
                }
            };
            let action_fn = format_ident!("{}action_{}", ACTION_PREFIX, usize::from(pidx));
            let lexer_var = format_ident!("{}lexer", ACTION_PREFIX);
            let span_var = format_ident!("{}span", ACTION_PREFIX);
            let ridx_var = format_ident!("{}ridx", ACTION_PREFIX);
            let storaget = str::parse::<TokenStream>(type_name::<StorageT>())?;
            let lexertypest = str::parse::<TokenStream>(type_name::<LexerTypesT>())?;
            let bind_parse_param = if !parse_param_unit {
                Some(quote! {let _ = #parse_paramname;})
            } else {
                None
            };

            // Iterate over all $-arguments and replace them with their respective
            // element from the argument vector (e.g. $1 is replaced by args[0]).
            let pre_action = grm.action(pidx).as_ref().ok_or_else(|| {
                let mut s = String::from("\n");
                let span = grm.prod_span(pidx);
                s.push_str(&diag.file_location_msg("Error", Some(span)));
                s.push_str("\n");
                s.push_str(&diag.underline_span_with_text(
                    span,
                    "Production is missing action code".to_string(),
                    '^',
                ));
                ErrorString(s)
            })?;
            let mut last = 0;
            let mut outs = String::new();
            loop {
                match pre_action[last..].find('$') {
                    Some(off) => {
                        if pre_action[last + off..].starts_with("$$") {
                            outs.push_str(&pre_action[last..last + off + "$".len()]);
                            last = last + off + "$$".len();
                        } else if pre_action[last + off..].starts_with("$lexer") {
                            outs.push_str(&pre_action[last..last + off]);
                            write!(outs, "{prefix}lexer", prefix = ACTION_PREFIX).ok();
                            last = last + off + "$lexer".len();
                        } else if pre_action[last + off..].starts_with("$span") {
                            outs.push_str(&pre_action[last..last + off]);
                            write!(outs, "{prefix}span", prefix = ACTION_PREFIX).ok();
                            last = last + off + "$span".len();
                        } else if last + off + 1 < pre_action.len()
                            && pre_action[last + off + 1..].starts_with(|c: char| c.is_numeric())
                        {
                            outs.push_str(&pre_action[last..last + off]);
                            write!(outs, "{prefix}arg_", prefix = ACTION_PREFIX).ok();
                            last = last + off + "$".len();
                        } else {
                            let span = grm.action_span(pidx).unwrap();
                            let inner_span =
                                Span::new(span.start() + last + off + "$".len(), span.end());
                            let mut s = String::from("\n");
                            s.push_str(&diag.file_location_msg("Error", Some(inner_span)));
                            s.push_str("\n");
                            s.push_str(&diag.underline_span_with_text(
                                inner_span,
                                "Unknown text following '$'".to_string(),
                                '^',
                            ));
                            return Err(ErrorString(s).into());
                        }
                    }
                    None => {
                        outs.push_str(&pre_action[last..]);
                        break;
                    }
                }
            }

            let action_body = str::parse::<TokenStream>(&outs)?;
            action_fns.extend(quote! {
                #[allow(clippy::too_many_arguments)]
                fn #action_fn #generics (
                    #ridx_var: ::cfgrammar::RIdx<#storaget>,
                    #lexer_var: &'lexer dyn ::lrpar::NonStreamingLexer<'input, #lexertypest>,
                    #span_var: ::cfgrammar::Span,
                    #parse_paramdef,
                    #(#args,)*
                ) #returnt
                #where_clause
                {
                    #bind_parse_param
                    #action_body
                }
            })
        }
        Ok(quote! {
            #programs
            #action_fns
        })
    }

    /// Return the `RIdx` of the %start rule in the grammar (which will not be the same as
    /// grm.start_rule_idx because the latter has an additional rule insert by cfgrammar
    /// which then calls the user's %start rule).
    fn user_start_ridx(&self, grm: &YaccGrammar<StorageT>) -> RIdx<StorageT> {
        debug_assert_eq!(grm.prod(grm.start_prod()).len(), 1);
        match grm.prod(grm.start_prod())[0] {
            Symbol::Rule(ridx) => ridx,
            _ => unreachable!(),
        }
    }
}

/// This function is called by generated files; it exists so that generated files don't require a
/// direct dependency on bincode.
#[doc(hidden)]
pub fn _reconstitute<StorageT: Decode<()> + Hash + PrimInt + Unsigned + 'static>(
    grm_buf: &[u8],
    stable_buf: &[u8],
) -> (YaccGrammar<StorageT>, StateTable<StorageT>) {
    let (grm, _) = decode_from_slice(grm_buf, bincode::config::standard()).unwrap();
    let (stable, _) = decode_from_slice(stable_buf, bincode::config::standard()).unwrap();
    (grm, stable)
}

/// An interface to the result of [CTParserBuilder::build()].
pub struct CTParser<StorageT = u32>
where
    StorageT: Eq + Hash,
{
    regenerated: bool,
    rule_ids: HashMap<String, StorageT>,
    yacc_grammar: YaccGrammar<StorageT>,
    grammar_src: String,
    grammar_path: PathBuf,
    conflicts: Option<(StateGraph<StorageT>, StateTable<StorageT>)>,
}

impl<StorageT> CTParser<StorageT>
where
    StorageT: 'static + Debug + Hash + PrimInt + Unsigned,
    usize: AsPrimitive<StorageT>,
{
    /// Returns `true` if this compile-time parser was regenerated or `false` if it was not.
    pub fn regenerated(&self) -> bool {
        self.regenerated
    }

    /// Returns a [HashMap] from lexeme string types to numeric types (e.g. `INT: 2`), suitable for
    /// handing to a lexer to coordinate the IDs of lexer and parser.
    pub fn token_map(&self) -> &HashMap<String, StorageT> {
        &self.rule_ids
    }

    /// If there are any conflicts in the grammar, return a tuple which allows users to inspect and
    /// pretty print them; otherwise returns `None`. If the grammar was not regenerated, this will
    /// always return `None`, even if the grammar actually has conflicts.
    ///
    /// **Note: The conflicts feature is currently unstable and may change in the future.**
    #[allow(private_interfaces)]
    pub fn conflicts(
        &self,
        _: crate::unstable::UnstableApi,
    ) -> Option<(
        &YaccGrammar<StorageT>,
        &StateGraph<StorageT>,
        &StateTable<StorageT>,
        &Conflicts<StorageT>,
    )> {
        if let Some((sgraph, stable)) = &self.conflicts {
            return Some((
                &self.yacc_grammar,
                sgraph,
                stable,
                stable.conflicts().unwrap(),
            ));
        }
        None
    }

    #[doc(hidden)]
    pub fn yacc_grammar(&self) -> &YaccGrammar<StorageT> {
        &self.yacc_grammar
    }
    #[doc(hidden)]
    pub fn grammar_src(&self) -> &str {
        &self.grammar_src
    }
    #[doc(hidden)]
    pub fn grammar_path(&self) -> &Path {
        self.grammar_path.as_path()
    }
}

/// Indents a multi-line string and trims any trailing newline.
/// This currently assumes that indentation on blank lines does not matter.
///
/// The algorithm used by this function is:
/// 1. Prefix `s` with the indentation, indenting the first line.
/// 2. Trim any trailing newlines.
/// 3. Replace all newlines with `\n{indent}`` to indent all lines after the first.
///
/// It is plausible that we should a step 4, but currently do not:
/// 4. Replace all `\n{indent}\n` with `\n\n`
fn indent(indent: &str, s: &str) -> String {
    format!("{indent}{}\n", s.trim_end_matches('\n')).replace('\n', &format!("\n{}", indent))
}

fn make_generics(parse_generics: Option<&str>) -> Result<Generics, Box<dyn Error>> {
    if let Some(parse_generics) = parse_generics {
        let tokens = str::parse::<TokenStream>(parse_generics)?;
        match syn::parse2(quote!(<'lexer, 'input: 'lexer, #tokens>)) {
            Ok(res) => Ok(res),
            Err(err) => Err(format!("unable to parse %parse-generics: {}", err).into()),
        }
    } else {
        Ok(parse_quote!(<'lexer, 'input: 'lexer>))
    }
}

// Tests dealing with the filesystem not supported under wasm32
#[cfg(all(not(target_arch = "wasm32"), test))]
mod test {
    use std::{fs::File, io::Write, path::PathBuf};

    use super::{CTConflictsError, CTParserBuilder};
    use crate::test_utils::TestLexerTypes;
    use cfgrammar::yacc::{YaccKind, YaccOriginalActionKind};
    use tempfile::TempDir;

    #[test]
    fn test_conflicts() {
        let temp = TempDir::new().unwrap();
        let mut file_path = PathBuf::from(temp.as_ref());
        file_path.push("grm.y");
        let mut f = File::create(&file_path).unwrap();
        let _ = f.write_all(
            "%start A
%%
A : 'a' 'b' | B 'b';
B : 'a' | C;
C : 'a';"
                .as_bytes(),
        );

        match CTParserBuilder::<TestLexerTypes>::new()
            .error_on_conflicts(false)
            .yacckind(YaccKind::Original(YaccOriginalActionKind::GenericParseTree))
            .grammar_path(file_path.to_str().unwrap())
            .output_path(file_path.with_extension("ignored"))
            .build()
            .unwrap()
            .conflicts(crate::unstable::UnstableApi)
        {
            Some((_, _, _, conflicts)) => {
                assert_eq!(conflicts.sr_len(), 1);
                assert_eq!(conflicts.rr_len(), 1);
            }
            None => panic!("Expected error data"),
        }
    }

    #[test]
    fn test_conflicts_error() {
        let temp = TempDir::new().unwrap();
        let mut file_path = PathBuf::from(temp.as_ref());
        file_path.push("grm.y");
        let mut f = File::create(&file_path).unwrap();
        let _ = f.write_all(
            "%start A
%%
A : 'a' 'b' | B 'b';
B : 'a' | C;
C : 'a';"
                .as_bytes(),
        );

        match CTParserBuilder::<TestLexerTypes>::new()
            .yacckind(YaccKind::Original(YaccOriginalActionKind::GenericParseTree))
            .grammar_path(file_path.to_str().unwrap())
            .output_path(file_path.with_extension("ignored"))
            .build()
        {
            Ok(_) => panic!("Expected error"),
            Err(e) => {
                let cs = e.downcast_ref::<CTConflictsError<u16>>();
                assert_eq!(cs.unwrap().stable.conflicts().unwrap().rr_len(), 1);
                assert_eq!(cs.unwrap().stable.conflicts().unwrap().sr_len(), 1);
            }
        }
    }

    #[test]
    fn test_expect_error() {
        let temp = TempDir::new().unwrap();
        let mut file_path = PathBuf::from(temp.as_ref());
        file_path.push("grm.y");
        let mut f = File::create(&file_path).unwrap();
        let _ = f.write_all(
            "%start A
%expect 2
%%
A: 'a' 'b' | B 'b';
B: 'a';"
                .as_bytes(),
        );

        match CTParserBuilder::<TestLexerTypes>::new()
            .yacckind(YaccKind::Original(YaccOriginalActionKind::GenericParseTree))
            .grammar_path(file_path.to_str().unwrap())
            .output_path(file_path.with_extension("ignored"))
            .build()
        {
            Ok(_) => panic!("Expected error"),
            Err(e) => {
                let cs = e.downcast_ref::<CTConflictsError<u16>>();
                assert_eq!(cs.unwrap().stable.conflicts().unwrap().rr_len(), 0);
                assert_eq!(cs.unwrap().stable.conflicts().unwrap().sr_len(), 1);
            }
        }
    }

    #[test]
    fn test_expectrr_error() {
        let temp = TempDir::new().unwrap();
        let mut file_path = PathBuf::from(temp.as_ref());
        file_path.push("grm.y");
        let mut f = File::create(&file_path).unwrap();
        let _ = f.write_all(
            "%start A
%expect 1
%expect-rr 2
%%
A : 'a' 'b' | B 'b';
B : 'a' | C;
C : 'a';"
                .as_bytes(),
        );

        match CTParserBuilder::<TestLexerTypes>::new()
            .yacckind(YaccKind::Original(YaccOriginalActionKind::GenericParseTree))
            .grammar_path(file_path.to_str().unwrap())
            .output_path(file_path.with_extension("ignored"))
            .build()
        {
            Ok(_) => panic!("Expected error"),
            Err(e) => {
                let cs = e.downcast_ref::<CTConflictsError<u16>>();
                assert_eq!(cs.unwrap().stable.conflicts().unwrap().rr_len(), 1);
                assert_eq!(cs.unwrap().stable.conflicts().unwrap().sr_len(), 1);
            }
        }
    }

    #[test]
    /// Tests a yacc .y filename containing a dash character leading to an invalid rust identifier
    /// when that dash is subsequently used as the default `CTParserBuilder::mod_name`.
    fn test_invalid_identifier_in_derived_mod_name() {
        let temp = TempDir::new().unwrap();
        let mut file_path = PathBuf::from(temp.as_ref());
        file_path.push("contains-a-dash.y");
        let mut f = File::create(&file_path).unwrap();
        let _ = f.write_all(
            "%start A
%%
A : 'a';"
                .as_bytes(),
        );
        match CTParserBuilder::<TestLexerTypes>::new()
            .yacckind(YaccKind::Original(YaccOriginalActionKind::GenericParseTree))
            .grammar_path(file_path.to_str().unwrap())
            .output_path(file_path.with_extension("ignored"))
            .build()
        {
            Ok(_) => panic!("Expected error"),
            Err(e) => {
                let err_string = e.to_string();
                assert_eq!(
                    err_string,
                    "CTParserBuilder::mod_name(\"contains-a-dash_y\") is not a valid rust identifier due to 'unexpected token'"
                );
            }
        }
    }

    #[cfg(test)]
    #[test]
    fn test_recoverer_header() -> Result<(), Box<dyn std::error::Error>> {
        use crate::RecoveryKind as RK;
        #[rustfmt::skip]
            let recovery_kinds = [
                //  Builder,          Header setting,     Expected result.
                // -----------       ------------------  -------------------
                (Some(RK::None),      Some(RK::None),     Some(RK::None)),
                (Some(RK::None),      Some(RK::CPCTPlus), Some(RK::None)),
                (Some(RK::CPCTPlus),  Some(RK::CPCTPlus), Some(RK::CPCTPlus)),
                (Some(RK::CPCTPlus),  Some(RK::None),     Some(RK::CPCTPlus)),
                (None,                Some(RK::CPCTPlus), Some(RK::CPCTPlus)),
                (None,                Some(RK::None),     Some(RK::None)),
                (None,                None,               Some(RK::CPCTPlus)),
                (Some(RK::None),      None,               Some(RK::None)),
                (Some(RK::CPCTPlus),  None,               Some(RK::CPCTPlus)),
            ];

        for (i, (builder_arg, header_arg, expected_rk)) in
            recovery_kinds.iter().cloned().enumerate()
        {
            let y_src = if let Some(header_arg) = header_arg {
                format!(
                    "\
                    %grmtools{{yacckind: Original(NoAction), recoverer: {}}} \
                    %% \
                    start: ; \
                    ",
                    match header_arg {
                        RK::None => "RecoveryKind::None",
                        RK::CPCTPlus => "RecoveryKind::CPCTPlus",
                    }
                )
            } else {
                r#"
                    %grmtools{yacckind: Original(NoAction)}
                    %%
                    Start: ;
                    "#
                .to_string()
            };
            let out_dir = std::env::var("OUT_DIR").unwrap();
            let y_path = format!("{out_dir}/recoverykind_test_{i}.y");
            let y_out_path = format!("{y_path}.rs");
            std::fs::File::create(y_path.clone()).unwrap();
            std::fs::write(y_path.clone(), y_src).unwrap();
            let mut cp_builder = CTParserBuilder::<TestLexerTypes>::new();
            cp_builder = cp_builder
                .output_path(y_out_path.clone())
                .grammar_path(y_path.clone());
            cp_builder = if let Some(builder_arg) = builder_arg {
                cp_builder.recoverer(builder_arg)
            } else {
                cp_builder
            }
            .inspect_recoverer(Box::new(move |rk| {
                if matches!(
                    (rk, expected_rk),
                    (RK::None, Some(RK::None)) | (RK::CPCTPlus, Some(RK::CPCTPlus))
                ) {
                    Ok(())
                } else {
                    panic!("Unexpected recovery kind")
                }
            }));
            cp_builder.build()?;
        }
        Ok(())
    }
}