cranelift_reader/

parser.rs

//! Parser for .clif files.

use crate::error::{Location, ParseError, ParseResult};
use crate::isaspec;
use crate::lexer::{LexError, Lexer, LocatedError, LocatedToken, Token};
use crate::run_command::{Comparison, Invocation, RunCommand};
use crate::sourcemap::SourceMap;
use crate::testcommand::TestCommand;
use crate::testfile::{Comment, Details, Feature, TestFile};
use cranelift_codegen::data_value::DataValue;
use cranelift_codegen::entity::{EntityRef, PrimaryMap};
use cranelift_codegen::ir::entities::{AnyEntity, DynamicType};
use cranelift_codegen::ir::immediates::{
    Ieee16, Ieee32, Ieee64, Ieee128, Imm64, Offset32, Uimm32, Uimm64,
};
use cranelift_codegen::ir::instructions::{InstructionData, InstructionFormat, VariableArgs};
use cranelift_codegen::ir::{self, StackSlotKey, UserExternalNameRef};
use cranelift_codegen::ir::{DebugTag, types::*};

use cranelift_codegen::ir::{
    AbiParam, ArgumentExtension, ArgumentPurpose, Block, BlockArg, Constant, ConstantData,
    DynamicStackSlot, DynamicStackSlotData, DynamicTypeData, ExtFuncData, ExternalName, FuncRef,
    Function, GlobalValue, GlobalValueData, JumpTableData, MemFlags, Opcode, SigRef, Signature,
    StackSlot, StackSlotData, StackSlotKind, UserFuncName, Value, types,
};
use cranelift_codegen::isa::{self, CallConv};
use cranelift_codegen::packed_option::ReservedValue;
use cranelift_codegen::{settings, settings::Configurable, timing};
use smallvec::SmallVec;
use std::mem;
use std::str::FromStr;
use std::{u16, u32};
use target_lexicon::Triple;

macro_rules! match_imm {
    ($signed:ty, $unsigned:ty, $parser:expr, $err_msg:expr) => {{
        if let Some(Token::Integer(text)) = $parser.token() {
            $parser.consume();
            let negative = text.starts_with('-');
            let positive = text.starts_with('+');
            let text = if negative || positive {
                // Strip sign prefix.
                &text[1..]
            } else {
                text
            };

            // Parse the text value; the lexer gives us raw text that looks like an integer.
            let value = if text.starts_with("0x") {
                // Skip underscores.
                let text = text.replace("_", "");
                // Parse it in hexadecimal form.
                <$unsigned>::from_str_radix(&text[2..], 16).map_err(|_| {
                    $parser.error(&format!(
                        "unable to parse '{}' value as a hexadecimal {} immediate",
                        &text[2..],
                        stringify!($unsigned),
                    ))
                })?
            } else {
                // Parse it as a signed type to check for overflow and other issues.
                text.parse()
                    .map_err(|_| $parser.error("expected decimal immediate"))?
            };

            // Apply sign if necessary.
            let signed = if negative {
                let value = value.wrapping_neg() as $signed;
                if value > 0 {
                    return Err($parser.error("negative number too small"));
                }
                value
            } else {
                value as $signed
            };

            Ok(signed)
        } else {
            err!($parser.loc, $err_msg)
        }
    }};
}

/// After some quick benchmarks a program should never have more than 100,000 blocks.
const MAX_BLOCKS_IN_A_FUNCTION: u32 = 100_000;

/// Parse the entire `text` into a list of functions.
///
/// Any test commands or target declarations are ignored.
pub fn parse_functions(text: &str) -> ParseResult<Vec<Function>> {
    let _tt = timing::parse_text();
    parse_test(text, ParseOptions::default())
        .map(|file| file.functions.into_iter().map(|(func, _)| func).collect())
}

/// Options for configuring the parsing of filetests.
pub struct ParseOptions<'a> {
    /// Compiler passes to run on the parsed functions.
    pub passes: Option<&'a [String]>,
    /// Target ISA for compiling the parsed functions, e.g. "x86_64 skylake".
    pub target: Option<&'a str>,
    /// Default calling convention used when none is specified for a parsed function.
    pub default_calling_convention: CallConv,
    /// Default for unwind-info setting (enabled or disabled).
    pub unwind_info: bool,
    /// Default for machine_code_cfg_info setting (enabled or disabled).
    pub machine_code_cfg_info: bool,
}

impl Default for ParseOptions<'_> {
    fn default() -> Self {
        Self {
            passes: None,
            target: None,
            default_calling_convention: CallConv::Fast,
            unwind_info: false,
            machine_code_cfg_info: false,
        }
    }
}

/// Parse the entire `text` as a test case file.
///
/// The returned `TestFile` contains direct references to substrings of `text`.
pub fn parse_test<'a>(text: &'a str, options: ParseOptions<'a>) -> ParseResult<TestFile<'a>> {
    let _tt = timing::parse_text();
    let mut parser = Parser::new(text);

    // Gather the preamble comments.
    parser.start_gathering_comments();

    let isa_spec: isaspec::IsaSpec;
    let commands: Vec<TestCommand<'a>>;

    // Check for specified passes and target, if present throw out test commands/targets specified
    // in file.
    match options.passes {
        Some(pass_vec) => {
            parser.parse_test_commands();
            commands = parser.parse_cmdline_passes(pass_vec);
            parser.parse_target_specs(&options)?;
            isa_spec = parser.parse_cmdline_target(options.target)?;
        }
        None => {
            commands = parser.parse_test_commands();
            isa_spec = parser.parse_target_specs(&options)?;
        }
    };
    let features = parser.parse_cranelift_features()?;

    // Decide between using the calling convention passed in the options or using the
    // host's calling convention--if any tests are to be run on the host we should default to the
    // host's calling convention.
    parser = if commands.iter().any(|tc| tc.command == "run") {
        let host_default_calling_convention = CallConv::triple_default(&Triple::host());
        parser.with_default_calling_convention(host_default_calling_convention)
    } else {
        parser.with_default_calling_convention(options.default_calling_convention)
    };

    parser.token();
    parser.claim_gathered_comments(AnyEntity::Function);

    let preamble_comments = parser.take_comments();
    let functions = parser.parse_function_list()?;

    Ok(TestFile {
        commands,
        isa_spec,
        features,
        preamble_comments,
        functions,
    })
}

/// Parse a CLIF comment `text` as a run command.
///
/// Return:
///  - `Ok(None)` if the comment is not intended to be a `RunCommand` (i.e. does not start with `run`
///    or `print`
///  - `Ok(Some(command))` if the comment is intended as a `RunCommand` and can be parsed to one
///  - `Err` otherwise.
pub fn parse_run_command(text: &str, signature: &Signature) -> ParseResult<Option<RunCommand>> {
    let _tt = timing::parse_text();
    // We remove leading spaces and semi-colons for convenience here instead of at the call sites
    // since this function will be attempting to parse a RunCommand from a CLIF comment.
    let trimmed_text = text.trim_start_matches(|c| c == ' ' || c == ';');
    let mut parser = Parser::new(trimmed_text);
    match parser.token() {
        Some(Token::Identifier("run")) | Some(Token::Identifier("print")) => {
            parser.parse_run_command(signature).map(|c| Some(c))
        }
        Some(_) | None => Ok(None),
    }
}

pub struct Parser<'a> {
    lex: Lexer<'a>,

    lex_error: Option<LexError>,

    /// Current lookahead token.
    lookahead: Option<Token<'a>>,

    /// Location of lookahead.
    loc: Location,

    /// Are we gathering any comments that we encounter?
    gathering_comments: bool,

    /// The gathered comments; claim them with `claim_gathered_comments`.
    gathered_comments: Vec<&'a str>,

    /// Comments collected so far.
    comments: Vec<Comment<'a>>,

    /// Maps inlined external names to a ref value, so they can be declared before parsing the rest
    /// of the function later.
    ///
    /// This maintains backward compatibility with previous ways for declaring external names.
    predeclared_external_names: PrimaryMap<UserExternalNameRef, ir::UserExternalName>,

    /// Default calling conventions; used when none is specified.
    default_calling_convention: CallConv,
}

/// Context for resolving references when parsing a single function.
struct Context {
    function: Function,
    map: SourceMap,

    /// Aliases to resolve once value definitions are known.
    aliases: Vec<Value>,
}

impl Context {
    fn new(f: Function) -> Self {
        Self {
            function: f,
            map: SourceMap::new(),
            aliases: Vec::new(),
        }
    }

    // Allocate a new stack slot.
    fn add_ss(&mut self, ss: StackSlot, data: StackSlotData, loc: Location) -> ParseResult<()> {
        self.map.def_ss(ss, loc)?;
        while self.function.sized_stack_slots.next_key().index() <= ss.index() {
            self.function.create_sized_stack_slot(StackSlotData::new(
                StackSlotKind::ExplicitSlot,
                0,
                0,
            ));
        }
        self.function.sized_stack_slots[ss] = data;
        Ok(())
    }

    // Resolve a reference to a stack slot.
    fn check_ss(&self, ss: StackSlot, loc: Location) -> ParseResult<()> {
        if !self.map.contains_ss(ss) {
            err!(loc, "undefined stack slot {}", ss)
        } else {
            Ok(())
        }
    }

    // Allocate a new stack slot.
    fn add_dss(
        &mut self,
        ss: DynamicStackSlot,
        data: DynamicStackSlotData,
        loc: Location,
    ) -> ParseResult<()> {
        self.map.def_dss(ss, loc)?;
        while self.function.dynamic_stack_slots.next_key().index() <= ss.index() {
            self.function
                .create_dynamic_stack_slot(DynamicStackSlotData::new(
                    StackSlotKind::ExplicitDynamicSlot,
                    data.dyn_ty,
                ));
        }
        self.function.dynamic_stack_slots[ss] = data;
        Ok(())
    }

    // Resolve a reference to a dynamic stack slot.
    fn check_dss(&self, dss: DynamicStackSlot, loc: Location) -> ParseResult<()> {
        if !self.map.contains_dss(dss) {
            err!(loc, "undefined dynamic stack slot {}", dss)
        } else {
            Ok(())
        }
    }

    // Allocate a new dynamic type.
    fn add_dt(&mut self, dt: DynamicType, data: DynamicTypeData, loc: Location) -> ParseResult<()> {
        self.map.def_dt(dt, loc)?;
        while self.function.dfg.dynamic_types.next_key().index() <= dt.index() {
            self.function.dfg.make_dynamic_ty(DynamicTypeData::new(
                data.base_vector_ty,
                data.dynamic_scale,
            ));
        }
        self.function.dfg.dynamic_types[dt] = data;
        Ok(())
    }

    // Allocate a global value slot.
    fn add_gv(&mut self, gv: GlobalValue, data: GlobalValueData, loc: Location) -> ParseResult<()> {
        self.map.def_gv(gv, loc)?;
        while self.function.global_values.next_key().index() <= gv.index() {
            self.function.create_global_value(GlobalValueData::Symbol {
                name: ExternalName::testcase(""),
                offset: Imm64::new(0),
                colocated: false,
                tls: false,
            });
        }
        self.function.global_values[gv] = data;
        Ok(())
    }

    // Resolve a reference to a global value.
    fn check_gv(&self, gv: GlobalValue, loc: Location) -> ParseResult<()> {
        if !self.map.contains_gv(gv) {
            err!(loc, "undefined global value {}", gv)
        } else {
            Ok(())
        }
    }

    // Allocate a new signature.
    fn add_sig(
        &mut self,
        sig: SigRef,
        data: Signature,
        loc: Location,
        defaultcc: CallConv,
    ) -> ParseResult<()> {
        self.map.def_sig(sig, loc)?;
        while self.function.dfg.signatures.next_key().index() <= sig.index() {
            self.function.import_signature(Signature::new(defaultcc));
        }
        self.function.dfg.signatures[sig] = data;
        Ok(())
    }

    // Resolve a reference to a signature.
    fn check_sig(&self, sig: SigRef, loc: Location) -> ParseResult<()> {
        if !self.map.contains_sig(sig) {
            err!(loc, "undefined signature {}", sig)
        } else {
            Ok(())
        }
    }

    // Allocate a new external function.
    fn add_fn(&mut self, fn_: FuncRef, data: ExtFuncData, loc: Location) -> ParseResult<()> {
        self.map.def_fn(fn_, loc)?;
        while self.function.dfg.ext_funcs.next_key().index() <= fn_.index() {
            self.function.import_function(ExtFuncData {
                name: ExternalName::testcase(""),
                signature: SigRef::reserved_value(),
                colocated: false,
                patchable: false,
            });
        }
        self.function.dfg.ext_funcs[fn_] = data;
        Ok(())
    }

    // Resolve a reference to a function.
    fn check_fn(&self, fn_: FuncRef, loc: Location) -> ParseResult<()> {
        if !self.map.contains_fn(fn_) {
            err!(loc, "undefined function {}", fn_)
        } else {
            Ok(())
        }
    }

    // Allocate a new constant.
    fn add_constant(
        &mut self,
        constant: Constant,
        data: ConstantData,
        loc: Location,
    ) -> ParseResult<()> {
        self.map.def_constant(constant, loc)?;
        self.function.dfg.constants.set(constant, data);
        Ok(())
    }

    // Configure the stack limit of the current function.
    fn add_stack_limit(&mut self, limit: GlobalValue, loc: Location) -> ParseResult<()> {
        if self.function.stack_limit.is_some() {
            return err!(loc, "stack limit defined twice");
        }
        self.function.stack_limit = Some(limit);
        Ok(())
    }

    // Resolve a reference to a constant.
    fn check_constant(&self, c: Constant, loc: Location) -> ParseResult<()> {
        if !self.map.contains_constant(c) {
            err!(loc, "undefined constant {}", c)
        } else {
            Ok(())
        }
    }

    // Allocate a new block.
    fn add_block(&mut self, block: Block, loc: Location) -> ParseResult<Block> {
        self.map.def_block(block, loc)?;
        while self.function.dfg.num_blocks() <= block.index() {
            self.function.dfg.make_block();
        }
        self.function.layout.append_block(block);
        Ok(block)
    }

    /// Set a block as cold.
    fn set_cold_block(&mut self, block: Block) {
        self.function.layout.set_cold(block);
    }
}

impl<'a> Parser<'a> {
    /// Create a new `Parser` which reads `text`. The referenced text must outlive the parser.
    pub fn new(text: &'a str) -> Self {
        Self {
            lex: Lexer::new(text),
            lex_error: None,
            lookahead: None,
            loc: Location { line_number: 0 },
            gathering_comments: false,
            gathered_comments: Vec::new(),
            comments: Vec::new(),
            default_calling_convention: CallConv::Fast,
            predeclared_external_names: Default::default(),
        }
    }

    /// Modify the default calling convention; returns a new parser with the changed calling
    /// convention.
    pub fn with_default_calling_convention(self, default_calling_convention: CallConv) -> Self {
        Self {
            default_calling_convention,
            ..self
        }
    }

    // Consume the current lookahead token and return it.
    fn consume(&mut self) -> Token<'a> {
        self.lookahead.take().expect("No token to consume")
    }

    // Consume the whole line following the current lookahead token.
    // Return the text of the line tail.
    fn consume_line(&mut self) -> &'a str {
        let rest = self.lex.rest_of_line();
        self.consume();
        rest
    }

    // Get the current lookahead token, after making sure there is one.
    fn token(&mut self) -> Option<Token<'a>> {
        while self.lookahead.is_none() {
            match self.lex.next() {
                Some(Ok(LocatedToken { token, location })) => {
                    match token {
                        Token::Comment(text) => {
                            if self.gathering_comments {
                                self.gathered_comments.push(text);
                            }
                        }
                        _ => self.lookahead = Some(token),
                    }
                    self.loc = location;
                }
                Some(Err(LocatedError { error, location })) => {
                    self.lex_error = Some(error);
                    self.loc = location;
                    break;
                }
                None => break,
            }
        }
        self.lookahead
    }

    // Enable gathering of all comments encountered.
    fn start_gathering_comments(&mut self) {
        debug_assert!(!self.gathering_comments);
        self.gathering_comments = true;
        debug_assert!(self.gathered_comments.is_empty());
    }

    // Claim the comments gathered up to the current position for the
    // given entity.
    fn claim_gathered_comments<E: Into<AnyEntity>>(&mut self, entity: E) {
        debug_assert!(self.gathering_comments);
        let entity = entity.into();
        self.comments.extend(
            self.gathered_comments
                .drain(..)
                .map(|text| Comment { entity, text }),
        );
        self.gathering_comments = false;
    }

    // Get the comments collected so far, clearing out the internal list.
    fn take_comments(&mut self) -> Vec<Comment<'a>> {
        debug_assert!(!self.gathering_comments);
        mem::replace(&mut self.comments, Vec::new())
    }

    // Match and consume a token without payload.
    fn match_token(&mut self, want: Token<'a>, err_msg: &str) -> ParseResult<Token<'a>> {
        if self.token() == Some(want) {
            Ok(self.consume())
        } else {
            err!(self.loc, err_msg)
        }
    }

    // If the next token is a `want`, consume it, otherwise do nothing.
    fn optional(&mut self, want: Token<'a>) -> bool {
        if self.token() == Some(want) {
            self.consume();
            true
        } else {
            false
        }
    }

    // Match and consume a specific identifier string.
    // Used for pseudo-keywords like "stack_slot" that only appear in certain contexts.
    fn match_identifier(&mut self, want: &'static str, err_msg: &str) -> ParseResult<Token<'a>> {
        if self.token() == Some(Token::Identifier(want)) {
            Ok(self.consume())
        } else {
            err!(self.loc, err_msg)
        }
    }

    // Match and consume a type.
    fn match_type(&mut self, err_msg: &str) -> ParseResult<Type> {
        if let Some(Token::Type(t)) = self.token() {
            self.consume();
            Ok(t)
        } else {
            err!(self.loc, err_msg)
        }
    }

    // Match and consume a stack slot reference.
    fn match_ss(&mut self, err_msg: &str) -> ParseResult<StackSlot> {
        if let Some(Token::StackSlot(ss)) = self.token() {
            self.consume();
            if let Some(ss) = StackSlot::with_number(ss) {
                return Ok(ss);
            }
        }
        err!(self.loc, err_msg)
    }

    // Match and consume a dynamic stack slot reference.
    fn match_dss(&mut self, err_msg: &str) -> ParseResult<DynamicStackSlot> {
        if let Some(Token::DynamicStackSlot(ss)) = self.token() {
            self.consume();
            if let Some(ss) = DynamicStackSlot::with_number(ss) {
                return Ok(ss);
            }
        }
        err!(self.loc, err_msg)
    }

    // Match and consume a dynamic type reference.
    fn match_dt(&mut self, err_msg: &str) -> ParseResult<DynamicType> {
        if let Some(Token::DynamicType(dt)) = self.token() {
            self.consume();
            if let Some(dt) = DynamicType::with_number(dt) {
                return Ok(dt);
            }
        }
        err!(self.loc, err_msg)
    }

    // Extract Type from DynamicType
    fn concrete_from_dt(&mut self, dt: DynamicType, ctx: &mut Context) -> Option<Type> {
        ctx.function.get_concrete_dynamic_ty(dt)
    }

    // Match and consume a global value reference.
    fn match_gv(&mut self, err_msg: &str) -> ParseResult<GlobalValue> {
        if let Some(Token::GlobalValue(gv)) = self.token() {
            self.consume();
            if let Some(gv) = GlobalValue::with_number(gv) {
                return Ok(gv);
            }
        }
        err!(self.loc, err_msg)
    }

    // Match and consume a function reference.
    fn match_fn(&mut self, err_msg: &str) -> ParseResult<FuncRef> {
        if let Some(Token::FuncRef(fnref)) = self.token() {
            self.consume();
            if let Some(fnref) = FuncRef::with_number(fnref) {
                return Ok(fnref);
            }
        }
        err!(self.loc, err_msg)
    }

    // Match and consume a signature reference.
    fn match_sig(&mut self, err_msg: &str) -> ParseResult<SigRef> {
        if let Some(Token::SigRef(sigref)) = self.token() {
            self.consume();
            if let Some(sigref) = SigRef::with_number(sigref) {
                return Ok(sigref);
            }
        }
        err!(self.loc, err_msg)
    }

    // Match and consume a constant reference.
    fn match_constant(&mut self) -> ParseResult<Constant> {
        if let Some(Token::Constant(c)) = self.token() {
            self.consume();
            if let Some(c) = Constant::with_number(c) {
                return Ok(c);
            }
        }
        err!(self.loc, "expected constant number: const«n»")
    }

    // Match and consume a stack limit token
    fn match_stack_limit(&mut self) -> ParseResult<()> {
        if let Some(Token::Identifier("stack_limit")) = self.token() {
            self.consume();
            return Ok(());
        }
        err!(self.loc, "expected identifier: stack_limit")
    }

    // Match and consume a block reference.
    fn match_block(&mut self, err_msg: &str) -> ParseResult<Block> {
        if let Some(Token::Block(block)) = self.token() {
            self.consume();
            Ok(block)
        } else {
            err!(self.loc, err_msg)
        }
    }

    // Match and consume a value reference.
    fn match_value(&mut self, err_msg: &str) -> ParseResult<Value> {
        if let Some(Token::Value(v)) = self.token() {
            self.consume();
            Ok(v)
        } else {
            err!(self.loc, err_msg)
        }
    }

    fn error(&self, message: &str) -> ParseError {
        ParseError {
            location: self.loc,
            message: message.to_string(),
            is_warning: false,
        }
    }

    // Match and consume an Imm64 immediate.
    fn match_imm64(&mut self, err_msg: &str) -> ParseResult<Imm64> {
        if let Some(Token::Integer(text)) = self.token() {
            self.consume();
            // Lexer just gives us raw text that looks like an integer.
            // Parse it as an Imm64 to check for overflow and other issues.
            text.parse().map_err(|e| self.error(e))
        } else {
            err!(self.loc, err_msg)
        }
    }

    // Match and consume a hexadeximal immediate
    fn match_hexadecimal_constant(&mut self, err_msg: &str) -> ParseResult<ConstantData> {
        if let Some(Token::Integer(text)) = self.token() {
            self.consume();
            text.parse().map_err(|e| {
                self.error(&format!(
                    "expected hexadecimal immediate, failed to parse: {e}"
                ))
            })
        } else {
            err!(self.loc, err_msg)
        }
    }

    // Match and consume either a hexadecimal Uimm128 immediate (e.g. 0x000102...) or its literal
    // list form (e.g. [0 1 2...]). For convenience, since uimm128 values are stored in the
    // `ConstantPool`, this returns `ConstantData`.
    fn match_uimm128(&mut self, controlling_type: Type) -> ParseResult<ConstantData> {
        let expected_size = controlling_type.bytes() as usize;
        let constant_data = if self.optional(Token::LBracket) {
            // parse using a list of values, e.g. vconst.i32x4 [0 1 2 3]
            let uimm128 = self.parse_literals_to_constant_data(controlling_type)?;
            self.match_token(Token::RBracket, "expected a terminating right bracket")?;
            uimm128
        } else {
            // parse using a hexadecimal value, e.g. 0x000102...
            let uimm128 =
                self.match_hexadecimal_constant("expected an immediate hexadecimal operand")?;
            uimm128.expand_to(expected_size)
        };

        if constant_data.len() == expected_size {
            Ok(constant_data)
        } else {
            Err(self.error(&format!(
                "expected parsed constant to have {expected_size} bytes"
            )))
        }
    }

    // Match and consume a Uimm64 immediate.
    fn match_uimm64(&mut self, err_msg: &str) -> ParseResult<Uimm64> {
        if let Some(Token::Integer(text)) = self.token() {
            self.consume();
            // Lexer just gives us raw text that looks like an integer.
            // Parse it as an Uimm64 to check for overflow and other issues.
            text.parse()
                .map_err(|_| self.error("expected u64 decimal immediate"))
        } else {
            err!(self.loc, err_msg)
        }
    }

    // Match and consume a Uimm32 immediate.
    fn match_uimm32(&mut self, err_msg: &str) -> ParseResult<Uimm32> {
        if let Some(Token::Integer(text)) = self.token() {
            self.consume();
            // Lexer just gives us raw text that looks like an integer.
            // Parse it as an Uimm32 to check for overflow and other issues.
            text.parse().map_err(|e| self.error(e))
        } else {
            err!(self.loc, err_msg)
        }
    }

    // Match and consume a u8 immediate.
    // This is used for lane numbers in SIMD vectors.
    fn match_uimm8(&mut self, err_msg: &str) -> ParseResult<u8> {
        if let Some(Token::Integer(text)) = self.token() {
            self.consume();
            // Lexer just gives us raw text that looks like an integer.
            if let Some(num) = text.strip_prefix("0x") {
                // Parse it as a u8 in hexadecimal form.
                u8::from_str_radix(num, 16)
                    .map_err(|_| self.error("unable to parse u8 as a hexadecimal immediate"))
            } else {
                // Parse it as a u8 to check for overflow and other issues.
                text.parse()
                    .map_err(|_| self.error("expected u8 decimal immediate"))
            }
        } else {
            err!(self.loc, err_msg)
        }
    }

    // Match and consume an i8 immediate.
    fn match_imm8(&mut self, err_msg: &str) -> ParseResult<i8> {
        match_imm!(i8, u8, self, err_msg)
    }

    // Match and consume a signed 16-bit immediate.
    fn match_imm16(&mut self, err_msg: &str) -> ParseResult<i16> {
        match_imm!(i16, u16, self, err_msg)
    }

    // Match and consume an i32 immediate.
    // This is used for stack argument byte offsets.
    fn match_imm32(&mut self, err_msg: &str) -> ParseResult<i32> {
        match_imm!(i32, u32, self, err_msg)
    }

    // Match and consume an i128 immediate.
    fn match_imm128(&mut self, err_msg: &str) -> ParseResult<i128> {
        match_imm!(i128, u128, self, err_msg)
    }

    // Match and consume an optional offset32 immediate.
    //
    // Note that this will match an empty string as an empty offset, and that if an offset is
    // present, it must contain a sign.
    fn optional_offset32(&mut self) -> ParseResult<Offset32> {
        if let Some(Token::Integer(text)) = self.token() {
            if text.starts_with('+') || text.starts_with('-') {
                self.consume();
                // Lexer just gives us raw text that looks like an integer.
                // Parse it as an `Offset32` to check for overflow and other issues.
                return text.parse().map_err(|e| self.error(e));
            }
        }
        // An offset32 operand can be absent.
        Ok(Offset32::new(0))
    }

    // Match and consume an optional offset32 immediate.
    //
    // Note that this will match an empty string as an empty offset, and that if an offset is
    // present, it must contain a sign.
    fn optional_offset_imm64(&mut self) -> ParseResult<Imm64> {
        if let Some(Token::Integer(text)) = self.token() {
            if text.starts_with('+') || text.starts_with('-') {
                self.consume();
                // Lexer just gives us raw text that looks like an integer.
                // Parse it as an `Offset32` to check for overflow and other issues.
                return text.parse().map_err(|e| self.error(e));
            }
        }
        // If no explicit offset is present, the offset is 0.
        Ok(Imm64::new(0))
    }

    // Match and consume an Ieee16 immediate.
    fn match_ieee16(&mut self, err_msg: &str) -> ParseResult<Ieee16> {
        if let Some(Token::Float(text)) = self.token() {
            self.consume();
            // Lexer just gives us raw text that looks like a float.
            // Parse it as an Ieee16 to check for the right number of digits and other issues.
            text.parse().map_err(|e| self.error(e))
        } else {
            err!(self.loc, err_msg)
        }
    }

    // Match and consume an Ieee32 immediate.
    fn match_ieee32(&mut self, err_msg: &str) -> ParseResult<Ieee32> {
        if let Some(Token::Float(text)) = self.token() {
            self.consume();
            // Lexer just gives us raw text that looks like a float.
            // Parse it as an Ieee32 to check for the right number of digits and other issues.
            text.parse().map_err(|e| self.error(e))
        } else {
            err!(self.loc, err_msg)
        }
    }

    // Match and consume an Ieee64 immediate.
    fn match_ieee64(&mut self, err_msg: &str) -> ParseResult<Ieee64> {
        if let Some(Token::Float(text)) = self.token() {
            self.consume();
            // Lexer just gives us raw text that looks like a float.
            // Parse it as an Ieee64 to check for the right number of digits and other issues.
            text.parse().map_err(|e| self.error(e))
        } else {
            err!(self.loc, err_msg)
        }
    }

    // Match and consume an Ieee128 immediate.
    fn match_ieee128(&mut self, err_msg: &str) -> ParseResult<Ieee128> {
        if let Some(Token::Float(text)) = self.token() {
            self.consume();
            // Lexer just gives us raw text that looks like a float.
            // Parse it as an Ieee128 to check for the right number of digits and other issues.
            text.parse().map_err(|e| self.error(e))
        } else {
            err!(self.loc, err_msg)
        }
    }

    // Match and consume an enumerated immediate, like one of the condition codes.
    fn match_enum<T: FromStr>(&mut self, err_msg: &str) -> ParseResult<T> {
        if let Some(Token::Identifier(text)) = self.token() {
            self.consume();
            text.parse().map_err(|_| self.error(err_msg))
        } else {
            err!(self.loc, err_msg)
        }
    }

    // Match and a consume a possibly empty sequence of memory operation flags.
    fn optional_memflags(&mut self) -> ParseResult<MemFlags> {
        let mut flags = MemFlags::new();
        while let Some(Token::Identifier(text)) = self.token() {
            match flags.set_by_name(text) {
                Ok(true) => {
                    self.consume();
                }
                Ok(false) => break,
                Err(msg) => return err!(self.loc, msg),
            }
        }
        Ok(flags)
    }

    // Match and consume an identifier.
    fn match_any_identifier(&mut self, err_msg: &str) -> ParseResult<&'a str> {
        if let Some(Token::Identifier(text)) = self.token() {
            self.consume();
            Ok(text)
        } else {
            err!(self.loc, err_msg)
        }
    }

    /// Parse an optional source location.
    ///
    /// Return an optional source location if no real location is present.
    fn optional_srcloc(&mut self) -> ParseResult<ir::SourceLoc> {
        if let Some(Token::SourceLoc(text)) = self.token() {
            match u32::from_str_radix(text, 16) {
                Ok(num) => {
                    self.consume();
                    Ok(ir::SourceLoc::new(num))
                }
                Err(_) => return err!(self.loc, "invalid source location: {}", text),
            }
        } else {
            Ok(Default::default())
        }
    }

    /// Parse an optional list of debug tags.
    fn optional_debug_tags(&mut self) -> ParseResult<Vec<DebugTag>> {
        if self.optional(Token::LAngle) {
            let mut tags = vec![];
            while !self.optional(Token::RAngle) {
                match self.token() {
                    Some(Token::Integer(_)) => {
                        let value: u32 = self.match_uimm32("expected a u32 value")?.into();
                        tags.push(DebugTag::User(value));
                    }
                    Some(Token::StackSlot(slot)) => {
                        self.consume();
                        tags.push(DebugTag::StackSlot(StackSlot::from_u32(slot)));
                    }
                    _ => {
                        return err!(
                            self.loc,
                            "expected integer user value or stack slot in debug tags"
                        );
                    }
                }
                if !self.optional(Token::Comma) {
                    self.match_token(Token::RAngle, "expected `,` or `>`")?;
                    break;
                }
            }
            Ok(tags)
        } else {
            Ok(vec![])
        }
    }

    /// Parse a list of literals (i.e. integers, floats, booleans); e.g. `0 1 2 3`, usually as
    /// part of something like `vconst.i32x4 [0 1 2 3]`.
    fn parse_literals_to_constant_data(&mut self, ty: Type) -> ParseResult<ConstantData> {
        macro_rules! consume {
            ( $ty:ident, $match_fn:expr ) => {{
                assert!($ty.is_vector());
                let mut data = ConstantData::default();
                for _ in 0..$ty.lane_count() {
                    data = data.append($match_fn);
                }
                data
            }};
        }

        if !ty.is_vector() && !ty.is_dynamic_vector() {
            err!(self.loc, "Expected a controlling vector type, not {}", ty)
        } else {
            let constant_data = match ty.lane_type() {
                I8 => consume!(ty, self.match_imm8("Expected an 8-bit integer")?),
                I16 => consume!(ty, self.match_imm16("Expected a 16-bit integer")?),
                I32 => consume!(ty, self.match_imm32("Expected a 32-bit integer")?),
                I64 => consume!(ty, self.match_imm64("Expected a 64-bit integer")?),
                F32 => consume!(ty, self.match_ieee32("Expected a 32-bit float")?),
                F64 => consume!(ty, self.match_ieee64("Expected a 64-bit float")?),
                _ => return err!(self.loc, "Expected a type of: float, int, bool"),
            };
            Ok(constant_data)
        }
    }

    /// Parse a list of test command passes specified in command line.
    pub fn parse_cmdline_passes(&mut self, passes: &'a [String]) -> Vec<TestCommand<'a>> {
        let mut list = Vec::new();
        for pass in passes {
            list.push(TestCommand::new(pass));
        }
        list
    }

    /// Parse a list of test commands.
    pub fn parse_test_commands(&mut self) -> Vec<TestCommand<'a>> {
        let mut list = Vec::new();
        while self.token() == Some(Token::Identifier("test")) {
            list.push(TestCommand::new(self.consume_line()));
        }
        list
    }

    /// Parse a target spec.
    ///
    /// Accept the target from the command line for pass command.
    ///
    fn parse_cmdline_target(&mut self, target_pass: Option<&str>) -> ParseResult<isaspec::IsaSpec> {
        // Were there any `target` commands specified?
        let mut specified_target = false;

        let mut targets = Vec::new();
        let flag_builder = settings::builder();

        if let Some(targ) = target_pass {
            let loc = self.loc;
            let triple = match Triple::from_str(targ) {
                Ok(triple) => triple,
                Err(err) => return err!(loc, err),
            };
            let isa_builder = match isa::lookup(triple) {
                Err(isa::LookupError::SupportDisabled) => {
                    return err!(loc, "support disabled target '{}'", targ);
                }
                Err(isa::LookupError::Unsupported) => {
                    return warn!(loc, "unsupported target '{}'", targ);
                }
                Ok(b) => b,
            };
            specified_target = true;

            // Construct a trait object with the aggregate settings.
            targets.push(
                isa_builder
                    .finish(settings::Flags::new(flag_builder.clone()))
                    .map_err(|e| ParseError {
                        location: loc,
                        message: format!("invalid ISA flags for '{targ}': {e:?}"),
                        is_warning: false,
                    })?,
            );
        }

        if !specified_target {
            // No `target` commands.
            Ok(isaspec::IsaSpec::None(settings::Flags::new(flag_builder)))
        } else {
            Ok(isaspec::IsaSpec::Some(targets))
        }
    }

    /// Parse a list of target specs.
    ///
    /// Accept a mix of `target` and `set` command lines. The `set` commands are cumulative.
    ///
    fn parse_target_specs(&mut self, options: &ParseOptions) -> ParseResult<isaspec::IsaSpec> {
        // Were there any `target` commands?
        let mut seen_target = false;
        // Location of last `set` command since the last `target`.
        let mut last_set_loc = None;

        let mut targets = Vec::new();
        let mut flag_builder = settings::builder();

        let bool_to_str = |val: bool| {
            if val { "true" } else { "false" }
        };

        // default to enabling cfg info
        flag_builder
            .set(
                "machine_code_cfg_info",
                bool_to_str(options.machine_code_cfg_info),
            )
            .expect("machine_code_cfg_info option should be present");

        flag_builder
            .set("unwind_info", bool_to_str(options.unwind_info))
            .expect("unwind_info option should be present");

        while let Some(Token::Identifier(command)) = self.token() {
            match command {
                "set" => {
                    last_set_loc = Some(self.loc);
                    isaspec::parse_options(
                        self.consume_line().trim().split_whitespace(),
                        &mut flag_builder,
                        self.loc,
                    )
                    .map_err(|err| ParseError::from(err))?;
                }
                "target" => {
                    let loc = self.loc;
                    // Grab the whole line so the lexer won't go looking for tokens on the
                    // following lines.
                    let mut words = self.consume_line().trim().split_whitespace().peekable();
                    // Look for `target foo`.
                    let target_name = match words.next() {
                        Some(w) => w,
                        None => return err!(loc, "expected target triple"),
                    };
                    let triple = match Triple::from_str(target_name) {
                        Ok(triple) => triple,
                        Err(err) => return err!(loc, err),
                    };
                    let mut isa_builder = match isa::lookup(triple) {
                        Err(isa::LookupError::SupportDisabled) => {
                            continue;
                        }
                        Err(isa::LookupError::Unsupported) => {
                            return warn!(loc, "unsupported target '{}'", target_name);
                        }
                        Ok(b) => b,
                    };
                    last_set_loc = None;
                    seen_target = true;
                    // Apply the target-specific settings to `isa_builder`.
                    isaspec::parse_options(words, &mut isa_builder, self.loc)?;

                    // Construct a trait object with the aggregate settings.
                    targets.push(
                        isa_builder
                            .finish(settings::Flags::new(flag_builder.clone()))
                            .map_err(|e| ParseError {
                                location: loc,
                                message: format!("invalid ISA flags for '{target_name}': {e:?}"),
                                is_warning: false,
                            })?,
                    );
                }
                _ => break,
            }
        }

        if !seen_target {
            // No `target` commands, but we allow for `set` commands.
            Ok(isaspec::IsaSpec::None(settings::Flags::new(flag_builder)))
        } else if let Some(loc) = last_set_loc {
            err!(
                loc,
                "dangling 'set' command after ISA specification has no effect."
            )
        } else {
            Ok(isaspec::IsaSpec::Some(targets))
        }
    }

    /// Parse a list of expected features that Cranelift should be compiled with, or without.
    pub fn parse_cranelift_features(&mut self) -> ParseResult<Vec<Feature<'a>>> {
        let mut list = Vec::new();
        while self.token() == Some(Token::Identifier("feature")) {
            self.consume();
            let has = !self.optional(Token::Bang);
            match (self.token(), has) {
                (Some(Token::String(flag)), true) => list.push(Feature::With(flag)),
                (Some(Token::String(flag)), false) => list.push(Feature::Without(flag)),
                (tok, _) => {
                    return err!(
                        self.loc,
                        format!("Expected feature flag string, got {:?}", tok)
                    );
                }
            }
            self.consume();
        }
        Ok(list)
    }

    /// Parse a list of function definitions.
    ///
    /// This is the top-level parse function matching the whole contents of a file.
    pub fn parse_function_list(&mut self) -> ParseResult<Vec<(Function, Details<'a>)>> {
        let mut list = Vec::new();
        while self.token().is_some() {
            list.push(self.parse_function()?);
        }
        if let Some(err) = self.lex_error {
            return match err {
                LexError::InvalidChar => err!(self.loc, "invalid character"),
            };
        }
        Ok(list)
    }

    // Parse a whole function definition.
    //
    // function ::= * "function" name signature "{" preamble function-body "}"
    //
    fn parse_function(&mut self) -> ParseResult<(Function, Details<'a>)> {
        // Begin gathering comments.
        // Make sure we don't include any comments before the `function` keyword.
        self.token();
        debug_assert!(self.comments.is_empty());
        self.start_gathering_comments();

        self.match_identifier("function", "expected 'function'")?;

        let location = self.loc;

        // function ::= "function" * name signature "{" preamble function-body "}"
        let name = self.parse_user_func_name()?;

        // function ::= "function" name * signature "{" preamble function-body "}"
        let sig = self.parse_signature()?;

        let mut ctx = Context::new(Function::with_name_signature(name, sig));

        // function ::= "function" name signature * "{" preamble function-body "}"
        self.match_token(Token::LBrace, "expected '{' before function body")?;

        self.token();
        self.claim_gathered_comments(AnyEntity::Function);

        // function ::= "function" name signature "{" * preamble function-body "}"
        self.parse_preamble(&mut ctx)?;
        // function ::= "function" name signature "{"  preamble * function-body "}"
        self.parse_function_body(&mut ctx)?;
        // function ::= "function" name signature "{" preamble function-body * "}"
        self.match_token(Token::RBrace, "expected '}' after function body")?;

        // Collect any comments following the end of the function, then stop gathering comments.
        self.start_gathering_comments();
        self.token();
        self.claim_gathered_comments(AnyEntity::Function);

        // Claim all the declared user-defined function names.
        for (user_func_ref, user_external_name) in
            std::mem::take(&mut self.predeclared_external_names)
        {
            let actual_ref = ctx
                .function
                .declare_imported_user_function(user_external_name);
            assert_eq!(user_func_ref, actual_ref);
        }

        let details = Details {
            location,
            comments: self.take_comments(),
            map: ctx.map,
        };

        Ok((ctx.function, details))
    }

    // Parse a user-defined function name
    //
    // For example, in a function decl, the parser would be in this state:
    //
    // function ::= "function" * name signature { ... }
    //
    fn parse_user_func_name(&mut self) -> ParseResult<UserFuncName> {
        match self.token() {
            Some(Token::Name(s)) => {
                self.consume();
                Ok(UserFuncName::testcase(s))
            }
            Some(Token::UserRef(namespace)) => {
                self.consume();
                match self.token() {
                    Some(Token::Colon) => {
                        self.consume();
                        match self.token() {
                            Some(Token::Integer(index_str)) => {
                                self.consume();
                                let index: u32 =
                                    u32::from_str_radix(index_str, 10).map_err(|_| {
                                        self.error("the integer given overflows the u32 type")
                                    })?;
                                Ok(UserFuncName::user(namespace, index))
                            }
                            _ => err!(self.loc, "expected integer"),
                        }
                    }
                    _ => {
                        err!(self.loc, "expected user function name in the form uX:Y")
                    }
                }
            }
            _ => err!(self.loc, "expected external name"),
        }
    }

    // Parse an external name.
    //
    // For example, in a function reference decl, the parser would be in this state:
    //
    // fn0 = * name signature
    //
    fn parse_external_name(&mut self) -> ParseResult<ExternalName> {
        match self.token() {
            Some(Token::Name(s)) => {
                self.consume();
                s.parse()
                    .map_err(|_| self.error("invalid test case or libcall name"))
            }

            Some(Token::UserNameRef(name_ref)) => {
                self.consume();
                Ok(ExternalName::user(UserExternalNameRef::new(
                    name_ref as usize,
                )))
            }

            Some(Token::UserRef(namespace)) => {
                self.consume();
                if let Some(Token::Colon) = self.token() {
                    self.consume();
                    match self.token() {
                        Some(Token::Integer(index_str)) => {
                            let index: u32 = u32::from_str_radix(index_str, 10).map_err(|_| {
                                self.error("the integer given overflows the u32 type")
                            })?;
                            self.consume();

                            // Deduplicate the reference (O(n), but should be fine for tests),
                            // to follow `FunctionParameters::declare_imported_user_function`,
                            // otherwise this will cause ref mismatches when asserted below.
                            let name_ref = self
                                .predeclared_external_names
                                .iter()
                                .find_map(|(reff, name)| {
                                    if name.index == index && name.namespace == namespace {
                                        Some(reff)
                                    } else {
                                        None
                                    }
                                })
                                .unwrap_or_else(|| {
                                    self.predeclared_external_names
                                        .push(ir::UserExternalName { namespace, index })
                                });

                            Ok(ExternalName::user(name_ref))
                        }
                        _ => err!(self.loc, "expected integer"),
                    }
                } else {
                    err!(self.loc, "expected colon")
                }
            }

            _ => err!(self.loc, "expected external name"),
        }
    }

    // Parse a function signature.
    //
    // signature ::=  * "(" [paramlist] ")" ["->" retlist] [callconv]
    //
    fn parse_signature(&mut self) -> ParseResult<Signature> {
        // Calling convention defaults to `fast`, but can be changed.
        let mut sig = Signature::new(self.default_calling_convention);

        self.match_token(Token::LPar, "expected function signature: ( args... )")?;
        // signature ::=  "(" * [abi-param-list] ")" ["->" retlist] [callconv]
        if self.token() != Some(Token::RPar) {
            sig.params = self.parse_abi_param_list()?;
        }
        self.match_token(Token::RPar, "expected ')' after function arguments")?;
        if self.optional(Token::Arrow) {
            sig.returns = self.parse_abi_param_list()?;
        }

        // The calling convention is optional.
        match self.token() {
            Some(Token::Identifier(text)) => match text.parse() {
                Ok(cc) => {
                    self.consume();
                    sig.call_conv = cc;
                }
                _ => return err!(self.loc, "unknown calling convention: {}", text),
            },
            _ => {}
        }

        Ok(sig)
    }

    // Parse list of function parameter / return value types.
    //
    // paramlist ::= * param { "," param }
    //
    fn parse_abi_param_list(&mut self) -> ParseResult<Vec<AbiParam>> {
        let mut list = Vec::new();

        // abi-param-list ::= * abi-param { "," abi-param }
        list.push(self.parse_abi_param()?);

        // abi-param-list ::= abi-param * { "," abi-param }
        while self.optional(Token::Comma) {
            // abi-param-list ::= abi-param { "," * abi-param }
            list.push(self.parse_abi_param()?);
        }

        Ok(list)
    }

    // Parse a single argument type with flags.
    fn parse_abi_param(&mut self) -> ParseResult<AbiParam> {
        // abi-param ::= * type { flag }
        let mut arg = AbiParam::new(self.match_type("expected parameter type")?);

        // abi-param ::= type * { flag }
        while let Some(Token::Identifier(s)) = self.token() {
            match s {
                "uext" => arg.extension = ArgumentExtension::Uext,
                "sext" => arg.extension = ArgumentExtension::Sext,
                "sarg" => {
                    self.consume();
                    self.match_token(Token::LPar, "expected '(' to begin sarg size")?;
                    let size = self.match_uimm32("expected byte-size in sarg decl")?;
                    self.match_token(Token::RPar, "expected ')' to end sarg size")?;
                    arg.purpose = ArgumentPurpose::StructArgument(size.into());
                    continue;
                }
                _ => {
                    if let Ok(purpose) = s.parse() {
                        arg.purpose = purpose;
                    } else {
                        break;
                    }
                }
            }
            self.consume();
        }

        Ok(arg)
    }

    // Parse the function preamble.
    //
    // preamble      ::= * { preamble-decl }
    // preamble-decl ::= * stack-slot-decl
    //                   * function-decl
    //                   * signature-decl
    //                   * jump-table-decl
    //                   * stack-limit-decl
    //
    // The parsed decls are added to `ctx` rather than returned.
    fn parse_preamble(&mut self, ctx: &mut Context) -> ParseResult<()> {
        loop {
            match self.token() {
                Some(Token::StackSlot(..)) => {
                    self.start_gathering_comments();
                    let loc = self.loc;
                    self.parse_stack_slot_decl()
                        .and_then(|(ss, dat)| ctx.add_ss(ss, dat, loc))
                }
                Some(Token::DynamicStackSlot(..)) => {
                    self.start_gathering_comments();
                    let loc = self.loc;
                    self.parse_dynamic_stack_slot_decl()
                        .and_then(|(dss, dat)| ctx.add_dss(dss, dat, loc))
                }
                Some(Token::DynamicType(..)) => {
                    self.start_gathering_comments();
                    let loc = self.loc;
                    self.parse_dynamic_type_decl()
                        .and_then(|(dt, dat)| ctx.add_dt(dt, dat, loc))
                }
                Some(Token::GlobalValue(..)) => {
                    self.start_gathering_comments();
                    self.parse_global_value_decl()
                        .and_then(|(gv, dat)| ctx.add_gv(gv, dat, self.loc))
                }
                Some(Token::SigRef(..)) => {
                    self.start_gathering_comments();
                    self.parse_signature_decl().and_then(|(sig, dat)| {
                        ctx.add_sig(sig, dat, self.loc, self.default_calling_convention)
                    })
                }
                Some(Token::FuncRef(..)) => {
                    self.start_gathering_comments();
                    self.parse_function_decl(ctx)
                        .and_then(|(fn_, dat)| ctx.add_fn(fn_, dat, self.loc))
                }
                Some(Token::Constant(..)) => {
                    self.start_gathering_comments();
                    self.parse_constant_decl()
                        .and_then(|(c, v)| ctx.add_constant(c, v, self.loc))
                }
                Some(Token::Identifier("stack_limit")) => {
                    self.start_gathering_comments();
                    self.parse_stack_limit_decl()
                        .and_then(|gv| ctx.add_stack_limit(gv, self.loc))
                }
                // More to come..
                _ => return Ok(()),
            }?;
        }
    }

    // Parse a stack slot decl.
    //
    // stack-slot-decl ::= * StackSlot(ss) "=" stack-slot-kind Bytes {"," stack-slot-flag}
    // stack-slot-kind ::= "explicit_slot"
    //                   | "spill_slot"
    //                   | "incoming_arg"
    //                   | "outgoing_arg"
    // stack-slot-flag ::= "align" "=" Bytes | "key" "=" uimm64
    fn parse_stack_slot_decl(&mut self) -> ParseResult<(StackSlot, StackSlotData)> {
        let ss = self.match_ss("expected stack slot number: ss«n»")?;
        self.match_token(Token::Equal, "expected '=' in stack slot declaration")?;
        let kind = self.match_enum("expected stack slot kind")?;

        // stack-slot-decl ::= StackSlot(ss) "=" stack-slot-kind * Bytes {"," stack-slot-flag}
        let bytes: i64 = self
            .match_imm64("expected byte-size in stack_slot decl")?
            .into();
        if bytes < 0 {
            return err!(self.loc, "negative stack slot size");
        }
        if bytes > i64::from(u32::MAX) {
            return err!(self.loc, "stack slot too large");
        }

        let mut align = 1;
        let mut key = None;

        while self.token() == Some(Token::Comma) {
            self.consume();
            match self.token() {
                Some(Token::Identifier("align")) => {
                    self.consume();
                    self.match_token(Token::Equal, "expected `=` after flag")?;
                    let align64: i64 = self
                        .match_imm64("expected alignment-size after `align` flag")?
                        .into();
                    align = u32::try_from(align64)
                        .map_err(|_| self.error("alignment must be a 32-bit unsigned integer"))?;
                }
                Some(Token::Identifier("key")) => {
                    self.consume();
                    self.match_token(Token::Equal, "expected `=` after flag")?;
                    let value = self.match_uimm64("expected `u64` value for `key` flag")?;
                    key = Some(StackSlotKey::new(value.into()));
                }
                _ => {
                    return Err(self.error("invalid flag for stack slot"));
                }
            }
        }

        if !align.is_power_of_two() {
            return err!(self.loc, "stack slot alignment is not a power of two");
        }
        let align_shift = u8::try_from(align.ilog2()).unwrap(); // Always succeeds: range 0..=31.

        let data = match key {
            Some(key) => StackSlotData::new_with_key(kind, bytes as u32, align_shift, key),
            None => StackSlotData::new(kind, bytes as u32, align_shift),
        };

        // Collect any trailing comments.
        self.token();
        self.claim_gathered_comments(ss);

        // TBD: stack-slot-decl ::= StackSlot(ss) "=" stack-slot-kind Bytes * {"," stack-slot-flag}
        Ok((ss, data))
    }

    fn parse_dynamic_stack_slot_decl(
        &mut self,
    ) -> ParseResult<(DynamicStackSlot, DynamicStackSlotData)> {
        let dss = self.match_dss("expected stack slot number: dss«n»")?;
        self.match_token(Token::Equal, "expected '=' in stack slot declaration")?;
        let kind = self.match_enum("expected stack slot kind")?;
        let dt = self.match_dt("expected dynamic type")?;
        let data = DynamicStackSlotData::new(kind, dt);
        // Collect any trailing comments.
        self.token();
        self.claim_gathered_comments(dss);

        // TBD: stack-slot-decl ::= StackSlot(ss) "=" stack-slot-kind Bytes * {"," stack-slot-flag}
        Ok((dss, data))
    }

    fn parse_dynamic_type_decl(&mut self) -> ParseResult<(DynamicType, DynamicTypeData)> {
        let dt = self.match_dt("expected dynamic type number: dt«n»")?;
        self.match_token(Token::Equal, "expected '=' in stack slot declaration")?;
        let vector_base_ty = self.match_type("expected base type")?;
        assert!(vector_base_ty.is_vector(), "expected vector type");
        self.match_token(
            Token::Multiply,
            "expected '*' followed by a dynamic scale value",
        )?;
        let dyn_scale = self.match_gv("expected dynamic scale global value")?;
        let data = DynamicTypeData::new(vector_base_ty, dyn_scale);
        // Collect any trailing comments.
        self.token();
        self.claim_gathered_comments(dt);
        Ok((dt, data))
    }

    // Parse a global value decl.
    //
    // global-val-decl ::= * GlobalValue(gv) "=" global-val-desc
    // global-val-desc ::= "vmctx"
    //                   | "load" "." type "notrap" "aligned" GlobalValue(base) [offset]
    //                   | "iadd_imm" "(" GlobalValue(base) ")" imm64
    //                   | "symbol" ["colocated"] name + imm64
    //                   | "dyn_scale_target_const" "." type
    //
    fn parse_global_value_decl(&mut self) -> ParseResult<(GlobalValue, GlobalValueData)> {
        let gv = self.match_gv("expected global value number: gv«n»")?;

        self.match_token(Token::Equal, "expected '=' in global value declaration")?;

        let data = match self.match_any_identifier("expected global value kind")? {
            "vmctx" => GlobalValueData::VMContext,
            "load" => {
                self.match_token(
                    Token::Dot,
                    "expected '.' followed by type in load global value decl",
                )?;
                let global_type = self.match_type("expected load type")?;
                let flags = self.optional_memflags()?;
                let base = self.match_gv("expected global value: gv«n»")?;
                let offset = self.optional_offset32()?;

                if !(flags.notrap() && flags.aligned()) {
                    return err!(self.loc, "global-value load must be notrap and aligned");
                }
                GlobalValueData::Load {
                    base,
                    offset,
                    global_type,
                    flags,
                }
            }
            "iadd_imm" => {
                self.match_token(
                    Token::Dot,
                    "expected '.' followed by type in iadd_imm global value decl",
                )?;
                let global_type = self.match_type("expected iadd type")?;
                let base = self.match_gv("expected global value: gv«n»")?;
                self.match_token(
                    Token::Comma,
                    "expected ',' followed by rhs in iadd_imm global value decl",
                )?;
                let offset = self.match_imm64("expected iadd_imm immediate")?;
                GlobalValueData::IAddImm {
                    base,
                    offset,
                    global_type,
                }
            }
            "symbol" => {
                let colocated = self.optional(Token::Identifier("colocated"));
                let tls = self.optional(Token::Identifier("tls"));
                let name = self.parse_external_name()?;
                let offset = self.optional_offset_imm64()?;
                GlobalValueData::Symbol {
                    name,
                    offset,
                    colocated,
                    tls,
                }
            }
            "dyn_scale_target_const" => {
                self.match_token(
                    Token::Dot,
                    "expected '.' followed by type in dynamic scale global value decl",
                )?;
                let vector_type = self.match_type("expected load type")?;
                assert!(vector_type.is_vector(), "Expected vector type");
                GlobalValueData::DynScaleTargetConst { vector_type }
            }
            other => return err!(self.loc, "Unknown global value kind '{}'", other),
        };

        // Collect any trailing comments.
        self.token();
        self.claim_gathered_comments(gv);

        Ok((gv, data))
    }

    // Parse a signature decl.
    //
    // signature-decl ::= SigRef(sigref) "=" signature
    //
    fn parse_signature_decl(&mut self) -> ParseResult<(SigRef, Signature)> {
        let sig = self.match_sig("expected signature number: sig«n»")?;
        self.match_token(Token::Equal, "expected '=' in signature decl")?;
        let data = self.parse_signature()?;

        // Collect any trailing comments.
        self.token();
        self.claim_gathered_comments(sig);

        Ok((sig, data))
    }

    // Parse a function decl.
    //
    // Two variants:
    //
    // function-decl ::= FuncRef(fnref) "=" ["colocated"] ["patchable"] name function-decl-sig
    // function-decl-sig ::= SigRef(sig) | signature
    //
    // The first variant allocates a new signature reference. The second references an existing
    // signature which must be declared first.
    //
    fn parse_function_decl(&mut self, ctx: &mut Context) -> ParseResult<(FuncRef, ExtFuncData)> {
        let fn_ = self.match_fn("expected function number: fn«n»")?;
        self.match_token(Token::Equal, "expected '=' in function decl")?;

        let loc = self.loc;

        // function-decl ::= FuncRef(fnref) "=" * ["colocated"] ["patchable"] name function-decl-sig
        let colocated = self.optional(Token::Identifier("colocated"));
        // function-decl ::= FuncRef(fnref) "=" ["colocated"] * ["patchable"] name function-decl-sig
        let patchable = self.optional(Token::Identifier("patchable"));

        // function-decl ::= FuncRef(fnref) "=" ["colocated"] ["patchable"] * name function-decl-sig
        let name = self.parse_external_name()?;

        // function-decl ::= FuncRef(fnref) "=" ["colocated"] ["patchable"] name * function-decl-sig
        let data = match self.token() {
            Some(Token::LPar) => {
                // function-decl ::= FuncRef(fnref) "=" ["colocated"] ["patchable"] name * signature
                let sig = self.parse_signature()?;
                let sigref = ctx.function.import_signature(sig);
                ctx.map
                    .def_entity(sigref.into(), loc)
                    .expect("duplicate SigRef entities created");
                ExtFuncData {
                    name,
                    signature: sigref,
                    colocated,
                    patchable,
                }
            }
            Some(Token::SigRef(sig_src)) => {
                let sig = match SigRef::with_number(sig_src) {
                    None => {
                        return err!(self.loc, "attempted to use invalid signature ss{}", sig_src);
                    }
                    Some(sig) => sig,
                };
                ctx.check_sig(sig, self.loc)?;
                self.consume();
                ExtFuncData {
                    name,
                    signature: sig,
                    colocated,
                    patchable,
                }
            }
            _ => return err!(self.loc, "expected 'function' or sig«n» in function decl"),
        };

        // Collect any trailing comments.
        self.token();
        self.claim_gathered_comments(fn_);

        Ok((fn_, data))
    }

    // Parse a jump table literal.
    //
    // jump-table-lit ::= "[" block(args) {"," block(args) } "]"
    //                  | "[]"
    fn parse_jump_table(
        &mut self,
        ctx: &mut Context,
        def: ir::BlockCall,
    ) -> ParseResult<ir::JumpTable> {
        self.match_token(Token::LBracket, "expected '[' before jump table contents")?;

        let mut data = Vec::new();

        match self.token() {
            Some(Token::Block(dest)) => {
                self.consume();
                let args = self.parse_opt_block_call_args()?;
                data.push(ctx.function.dfg.block_call(dest, &args));

                loop {
                    match self.token() {
                        Some(Token::Comma) => {
                            self.consume();
                            if let Some(Token::Block(dest)) = self.token() {
                                self.consume();
                                let args = self.parse_opt_block_call_args()?;
                                data.push(ctx.function.dfg.block_call(dest, &args));
                            } else {
                                return err!(self.loc, "expected jump_table entry");
                            }
                        }
                        Some(Token::RBracket) => break,
                        _ => return err!(self.loc, "expected ']' after jump table contents"),
                    }
                }
            }
            Some(Token::RBracket) => (),
            _ => return err!(self.loc, "expected jump_table entry"),
        }

        self.consume();

        Ok(ctx
            .function
            .dfg
            .jump_tables
            .push(JumpTableData::new(def, &data)))
    }

    // Parse an exception-table decl.
    //
    // exception-table ::= * SigRef(sig) "," BlockCall "," "[" (exception-table-entry ( "," exception-table-entry )*)? "]"
    // exception-table-entry ::=   ExceptionTag(tag) ":" BlockCall
    //                           | "default" ":" BlockCall
    //                           | "context" value
    fn parse_exception_table(&mut self, ctx: &mut Context) -> ParseResult<ir::ExceptionTable> {
        let sig = self.match_sig("expected signature of called function")?;
        self.match_token(Token::Comma, "expected comma after signature argument")?;

        let mut handlers = vec![];

        let block_num = self.match_block("expected branch destination block")?;
        let args = self.parse_opt_block_call_args()?;
        let normal_return = ctx.function.dfg.block_call(block_num, &args);

        self.match_token(
            Token::Comma,
            "expected comma after normal-return destination",
        )?;

        self.match_token(
            Token::LBracket,
            "expected an open-bracket for exception table list",
        )?;
        loop {
            match self.token() {
                Some(Token::RBracket) => {
                    break;
                }
                Some(Token::ExceptionTag(tag)) => {
                    self.consume();
                    self.match_token(Token::Colon, "expected ':' after exception tag")?;
                    let tag = ir::ExceptionTag::from_u32(tag);
                    let block_num = self.match_block("expected branch destination block")?;
                    let args = self.parse_opt_block_call_args()?;
                    let block_call = ctx.function.dfg.block_call(block_num, &args);
                    handlers.push(ir::ExceptionTableItem::Tag(tag, block_call));
                }
                Some(Token::Identifier("default")) => {
                    self.consume();
                    self.match_token(Token::Colon, "expected ':' after 'default'")?;
                    let block_num = self.match_block("expected branch destination block")?;
                    let args = self.parse_opt_block_call_args()?;
                    let block_call = ctx.function.dfg.block_call(block_num, &args);
                    handlers.push(ir::ExceptionTableItem::Default(block_call));
                }
                Some(Token::Identifier("context")) => {
                    self.consume();
                    let val = self.match_value("expected value for exception-handler context")?;
                    handlers.push(ir::ExceptionTableItem::Context(val));
                }
                _ => return err!(self.loc, "invalid token"),
            }

            if let Some(Token::Comma) = self.token() {
                self.consume();
            } else {
                break;
            }
        }
        self.match_token(Token::RBracket, "expected closing bracket")?;

        Ok(ctx
            .function
            .dfg
            .exception_tables
            .push(ir::ExceptionTableData::new(sig, normal_return, handlers)))
    }

    // Parse a constant decl.
    //
    // constant-decl ::= * Constant(c) "=" ty? "[" literal {"," literal} "]"
    fn parse_constant_decl(&mut self) -> ParseResult<(Constant, ConstantData)> {
        let name = self.match_constant()?;
        self.match_token(Token::Equal, "expected '=' in constant decl")?;
        let data = if let Some(Token::Type(_)) = self.token() {
            let ty = self.match_type("expected type of constant")?;
            self.match_uimm128(ty)
        } else {
            self.match_hexadecimal_constant("expected an immediate hexadecimal operand")
        }?;

        // Collect any trailing comments.
        self.token();
        self.claim_gathered_comments(name);

        Ok((name, data))
    }

    // Parse a stack limit decl
    //
    // stack-limit-decl ::= * StackLimit "=" GlobalValue(gv)
    fn parse_stack_limit_decl(&mut self) -> ParseResult<GlobalValue> {
        self.match_stack_limit()?;
        self.match_token(Token::Equal, "expected '=' in stack limit decl")?;
        let limit = match self.token() {
            Some(Token::GlobalValue(base_num)) => match GlobalValue::with_number(base_num) {
                Some(gv) => gv,
                None => return err!(self.loc, "invalid global value number for stack limit"),
            },
            _ => return err!(self.loc, "expected global value"),
        };
        self.consume();

        // Collect any trailing comments.
        self.token();
        self.claim_gathered_comments(AnyEntity::StackLimit);

        Ok(limit)
    }

    // Parse a function body, add contents to `ctx`.
    //
    // function-body ::= * { extended-basic-block }
    //
    fn parse_function_body(&mut self, ctx: &mut Context) -> ParseResult<()> {
        while self.token() != Some(Token::RBrace) {
            self.parse_basic_block(ctx)?;
        }

        // Now that we've seen all defined values in the function, ensure that
        // all references refer to a definition.
        for block in &ctx.function.layout {
            for inst in ctx.function.layout.block_insts(block) {
                for value in ctx.function.dfg.inst_values(inst) {
                    if !ctx.map.contains_value(value) {
                        return err!(
                            ctx.map.location(AnyEntity::Inst(inst)).unwrap(),
                            "undefined operand value {}",
                            value
                        );
                    }
                }
            }
        }

        for alias in &ctx.aliases {
            if !ctx.function.dfg.set_alias_type_for_parser(*alias) {
                let loc = ctx.map.location(AnyEntity::Value(*alias)).unwrap();
                return err!(loc, "alias cycle involving {}", alias);
            }
        }

        Ok(())
    }

    // Parse a basic block, add contents to `ctx`.
    //
    // extended-basic-block ::= * block-header { instruction }
    // block-header         ::= Block(block) [block-params] [block-flags] ":"
    // block-flags          ::= [Cold]
    //
    fn parse_basic_block(&mut self, ctx: &mut Context) -> ParseResult<()> {
        // Collect comments for the next block.
        self.start_gathering_comments();

        let block_num = self.match_block("expected block header")?;
        let block = ctx.add_block(block_num, self.loc)?;

        if block_num.as_u32() >= MAX_BLOCKS_IN_A_FUNCTION {
            return Err(self.error("too many blocks"));
        }

        if self.token() == Some(Token::LPar) {
            self.parse_block_params(ctx, block)?;
        }

        if self.optional(Token::Cold) {
            ctx.set_cold_block(block);
        }

        self.match_token(Token::Colon, "expected ':' after block parameters")?;

        // Collect any trailing comments.
        self.token();
        self.claim_gathered_comments(block);

        // extended-basic-block ::= block-header * { instruction }
        while match self.token() {
            Some(Token::Value(_))
            | Some(Token::Identifier(_))
            | Some(Token::LBracket)
            | Some(Token::SourceLoc(_))
            | Some(Token::LAngle) => true,
            _ => false,
        } {
            let srcloc = self.optional_srcloc()?;

            let debug_tags = self.optional_debug_tags()?;

            // We need to parse instruction results here because they are shared
            // between the parsing of value aliases and the parsing of instructions.
            //
            // inst-results ::= Value(v) { "," Value(v) }
            let results = self.parse_inst_results()?;

            for result in &results {
                while ctx.function.dfg.num_values() <= result.index() {
                    ctx.function.dfg.make_invalid_value_for_parser();
                }
            }

            match self.token() {
                Some(Token::Arrow) => {
                    self.consume();
                    self.parse_value_alias(&results, ctx)?;
                }
                Some(Token::Equal) => {
                    self.consume();
                    self.parse_instruction(&results, srcloc, debug_tags, ctx, block)?;
                }
                _ if !results.is_empty() => return err!(self.loc, "expected -> or ="),
                _ => self.parse_instruction(&results, srcloc, debug_tags, ctx, block)?,
            }
        }

        Ok(())
    }

    // Parse parenthesized list of block parameters.
    //
    // block-params ::= * "(" ( block-param { "," block-param } )? ")"
    fn parse_block_params(&mut self, ctx: &mut Context, block: Block) -> ParseResult<()> {
        // block-params ::= * "(" ( block-param { "," block-param } )? ")"
        self.match_token(Token::LPar, "expected '(' before block parameters")?;

        // block-params ::= "(" * ")"
        if self.token() == Some(Token::RPar) {
            self.consume();
            return Ok(());
        }

        // block-params ::= "(" * block-param { "," block-param } ")"
        self.parse_block_param(ctx, block)?;

        // block-params ::= "(" block-param * { "," block-param } ")"
        while self.optional(Token::Comma) {
            // block-params ::= "(" block-param { "," * block-param } ")"
            self.parse_block_param(ctx, block)?;
        }

        // block-params ::= "(" block-param { "," block-param } * ")"
        self.match_token(Token::RPar, "expected ')' after block parameters")?;

        Ok(())
    }

    // Parse a single block parameter declaration, and append it to `block`.
    //
    // block-param ::= * Value(v) ":" Type(t) arg-loc?
    // arg-loc ::= "[" value-location "]"
    //
    fn parse_block_param(&mut self, ctx: &mut Context, block: Block) -> ParseResult<()> {
        // block-param ::= * Value(v) ":" Type(t) arg-loc?
        let v = self.match_value("block argument must be a value")?;
        let v_location = self.loc;
        self.match_token(Token::Colon, "expected ':' after block argument")?;
        // block-param ::= Value(v) ":" * Type(t) arg-loc?
        while ctx.function.dfg.num_values() <= v.index() {
            ctx.function.dfg.make_invalid_value_for_parser();
        }

        let t = self.match_type("expected block argument type")?;
        // Allocate the block argument.
        ctx.function.dfg.append_block_param_for_parser(block, t, v);
        ctx.map.def_value(v, v_location)?;

        Ok(())
    }

    // Parse instruction results and return them.
    //
    // inst-results ::= Value(v) { "," Value(v) }
    //
    fn parse_inst_results(&mut self) -> ParseResult<SmallVec<[Value; 1]>> {
        // Result value numbers.
        let mut results = SmallVec::new();

        // instruction  ::=  * [inst-results "="] Opcode(opc) ["." Type] ...
        // inst-results ::= * Value(v) { "," Value(v) }
        if let Some(Token::Value(v)) = self.token() {
            self.consume();

            results.push(v);

            // inst-results ::= Value(v) * { "," Value(v) }
            while self.optional(Token::Comma) {
                // inst-results ::= Value(v) { "," * Value(v) }
                let v = self.match_value("expected result value")?;
                results.push(v);
            }
        }

        Ok(results)
    }

    // Parse a value alias, and append it to `block`.
    //
    // value_alias ::= [inst-results] "->" Value(v)
    //
    fn parse_value_alias(&mut self, results: &[Value], ctx: &mut Context) -> ParseResult<()> {
        if results.len() != 1 {
            return err!(self.loc, "wrong number of aliases");
        }
        let result = results[0];
        let dest = self.match_value("expected value alias")?;

        // Allow duplicate definitions of aliases, as long as they are identical.
        if ctx.map.contains_value(result) {
            if let Some(old) = ctx.function.dfg.value_alias_dest_for_serialization(result) {
                if old != dest {
                    return err!(
                        self.loc,
                        "value {} is already defined as an alias with destination {}",
                        result,
                        old
                    );
                }
            } else {
                return err!(self.loc, "value {} is already defined");
            }
        } else {
            ctx.map.def_value(result, self.loc)?;
        }

        if !ctx.map.contains_value(dest) {
            return err!(self.loc, "value {} is not yet defined", dest);
        }

        ctx.function
            .dfg
            .make_value_alias_for_serialization(dest, result);

        ctx.aliases.push(result);
        Ok(())
    }

    // Parse an instruction, append it to `block`.
    //
    // instruction ::= [inst-results "="] Opcode(opc) ["." Type] ...
    //
    fn parse_instruction(
        &mut self,
        results: &[Value],
        srcloc: ir::SourceLoc,
        debug_tags: Vec<DebugTag>,
        ctx: &mut Context,
        block: Block,
    ) -> ParseResult<()> {
        // Define the result values.
        for val in results {
            ctx.map.def_value(*val, self.loc)?;
        }

        // Collect comments for the next instruction.
        self.start_gathering_comments();

        // instruction ::=  [inst-results "="] * Opcode(opc) ["." Type] ...
        let opcode = if let Some(Token::Identifier(text)) = self.token() {
            match text.parse() {
                Ok(opc) => opc,
                Err(msg) => return err!(self.loc, "{}: '{}'", msg, text),
            }
        } else {
            return err!(self.loc, "expected instruction opcode");
        };
        let opcode_loc = self.loc;
        self.consume();

        // Look for a controlling type variable annotation.
        // instruction ::=  [inst-results "="] Opcode(opc) * ["." Type] ...
        let explicit_ctrl_type = if self.optional(Token::Dot) {
            if let Some(Token::Type(_t)) = self.token() {
                Some(self.match_type("expected type after 'opcode.'")?)
            } else {
                let dt = self.match_dt("expected dynamic type")?;
                self.concrete_from_dt(dt, ctx)
            }
        } else {
            None
        };

        // instruction ::=  [inst-results "="] Opcode(opc) ["." Type] * ...
        let inst_data = self.parse_inst_operands(ctx, opcode, explicit_ctrl_type)?;

        let ctrl_typevar = self.infer_typevar(ctx, opcode, explicit_ctrl_type, &inst_data)?;
        let inst = ctx.function.dfg.make_inst(inst_data);

        // Attach stack map, if present.
        if self.optional(Token::Comma) {
            self.match_token(
                Token::Identifier("stack_map"),
                "expected `stack_map = [...]`",
            )?;
            if !opcode.is_call() || opcode.is_return() {
                return err!(
                    self.loc,
                    "stack map can only be attached to a (non-tail) call"
                );
            }

            self.match_token(Token::Equal, "expected `= [...]`")?;
            self.match_token(Token::LBracket, "expected `[...]`")?;
            while !self.optional(Token::RBracket) {
                let ty = self.match_type("expected `<type> @ <slot> + <offset>`")?;
                self.match_token(Token::At, "expected `@ <slot> + <offset>`")?;
                let slot = self.match_ss("expected `<slot> + <offset>`")?;
                let offset: u32 = match self.token() {
                    Some(Token::Integer(s)) if s.starts_with('+') => {
                        self.match_uimm32("expected a u32 offset")?.into()
                    }
                    _ => {
                        self.match_token(Token::Plus, "expected `+ <offset>`")?;
                        self.match_uimm32("expected a u32 offset")?.into()
                    }
                };
                ctx.function
                    .dfg
                    .append_user_stack_map_entry(inst, ir::UserStackMapEntry { ty, slot, offset });
                if !self.optional(Token::Comma) {
                    self.match_token(Token::RBracket, "expected `,` or `]`")?;
                    break;
                }
            }
        }

        // We're done parsing the instruction data itself.
        //
        // We still need to check that the number of result values in
        // the source matches the opcode or function call
        // signature. We also need to create values with the right
        // type for all the instruction results.
        let num_results =
            ctx.function
                .dfg
                .make_inst_results_for_parser(inst, ctrl_typevar, results);
        ctx.function.layout.append_inst(inst, block);
        ctx.map
            .def_entity(inst.into(), opcode_loc)
            .expect("duplicate inst references created");

        if !srcloc.is_default() {
            ctx.function.set_srcloc(inst, srcloc);
        }
        if !debug_tags.is_empty() {
            ctx.function.debug_tags.set(inst, debug_tags);
        }

        if results.len() != num_results {
            return err!(
                self.loc,
                "instruction produces {} result values, {} given",
                num_results,
                results.len()
            );
        }

        // Collect any trailing comments.
        self.token();
        self.claim_gathered_comments(inst);

        Ok(())
    }

    // Type inference for polymorphic instructions.
    //
    // The controlling type variable can be specified explicitly as 'splat.i32x4 v5', or it can be
    // inferred from `inst_data.typevar_operand` for some opcodes.
    //
    // Returns the controlling typevar for a polymorphic opcode, or `INVALID` for a non-polymorphic
    // opcode.
    fn infer_typevar(
        &self,
        ctx: &Context,
        opcode: Opcode,
        explicit_ctrl_type: Option<Type>,
        inst_data: &InstructionData,
    ) -> ParseResult<Type> {
        let constraints = opcode.constraints();
        let ctrl_type = match explicit_ctrl_type {
            Some(t) => t,
            None => {
                if constraints.use_typevar_operand() {
                    // This is an opcode that supports type inference, AND there was no
                    // explicit type specified. Look up `ctrl_value` to see if it was defined
                    // already.
                    // TBD: If it is defined in another block, the type should have been
                    // specified explicitly. It is unfortunate that the correctness of IR
                    // depends on the layout of the blocks.
                    let ctrl_src_value = inst_data
                        .typevar_operand(&ctx.function.dfg.value_lists)
                        .expect("Constraints <-> Format inconsistency");
                    if !ctx.map.contains_value(ctrl_src_value) {
                        return err!(
                            self.loc,
                            "type variable required for polymorphic opcode, e.g. '{}.{}'; \
                             can't infer from {} which is not yet defined",
                            opcode,
                            constraints.ctrl_typeset().unwrap().example(),
                            ctrl_src_value
                        );
                    }
                    if !ctx.function.dfg.value_is_valid_for_parser(ctrl_src_value) {
                        return err!(
                            self.loc,
                            "type variable required for polymorphic opcode, e.g. '{}.{}'; \
                             can't infer from {} which is not yet resolved",
                            opcode,
                            constraints.ctrl_typeset().unwrap().example(),
                            ctrl_src_value
                        );
                    }
                    ctx.function.dfg.value_type(ctrl_src_value)
                } else if constraints.is_polymorphic() {
                    // This opcode does not support type inference, so the explicit type
                    // variable is required.
                    return err!(
                        self.loc,
                        "type variable required for polymorphic opcode, e.g. '{}.{}'",
                        opcode,
                        constraints.ctrl_typeset().unwrap().example()
                    );
                } else {
                    // This is a non-polymorphic opcode. No typevar needed.
                    INVALID
                }
            }
        };

        // Verify that `ctrl_type` is valid for the controlling type variable. We don't want to
        // attempt deriving types from an incorrect basis.
        // This is not a complete type check. The verifier does that.
        if let Some(typeset) = constraints.ctrl_typeset() {
            // This is a polymorphic opcode.
            if !typeset.contains(ctrl_type) {
                return err!(
                    self.loc,
                    "{} is not a valid typevar for {}",
                    ctrl_type,
                    opcode
                );
            }
        // Treat it as a syntax error to specify a typevar on a non-polymorphic opcode.
        } else if ctrl_type != INVALID {
            return err!(self.loc, "{} does not take a typevar", opcode);
        }

        Ok(ctrl_type)
    }

    // Parse comma-separated value list into a VariableArgs struct.
    //
    // value_list ::= [ value { "," value } ]
    //
    fn parse_value_list(&mut self) -> ParseResult<VariableArgs> {
        let mut args = VariableArgs::new();

        if let Some(Token::Value(v)) = self.token() {
            args.push(v);
            self.consume();
        } else {
            return Ok(args);
        }

        while self.optional(Token::Comma) {
            args.push(self.match_value("expected value in argument list")?);
        }

        Ok(args)
    }

    /// Parse an optional list of block-call arguments enclosed in
    /// parentheses.
    fn parse_opt_block_call_args(&mut self) -> ParseResult<Vec<BlockArg>> {
        if !self.optional(Token::LPar) {
            return Ok(vec![]);
        }

        let mut args = vec![];
        while self.token() != Some(Token::RPar) {
            args.push(self.parse_block_call_arg()?);
            if self.token() == Some(Token::Comma) {
                self.consume();
            } else {
                break;
            }
        }

        self.match_token(Token::RPar, "expected ')' after arguments")?;

        Ok(args)
    }

    fn parse_block_call_arg(&mut self) -> ParseResult<BlockArg> {
        match self.token() {
            Some(Token::Value(v)) => {
                self.consume();
                Ok(BlockArg::Value(v))
            }
            Some(Token::TryCallRet(i)) => {
                self.consume();
                Ok(BlockArg::TryCallRet(i))
            }
            Some(Token::TryCallExn(i)) => {
                self.consume();
                Ok(BlockArg::TryCallExn(i))
            }
            tok => Err(self.error(&format!("unexpected token: {tok:?}"))),
        }
    }

    /// Parse a CLIF run command.
    ///
    /// run-command ::= "run" [":" invocation comparison expected]
    ///               \ "print" [":" invocation]
    fn parse_run_command(&mut self, sig: &Signature) -> ParseResult<RunCommand> {
        // skip semicolon
        match self.token() {
            Some(Token::Identifier("run")) => {
                self.consume();
                if self.optional(Token::Colon) {
                    let invocation = self.parse_run_invocation(sig)?;
                    let comparison = self.parse_run_comparison()?;
                    let expected = self.parse_run_returns(sig)?;
                    Ok(RunCommand::Run(invocation, comparison, expected))
                } else if sig.params.is_empty()
                    && sig.returns.len() == 1
                    && sig.returns[0].value_type.is_int()
                {
                    // To match the existing run behavior that does not require an explicit
                    // invocation, we create an invocation from a function like `() -> i*` and
                    // require the result to be non-zero.
                    let invocation = Invocation::new("default", vec![]);
                    let expected = vec![DataValue::I8(0)];
                    let comparison = Comparison::NotEquals;
                    Ok(RunCommand::Run(invocation, comparison, expected))
                } else {
                    Err(self.error("unable to parse the run command"))
                }
            }
            Some(Token::Identifier("print")) => {
                self.consume();
                if self.optional(Token::Colon) {
                    Ok(RunCommand::Print(self.parse_run_invocation(sig)?))
                } else if sig.params.is_empty() {
                    // To allow printing of functions like `() -> *`, we create a no-arg invocation.
                    let invocation = Invocation::new("default", vec![]);
                    Ok(RunCommand::Print(invocation))
                } else {
                    Err(self.error("unable to parse the print command"))
                }
            }
            _ => Err(self.error("expected a 'run:' or 'print:' command")),
        }
    }

    /// Parse the invocation of a CLIF function.
    ///
    /// This is different from parsing a CLIF `call`; it is used in parsing run commands like
    /// `run: %fn(42, 4.2) == false`.
    ///
    /// invocation ::= name "(" [data-value-list] ")"
    fn parse_run_invocation(&mut self, sig: &Signature) -> ParseResult<Invocation> {
        if let Some(Token::Name(name)) = self.token() {
            self.consume();
            self.match_token(
                Token::LPar,
                "expected invocation parentheses, e.g. %fn(...)",
            )?;

            let arg_types = sig
                .params
                .iter()
                .map(|abi| abi.value_type)
                .collect::<Vec<_>>();
            let args = self.parse_data_value_list(&arg_types)?;

            self.match_token(
                Token::RPar,
                "expected invocation parentheses, e.g. %fn(...)",
            )?;
            Ok(Invocation::new(name, args))
        } else {
            Err(self.error("expected a function name, e.g. %my_fn"))
        }
    }

    /// Parse a comparison operator for run commands.
    ///
    /// comparison ::= "==" | "!="
    fn parse_run_comparison(&mut self) -> ParseResult<Comparison> {
        if self.optional(Token::Equal) {
            self.match_token(Token::Equal, "expected another =")?;
            Ok(Comparison::Equals)
        } else if self.optional(Token::Bang) {
            self.match_token(Token::Equal, "expected a =")?;
            Ok(Comparison::NotEquals)
        } else {
            Err(self.error("unable to parse a valid comparison operator"))
        }
    }

    /// Parse the expected return values of a run invocation.
    ///
    /// expected ::= "[" "]"
    ///            | data-value
    ///            | "[" data-value-list "]"
    fn parse_run_returns(&mut self, sig: &Signature) -> ParseResult<Vec<DataValue>> {
        if sig.returns.len() != 1 {
            self.match_token(Token::LBracket, "expected a left bracket [")?;
        }

        let returns = self
            .parse_data_value_list(&sig.returns.iter().map(|a| a.value_type).collect::<Vec<_>>())?;

        if sig.returns.len() != 1 {
            self.match_token(Token::RBracket, "expected a right bracket ]")?;
        }
        Ok(returns)
    }

    /// Parse a comma-separated list of data values.
    ///
    /// data-value-list ::= [data-value {"," data-value-list}]
    fn parse_data_value_list(&mut self, types: &[Type]) -> ParseResult<Vec<DataValue>> {
        let mut values = vec![];
        for ty in types.iter().take(1) {
            values.push(self.parse_data_value(*ty)?);
        }
        for ty in types.iter().skip(1) {
            self.match_token(
                Token::Comma,
                "expected a comma between invocation arguments",
            )?;
            values.push(self.parse_data_value(*ty)?);
        }
        Ok(values)
    }

    /// Parse a data value; e.g. `42`, `4.2`, `true`.
    ///
    /// data-value-list ::= [data-value {"," data-value-list}]
    fn parse_data_value(&mut self, ty: Type) -> ParseResult<DataValue> {
        let dv = match ty {
            I8 => DataValue::from(self.match_imm8("expected a i8")?),
            I16 => DataValue::from(self.match_imm16("expected an i16")?),
            I32 => DataValue::from(self.match_imm32("expected an i32")?),
            I64 => DataValue::from(Into::<i64>::into(self.match_imm64("expected an i64")?)),
            I128 => DataValue::from(self.match_imm128("expected an i128")?),
            F16 => DataValue::from(self.match_ieee16("expected an f16")?),
            F32 => DataValue::from(self.match_ieee32("expected an f32")?),
            F64 => DataValue::from(self.match_ieee64("expected an f64")?),
            F128 => DataValue::from(self.match_ieee128("expected an f128")?),
            _ if (ty.is_vector() || ty.is_dynamic_vector()) => {
                let as_vec = self.match_uimm128(ty)?.into_vec();
                let slice = as_vec.as_slice();
                match slice.len() {
                    16 => DataValue::V128(slice.try_into().unwrap()),
                    8 => DataValue::V64(slice.try_into().unwrap()),
                    4 => DataValue::V32(slice.try_into().unwrap()),
                    2 => DataValue::V16(slice.try_into().unwrap()),
                    _ => {
                        return Err(
                            self.error("vectors larger than 128 bits are not currently supported")
                        );
                    }
                }
            }
            _ => return Err(self.error(&format!("don't know how to parse data values of: {ty}"))),
        };
        Ok(dv)
    }

    // Parse the operands following the instruction opcode.
    // This depends on the format of the opcode.
    fn parse_inst_operands(
        &mut self,
        ctx: &mut Context,
        opcode: Opcode,
        explicit_control_type: Option<Type>,
    ) -> ParseResult<InstructionData> {
        let idata = match opcode.format() {
            InstructionFormat::Unary => InstructionData::Unary {
                opcode,
                arg: self.match_value("expected SSA value operand")?,
            },
            InstructionFormat::UnaryImm => {
                let msg = |bits| format!("expected immediate {bits}-bit integer operand");
                let unsigned = match explicit_control_type {
                    Some(types::I8) => self.match_imm8(&msg(8))? as u8 as i64,
                    Some(types::I16) => self.match_imm16(&msg(16))? as u16 as i64,
                    Some(types::I32) => self.match_imm32(&msg(32))? as u32 as i64,
                    Some(types::I64) => self.match_imm64(&msg(64))?.bits(),
                    _ => {
                        return err!(
                            self.loc,
                            "expected one of the following type: i8, i16, i32 or i64"
                        );
                    }
                };
                InstructionData::UnaryImm {
                    opcode,
                    imm: Imm64::new(unsigned),
                }
            }
            InstructionFormat::UnaryIeee16 => InstructionData::UnaryIeee16 {
                opcode,
                imm: self.match_ieee16("expected immediate 16-bit float operand")?,
            },
            InstructionFormat::UnaryIeee32 => InstructionData::UnaryIeee32 {
                opcode,
                imm: self.match_ieee32("expected immediate 32-bit float operand")?,
            },
            InstructionFormat::UnaryIeee64 => InstructionData::UnaryIeee64 {
                opcode,
                imm: self.match_ieee64("expected immediate 64-bit float operand")?,
            },
            InstructionFormat::UnaryConst => {
                let constant_handle = if let Some(Token::Constant(_)) = self.token() {
                    // If handed a `const?`, use that.
                    let c = self.match_constant()?;
                    ctx.check_constant(c, self.loc)?;
                    c
                } else if opcode == Opcode::F128const {
                    let ieee128 = self.match_ieee128("expected immediate 128-bit float operand")?;
                    ctx.function.dfg.constants.insert(ieee128.into())
                } else if let Some(controlling_type) = explicit_control_type {
                    // If an explicit control type is present, we expect a sized value and insert
                    // it in the constant pool.
                    let uimm128 = self.match_uimm128(controlling_type)?;
                    ctx.function.dfg.constants.insert(uimm128)
                } else {
                    return err!(
                        self.loc,
                        "Expected either a const entity or a typed value, e.g. inst.i32x4 [...]"
                    );
                };
                InstructionData::UnaryConst {
                    opcode,
                    constant_handle,
                }
            }
            InstructionFormat::UnaryGlobalValue => {
                let gv = self.match_gv("expected global value")?;
                ctx.check_gv(gv, self.loc)?;
                InstructionData::UnaryGlobalValue {
                    opcode,
                    global_value: gv,
                }
            }
            InstructionFormat::Binary => {
                let lhs = self.match_value("expected SSA value first operand")?;
                self.match_token(Token::Comma, "expected ',' between operands")?;
                let rhs = self.match_value("expected SSA value second operand")?;
                InstructionData::Binary {
                    opcode,
                    args: [lhs, rhs],
                }
            }
            InstructionFormat::BinaryImm8 => {
                let arg = self.match_value("expected SSA value first operand")?;
                self.match_token(Token::Comma, "expected ',' between operands")?;
                let imm = self.match_uimm8("expected unsigned 8-bit immediate")?;
                InstructionData::BinaryImm8 { opcode, arg, imm }
            }
            InstructionFormat::BinaryImm64 => {
                let lhs = self.match_value("expected SSA value first operand")?;
                self.match_token(Token::Comma, "expected ',' between operands")?;
                let rhs = self.match_imm64("expected immediate integer second operand")?;
                InstructionData::BinaryImm64 {
                    opcode,
                    arg: lhs,
                    imm: rhs,
                }
            }
            InstructionFormat::Ternary => {
                // Names here refer to the `select` instruction.
                // This format is also use by `fma`.
                let ctrl_arg = self.match_value("expected SSA value control operand")?;
                self.match_token(Token::Comma, "expected ',' between operands")?;
                let true_arg = self.match_value("expected SSA value true operand")?;
                self.match_token(Token::Comma, "expected ',' between operands")?;
                let false_arg = self.match_value("expected SSA value false operand")?;
                InstructionData::Ternary {
                    opcode,
                    args: [ctrl_arg, true_arg, false_arg],
                }
            }
            InstructionFormat::MultiAry => {
                let args = self.parse_value_list()?;
                InstructionData::MultiAry {
                    opcode,
                    args: args.into_value_list(&[], &mut ctx.function.dfg.value_lists),
                }
            }
            InstructionFormat::NullAry => InstructionData::NullAry { opcode },
            InstructionFormat::Jump => {
                // Parse the destination block number.
                let block_num = self.match_block("expected jump destination block")?;
                let args = self.parse_opt_block_call_args()?;
                let destination = ctx.function.dfg.block_call(block_num, &args);
                InstructionData::Jump {
                    opcode,
                    destination,
                }
            }
            InstructionFormat::Brif => {
                let arg = self.match_value("expected SSA value control operand")?;
                self.match_token(Token::Comma, "expected ',' between operands")?;
                let block_then = {
                    let block_num = self.match_block("expected branch then block")?;
                    let args = self.parse_opt_block_call_args()?;
                    ctx.function.dfg.block_call(block_num, &args)
                };
                self.match_token(Token::Comma, "expected ',' between operands")?;
                let block_else = {
                    let block_num = self.match_block("expected branch else block")?;
                    let args = self.parse_opt_block_call_args()?;
                    ctx.function.dfg.block_call(block_num, &args)
                };
                InstructionData::Brif {
                    opcode,
                    arg,
                    blocks: [block_then, block_else],
                }
            }
            InstructionFormat::BranchTable => {
                let arg = self.match_value("expected SSA value operand")?;
                self.match_token(Token::Comma, "expected ',' between operands")?;
                let block_num = self.match_block("expected branch destination block")?;
                let args = self.parse_opt_block_call_args()?;
                let destination = ctx.function.dfg.block_call(block_num, &args);
                self.match_token(Token::Comma, "expected ',' between operands")?;
                let table = self.parse_jump_table(ctx, destination)?;
                InstructionData::BranchTable { opcode, arg, table }
            }
            InstructionFormat::TernaryImm8 => {
                let lhs = self.match_value("expected SSA value first operand")?;
                self.match_token(Token::Comma, "expected ',' between operands")?;
                let rhs = self.match_value("expected SSA value last operand")?;
                self.match_token(Token::Comma, "expected ',' between operands")?;
                let imm = self.match_uimm8("expected 8-bit immediate")?;
                InstructionData::TernaryImm8 {
                    opcode,
                    imm,
                    args: [lhs, rhs],
                }
            }
            InstructionFormat::Shuffle => {
                let a = self.match_value("expected SSA value first operand")?;
                self.match_token(Token::Comma, "expected ',' between operands")?;
                let b = self.match_value("expected SSA value second operand")?;
                self.match_token(Token::Comma, "expected ',' between operands")?;
                let uimm128 = self.match_uimm128(I8X16)?;
                let imm = ctx.function.dfg.immediates.push(uimm128);
                InstructionData::Shuffle {
                    opcode,
                    imm,
                    args: [a, b],
                }
            }
            InstructionFormat::IntCompare => {
                let cond = self.match_enum("expected intcc condition code")?;
                let lhs = self.match_value("expected SSA value first operand")?;
                self.match_token(Token::Comma, "expected ',' between operands")?;
                let rhs = self.match_value("expected SSA value second operand")?;
                InstructionData::IntCompare {
                    opcode,
                    cond,
                    args: [lhs, rhs],
                }
            }
            InstructionFormat::IntCompareImm => {
                let cond = self.match_enum("expected intcc condition code")?;
                let lhs = self.match_value("expected SSA value first operand")?;
                self.match_token(Token::Comma, "expected ',' between operands")?;
                let rhs = self.match_imm64("expected immediate second operand")?;
                InstructionData::IntCompareImm {
                    opcode,
                    cond,
                    arg: lhs,
                    imm: rhs,
                }
            }
            InstructionFormat::FloatCompare => {
                let cond = self.match_enum("expected floatcc condition code")?;
                let lhs = self.match_value("expected SSA value first operand")?;
                self.match_token(Token::Comma, "expected ',' between operands")?;
                let rhs = self.match_value("expected SSA value second operand")?;
                InstructionData::FloatCompare {
                    opcode,
                    cond,
                    args: [lhs, rhs],
                }
            }
            InstructionFormat::Call => {
                let func_ref = self.match_fn("expected function reference")?;
                ctx.check_fn(func_ref, self.loc)?;
                self.match_token(Token::LPar, "expected '(' before arguments")?;
                let args = self.parse_value_list()?;
                self.match_token(Token::RPar, "expected ')' after arguments")?;
                InstructionData::Call {
                    opcode,
                    func_ref,
                    args: args.into_value_list(&[], &mut ctx.function.dfg.value_lists),
                }
            }
            InstructionFormat::CallIndirect => {
                let sig_ref = self.match_sig("expected signature reference")?;
                ctx.check_sig(sig_ref, self.loc)?;
                self.match_token(Token::Comma, "expected ',' between operands")?;
                let callee = self.match_value("expected SSA value callee operand")?;
                self.match_token(Token::LPar, "expected '(' before arguments")?;
                let args = self.parse_value_list()?;
                self.match_token(Token::RPar, "expected ')' after arguments")?;
                InstructionData::CallIndirect {
                    opcode,
                    sig_ref,
                    args: args.into_value_list(&[callee], &mut ctx.function.dfg.value_lists),
                }
            }
            InstructionFormat::TryCall => {
                let func_ref = self.match_fn("expected function reference")?;
                ctx.check_fn(func_ref, self.loc)?;
                self.match_token(Token::LPar, "expected '(' before arguments")?;
                let args = self.parse_value_list()?;
                self.match_token(Token::RPar, "expected ')' after arguments")?;
                self.match_token(Token::Comma, "expected ',' after argument list")?;
                let exception = self.parse_exception_table(ctx)?;
                InstructionData::TryCall {
                    opcode,
                    func_ref,
                    args: args.into_value_list(&[], &mut ctx.function.dfg.value_lists),
                    exception,
                }
            }
            InstructionFormat::TryCallIndirect => {
                let callee = self.match_value("expected SSA value callee operand")?;
                self.match_token(Token::LPar, "expected '(' before arguments")?;
                let args = self.parse_value_list()?;
                self.match_token(Token::RPar, "expected ')' after arguments")?;
                self.match_token(Token::Comma, "expected ',' after argument list")?;
                let exception = self.parse_exception_table(ctx)?;
                InstructionData::TryCallIndirect {
                    opcode,
                    args: args.into_value_list(&[callee], &mut ctx.function.dfg.value_lists),
                    exception,
                }
            }
            InstructionFormat::FuncAddr => {
                let func_ref = self.match_fn("expected function reference")?;
                ctx.check_fn(func_ref, self.loc)?;
                InstructionData::FuncAddr { opcode, func_ref }
            }
            InstructionFormat::StackLoad => {
                let ss = self.match_ss("expected stack slot number: ss«n»")?;
                ctx.check_ss(ss, self.loc)?;
                let offset = self.optional_offset32()?;
                InstructionData::StackLoad {
                    opcode,
                    stack_slot: ss,
                    offset,
                }
            }
            InstructionFormat::StackStore => {
                let arg = self.match_value("expected SSA value operand")?;
                self.match_token(Token::Comma, "expected ',' between operands")?;
                let ss = self.match_ss("expected stack slot number: ss«n»")?;
                ctx.check_ss(ss, self.loc)?;
                let offset = self.optional_offset32()?;
                InstructionData::StackStore {
                    opcode,
                    arg,
                    stack_slot: ss,
                    offset,
                }
            }
            InstructionFormat::DynamicStackLoad => {
                let dss = self.match_dss("expected dynamic stack slot number: dss«n»")?;
                ctx.check_dss(dss, self.loc)?;
                InstructionData::DynamicStackLoad {
                    opcode,
                    dynamic_stack_slot: dss,
                }
            }
            InstructionFormat::DynamicStackStore => {
                let arg = self.match_value("expected SSA value operand")?;
                self.match_token(Token::Comma, "expected ',' between operands")?;
                let dss = self.match_dss("expected dynamic stack slot number: dss«n»")?;
                ctx.check_dss(dss, self.loc)?;
                InstructionData::DynamicStackStore {
                    opcode,
                    arg,
                    dynamic_stack_slot: dss,
                }
            }
            InstructionFormat::Load => {
                let flags = self.optional_memflags()?;
                let addr = self.match_value("expected SSA value address")?;
                let offset = self.optional_offset32()?;
                InstructionData::Load {
                    opcode,
                    flags,
                    arg: addr,
                    offset,
                }
            }
            InstructionFormat::Store => {
                let flags = self.optional_memflags()?;
                let arg = self.match_value("expected SSA value operand")?;
                self.match_token(Token::Comma, "expected ',' between operands")?;
                let addr = self.match_value("expected SSA value address")?;
                let offset = self.optional_offset32()?;
                InstructionData::Store {
                    opcode,
                    flags,
                    args: [arg, addr],
                    offset,
                }
            }
            InstructionFormat::Trap => {
                let code = self.match_enum("expected trap code")?;
                InstructionData::Trap { opcode, code }
            }
            InstructionFormat::CondTrap => {
                let arg = self.match_value("expected SSA value operand")?;
                self.match_token(Token::Comma, "expected ',' between operands")?;
                let code = self.match_enum("expected trap code")?;
                InstructionData::CondTrap { opcode, arg, code }
            }
            InstructionFormat::AtomicCas => {
                let flags = self.optional_memflags()?;
                let addr = self.match_value("expected SSA value address")?;
                self.match_token(Token::Comma, "expected ',' between operands")?;
                let expected = self.match_value("expected SSA value address")?;
                self.match_token(Token::Comma, "expected ',' between operands")?;
                let replacement = self.match_value("expected SSA value address")?;
                InstructionData::AtomicCas {
                    opcode,
                    flags,
                    args: [addr, expected, replacement],
                }
            }
            InstructionFormat::AtomicRmw => {
                let flags = self.optional_memflags()?;
                let op = self.match_enum("expected AtomicRmwOp")?;
                let addr = self.match_value("expected SSA value address")?;
                self.match_token(Token::Comma, "expected ',' between operands")?;
                let arg2 = self.match_value("expected SSA value address")?;
                InstructionData::AtomicRmw {
                    opcode,
                    flags,
                    op,
                    args: [addr, arg2],
                }
            }
            InstructionFormat::LoadNoOffset => {
                let flags = self.optional_memflags()?;
                let addr = self.match_value("expected SSA value address")?;
                InstructionData::LoadNoOffset {
                    opcode,
                    flags,
                    arg: addr,
                }
            }
            InstructionFormat::StoreNoOffset => {
                let flags = self.optional_memflags()?;
                let arg = self.match_value("expected SSA value operand")?;
                self.match_token(Token::Comma, "expected ',' between operands")?;
                let addr = self.match_value("expected SSA value address")?;
                InstructionData::StoreNoOffset {
                    opcode,
                    flags,
                    args: [arg, addr],
                }
            }
            InstructionFormat::IntAddTrap => {
                let a = self.match_value("expected SSA value operand")?;
                self.match_token(Token::Comma, "expected ',' between operands")?;
                let b = self.match_value("expected SSA value operand")?;
                self.match_token(Token::Comma, "expected ',' between operands")?;
                let code = self.match_enum("expected trap code")?;
                InstructionData::IntAddTrap {
                    opcode,
                    args: [a, b],
                    code,
                }
            }
            InstructionFormat::ExceptionHandlerAddress => {
                let block = self.match_block("expected block")?;
                self.match_token(Token::Comma, "expected ',' between operands")?;
                let imm = self.match_imm64("expected immediate handler index")?;
                InstructionData::ExceptionHandlerAddress { opcode, block, imm }
            }
        };
        Ok(idata)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::isaspec::IsaSpec;

    #[test]
    fn argument_type() {
        let mut p = Parser::new("i32 sext");
        let arg = p.parse_abi_param().unwrap();
        assert_eq!(arg.value_type, types::I32);
        assert_eq!(arg.extension, ArgumentExtension::Sext);
        assert_eq!(arg.purpose, ArgumentPurpose::Normal);
        let ParseError {
            location,
            message,
            is_warning,
        } = p.parse_abi_param().unwrap_err();
        assert_eq!(location.line_number, 1);
        assert_eq!(message, "expected parameter type");
        assert!(!is_warning);
    }

    #[test]
    fn aliases() {
        let (func, details) = Parser::new(
            "function %qux() system_v {
                                           block0:
                                             v4 = iconst.i8 6
                                             v3 -> v4
                                             v1 = iadd_imm v3, 17
                                           }",
        )
        .parse_function()
        .unwrap();
        assert_eq!(func.name.to_string(), "%qux");
        let v4 = details.map.lookup_str("v4").unwrap();
        assert_eq!(v4.to_string(), "v4");
        let v3 = details.map.lookup_str("v3").unwrap();
        assert_eq!(v3.to_string(), "v3");
        match v3 {
            AnyEntity::Value(v3) => {
                let aliased_to = func.dfg.resolve_aliases(v3);
                assert_eq!(aliased_to.to_string(), "v4");
            }
            _ => panic!("expected value: {v3}"),
        }
    }

    #[test]
    fn signature() {
        let sig = Parser::new("()system_v").parse_signature().unwrap();
        assert_eq!(sig.params.len(), 0);
        assert_eq!(sig.returns.len(), 0);
        assert_eq!(sig.call_conv, CallConv::SystemV);

        let sig2 =
            Parser::new("(i8 uext, f16, f32, f64, f128, i32 sret) -> i32 sext, f64 system_v")
                .parse_signature()
                .unwrap();
        assert_eq!(
            sig2.to_string(),
            "(i8 uext, f16, f32, f64, f128, i32 sret) -> i32 sext, f64 system_v"
        );
        assert_eq!(sig2.call_conv, CallConv::SystemV);

        // Old-style signature without a calling convention.
        assert_eq!(
            Parser::new("()").parse_signature().unwrap().to_string(),
            "() fast"
        );
        assert_eq!(
            Parser::new("() notacc")
                .parse_signature()
                .unwrap_err()
                .to_string(),
            "1: unknown calling convention: notacc"
        );

        // `void` is not recognized as a type by the lexer. It should not appear in files.
        assert_eq!(
            Parser::new("() -> void")
                .parse_signature()
                .unwrap_err()
                .to_string(),
            "1: expected parameter type"
        );
        assert_eq!(
            Parser::new("i8 -> i8")
                .parse_signature()
                .unwrap_err()
                .to_string(),
            "1: expected function signature: ( args... )"
        );
        assert_eq!(
            Parser::new("(i8 -> i8")
                .parse_signature()
                .unwrap_err()
                .to_string(),
            "1: expected ')' after function arguments"
        );
    }

    #[test]
    fn stack_slot_decl() {
        let (func, _) = Parser::new(
            "function %foo() system_v {
                                       ss3 = explicit_slot 13
                                       ss1 = explicit_slot 1
                                     }",
        )
        .parse_function()
        .unwrap();
        assert_eq!(func.name.to_string(), "%foo");
        let mut iter = func.sized_stack_slots.keys();
        let _ss0 = iter.next().unwrap();
        let ss1 = iter.next().unwrap();
        assert_eq!(ss1.to_string(), "ss1");
        assert_eq!(
            func.sized_stack_slots[ss1].kind,
            StackSlotKind::ExplicitSlot
        );
        assert_eq!(func.sized_stack_slots[ss1].size, 1);
        let _ss2 = iter.next().unwrap();
        let ss3 = iter.next().unwrap();
        assert_eq!(ss3.to_string(), "ss3");
        assert_eq!(
            func.sized_stack_slots[ss3].kind,
            StackSlotKind::ExplicitSlot
        );
        assert_eq!(func.sized_stack_slots[ss3].size, 13);
        assert_eq!(iter.next(), None);

        // Catch duplicate definitions.
        assert_eq!(
            Parser::new(
                "function %bar() system_v {
                                    ss1  = explicit_slot 13
                                    ss1  = explicit_slot 1
                                }",
            )
            .parse_function()
            .unwrap_err()
            .to_string(),
            "3: duplicate entity: ss1"
        );
    }

    #[test]
    fn block_header() {
        let (func, _) = Parser::new(
            "function %blocks() system_v {
                                     block0:
                                     block4(v3: i32):
                                     }",
        )
        .parse_function()
        .unwrap();
        assert_eq!(func.name.to_string(), "%blocks");

        let mut blocks = func.layout.blocks();

        let block0 = blocks.next().unwrap();
        assert_eq!(func.dfg.block_params(block0), &[]);

        let block4 = blocks.next().unwrap();
        let block4_args = func.dfg.block_params(block4);
        assert_eq!(block4_args.len(), 1);
        assert_eq!(func.dfg.value_type(block4_args[0]), types::I32);
    }

    #[test]
    fn duplicate_block() {
        let ParseError {
            location,
            message,
            is_warning,
        } = Parser::new(
            "function %blocks() system_v {
                block0:
                block0:
                    return 2",
        )
        .parse_function()
        .unwrap_err();

        assert_eq!(location.line_number, 3);
        assert_eq!(message, "duplicate entity: block0");
        assert!(!is_warning);
    }

    #[test]
    fn number_of_blocks() {
        let ParseError {
            location,
            message,
            is_warning,
        } = Parser::new(
            "function %a() {
                block100000:",
        )
        .parse_function()
        .unwrap_err();

        assert_eq!(location.line_number, 2);
        assert_eq!(message, "too many blocks");
        assert!(!is_warning);
    }

    #[test]
    fn duplicate_ss() {
        let ParseError {
            location,
            message,
            is_warning,
        } = Parser::new(
            "function %blocks() system_v {
                ss0 = explicit_slot 8
                ss0 = explicit_slot 8",
        )
        .parse_function()
        .unwrap_err();

        assert_eq!(location.line_number, 3);
        assert_eq!(message, "duplicate entity: ss0");
        assert!(!is_warning);
    }

    #[test]
    fn duplicate_gv() {
        let ParseError {
            location,
            message,
            is_warning,
        } = Parser::new(
            "function %blocks() system_v {
                gv0 = vmctx
                gv0 = vmctx",
        )
        .parse_function()
        .unwrap_err();

        assert_eq!(location.line_number, 3);
        assert_eq!(message, "duplicate entity: gv0");
        assert!(!is_warning);
    }

    #[test]
    fn duplicate_sig() {
        let ParseError {
            location,
            message,
            is_warning,
        } = Parser::new(
            "function %blocks() system_v {
                sig0 = ()
                sig0 = ()",
        )
        .parse_function()
        .unwrap_err();

        assert_eq!(location.line_number, 3);
        assert_eq!(message, "duplicate entity: sig0");
        assert!(!is_warning);
    }

    #[test]
    fn duplicate_fn() {
        let ParseError {
            location,
            message,
            is_warning,
        } = Parser::new(
            "function %blocks() system_v {
                sig0 = ()
                fn0 = %foo sig0
                fn0 = %foo sig0",
        )
        .parse_function()
        .unwrap_err();

        assert_eq!(location.line_number, 4);
        assert_eq!(message, "duplicate entity: fn0");
        assert!(!is_warning);
    }

    #[test]
    fn comments() {
        let (func, Details { comments, .. }) = Parser::new(
            "; before
                         function %comment() system_v { ; decl
                            ss10  = explicit_slot 13 ; stackslot.
                            ; Still stackslot.
                         block0: ; Basic block
                         trap user42; Instruction
                         } ; Trailing.
                         ; More trailing.",
        )
        .parse_function()
        .unwrap();
        assert_eq!(func.name.to_string(), "%comment");
        assert_eq!(comments.len(), 7); // no 'before' comment.
        assert_eq!(
            comments[0],
            Comment {
                entity: AnyEntity::Function,
                text: "; decl",
            }
        );
        assert_eq!(comments[1].entity.to_string(), "ss10");
        assert_eq!(comments[2].entity.to_string(), "ss10");
        assert_eq!(comments[2].text, "; Still stackslot.");
        assert_eq!(comments[3].entity.to_string(), "block0");
        assert_eq!(comments[3].text, "; Basic block");

        assert_eq!(comments[4].entity.to_string(), "inst0");
        assert_eq!(comments[4].text, "; Instruction");

        assert_eq!(comments[5].entity, AnyEntity::Function);
        assert_eq!(comments[6].entity, AnyEntity::Function);
    }

    #[test]
    fn test_file() {
        let tf = parse_test(
            r#"; before
                             test cfg option=5
                             test verify
                             set unwind_info=false
                             feature "foo"
                             feature !"bar"
                             ; still preamble
                             function %comment() system_v {}"#,
            ParseOptions::default(),
        )
        .unwrap();
        assert_eq!(tf.commands.len(), 2);
        assert_eq!(tf.commands[0].command, "cfg");
        assert_eq!(tf.commands[1].command, "verify");
        match tf.isa_spec {
            IsaSpec::None(s) => {
                assert!(s.enable_verifier());
                assert!(!s.unwind_info());
            }
            _ => panic!("unexpected ISAs"),
        }
        assert_eq!(tf.features[0], Feature::With(&"foo"));
        assert_eq!(tf.features[1], Feature::Without(&"bar"));
        assert_eq!(tf.preamble_comments.len(), 2);
        assert_eq!(tf.preamble_comments[0].text, "; before");
        assert_eq!(tf.preamble_comments[1].text, "; still preamble");
        assert_eq!(tf.functions.len(), 1);
        assert_eq!(tf.functions[0].0.name.to_string(), "%comment");
    }

    #[test]
    fn isa_spec() {
        assert!(
            parse_test(
                "target
                            function %foo() system_v {}",
                ParseOptions::default()
            )
            .is_err()
        );

        assert!(
            parse_test(
                "target x86_64
                            set unwind_info=false
                            function %foo() system_v {}",
                ParseOptions::default()
            )
            .is_err()
        );

        match parse_test(
            "set unwind_info=false
                          target x86_64
                          function %foo() system_v {}",
            ParseOptions::default(),
        )
        .unwrap()
        .isa_spec
        {
            IsaSpec::None(_) => panic!("Expected some ISA"),
            IsaSpec::Some(v) => {
                assert_eq!(v.len(), 1);
                assert!(v[0].name() == "x64" || v[0].name() == "x86");
            }
        }
    }

    #[test]
    fn user_function_name() {
        // Valid characters in the name:
        let func = Parser::new(
            "function u1:2() system_v {
                                           block0:
                                             trap int_divz
                                           }",
        )
        .parse_function()
        .unwrap()
        .0;
        assert_eq!(func.name.to_string(), "u1:2");

        // Invalid characters in the name:
        let mut parser = Parser::new(
            "function u123:abc() system_v {
                                           block0:
                                             trap stk_ovf
                                           }",
        );
        assert!(parser.parse_function().is_err());

        // Incomplete function names should not be valid:
        let mut parser = Parser::new(
            "function u() system_v {
                                           block0:
                                             trap int_ovf
                                           }",
        );
        assert!(parser.parse_function().is_err());

        let mut parser = Parser::new(
            "function u0() system_v {
                                           block0:
                                             trap int_ovf
                                           }",
        );
        assert!(parser.parse_function().is_err());

        let mut parser = Parser::new(
            "function u0:() system_v {
                                           block0:
                                             trap int_ovf
                                           }",
        );
        assert!(parser.parse_function().is_err());
    }

    #[test]
    fn change_default_calling_convention() {
        let code = "function %test() {
        block0:
            return
        }";

        // By default the parser will use the fast calling convention if none is specified.
        let mut parser = Parser::new(code);
        assert_eq!(
            parser.parse_function().unwrap().0.signature.call_conv,
            CallConv::Fast
        );

        // However, we can specify a different calling convention to be the default.
        let mut parser = Parser::new(code).with_default_calling_convention(CallConv::PreserveAll);
        assert_eq!(
            parser.parse_function().unwrap().0.signature.call_conv,
            CallConv::PreserveAll
        );
    }

    #[test]
    fn u8_as_hex() {
        fn parse_as_uimm8(text: &str) -> ParseResult<u8> {
            Parser::new(text).match_uimm8("unable to parse u8")
        }

        assert_eq!(parse_as_uimm8("0").unwrap(), 0);
        assert_eq!(parse_as_uimm8("0xff").unwrap(), 255);
        assert!(parse_as_uimm8("-1").is_err());
        assert!(parse_as_uimm8("0xffa").is_err());
    }

    #[test]
    fn i16_as_hex() {
        fn parse_as_imm16(text: &str) -> ParseResult<i16> {
            Parser::new(text).match_imm16("unable to parse i16")
        }

        assert_eq!(parse_as_imm16("0x8000").unwrap(), -32768);
        assert_eq!(parse_as_imm16("0xffff").unwrap(), -1);
        assert_eq!(parse_as_imm16("0").unwrap(), 0);
        assert_eq!(parse_as_imm16("0x7fff").unwrap(), 32767);
        assert_eq!(
            parse_as_imm16("-0x0001").unwrap(),
            parse_as_imm16("0xffff").unwrap()
        );
        assert_eq!(
            parse_as_imm16("-0x7fff").unwrap(),
            parse_as_imm16("0x8001").unwrap()
        );
        assert!(parse_as_imm16("0xffffa").is_err());
    }

    #[test]
    fn i32_as_hex() {
        fn parse_as_imm32(text: &str) -> ParseResult<i32> {
            Parser::new(text).match_imm32("unable to parse i32")
        }

        assert_eq!(parse_as_imm32("0x80000000").unwrap(), -2147483648);
        assert_eq!(parse_as_imm32("0xffffffff").unwrap(), -1);
        assert_eq!(parse_as_imm32("0").unwrap(), 0);
        assert_eq!(parse_as_imm32("0x7fffffff").unwrap(), 2147483647);
        assert_eq!(
            parse_as_imm32("-0x00000001").unwrap(),
            parse_as_imm32("0xffffffff").unwrap()
        );
        assert_eq!(
            parse_as_imm32("-0x7fffffff").unwrap(),
            parse_as_imm32("0x80000001").unwrap()
        );
        assert!(parse_as_imm32("0xffffffffa").is_err());
    }

    #[test]
    fn i64_as_hex() {
        fn parse_as_imm64(text: &str) -> ParseResult<Imm64> {
            Parser::new(text).match_imm64("unable to parse Imm64")
        }

        assert_eq!(
            parse_as_imm64("0x8000000000000000").unwrap(),
            Imm64::new(-9223372036854775808)
        );
        assert_eq!(
            parse_as_imm64("0xffffffffffffffff").unwrap(),
            Imm64::new(-1)
        );
        assert_eq!(parse_as_imm64("0").unwrap(), Imm64::new(0));
        assert_eq!(
            parse_as_imm64("0x7fffffffffffffff").unwrap(),
            Imm64::new(9223372036854775807)
        );
        assert_eq!(
            parse_as_imm64("-0x0000000000000001").unwrap(),
            parse_as_imm64("0xffffffffffffffff").unwrap()
        );
        assert_eq!(
            parse_as_imm64("-0x7fffffffffffffff").unwrap(),
            parse_as_imm64("0x8000000000000001").unwrap()
        );
        assert!(parse_as_imm64("0xffffffffffffffffa").is_err());
    }

    #[test]
    fn uimm128() {
        macro_rules! parse_as_constant_data {
            ($text:expr, $type:expr) => {{ Parser::new($text).parse_literals_to_constant_data($type) }};
        }
        macro_rules! can_parse_as_constant_data {
            ($text:expr, $type:expr) => {{ assert!(parse_as_constant_data!($text, $type).is_ok()) }};
        }
        macro_rules! cannot_parse_as_constant_data {
            ($text:expr, $type:expr) => {{ assert!(parse_as_constant_data!($text, $type).is_err()) }};
        }

        can_parse_as_constant_data!("1 2 3 4", I32X4);
        can_parse_as_constant_data!("1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16", I8X16);
        can_parse_as_constant_data!("0x1.1 0x2.2 0x3.3 0x4.4", F32X4);
        can_parse_as_constant_data!("0x0 0x1 0x2 0x3", I32X4);
        can_parse_as_constant_data!("-1 0 -1 0 -1 0 -1 0", I16X8);
        can_parse_as_constant_data!("0 -1", I64X2);
        can_parse_as_constant_data!("-1 0", I64X2);
        can_parse_as_constant_data!("-1 -1 -1 -1 -1", I32X4); // note that parse_literals_to_constant_data will leave extra tokens unconsumed

        cannot_parse_as_constant_data!("1 2 3", I32X4);
        cannot_parse_as_constant_data!(" ", F32X4);
    }

    #[test]
    fn parse_constant_from_booleans() {
        let c = Parser::new("-1 0 -1 0")
            .parse_literals_to_constant_data(I32X4)
            .unwrap();
        assert_eq!(
            c.into_vec(),
            [
                0xFF, 0xFF, 0xFF, 0xFF, 0, 0, 0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0, 0, 0, 0
            ]
        )
    }

    #[test]
    fn parse_unbounded_constants() {
        // Unlike match_uimm128, match_hexadecimal_constant can parse byte sequences of any size:
        assert_eq!(
            Parser::new("0x0100")
                .match_hexadecimal_constant("err message")
                .unwrap(),
            vec![0, 1].into()
        );

        // Only parse hexadecimal constants:
        assert!(
            Parser::new("228")
                .match_hexadecimal_constant("err message")
                .is_err()
        );
    }

    #[test]
    fn parse_run_commands() {
        // Helper for creating signatures.
        fn sig(ins: &[Type], outs: &[Type]) -> Signature {
            let mut sig = Signature::new(CallConv::Fast);
            for i in ins {
                sig.params.push(AbiParam::new(*i));
            }
            for o in outs {
                sig.returns.push(AbiParam::new(*o));
            }
            sig
        }

        // Helper for parsing run commands.
        fn parse(text: &str, sig: &Signature) -> ParseResult<RunCommand> {
            Parser::new(text).parse_run_command(sig)
        }

        // Check that we can parse and display the same set of run commands.
        fn assert_roundtrip(text: &str, sig: &Signature) {
            assert_eq!(parse(text, sig).unwrap().to_string(), text);
        }
        assert_roundtrip("run: %fn0() == 42", &sig(&[], &[I32]));
        assert_roundtrip(
            "run: %fn0(8, 16, 32, 64) == 1",
            &sig(&[I8, I16, I32, I64], &[I8]),
        );
        assert_roundtrip(
            "run: %my_func(1) == 0x0f0e0d0c0b0a09080706050403020100",
            &sig(&[I32], &[I8X16]),
        );

        // Verify that default invocations are created when not specified.
        assert_eq!(
            parse("run", &sig(&[], &[I32])).unwrap().to_string(),
            "run: %default() != 0"
        );
        assert_eq!(
            parse("print", &sig(&[], &[F32X4, I16X8]))
                .unwrap()
                .to_string(),
            "print: %default()"
        );

        // Demonstrate some unparsable cases.
        assert!(parse("print", &sig(&[I32], &[I32])).is_err());
        assert!(parse("print:", &sig(&[], &[])).is_err());
        assert!(parse("run: ", &sig(&[], &[])).is_err());
    }

    #[test]
    fn parse_data_values() {
        fn parse(text: &str, ty: Type) -> DataValue {
            Parser::new(text).parse_data_value(ty).unwrap()
        }

        assert_eq!(parse("8", I8).to_string(), "8");
        assert_eq!(parse("16", I16).to_string(), "16");
        assert_eq!(parse("32", I32).to_string(), "32");
        assert_eq!(parse("64", I64).to_string(), "64");
        assert_eq!(
            parse("0x01234567_01234567_01234567_01234567", I128).to_string(),
            "1512366032949150931280199141537564007"
        );
        assert_eq!(parse("1234567", I128).to_string(), "1234567");
        assert_eq!(parse("0x16.1", F16).to_string(), "0x1.610p4");
        assert_eq!(parse("0x32.32", F32).to_string(), "0x1.919000p5");
        assert_eq!(parse("0x64.64", F64).to_string(), "0x1.9190000000000p6");
        assert_eq!(
            parse("0x128.128", F128).to_string(),
            "0x1.2812800000000000000000000000p8"
        );
        assert_eq!(
            parse("[0 1 2 3]", I32X4).to_string(),
            "0x00000003000000020000000100000000"
        );
        assert_eq!(parse("[1 2]", I32X2).to_string(), "0x0000000200000001");
        assert_eq!(parse("[1 2 3 4]", I8X4).to_string(), "0x04030201");
        assert_eq!(parse("[1 2]", I8X2).to_string(), "0x0201");
    }

    #[test]
    fn parse_cold_blocks() {
        let code = "function %test() {
        block0 cold:
            return
        block1(v0: i32) cold:
            return
        block2(v1: i32):
            return
        }";

        let mut parser = Parser::new(code);
        let func = parser.parse_function().unwrap().0;
        assert_eq!(func.layout.blocks().count(), 3);
        assert!(func.layout.is_cold(Block::from_u32(0)));
        assert!(func.layout.is_cold(Block::from_u32(1)));
        assert!(!func.layout.is_cold(Block::from_u32(2)));
    }
}