cranelift_codegen_meta/shared/
instructions.rs

1#![expect(non_snake_case, reason = "DSL style here")]
2
3use crate::cdsl::instructions::{
4    AllInstructions, InstructionBuilder as Inst, InstructionGroupBuilder,
5};
6use crate::cdsl::operands::Operand;
7use crate::cdsl::types::{LaneType, ValueType};
8use crate::cdsl::typevar::{Interval, TypeSetBuilder, TypeVar};
9use crate::shared::formats::Formats;
10use crate::shared::types;
11use crate::shared::{entities::EntityRefs, immediates::Immediates};
12
13#[inline(never)]
14fn define_control_flow(
15    ig: &mut InstructionGroupBuilder,
16    formats: &Formats,
17    imm: &Immediates,
18    entities: &EntityRefs,
19) {
20    ig.push(
21        Inst::new(
22            "jump",
23            r#"
24        Jump.
25
26        Unconditionally jump to a basic block, passing the specified
27        block arguments. The number and types of arguments must match the
28        destination block.
29        "#,
30            &formats.jump,
31        )
32        .operands_in(vec![
33            Operand::new("block_call", &entities.block_call)
34                .with_doc("Destination basic block, with its arguments provided"),
35        ])
36        .branches(),
37    );
38
39    let ScalarTruthy = &TypeVar::new(
40        "ScalarTruthy",
41        "A scalar truthy type",
42        TypeSetBuilder::new().ints(Interval::All).build(),
43    );
44
45    ig.push(
46        Inst::new(
47            "brif",
48            r#"
49        Conditional branch when cond is non-zero.
50
51        Take the ``then`` branch when ``c != 0``, and the ``else`` branch otherwise.
52        "#,
53            &formats.brif,
54        )
55        .operands_in(vec![
56            Operand::new("c", ScalarTruthy).with_doc("Controlling value to test"),
57            Operand::new("block_then", &entities.block_then).with_doc("Then block"),
58            Operand::new("block_else", &entities.block_else).with_doc("Else block"),
59        ])
60        .branches(),
61    );
62
63    {
64        let _i32 = &TypeVar::new(
65            "i32",
66            "A 32 bit scalar integer type",
67            TypeSetBuilder::new().ints(32..32).build(),
68        );
69
70        ig.push(
71            Inst::new(
72                "br_table",
73                r#"
74        Indirect branch via jump table.
75
76        Use ``x`` as an unsigned index into the jump table ``JT``. If a jump
77        table entry is found, branch to the corresponding block. If no entry was
78        found or the index is out-of-bounds, branch to the default block of the
79        table.
80
81        Note that this branch instruction can't pass arguments to the targeted
82        blocks. Split critical edges as needed to work around this.
83
84        Do not confuse this with "tables" in WebAssembly. ``br_table`` is for
85        jump tables with destinations within the current function only -- think
86        of a ``match`` in Rust or a ``switch`` in C.  If you want to call a
87        function in a dynamic library, that will typically use
88        ``call_indirect``.
89        "#,
90                &formats.branch_table,
91            )
92            .operands_in(vec![
93                Operand::new("x", _i32).with_doc("i32 index into jump table"),
94                Operand::new("JT", &entities.jump_table),
95            ])
96            .branches(),
97        );
98    }
99
100    let iAddr = &TypeVar::new(
101        "iAddr",
102        "An integer address type",
103        TypeSetBuilder::new().ints(32..64).build(),
104    );
105
106    ig.push(
107        Inst::new(
108            "debugtrap",
109            r#"
110        Encodes an assembly debug trap.
111        "#,
112            &formats.nullary,
113        )
114        .other_side_effects()
115        .can_load()
116        .can_store(),
117    );
118
119    ig.push(
120        Inst::new(
121            "trap",
122            r#"
123        Terminate execution unconditionally.
124        "#,
125            &formats.trap,
126        )
127        .operands_in(vec![Operand::new("code", &imm.trapcode)])
128        .can_trap()
129        .terminates_block(),
130    );
131
132    ig.push(
133        Inst::new(
134            "trapz",
135            r#"
136        Trap when zero.
137
138        if ``c`` is non-zero, execution continues at the following instruction.
139        "#,
140            &formats.cond_trap,
141        )
142        .operands_in(vec![
143            Operand::new("c", ScalarTruthy).with_doc("Controlling value to test"),
144            Operand::new("code", &imm.trapcode),
145        ])
146        .can_trap()
147        // When one `trapz` dominates another `trapz` and they have identical
148        // conditions and trap codes, it is safe to deduplicate them (like GVN,
149        // although there is not actually any value being numbered). Either the
150        // first `trapz` raised a trap and execution halted, or it didn't and
151        // therefore the dominated `trapz` will not raise a trap either.
152        .side_effects_idempotent(),
153    );
154
155    ig.push(
156        Inst::new(
157            "trapnz",
158            r#"
159        Trap when non-zero.
160
161        If ``c`` is zero, execution continues at the following instruction.
162        "#,
163            &formats.cond_trap,
164        )
165        .operands_in(vec![
166            Operand::new("c", ScalarTruthy).with_doc("Controlling value to test"),
167            Operand::new("code", &imm.trapcode),
168        ])
169        .can_trap()
170        // See the above comment for `trapz` and idempotent side effects.
171        .side_effects_idempotent(),
172    );
173
174    ig.push(
175        Inst::new(
176            "return",
177            r#"
178        Return from the function.
179
180        Unconditionally transfer control to the calling function, passing the
181        provided return values. The list of return values must match the
182        function signature's return types.
183        "#,
184            &formats.multiary,
185        )
186        .operands_in(vec![
187            Operand::new("rvals", &entities.varargs).with_doc("return values"),
188        ])
189        .returns(),
190    );
191
192    ig.push(
193        Inst::new(
194            "call",
195            r#"
196        Direct function call.
197
198        Call a function which has been declared in the preamble. The argument
199        types must match the function's signature.
200        "#,
201            &formats.call,
202        )
203        .operands_in(vec![
204            Operand::new("FN", &entities.func_ref)
205                .with_doc("function to call, declared by `function`"),
206            Operand::new("args", &entities.varargs).with_doc("call arguments"),
207        ])
208        .operands_out(vec![
209            Operand::new("rvals", &entities.varargs).with_doc("return values"),
210        ])
211        .call(),
212    );
213
214    ig.push(
215        Inst::new(
216            "call_indirect",
217            r#"
218        Indirect function call.
219
220        Call the function pointed to by `callee` with the given arguments. The
221        called function must match the specified signature.
222
223        Note that this is different from WebAssembly's ``call_indirect``; the
224        callee is a native address, rather than a table index. For WebAssembly,
225        `table_addr` and `load` are used to obtain a native address
226        from a table.
227        "#,
228            &formats.call_indirect,
229        )
230        .operands_in(vec![
231            Operand::new("SIG", &entities.sig_ref).with_doc("function signature"),
232            Operand::new("callee", iAddr).with_doc("address of function to call"),
233            Operand::new("args", &entities.varargs).with_doc("call arguments"),
234        ])
235        .operands_out(vec![
236            Operand::new("rvals", &entities.varargs).with_doc("return values"),
237        ])
238        .call(),
239    );
240
241    ig.push(
242        Inst::new(
243            "return_call",
244            r#"
245        Direct tail call.
246
247        Tail call a function which has been declared in the preamble. The
248        argument types must match the function's signature, the caller and
249        callee calling conventions must be the same, and must be a calling
250        convention that supports tail calls.
251
252        This instruction is a block terminator.
253        "#,
254            &formats.call,
255        )
256        .operands_in(vec![
257            Operand::new("FN", &entities.func_ref)
258                .with_doc("function to call, declared by `function`"),
259            Operand::new("args", &entities.varargs).with_doc("call arguments"),
260        ])
261        .returns()
262        .call(),
263    );
264
265    ig.push(
266        Inst::new(
267            "return_call_indirect",
268            r#"
269        Indirect tail call.
270
271        Call the function pointed to by `callee` with the given arguments. The
272        argument types must match the function's signature, the caller and
273        callee calling conventions must be the same, and must be a calling
274        convention that supports tail calls.
275
276        This instruction is a block terminator.
277
278        Note that this is different from WebAssembly's ``tail_call_indirect``;
279        the callee is a native address, rather than a table index. For
280        WebAssembly, `table_addr` and `load` are used to obtain a native address
281        from a table.
282        "#,
283            &formats.call_indirect,
284        )
285        .operands_in(vec![
286            Operand::new("SIG", &entities.sig_ref).with_doc("function signature"),
287            Operand::new("callee", iAddr).with_doc("address of function to call"),
288            Operand::new("args", &entities.varargs).with_doc("call arguments"),
289        ])
290        .returns()
291        .call(),
292    );
293
294    ig.push(
295        Inst::new(
296            "func_addr",
297            r#"
298        Get the address of a function.
299
300        Compute the absolute address of a function declared in the preamble.
301        The returned address can be used as a ``callee`` argument to
302        `call_indirect`. This is also a method for calling functions that
303        are too far away to be addressable by a direct `call`
304        instruction.
305        "#,
306            &formats.func_addr,
307        )
308        .operands_in(vec![
309            Operand::new("FN", &entities.func_ref)
310                .with_doc("function to call, declared by `function`"),
311        ])
312        .operands_out(vec![Operand::new("addr", iAddr)]),
313    );
314
315    ig.push(
316        Inst::new(
317            "try_call",
318            r#"
319        Call a function, catching the specified exceptions.
320
321        Call the function pointed to by `callee` with the given arguments. On
322        normal return, branch to the first target, with function returns
323        available as `retN` block arguments. On exceptional return,
324        look up the thrown exception tag in the provided exception table;
325        if the tag matches one of the targets, branch to the matching
326        target with the exception payloads available as `exnN` block arguments.
327        If no tag matches, then propagate the exception up the stack.
328
329        It is the Cranelift embedder's responsibility to define the meaning
330        of tags: they are accepted by this instruction and passed through
331        to unwind metadata tables in Cranelift's output. Actual unwinding is
332        outside the purview of the core Cranelift compiler.
333
334        Payload values on exception are passed in fixed register(s) that are
335        defined by the platform and ABI. See the documentation on `CallConv`
336        for details.
337        "#,
338            &formats.try_call,
339        )
340        .operands_in(vec![
341            Operand::new("callee", &entities.func_ref)
342                .with_doc("function to call, declared by `function`"),
343            Operand::new("args", &entities.varargs).with_doc("call arguments"),
344            Operand::new("ET", &entities.exception_table).with_doc("exception table"),
345        ])
346        .call()
347        .branches(),
348    );
349
350    ig.push(
351        Inst::new(
352            "try_call_indirect",
353            r#"
354        Call a function, catching the specified exceptions.
355
356        Call the function pointed to by `callee` with the given arguments. On
357        normal return, branch to the first target, with function returns
358        available as `retN` block arguments. On exceptional return,
359        look up the thrown exception tag in the provided exception table;
360        if the tag matches one of the targets, branch to the matching
361        target with the exception payloads available as `exnN` block arguments.
362        If no tag matches, then propagate the exception up the stack.
363
364        It is the Cranelift embedder's responsibility to define the meaning
365        of tags: they are accepted by this instruction and passed through
366        to unwind metadata tables in Cranelift's output. Actual unwinding is
367        outside the purview of the core Cranelift compiler.
368
369        Payload values on exception are passed in fixed register(s) that are
370        defined by the platform and ABI. See the documentation on `CallConv`
371        for details.
372        "#,
373            &formats.try_call_indirect,
374        )
375        .operands_in(vec![
376            Operand::new("callee", iAddr).with_doc("address of function to call"),
377            Operand::new("args", &entities.varargs).with_doc("call arguments"),
378            Operand::new("ET", &entities.exception_table).with_doc("exception table"),
379        ])
380        .call()
381        .branches(),
382    );
383}
384
385#[inline(never)]
386fn define_simd_lane_access(
387    ig: &mut InstructionGroupBuilder,
388    formats: &Formats,
389    imm: &Immediates,
390    _: &EntityRefs,
391) {
392    let TxN = &TypeVar::new(
393        "TxN",
394        "A SIMD vector type",
395        TypeSetBuilder::new()
396            .ints(Interval::All)
397            .floats(Interval::All)
398            .simd_lanes(Interval::All)
399            .dynamic_simd_lanes(Interval::All)
400            .includes_scalars(false)
401            .build(),
402    );
403
404    ig.push(
405        Inst::new(
406            "splat",
407            r#"
408        Vector splat.
409
410        Return a vector whose lanes are all ``x``.
411        "#,
412            &formats.unary,
413        )
414        .operands_in(vec![
415            Operand::new("x", &TxN.lane_of()).with_doc("Value to splat to all lanes"),
416        ])
417        .operands_out(vec![Operand::new("a", TxN)]),
418    );
419
420    let I8x16 = &TypeVar::new(
421        "I8x16",
422        "A SIMD vector type consisting of 16 lanes of 8-bit integers",
423        TypeSetBuilder::new()
424            .ints(8..8)
425            .simd_lanes(16..16)
426            .includes_scalars(false)
427            .build(),
428    );
429
430    ig.push(
431        Inst::new(
432            "swizzle",
433            r#"
434        Vector swizzle.
435
436        Returns a new vector with byte-width lanes selected from the lanes of the first input
437        vector ``x`` specified in the second input vector ``s``. The indices ``i`` in range
438        ``[0, 15]`` select the ``i``-th element of ``x``. For indices outside of the range the
439        resulting lane is 0. Note that this operates on byte-width lanes.
440        "#,
441            &formats.binary,
442        )
443        .operands_in(vec![
444            Operand::new("x", I8x16).with_doc("Vector to modify by re-arranging lanes"),
445            Operand::new("y", I8x16).with_doc("Mask for re-arranging lanes"),
446        ])
447        .operands_out(vec![Operand::new("a", I8x16)]),
448    );
449
450    ig.push(
451        Inst::new(
452            "x86_pshufb",
453            r#"
454        A vector swizzle lookalike which has the semantics of `pshufb` on x64.
455
456        This instruction will permute the 8-bit lanes of `x` with the indices
457        specified in `y`. Each lane in the mask, `y`, uses the bottom four
458        bits for selecting the lane from `x` unless the most significant bit
459        is set, in which case the lane is zeroed. The output vector will have
460        the following contents when the element of `y` is in these ranges:
461
462        * `[0, 127]` -> `x[y[i] % 16]`
463        * `[128, 255]` -> 0
464        "#,
465            &formats.binary,
466        )
467        .operands_in(vec![
468            Operand::new("x", I8x16).with_doc("Vector to modify by re-arranging lanes"),
469            Operand::new("y", I8x16).with_doc("Mask for re-arranging lanes"),
470        ])
471        .operands_out(vec![Operand::new("a", I8x16)]),
472    );
473
474    ig.push(
475        Inst::new(
476            "insertlane",
477            r#"
478        Insert ``y`` as lane ``Idx`` in x.
479
480        The lane index, ``Idx``, is an immediate value, not an SSA value. It
481        must indicate a valid lane index for the type of ``x``.
482        "#,
483            &formats.ternary_imm8,
484        )
485        .operands_in(vec![
486            Operand::new("x", TxN).with_doc("The vector to modify"),
487            Operand::new("y", &TxN.lane_of()).with_doc("New lane value"),
488            Operand::new("Idx", &imm.uimm8).with_doc("Lane index"),
489        ])
490        .operands_out(vec![Operand::new("a", TxN)]),
491    );
492
493    ig.push(
494        Inst::new(
495            "extractlane",
496            r#"
497        Extract lane ``Idx`` from ``x``.
498
499        The lane index, ``Idx``, is an immediate value, not an SSA value. It
500        must indicate a valid lane index for the type of ``x``. Note that the upper bits of ``a``
501        may or may not be zeroed depending on the ISA but the type system should prevent using
502        ``a`` as anything other than the extracted value.
503        "#,
504            &formats.binary_imm8,
505        )
506        .operands_in(vec![
507            Operand::new("x", TxN),
508            Operand::new("Idx", &imm.uimm8).with_doc("Lane index"),
509        ])
510        .operands_out(vec![Operand::new("a", &TxN.lane_of())]),
511    );
512}
513
514#[inline(never)]
515fn define_simd_arithmetic(
516    ig: &mut InstructionGroupBuilder,
517    formats: &Formats,
518    _: &Immediates,
519    _: &EntityRefs,
520) {
521    let Int = &TypeVar::new(
522        "Int",
523        "A scalar or vector integer type",
524        TypeSetBuilder::new()
525            .ints(Interval::All)
526            .simd_lanes(Interval::All)
527            .build(),
528    );
529
530    ig.push(
531        Inst::new(
532            "smin",
533            r#"
534        Signed integer minimum.
535        "#,
536            &formats.binary,
537        )
538        .operands_in(vec![Operand::new("x", Int), Operand::new("y", Int)])
539        .operands_out(vec![Operand::new("a", Int)]),
540    );
541
542    ig.push(
543        Inst::new(
544            "umin",
545            r#"
546        Unsigned integer minimum.
547        "#,
548            &formats.binary,
549        )
550        .operands_in(vec![Operand::new("x", Int), Operand::new("y", Int)])
551        .operands_out(vec![Operand::new("a", Int)]),
552    );
553
554    ig.push(
555        Inst::new(
556            "smax",
557            r#"
558        Signed integer maximum.
559        "#,
560            &formats.binary,
561        )
562        .operands_in(vec![Operand::new("x", Int), Operand::new("y", Int)])
563        .operands_out(vec![Operand::new("a", Int)]),
564    );
565
566    ig.push(
567        Inst::new(
568            "umax",
569            r#"
570        Unsigned integer maximum.
571        "#,
572            &formats.binary,
573        )
574        .operands_in(vec![Operand::new("x", Int), Operand::new("y", Int)])
575        .operands_out(vec![Operand::new("a", Int)]),
576    );
577
578    let IxN = &TypeVar::new(
579        "IxN",
580        "A SIMD vector type containing integers",
581        TypeSetBuilder::new()
582            .ints(Interval::All)
583            .simd_lanes(Interval::All)
584            .includes_scalars(false)
585            .build(),
586    );
587
588    ig.push(
589        Inst::new(
590            "avg_round",
591            r#"
592        Unsigned average with rounding: `a := (x + y + 1) // 2`
593
594        The addition does not lose any information (such as from overflow).
595        "#,
596            &formats.binary,
597        )
598        .operands_in(vec![Operand::new("x", IxN), Operand::new("y", IxN)])
599        .operands_out(vec![Operand::new("a", IxN)]),
600    );
601
602    ig.push(
603        Inst::new(
604            "uadd_sat",
605            r#"
606        Add with unsigned saturation.
607
608        This is similar to `iadd` but the operands are interpreted as unsigned integers and their
609        summed result, instead of wrapping, will be saturated to the highest unsigned integer for
610        the controlling type (e.g. `0xFF` for i8).
611        "#,
612            &formats.binary,
613        )
614        .operands_in(vec![Operand::new("x", IxN), Operand::new("y", IxN)])
615        .operands_out(vec![Operand::new("a", IxN)]),
616    );
617
618    ig.push(
619        Inst::new(
620            "sadd_sat",
621            r#"
622        Add with signed saturation.
623
624        This is similar to `iadd` but the operands are interpreted as signed integers and their
625        summed result, instead of wrapping, will be saturated to the lowest or highest
626        signed integer for the controlling type (e.g. `0x80` or `0x7F` for i8). For example,
627        since an `sadd_sat.i8` of `0x70` and `0x70` is greater than `0x7F`, the result will be
628        clamped to `0x7F`.
629        "#,
630            &formats.binary,
631        )
632        .operands_in(vec![Operand::new("x", IxN), Operand::new("y", IxN)])
633        .operands_out(vec![Operand::new("a", IxN)]),
634    );
635
636    ig.push(
637        Inst::new(
638            "usub_sat",
639            r#"
640        Subtract with unsigned saturation.
641
642        This is similar to `isub` but the operands are interpreted as unsigned integers and their
643        difference, instead of wrapping, will be saturated to the lowest unsigned integer for
644        the controlling type (e.g. `0x00` for i8).
645        "#,
646            &formats.binary,
647        )
648        .operands_in(vec![Operand::new("x", IxN), Operand::new("y", IxN)])
649        .operands_out(vec![Operand::new("a", IxN)]),
650    );
651
652    ig.push(
653        Inst::new(
654            "ssub_sat",
655            r#"
656        Subtract with signed saturation.
657
658        This is similar to `isub` but the operands are interpreted as signed integers and their
659        difference, instead of wrapping, will be saturated to the lowest or highest
660        signed integer for the controlling type (e.g. `0x80` or `0x7F` for i8).
661        "#,
662            &formats.binary,
663        )
664        .operands_in(vec![Operand::new("x", IxN), Operand::new("y", IxN)])
665        .operands_out(vec![Operand::new("a", IxN)]),
666    );
667}
668
669pub(crate) fn define(
670    all_instructions: &mut AllInstructions,
671    formats: &Formats,
672    imm: &Immediates,
673    entities: &EntityRefs,
674) {
675    let mut ig = InstructionGroupBuilder::new(all_instructions);
676
677    define_control_flow(&mut ig, formats, imm, entities);
678    define_simd_lane_access(&mut ig, formats, imm, entities);
679    define_simd_arithmetic(&mut ig, formats, imm, entities);
680
681    // Operand kind shorthands.
682    let i8: &TypeVar = &ValueType::from(LaneType::from(types::Int::I8)).into();
683    let f16_: &TypeVar = &ValueType::from(LaneType::from(types::Float::F16)).into();
684    let f32_: &TypeVar = &ValueType::from(LaneType::from(types::Float::F32)).into();
685    let f64_: &TypeVar = &ValueType::from(LaneType::from(types::Float::F64)).into();
686    let f128_: &TypeVar = &ValueType::from(LaneType::from(types::Float::F128)).into();
687
688    // Starting definitions.
689    let Int = &TypeVar::new(
690        "Int",
691        "A scalar or vector integer type",
692        TypeSetBuilder::new()
693            .ints(Interval::All)
694            .simd_lanes(Interval::All)
695            .dynamic_simd_lanes(Interval::All)
696            .build(),
697    );
698
699    let NarrowInt = &TypeVar::new(
700        "NarrowInt",
701        "An integer type of width up to `i64`",
702        TypeSetBuilder::new().ints(8..64).build(),
703    );
704
705    let ScalarTruthy = &TypeVar::new(
706        "ScalarTruthy",
707        "A scalar truthy type",
708        TypeSetBuilder::new().ints(Interval::All).build(),
709    );
710
711    let iB = &TypeVar::new(
712        "iB",
713        "A scalar integer type",
714        TypeSetBuilder::new().ints(Interval::All).build(),
715    );
716
717    let iSwappable = &TypeVar::new(
718        "iSwappable",
719        "A multi byte scalar integer type",
720        TypeSetBuilder::new().ints(16..128).build(),
721    );
722
723    let iAddr = &TypeVar::new(
724        "iAddr",
725        "An integer address type",
726        TypeSetBuilder::new().ints(32..64).build(),
727    );
728
729    let TxN = &TypeVar::new(
730        "TxN",
731        "A SIMD vector type",
732        TypeSetBuilder::new()
733            .ints(Interval::All)
734            .floats(Interval::All)
735            .simd_lanes(Interval::All)
736            .includes_scalars(false)
737            .build(),
738    );
739    let Any = &TypeVar::new(
740        "Any",
741        "Any integer, float, or reference scalar or vector type",
742        TypeSetBuilder::new()
743            .ints(Interval::All)
744            .floats(Interval::All)
745            .simd_lanes(Interval::All)
746            .includes_scalars(true)
747            .build(),
748    );
749
750    let Mem = &TypeVar::new(
751        "Mem",
752        "Any type that can be stored in memory",
753        TypeSetBuilder::new()
754            .ints(Interval::All)
755            .floats(Interval::All)
756            .simd_lanes(Interval::All)
757            .dynamic_simd_lanes(Interval::All)
758            .build(),
759    );
760
761    let MemTo = &TypeVar::copy_from(Mem, "MemTo".to_string());
762
763    ig.push(
764        Inst::new(
765            "load",
766            r#"
767        Load from memory at ``p + Offset``.
768
769        This is a polymorphic instruction that can load any value type which
770        has a memory representation.
771        "#,
772            &formats.load,
773        )
774        .operands_in(vec![
775            Operand::new("MemFlags", &imm.memflags),
776            Operand::new("p", iAddr),
777            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
778        ])
779        .operands_out(vec![Operand::new("a", Mem).with_doc("Value loaded")])
780        .can_load(),
781    );
782
783    ig.push(
784        Inst::new(
785            "store",
786            r#"
787        Store ``x`` to memory at ``p + Offset``.
788
789        This is a polymorphic instruction that can store any value type with a
790        memory representation.
791        "#,
792            &formats.store,
793        )
794        .operands_in(vec![
795            Operand::new("MemFlags", &imm.memflags),
796            Operand::new("x", Mem).with_doc("Value to be stored"),
797            Operand::new("p", iAddr),
798            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
799        ])
800        .can_store(),
801    );
802
803    let iExt8 = &TypeVar::new(
804        "iExt8",
805        "An integer type with more than 8 bits",
806        TypeSetBuilder::new().ints(16..64).build(),
807    );
808
809    ig.push(
810        Inst::new(
811            "uload8",
812            r#"
813        Load 8 bits from memory at ``p + Offset`` and zero-extend.
814
815        This is equivalent to ``load.i8`` followed by ``uextend``.
816        "#,
817            &formats.load,
818        )
819        .operands_in(vec![
820            Operand::new("MemFlags", &imm.memflags),
821            Operand::new("p", iAddr),
822            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
823        ])
824        .operands_out(vec![Operand::new("a", iExt8)])
825        .can_load(),
826    );
827
828    ig.push(
829        Inst::new(
830            "sload8",
831            r#"
832        Load 8 bits from memory at ``p + Offset`` and sign-extend.
833
834        This is equivalent to ``load.i8`` followed by ``sextend``.
835        "#,
836            &formats.load,
837        )
838        .operands_in(vec![
839            Operand::new("MemFlags", &imm.memflags),
840            Operand::new("p", iAddr),
841            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
842        ])
843        .operands_out(vec![Operand::new("a", iExt8)])
844        .can_load(),
845    );
846
847    ig.push(
848        Inst::new(
849            "istore8",
850            r#"
851        Store the low 8 bits of ``x`` to memory at ``p + Offset``.
852
853        This is equivalent to ``ireduce.i8`` followed by ``store.i8``.
854        "#,
855            &formats.store,
856        )
857        .operands_in(vec![
858            Operand::new("MemFlags", &imm.memflags),
859            Operand::new("x", iExt8),
860            Operand::new("p", iAddr),
861            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
862        ])
863        .can_store(),
864    );
865
866    let iExt16 = &TypeVar::new(
867        "iExt16",
868        "An integer type with more than 16 bits",
869        TypeSetBuilder::new().ints(32..64).build(),
870    );
871
872    ig.push(
873        Inst::new(
874            "uload16",
875            r#"
876        Load 16 bits from memory at ``p + Offset`` and zero-extend.
877
878        This is equivalent to ``load.i16`` followed by ``uextend``.
879        "#,
880            &formats.load,
881        )
882        .operands_in(vec![
883            Operand::new("MemFlags", &imm.memflags),
884            Operand::new("p", iAddr),
885            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
886        ])
887        .operands_out(vec![Operand::new("a", iExt16)])
888        .can_load(),
889    );
890
891    ig.push(
892        Inst::new(
893            "sload16",
894            r#"
895        Load 16 bits from memory at ``p + Offset`` and sign-extend.
896
897        This is equivalent to ``load.i16`` followed by ``sextend``.
898        "#,
899            &formats.load,
900        )
901        .operands_in(vec![
902            Operand::new("MemFlags", &imm.memflags),
903            Operand::new("p", iAddr),
904            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
905        ])
906        .operands_out(vec![Operand::new("a", iExt16)])
907        .can_load(),
908    );
909
910    ig.push(
911        Inst::new(
912            "istore16",
913            r#"
914        Store the low 16 bits of ``x`` to memory at ``p + Offset``.
915
916        This is equivalent to ``ireduce.i16`` followed by ``store.i16``.
917        "#,
918            &formats.store,
919        )
920        .operands_in(vec![
921            Operand::new("MemFlags", &imm.memflags),
922            Operand::new("x", iExt16),
923            Operand::new("p", iAddr),
924            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
925        ])
926        .can_store(),
927    );
928
929    let iExt32 = &TypeVar::new(
930        "iExt32",
931        "An integer type with more than 32 bits",
932        TypeSetBuilder::new().ints(64..64).build(),
933    );
934
935    ig.push(
936        Inst::new(
937            "uload32",
938            r#"
939        Load 32 bits from memory at ``p + Offset`` and zero-extend.
940
941        This is equivalent to ``load.i32`` followed by ``uextend``.
942        "#,
943            &formats.load,
944        )
945        .operands_in(vec![
946            Operand::new("MemFlags", &imm.memflags),
947            Operand::new("p", iAddr),
948            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
949        ])
950        .operands_out(vec![Operand::new("a", iExt32)])
951        .can_load(),
952    );
953
954    ig.push(
955        Inst::new(
956            "sload32",
957            r#"
958        Load 32 bits from memory at ``p + Offset`` and sign-extend.
959
960        This is equivalent to ``load.i32`` followed by ``sextend``.
961        "#,
962            &formats.load,
963        )
964        .operands_in(vec![
965            Operand::new("MemFlags", &imm.memflags),
966            Operand::new("p", iAddr),
967            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
968        ])
969        .operands_out(vec![Operand::new("a", iExt32)])
970        .can_load(),
971    );
972
973    ig.push(
974        Inst::new(
975            "istore32",
976            r#"
977        Store the low 32 bits of ``x`` to memory at ``p + Offset``.
978
979        This is equivalent to ``ireduce.i32`` followed by ``store.i32``.
980        "#,
981            &formats.store,
982        )
983        .operands_in(vec![
984            Operand::new("MemFlags", &imm.memflags),
985            Operand::new("x", iExt32),
986            Operand::new("p", iAddr),
987            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
988        ])
989        .can_store(),
990    );
991    ig.push(
992        Inst::new(
993            "stack_switch",
994            r#"
995        Suspends execution of the current stack and resumes execution of another
996        one.
997
998        The target stack to switch to is identified by the data stored at
999        ``load_context_ptr``. Before switching, this instruction stores
1000        analogous information about the
1001        current (i.e., original) stack at ``store_context_ptr``, to
1002        enabled switching back to the original stack at a later point.
1003
1004        The size, alignment and layout of the information stored at
1005        ``load_context_ptr`` and ``store_context_ptr`` is platform-dependent.
1006        The instruction assumes that ``load_context_ptr`` and
1007        ``store_context_ptr`` are valid pointers to memory with said layout and
1008        alignment, and does not perform any checks on these pointers or the data
1009        stored there.
1010
1011        The instruction is experimental and only supported on x64 Linux at the
1012        moment.
1013
1014        When switching from a stack A to a stack B, one of the following cases
1015        must apply:
1016        1. Stack B was previously suspended using a ``stack_switch`` instruction.
1017        2. Stack B is a newly initialized stack. The necessary initialization is
1018        platform-dependent and will generally involve running some kind of
1019        trampoline to start execution of a function on the new stack.
1020
1021        In both cases, the ``in_payload`` argument of the ``stack_switch``
1022        instruction executed on A is passed to stack B. In the first case above,
1023        it will be the result value of the earlier ``stack_switch`` instruction
1024        executed on stack B. In the second case, the value will be accessible to
1025        the trampoline in a platform-dependent register.
1026
1027        The pointers ``load_context_ptr`` and ``store_context_ptr`` are allowed
1028        to be equal; the instruction ensures that all data is loaded from the
1029        former before writing to the latter.
1030
1031        Stack switching is one-shot in the sense that each ``stack_switch``
1032        operation effectively consumes the context identified by
1033        ``load_context_ptr``. In other words, performing two ``stack_switches``
1034        using the same ``load_context_ptr`` causes undefined behavior, unless
1035        the context at ``load_context_ptr`` is overwritten by another
1036        `stack_switch` in between.
1037        "#,
1038            &formats.ternary,
1039        )
1040        .operands_in(vec![
1041            Operand::new("store_context_ptr", iAddr),
1042            Operand::new("load_context_ptr", iAddr),
1043            Operand::new("in_payload0", iAddr),
1044        ])
1045        .operands_out(vec![Operand::new("out_payload0", iAddr)])
1046        .other_side_effects()
1047        .can_load()
1048        .can_store()
1049        .call(),
1050    );
1051
1052    let I16x8 = &TypeVar::new(
1053        "I16x8",
1054        "A SIMD vector with exactly 8 lanes of 16-bit values",
1055        TypeSetBuilder::new()
1056            .ints(16..16)
1057            .simd_lanes(8..8)
1058            .includes_scalars(false)
1059            .build(),
1060    );
1061
1062    ig.push(
1063        Inst::new(
1064            "uload8x8",
1065            r#"
1066        Load an 8x8 vector (64 bits) from memory at ``p + Offset`` and zero-extend into an i16x8
1067        vector.
1068        "#,
1069            &formats.load,
1070        )
1071        .operands_in(vec![
1072            Operand::new("MemFlags", &imm.memflags),
1073            Operand::new("p", iAddr),
1074            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
1075        ])
1076        .operands_out(vec![Operand::new("a", I16x8).with_doc("Value loaded")])
1077        .can_load(),
1078    );
1079
1080    ig.push(
1081        Inst::new(
1082            "sload8x8",
1083            r#"
1084        Load an 8x8 vector (64 bits) from memory at ``p + Offset`` and sign-extend into an i16x8
1085        vector.
1086        "#,
1087            &formats.load,
1088        )
1089        .operands_in(vec![
1090            Operand::new("MemFlags", &imm.memflags),
1091            Operand::new("p", iAddr),
1092            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
1093        ])
1094        .operands_out(vec![Operand::new("a", I16x8).with_doc("Value loaded")])
1095        .can_load(),
1096    );
1097
1098    let I32x4 = &TypeVar::new(
1099        "I32x4",
1100        "A SIMD vector with exactly 4 lanes of 32-bit values",
1101        TypeSetBuilder::new()
1102            .ints(32..32)
1103            .simd_lanes(4..4)
1104            .includes_scalars(false)
1105            .build(),
1106    );
1107
1108    ig.push(
1109        Inst::new(
1110            "uload16x4",
1111            r#"
1112        Load a 16x4 vector (64 bits) from memory at ``p + Offset`` and zero-extend into an i32x4
1113        vector.
1114        "#,
1115            &formats.load,
1116        )
1117        .operands_in(vec![
1118            Operand::new("MemFlags", &imm.memflags),
1119            Operand::new("p", iAddr),
1120            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
1121        ])
1122        .operands_out(vec![Operand::new("a", I32x4).with_doc("Value loaded")])
1123        .can_load(),
1124    );
1125
1126    ig.push(
1127        Inst::new(
1128            "sload16x4",
1129            r#"
1130        Load a 16x4 vector (64 bits) from memory at ``p + Offset`` and sign-extend into an i32x4
1131        vector.
1132        "#,
1133            &formats.load,
1134        )
1135        .operands_in(vec![
1136            Operand::new("MemFlags", &imm.memflags),
1137            Operand::new("p", iAddr),
1138            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
1139        ])
1140        .operands_out(vec![Operand::new("a", I32x4).with_doc("Value loaded")])
1141        .can_load(),
1142    );
1143
1144    let I64x2 = &TypeVar::new(
1145        "I64x2",
1146        "A SIMD vector with exactly 2 lanes of 64-bit values",
1147        TypeSetBuilder::new()
1148            .ints(64..64)
1149            .simd_lanes(2..2)
1150            .includes_scalars(false)
1151            .build(),
1152    );
1153
1154    ig.push(
1155        Inst::new(
1156            "uload32x2",
1157            r#"
1158        Load an 32x2 vector (64 bits) from memory at ``p + Offset`` and zero-extend into an i64x2
1159        vector.
1160        "#,
1161            &formats.load,
1162        )
1163        .operands_in(vec![
1164            Operand::new("MemFlags", &imm.memflags),
1165            Operand::new("p", iAddr),
1166            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
1167        ])
1168        .operands_out(vec![Operand::new("a", I64x2).with_doc("Value loaded")])
1169        .can_load(),
1170    );
1171
1172    ig.push(
1173        Inst::new(
1174            "sload32x2",
1175            r#"
1176        Load a 32x2 vector (64 bits) from memory at ``p + Offset`` and sign-extend into an i64x2
1177        vector.
1178        "#,
1179            &formats.load,
1180        )
1181        .operands_in(vec![
1182            Operand::new("MemFlags", &imm.memflags),
1183            Operand::new("p", iAddr),
1184            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
1185        ])
1186        .operands_out(vec![Operand::new("a", I64x2).with_doc("Value loaded")])
1187        .can_load(),
1188    );
1189
1190    ig.push(
1191        Inst::new(
1192            "stack_load",
1193            r#"
1194        Load a value from a stack slot at the constant offset.
1195
1196        This is a polymorphic instruction that can load any value type which
1197        has a memory representation.
1198
1199        The offset is an immediate constant, not an SSA value. The memory
1200        access cannot go out of bounds, i.e.
1201        `sizeof(a) + Offset <= sizeof(SS)`.
1202        "#,
1203            &formats.stack_load,
1204        )
1205        .operands_in(vec![
1206            Operand::new("SS", &entities.stack_slot),
1207            Operand::new("Offset", &imm.offset32).with_doc("In-bounds offset into stack slot"),
1208        ])
1209        .operands_out(vec![Operand::new("a", Mem).with_doc("Value loaded")])
1210        .can_load(),
1211    );
1212
1213    ig.push(
1214        Inst::new(
1215            "stack_store",
1216            r#"
1217        Store a value to a stack slot at a constant offset.
1218
1219        This is a polymorphic instruction that can store any value type with a
1220        memory representation.
1221
1222        The offset is an immediate constant, not an SSA value. The memory
1223        access cannot go out of bounds, i.e.
1224        `sizeof(a) + Offset <= sizeof(SS)`.
1225        "#,
1226            &formats.stack_store,
1227        )
1228        .operands_in(vec![
1229            Operand::new("x", Mem).with_doc("Value to be stored"),
1230            Operand::new("SS", &entities.stack_slot),
1231            Operand::new("Offset", &imm.offset32).with_doc("In-bounds offset into stack slot"),
1232        ])
1233        .can_store(),
1234    );
1235
1236    ig.push(
1237        Inst::new(
1238            "stack_addr",
1239            r#"
1240        Get the address of a stack slot.
1241
1242        Compute the absolute address of a byte in a stack slot. The offset must
1243        refer to a byte inside the stack slot:
1244        `0 <= Offset < sizeof(SS)`.
1245        "#,
1246            &formats.stack_load,
1247        )
1248        .operands_in(vec![
1249            Operand::new("SS", &entities.stack_slot),
1250            Operand::new("Offset", &imm.offset32).with_doc("In-bounds offset into stack slot"),
1251        ])
1252        .operands_out(vec![Operand::new("addr", iAddr)]),
1253    );
1254
1255    ig.push(
1256        Inst::new(
1257            "dynamic_stack_load",
1258            r#"
1259        Load a value from a dynamic stack slot.
1260
1261        This is a polymorphic instruction that can load any value type which
1262        has a memory representation.
1263        "#,
1264            &formats.dynamic_stack_load,
1265        )
1266        .operands_in(vec![Operand::new("DSS", &entities.dynamic_stack_slot)])
1267        .operands_out(vec![Operand::new("a", Mem).with_doc("Value loaded")])
1268        .can_load(),
1269    );
1270
1271    ig.push(
1272        Inst::new(
1273            "dynamic_stack_store",
1274            r#"
1275        Store a value to a dynamic stack slot.
1276
1277        This is a polymorphic instruction that can store any dynamic value type with a
1278        memory representation.
1279        "#,
1280            &formats.dynamic_stack_store,
1281        )
1282        .operands_in(vec![
1283            Operand::new("x", Mem).with_doc("Value to be stored"),
1284            Operand::new("DSS", &entities.dynamic_stack_slot),
1285        ])
1286        .can_store(),
1287    );
1288
1289    ig.push(
1290        Inst::new(
1291            "dynamic_stack_addr",
1292            r#"
1293        Get the address of a dynamic stack slot.
1294
1295        Compute the absolute address of the first byte of a dynamic stack slot.
1296        "#,
1297            &formats.dynamic_stack_load,
1298        )
1299        .operands_in(vec![Operand::new("DSS", &entities.dynamic_stack_slot)])
1300        .operands_out(vec![Operand::new("addr", iAddr)]),
1301    );
1302
1303    ig.push(
1304        Inst::new(
1305            "global_value",
1306            r#"
1307        Compute the value of global GV.
1308        "#,
1309            &formats.unary_global_value,
1310        )
1311        .operands_in(vec![Operand::new("GV", &entities.global_value)])
1312        .operands_out(vec![Operand::new("a", Mem).with_doc("Value loaded")]),
1313    );
1314
1315    ig.push(
1316        Inst::new(
1317            "symbol_value",
1318            r#"
1319        Compute the value of global GV, which is a symbolic value.
1320        "#,
1321            &formats.unary_global_value,
1322        )
1323        .operands_in(vec![Operand::new("GV", &entities.global_value)])
1324        .operands_out(vec![Operand::new("a", Mem).with_doc("Value loaded")]),
1325    );
1326
1327    ig.push(
1328        Inst::new(
1329            "tls_value",
1330            r#"
1331        Compute the value of global GV, which is a TLS (thread local storage) value.
1332        "#,
1333            &formats.unary_global_value,
1334        )
1335        .operands_in(vec![Operand::new("GV", &entities.global_value)])
1336        .operands_out(vec![Operand::new("a", Mem).with_doc("Value loaded")]),
1337    );
1338
1339    // Note this instruction is marked as having other side-effects, so GVN won't try to hoist it,
1340    // which would result in it being subject to spilling. While not hoisting would generally hurt
1341    // performance, since a computed value used many times may need to be regenerated before each
1342    // use, it is not the case here: this instruction doesn't generate any code.  That's because,
1343    // by definition the pinned register is never used by the register allocator, but is written to
1344    // and read explicitly and exclusively by set_pinned_reg and get_pinned_reg.
1345    ig.push(
1346        Inst::new(
1347            "get_pinned_reg",
1348            r#"
1349            Gets the content of the pinned register, when it's enabled.
1350        "#,
1351            &formats.nullary,
1352        )
1353        .operands_out(vec![Operand::new("addr", iAddr)])
1354        .other_side_effects(),
1355    );
1356
1357    ig.push(
1358        Inst::new(
1359            "set_pinned_reg",
1360            r#"
1361        Sets the content of the pinned register, when it's enabled.
1362        "#,
1363            &formats.unary,
1364        )
1365        .operands_in(vec![Operand::new("addr", iAddr)])
1366        .other_side_effects(),
1367    );
1368
1369    ig.push(
1370        Inst::new(
1371            "get_frame_pointer",
1372            r#"
1373        Get the address in the frame pointer register.
1374
1375        Usage of this instruction requires setting `preserve_frame_pointers` to `true`.
1376        "#,
1377            &formats.nullary,
1378        )
1379        .operands_out(vec![Operand::new("addr", iAddr)]),
1380    );
1381
1382    ig.push(
1383        Inst::new(
1384            "get_stack_pointer",
1385            r#"
1386        Get the address in the stack pointer register.
1387        "#,
1388            &formats.nullary,
1389        )
1390        .operands_out(vec![Operand::new("addr", iAddr)]),
1391    );
1392
1393    ig.push(
1394        Inst::new(
1395            "get_return_address",
1396            r#"
1397        Get the PC where this function will transfer control to when it returns.
1398
1399        Usage of this instruction requires setting `preserve_frame_pointers` to `true`.
1400        "#,
1401            &formats.nullary,
1402        )
1403        .operands_out(vec![Operand::new("addr", iAddr)]),
1404    );
1405
1406    ig.push(
1407        Inst::new(
1408            "get_exception_handler_address",
1409            r#"
1410        Get the handler PC for the given exceptional edge for an
1411        exception return from the given `try_call`-terminated block.
1412
1413        This instruction provides the PC for the handler resume point,
1414        as defined by the exception-handling aspect of the given
1415        callee ABI, for a return from the given calling block.  It can
1416        be used when the exception unwind mechanism requires manual
1417        plumbing for this information which must be set up before the call
1418        itself: for example, if the resume address needs to be stored in
1419        some context structure for a runtime to resume to on error.
1420
1421        The given caller block must end in a `try_call` and the given
1422        exception-handling block must be one of its exceptional
1423        successors in the associated exception-handling table. The
1424        returned PC is *only* valid to resume to when the `try_call`
1425        is on the stack having called the callee; in other words, when
1426        a normal exception unwinder might otherwise resume to that
1427        handler.
1428        "#,
1429            &formats.exception_handler_address,
1430        )
1431        .operands_in(vec![
1432            Operand::new("block", &entities.raw_block),
1433            Operand::new("index", &imm.imm64),
1434        ])
1435        .operands_out(vec![Operand::new("addr", iAddr)]),
1436    );
1437
1438    ig.push(
1439        Inst::new(
1440            "iconst",
1441            r#"
1442        Integer constant.
1443
1444        Create a scalar integer SSA value with an immediate constant value, or
1445        an integer vector where all the lanes have the same value.
1446        "#,
1447            &formats.unary_imm,
1448        )
1449        .operands_in(vec![Operand::new("N", &imm.imm64)])
1450        .operands_out(vec![
1451            Operand::new("a", NarrowInt).with_doc("A constant integer scalar or vector value"),
1452        ]),
1453    );
1454
1455    ig.push(
1456        Inst::new(
1457            "f16const",
1458            r#"
1459        Floating point constant.
1460
1461        Create a `f16` SSA value with an immediate constant value.
1462        "#,
1463            &formats.unary_ieee16,
1464        )
1465        .operands_in(vec![Operand::new("N", &imm.ieee16)])
1466        .operands_out(vec![
1467            Operand::new("a", f16_).with_doc("A constant f16 scalar value"),
1468        ]),
1469    );
1470
1471    ig.push(
1472        Inst::new(
1473            "f32const",
1474            r#"
1475        Floating point constant.
1476
1477        Create a `f32` SSA value with an immediate constant value.
1478        "#,
1479            &formats.unary_ieee32,
1480        )
1481        .operands_in(vec![Operand::new("N", &imm.ieee32)])
1482        .operands_out(vec![
1483            Operand::new("a", f32_).with_doc("A constant f32 scalar value"),
1484        ]),
1485    );
1486
1487    ig.push(
1488        Inst::new(
1489            "f64const",
1490            r#"
1491        Floating point constant.
1492
1493        Create a `f64` SSA value with an immediate constant value.
1494        "#,
1495            &formats.unary_ieee64,
1496        )
1497        .operands_in(vec![Operand::new("N", &imm.ieee64)])
1498        .operands_out(vec![
1499            Operand::new("a", f64_).with_doc("A constant f64 scalar value"),
1500        ]),
1501    );
1502
1503    ig.push(
1504        Inst::new(
1505            "f128const",
1506            r#"
1507        Floating point constant.
1508
1509        Create a `f128` SSA value with an immediate constant value.
1510        "#,
1511            &formats.unary_const,
1512        )
1513        .operands_in(vec![Operand::new("N", &entities.pool_constant)])
1514        .operands_out(vec![
1515            Operand::new("a", f128_).with_doc("A constant f128 scalar value"),
1516        ]),
1517    );
1518
1519    ig.push(
1520        Inst::new(
1521            "vconst",
1522            r#"
1523        SIMD vector constant.
1524
1525        Construct a vector with the given immediate bytes.
1526        "#,
1527            &formats.unary_const,
1528        )
1529        .operands_in(vec![
1530            Operand::new("N", &entities.pool_constant)
1531                .with_doc("The 16 immediate bytes of a 128-bit vector"),
1532        ])
1533        .operands_out(vec![
1534            Operand::new("a", TxN).with_doc("A constant vector value"),
1535        ]),
1536    );
1537
1538    let Tx16 = &TypeVar::new(
1539        "Tx16",
1540        "A SIMD vector with exactly 16 lanes of 8-bit values; eventually this may support other \
1541         lane counts and widths",
1542        TypeSetBuilder::new()
1543            .ints(8..8)
1544            .simd_lanes(16..16)
1545            .includes_scalars(false)
1546            .build(),
1547    );
1548
1549    ig.push(
1550        Inst::new(
1551            "shuffle",
1552            r#"
1553        SIMD vector shuffle.
1554
1555        Shuffle two vectors using the given immediate bytes. For each of the 16 bytes of the
1556        immediate, a value i of 0-15 selects the i-th element of the first vector and a value i of
1557        16-31 selects the (i-16)th element of the second vector. Immediate values outside of the
1558        0-31 range are not valid.
1559        "#,
1560            &formats.shuffle,
1561        )
1562        .operands_in(vec![
1563            Operand::new("a", Tx16).with_doc("A vector value"),
1564            Operand::new("b", Tx16).with_doc("A vector value"),
1565            Operand::new("mask", &entities.uimm128)
1566                .with_doc("The 16 immediate bytes used for selecting the elements to shuffle"),
1567        ])
1568        .operands_out(vec![Operand::new("a", Tx16).with_doc("A vector value")]),
1569    );
1570
1571    ig.push(Inst::new(
1572        "nop",
1573        r#"
1574        Just a dummy instruction.
1575
1576        Note: this doesn't compile to a machine code nop.
1577        "#,
1578        &formats.nullary,
1579    ));
1580
1581    ig.push(
1582        Inst::new(
1583            "select",
1584            r#"
1585        Conditional select.
1586
1587        This instruction selects whole values. Use `bitselect` to choose each
1588        bit according to a mask.
1589        "#,
1590            &formats.ternary,
1591        )
1592        .operands_in(vec![
1593            Operand::new("c", ScalarTruthy).with_doc("Controlling value to test"),
1594            Operand::new("x", Any).with_doc("Value to use when `c` is true"),
1595            Operand::new("y", Any).with_doc("Value to use when `c` is false"),
1596        ])
1597        .operands_out(vec![Operand::new("a", Any)]),
1598    );
1599
1600    ig.push(
1601        Inst::new(
1602            "select_spectre_guard",
1603            r#"
1604            Conditional select intended for Spectre guards.
1605
1606            This operation is semantically equivalent to a select instruction.
1607            However, this instruction prohibits all speculation on the
1608            controlling value when determining which input to use as the result.
1609            As such, it is suitable for use in Spectre guards.
1610
1611            For example, on a target which may speculatively execute branches,
1612            the lowering of this instruction is guaranteed to not conditionally
1613            branch. Instead it will typically lower to a conditional move
1614            instruction. (No Spectre-vulnerable processors are known to perform
1615            value speculation on conditional move instructions.)
1616
1617            Ensure that the instruction you're trying to protect from Spectre
1618            attacks has a data dependency on the result of this instruction.
1619            That prevents an out-of-order CPU from evaluating that instruction
1620            until the result of this one is known, which in turn will be blocked
1621            until the controlling value is known.
1622
1623            Typical usage is to use a bounds-check as the controlling value,
1624            and select between either a null pointer if the bounds-check
1625            fails, or an in-bounds address otherwise, so that dereferencing
1626            the resulting address with a load or store instruction will trap if
1627            the bounds-check failed. When this instruction is used in this way,
1628            any microarchitectural side effects of the memory access will only
1629            occur after the bounds-check finishes, which ensures that no Spectre
1630            vulnerability will exist.
1631
1632            Optimization opportunities for this instruction are limited compared
1633            to a normal select instruction, but it is allowed to be replaced
1634            by other values which are functionally equivalent as long as doing
1635            so does not introduce any new opportunities to speculate on the
1636            controlling value.
1637            "#,
1638            &formats.ternary,
1639        )
1640        .operands_in(vec![
1641            Operand::new("c", ScalarTruthy).with_doc("Controlling value to test"),
1642            Operand::new("x", Any).with_doc("Value to use when `c` is true"),
1643            Operand::new("y", Any).with_doc("Value to use when `c` is false"),
1644        ])
1645        .operands_out(vec![Operand::new("a", Any)]),
1646    );
1647
1648    ig.push(
1649        Inst::new(
1650            "bitselect",
1651            r#"
1652        Conditional select of bits.
1653
1654        For each bit in `c`, this instruction selects the corresponding bit from `x` if the bit
1655        in `x` is 1 and the corresponding bit from `y` if the bit in `c` is 0. See also:
1656        `select`.
1657        "#,
1658            &formats.ternary,
1659        )
1660        .operands_in(vec![
1661            Operand::new("c", Any).with_doc("Controlling value to test"),
1662            Operand::new("x", Any).with_doc("Value to use when `c` is true"),
1663            Operand::new("y", Any).with_doc("Value to use when `c` is false"),
1664        ])
1665        .operands_out(vec![Operand::new("a", Any)]),
1666    );
1667
1668    ig.push(
1669        Inst::new(
1670            "x86_blendv",
1671            r#"
1672        A bitselect-lookalike instruction except with the semantics of
1673        `blendv`-related instructions on x86.
1674
1675        This instruction will use the top bit of each lane in `c`, the condition
1676        mask. If the bit is 1 then the corresponding lane from `x` is chosen.
1677        Otherwise the corresponding lane from `y` is chosen.
1678
1679            "#,
1680            &formats.ternary,
1681        )
1682        .operands_in(vec![
1683            Operand::new("c", Any).with_doc("Controlling value to test"),
1684            Operand::new("x", Any).with_doc("Value to use when `c` is true"),
1685            Operand::new("y", Any).with_doc("Value to use when `c` is false"),
1686        ])
1687        .operands_out(vec![Operand::new("a", Any)]),
1688    );
1689
1690    ig.push(
1691        Inst::new(
1692            "vany_true",
1693            r#"
1694        Reduce a vector to a scalar boolean.
1695
1696        Return a scalar boolean true if any lane in ``a`` is non-zero, false otherwise.
1697        "#,
1698            &formats.unary,
1699        )
1700        .operands_in(vec![Operand::new("a", TxN)])
1701        .operands_out(vec![Operand::new("s", i8)]),
1702    );
1703
1704    ig.push(
1705        Inst::new(
1706            "vall_true",
1707            r#"
1708        Reduce a vector to a scalar boolean.
1709
1710        Return a scalar boolean true if all lanes in ``i`` are non-zero, false otherwise.
1711        "#,
1712            &formats.unary,
1713        )
1714        .operands_in(vec![Operand::new("a", TxN)])
1715        .operands_out(vec![Operand::new("s", i8)]),
1716    );
1717
1718    ig.push(
1719        Inst::new(
1720            "vhigh_bits",
1721            r#"
1722        Reduce a vector to a scalar integer.
1723
1724        Return a scalar integer, consisting of the concatenation of the most significant bit
1725        of each lane of ``a``.
1726        "#,
1727            &formats.unary,
1728        )
1729        .operands_in(vec![Operand::new("a", TxN)])
1730        .operands_out(vec![Operand::new("x", NarrowInt)]),
1731    );
1732
1733    ig.push(
1734        Inst::new(
1735            "icmp",
1736            r#"
1737        Integer comparison.
1738
1739        The condition code determines if the operands are interpreted as signed
1740        or unsigned integers.
1741
1742        | Signed | Unsigned | Condition             |
1743        |--------|----------|-----------------------|
1744        | eq     | eq       | Equal                 |
1745        | ne     | ne       | Not equal             |
1746        | slt    | ult      | Less than             |
1747        | sge    | uge      | Greater than or equal |
1748        | sgt    | ugt      | Greater than          |
1749        | sle    | ule      | Less than or equal    |
1750
1751        When this instruction compares integer vectors, it returns a vector of
1752        lane-wise comparisons.
1753
1754        When comparing scalars, the result is:
1755            - `1` if the condition holds.
1756            - `0` if the condition does not hold.
1757
1758        When comparing vectors, the result is:
1759            - `-1` (i.e. all ones) in each lane where the condition holds.
1760            - `0` in each lane where the condition does not hold.
1761        "#,
1762            &formats.int_compare,
1763        )
1764        .operands_in(vec![
1765            Operand::new("Cond", &imm.intcc),
1766            Operand::new("x", Int),
1767            Operand::new("y", Int),
1768        ])
1769        .operands_out(vec![Operand::new("a", &Int.as_truthy())]),
1770    );
1771
1772    ig.push(
1773        Inst::new(
1774            "icmp_imm",
1775            r#"
1776        Compare scalar integer to a constant.
1777
1778        This is the same as the `icmp` instruction, except one operand is
1779        a sign extended 64 bit immediate constant.
1780
1781        This instruction can only compare scalars. Use `icmp` for
1782        lane-wise vector comparisons.
1783        "#,
1784            &formats.int_compare_imm,
1785        )
1786        .operands_in(vec![
1787            Operand::new("Cond", &imm.intcc),
1788            Operand::new("x", iB),
1789            Operand::new("Y", &imm.imm64),
1790        ])
1791        .operands_out(vec![Operand::new("a", i8)]),
1792    );
1793
1794    ig.push(
1795        Inst::new(
1796            "iadd",
1797            r#"
1798        Wrapping integer addition: `a := x + y \pmod{2^B}`.
1799
1800        This instruction does not depend on the signed/unsigned interpretation
1801        of the operands.
1802        "#,
1803            &formats.binary,
1804        )
1805        .operands_in(vec![Operand::new("x", Int), Operand::new("y", Int)])
1806        .operands_out(vec![Operand::new("a", Int)]),
1807    );
1808
1809    ig.push(
1810        Inst::new(
1811            "isub",
1812            r#"
1813        Wrapping integer subtraction: `a := x - y \pmod{2^B}`.
1814
1815        This instruction does not depend on the signed/unsigned interpretation
1816        of the operands.
1817        "#,
1818            &formats.binary,
1819        )
1820        .operands_in(vec![Operand::new("x", Int), Operand::new("y", Int)])
1821        .operands_out(vec![Operand::new("a", Int)]),
1822    );
1823
1824    ig.push(
1825        Inst::new(
1826            "ineg",
1827            r#"
1828        Integer negation: `a := -x \pmod{2^B}`.
1829        "#,
1830            &formats.unary,
1831        )
1832        .operands_in(vec![Operand::new("x", Int)])
1833        .operands_out(vec![Operand::new("a", Int)]),
1834    );
1835
1836    ig.push(
1837        Inst::new(
1838            "iabs",
1839            r#"
1840        Integer absolute value with wrapping: `a := |x|`.
1841        "#,
1842            &formats.unary,
1843        )
1844        .operands_in(vec![Operand::new("x", Int)])
1845        .operands_out(vec![Operand::new("a", Int)]),
1846    );
1847
1848    ig.push(
1849        Inst::new(
1850            "imul",
1851            r#"
1852        Wrapping integer multiplication: `a := x y \pmod{2^B}`.
1853
1854        This instruction does not depend on the signed/unsigned interpretation
1855        of the operands.
1856
1857        Polymorphic over all integer types (vector and scalar).
1858        "#,
1859            &formats.binary,
1860        )
1861        .operands_in(vec![Operand::new("x", Int), Operand::new("y", Int)])
1862        .operands_out(vec![Operand::new("a", Int)]),
1863    );
1864
1865    ig.push(
1866        Inst::new(
1867            "umulhi",
1868            r#"
1869        Unsigned integer multiplication, producing the high half of a
1870        double-length result.
1871
1872        Polymorphic over all integer types (vector and scalar).
1873        "#,
1874            &formats.binary,
1875        )
1876        .operands_in(vec![Operand::new("x", Int), Operand::new("y", Int)])
1877        .operands_out(vec![Operand::new("a", Int)]),
1878    );
1879
1880    ig.push(
1881        Inst::new(
1882            "smulhi",
1883            r#"
1884        Signed integer multiplication, producing the high half of a
1885        double-length result.
1886
1887        Polymorphic over all integer types (vector and scalar).
1888        "#,
1889            &formats.binary,
1890        )
1891        .operands_in(vec![Operand::new("x", Int), Operand::new("y", Int)])
1892        .operands_out(vec![Operand::new("a", Int)]),
1893    );
1894
1895    let I16or32 = &TypeVar::new(
1896        "I16or32",
1897        "A vector integer type with 16- or 32-bit numbers",
1898        TypeSetBuilder::new().ints(16..32).simd_lanes(4..8).build(),
1899    );
1900
1901    ig.push(
1902        Inst::new(
1903            "sqmul_round_sat",
1904            r#"
1905        Fixed-point multiplication of numbers in the QN format, where N + 1
1906        is the number bitwidth:
1907        `a := signed_saturate((x * y + (1 << (Q - 1))) >> Q)`
1908
1909        Polymorphic over all integer vector types with 16- or 32-bit numbers.
1910        "#,
1911            &formats.binary,
1912        )
1913        .operands_in(vec![Operand::new("x", I16or32), Operand::new("y", I16or32)])
1914        .operands_out(vec![Operand::new("a", I16or32)]),
1915    );
1916
1917    ig.push(
1918        Inst::new(
1919            "x86_pmulhrsw",
1920            r#"
1921        A similar instruction to `sqmul_round_sat` except with the semantics
1922        of x86's `pmulhrsw` instruction.
1923
1924        This is the same as `sqmul_round_sat` except when both input lanes are
1925        `i16::MIN`.
1926        "#,
1927            &formats.binary,
1928        )
1929        .operands_in(vec![Operand::new("x", I16or32), Operand::new("y", I16or32)])
1930        .operands_out(vec![Operand::new("a", I16or32)]),
1931    );
1932
1933    // Integer division and remainder are scalar-only; most
1934    // hardware does not directly support vector integer division.
1935
1936    ig.push(
1937        Inst::new(
1938            "udiv",
1939            r#"
1940        Unsigned integer division: `a := \lfloor {x \over y} \rfloor`.
1941
1942        This operation traps if the divisor is zero.
1943        "#,
1944            &formats.binary,
1945        )
1946        .operands_in(vec![Operand::new("x", iB), Operand::new("y", iB)])
1947        .operands_out(vec![Operand::new("a", iB)])
1948        .can_trap()
1949        .side_effects_idempotent(),
1950    );
1951
1952    ig.push(
1953        Inst::new(
1954            "sdiv",
1955            r#"
1956        Signed integer division rounded toward zero: `a := sign(xy)
1957        \lfloor {|x| \over |y|}\rfloor`.
1958
1959        This operation traps if the divisor is zero, or if the result is not
1960        representable in `B` bits two's complement. This only happens
1961        when `x = -2^{B-1}, y = -1`.
1962        "#,
1963            &formats.binary,
1964        )
1965        .operands_in(vec![Operand::new("x", iB), Operand::new("y", iB)])
1966        .operands_out(vec![Operand::new("a", iB)])
1967        .can_trap()
1968        .side_effects_idempotent(),
1969    );
1970
1971    ig.push(
1972        Inst::new(
1973            "urem",
1974            r#"
1975        Unsigned integer remainder.
1976
1977        This operation traps if the divisor is zero.
1978        "#,
1979            &formats.binary,
1980        )
1981        .operands_in(vec![Operand::new("x", iB), Operand::new("y", iB)])
1982        .operands_out(vec![Operand::new("a", iB)])
1983        .can_trap()
1984        .side_effects_idempotent(),
1985    );
1986
1987    ig.push(
1988        Inst::new(
1989            "srem",
1990            r#"
1991        Signed integer remainder. The result has the sign of the dividend.
1992
1993        This operation traps if the divisor is zero.
1994        "#,
1995            &formats.binary,
1996        )
1997        .operands_in(vec![Operand::new("x", iB), Operand::new("y", iB)])
1998        .operands_out(vec![Operand::new("a", iB)])
1999        .can_trap()
2000        .side_effects_idempotent(),
2001    );
2002
2003    ig.push(
2004        Inst::new(
2005            "iadd_imm",
2006            r#"
2007        Add immediate integer.
2008
2009        Same as `iadd`, but one operand is a sign extended 64 bit immediate constant.
2010
2011        Polymorphic over all scalar integer types, but does not support vector
2012        types.
2013        "#,
2014            &formats.binary_imm64,
2015        )
2016        .operands_in(vec![Operand::new("x", iB), Operand::new("Y", &imm.imm64)])
2017        .operands_out(vec![Operand::new("a", iB)]),
2018    );
2019
2020    ig.push(
2021        Inst::new(
2022            "imul_imm",
2023            r#"
2024        Integer multiplication by immediate constant.
2025
2026        Same as `imul`, but one operand is a sign extended 64 bit immediate constant.
2027
2028        Polymorphic over all scalar integer types, but does not support vector
2029        types.
2030        "#,
2031            &formats.binary_imm64,
2032        )
2033        .operands_in(vec![Operand::new("x", iB), Operand::new("Y", &imm.imm64)])
2034        .operands_out(vec![Operand::new("a", iB)]),
2035    );
2036
2037    ig.push(
2038        Inst::new(
2039            "udiv_imm",
2040            r#"
2041        Unsigned integer division by an immediate constant.
2042
2043        Same as `udiv`, but one operand is a zero extended 64 bit immediate constant.
2044
2045        This operation traps if the divisor is zero.
2046        "#,
2047            &formats.binary_imm64,
2048        )
2049        .operands_in(vec![Operand::new("x", iB), Operand::new("Y", &imm.imm64)])
2050        .operands_out(vec![Operand::new("a", iB)]),
2051    );
2052
2053    ig.push(
2054        Inst::new(
2055            "sdiv_imm",
2056            r#"
2057        Signed integer division by an immediate constant.
2058
2059        Same as `sdiv`, but one operand is a sign extended 64 bit immediate constant.
2060
2061        This operation traps if the divisor is zero, or if the result is not
2062        representable in `B` bits two's complement. This only happens
2063        when `x = -2^{B-1}, Y = -1`.
2064        "#,
2065            &formats.binary_imm64,
2066        )
2067        .operands_in(vec![Operand::new("x", iB), Operand::new("Y", &imm.imm64)])
2068        .operands_out(vec![Operand::new("a", iB)]),
2069    );
2070
2071    ig.push(
2072        Inst::new(
2073            "urem_imm",
2074            r#"
2075        Unsigned integer remainder with immediate divisor.
2076
2077        Same as `urem`, but one operand is a zero extended 64 bit immediate constant.
2078
2079        This operation traps if the divisor is zero.
2080        "#,
2081            &formats.binary_imm64,
2082        )
2083        .operands_in(vec![Operand::new("x", iB), Operand::new("Y", &imm.imm64)])
2084        .operands_out(vec![Operand::new("a", iB)]),
2085    );
2086
2087    ig.push(
2088        Inst::new(
2089            "srem_imm",
2090            r#"
2091        Signed integer remainder with immediate divisor.
2092
2093        Same as `srem`, but one operand is a sign extended 64 bit immediate constant.
2094
2095        This operation traps if the divisor is zero.
2096        "#,
2097            &formats.binary_imm64,
2098        )
2099        .operands_in(vec![Operand::new("x", iB), Operand::new("Y", &imm.imm64)])
2100        .operands_out(vec![Operand::new("a", iB)]),
2101    );
2102
2103    ig.push(
2104        Inst::new(
2105            "irsub_imm",
2106            r#"
2107        Immediate reverse wrapping subtraction: `a := Y - x \pmod{2^B}`.
2108
2109        The immediate operand is a sign extended 64 bit constant.
2110
2111        Also works as integer negation when `Y = 0`. Use `iadd_imm`
2112        with a negative immediate operand for the reverse immediate
2113        subtraction.
2114
2115        Polymorphic over all scalar integer types, but does not support vector
2116        types.
2117        "#,
2118            &formats.binary_imm64,
2119        )
2120        .operands_in(vec![Operand::new("x", iB), Operand::new("Y", &imm.imm64)])
2121        .operands_out(vec![Operand::new("a", iB)]),
2122    );
2123
2124    ig.push(
2125        Inst::new(
2126            "sadd_overflow_cin",
2127            r#"
2128        Add signed integers with carry in and overflow out.
2129
2130        Same as `sadd_overflow` with an additional carry input. The `c_in` type
2131        is interpreted as 1 if it's nonzero or 0 if it's zero.
2132        "#,
2133            &formats.ternary,
2134        )
2135        .operands_in(vec![
2136            Operand::new("x", iB),
2137            Operand::new("y", iB),
2138            Operand::new("c_in", i8).with_doc("Input carry flag"),
2139        ])
2140        .operands_out(vec![
2141            Operand::new("a", iB),
2142            Operand::new("c_out", i8).with_doc("Output carry flag"),
2143        ]),
2144    );
2145
2146    ig.push(
2147        Inst::new(
2148            "uadd_overflow_cin",
2149            r#"
2150        Add unsigned integers with carry in and overflow out.
2151
2152        Same as `uadd_overflow` with an additional carry input. The `c_in` type
2153        is interpreted as 1 if it's nonzero or 0 if it's zero.
2154        "#,
2155            &formats.ternary,
2156        )
2157        .operands_in(vec![
2158            Operand::new("x", iB),
2159            Operand::new("y", iB),
2160            Operand::new("c_in", i8).with_doc("Input carry flag"),
2161        ])
2162        .operands_out(vec![
2163            Operand::new("a", iB),
2164            Operand::new("c_out", i8).with_doc("Output carry flag"),
2165        ]),
2166    );
2167
2168    {
2169        let of_out = Operand::new("of", i8).with_doc("Overflow flag");
2170        ig.push(
2171            Inst::new(
2172                "uadd_overflow",
2173                r#"
2174            Add integers unsigned with overflow out.
2175            ``of`` is set when the addition overflowed.
2176            ```text
2177                a &= x + y \pmod 2^B \\
2178                of &= x+y >= 2^B
2179            ```
2180            Polymorphic over all scalar integer types, but does not support vector
2181            types.
2182            "#,
2183                &formats.binary,
2184            )
2185            .operands_in(vec![Operand::new("x", iB), Operand::new("y", iB)])
2186            .operands_out(vec![Operand::new("a", iB), of_out.clone()]),
2187        );
2188
2189        ig.push(
2190            Inst::new(
2191                "sadd_overflow",
2192                r#"
2193            Add integers signed with overflow out.
2194            ``of`` is set when the addition over- or underflowed.
2195            Polymorphic over all scalar integer types, but does not support vector
2196            types.
2197            "#,
2198                &formats.binary,
2199            )
2200            .operands_in(vec![Operand::new("x", iB), Operand::new("y", iB)])
2201            .operands_out(vec![Operand::new("a", iB), of_out.clone()]),
2202        );
2203
2204        ig.push(
2205            Inst::new(
2206                "usub_overflow",
2207                r#"
2208            Subtract integers unsigned with overflow out.
2209            ``of`` is set when the subtraction underflowed.
2210            ```text
2211                a &= x - y \pmod 2^B \\
2212                of &= x - y < 0
2213            ```
2214            Polymorphic over all scalar integer types, but does not support vector
2215            types.
2216            "#,
2217                &formats.binary,
2218            )
2219            .operands_in(vec![Operand::new("x", iB), Operand::new("y", iB)])
2220            .operands_out(vec![Operand::new("a", iB), of_out.clone()]),
2221        );
2222
2223        ig.push(
2224            Inst::new(
2225                "ssub_overflow",
2226                r#"
2227            Subtract integers signed with overflow out.
2228            ``of`` is set when the subtraction over- or underflowed.
2229            Polymorphic over all scalar integer types, but does not support vector
2230            types.
2231            "#,
2232                &formats.binary,
2233            )
2234            .operands_in(vec![Operand::new("x", iB), Operand::new("y", iB)])
2235            .operands_out(vec![Operand::new("a", iB), of_out.clone()]),
2236        );
2237
2238        {
2239            let NarrowScalar = &TypeVar::new(
2240                "NarrowScalar",
2241                "A scalar integer type up to 64 bits",
2242                TypeSetBuilder::new().ints(8..64).build(),
2243            );
2244
2245            ig.push(
2246                Inst::new(
2247                    "umul_overflow",
2248                    r#"
2249                Multiply integers unsigned with overflow out.
2250                ``of`` is set when the multiplication overflowed.
2251                ```text
2252                    a &= x * y \pmod 2^B \\
2253                    of &= x * y > 2^B
2254                ```
2255                Polymorphic over all scalar integer types except i128, but does not support vector
2256                types.
2257                "#,
2258                    &formats.binary,
2259                )
2260                .operands_in(vec![
2261                    Operand::new("x", NarrowScalar),
2262                    Operand::new("y", NarrowScalar),
2263                ])
2264                .operands_out(vec![Operand::new("a", NarrowScalar), of_out.clone()]),
2265            );
2266
2267            ig.push(
2268                Inst::new(
2269                    "smul_overflow",
2270                    r#"
2271                Multiply integers signed with overflow out.
2272                ``of`` is set when the multiplication over- or underflowed.
2273                Polymorphic over all scalar integer types except i128, but does not support vector
2274                types.
2275                "#,
2276                    &formats.binary,
2277                )
2278                .operands_in(vec![
2279                    Operand::new("x", NarrowScalar),
2280                    Operand::new("y", NarrowScalar),
2281                ])
2282                .operands_out(vec![Operand::new("a", NarrowScalar), of_out.clone()]),
2283            );
2284        }
2285    }
2286
2287    let i32_64 = &TypeVar::new(
2288        "i32_64",
2289        "A 32 or 64-bit scalar integer type",
2290        TypeSetBuilder::new().ints(32..64).build(),
2291    );
2292
2293    ig.push(
2294        Inst::new(
2295            "uadd_overflow_trap",
2296            r#"
2297        Unsigned addition of x and y, trapping if the result overflows.
2298
2299        Accepts 32 or 64-bit integers, and does not support vector types.
2300        "#,
2301            &formats.int_add_trap,
2302        )
2303        .operands_in(vec![
2304            Operand::new("x", i32_64),
2305            Operand::new("y", i32_64),
2306            Operand::new("code", &imm.trapcode),
2307        ])
2308        .operands_out(vec![Operand::new("a", i32_64)])
2309        .can_trap()
2310        .side_effects_idempotent(),
2311    );
2312
2313    ig.push(
2314        Inst::new(
2315            "ssub_overflow_bin",
2316            r#"
2317        Subtract signed integers with borrow in and overflow out.
2318
2319        Same as `ssub_overflow` with an additional borrow input. The `b_in` type
2320        is interpreted as 1 if it's nonzero or 0 if it's zero. The computation
2321        performed here is `x - (y + (b_in != 0))`.
2322        "#,
2323            &formats.ternary,
2324        )
2325        .operands_in(vec![
2326            Operand::new("x", iB),
2327            Operand::new("y", iB),
2328            Operand::new("b_in", i8).with_doc("Input borrow flag"),
2329        ])
2330        .operands_out(vec![
2331            Operand::new("a", iB),
2332            Operand::new("b_out", i8).with_doc("Output borrow flag"),
2333        ]),
2334    );
2335
2336    ig.push(
2337        Inst::new(
2338            "usub_overflow_bin",
2339            r#"
2340        Subtract unsigned integers with borrow in and overflow out.
2341
2342        Same as `usub_overflow` with an additional borrow input. The `b_in` type
2343        is interpreted as 1 if it's nonzero or 0 if it's zero. The computation
2344        performed here is `x - (y + (b_in != 0))`.
2345        "#,
2346            &formats.ternary,
2347        )
2348        .operands_in(vec![
2349            Operand::new("x", iB),
2350            Operand::new("y", iB),
2351            Operand::new("b_in", i8).with_doc("Input borrow flag"),
2352        ])
2353        .operands_out(vec![
2354            Operand::new("a", iB),
2355            Operand::new("b_out", i8).with_doc("Output borrow flag"),
2356        ]),
2357    );
2358
2359    let bits = &TypeVar::new(
2360        "bits",
2361        "Any integer, float, or vector type",
2362        TypeSetBuilder::new()
2363            .ints(Interval::All)
2364            .floats(Interval::All)
2365            .simd_lanes(Interval::All)
2366            .includes_scalars(true)
2367            .build(),
2368    );
2369
2370    ig.push(
2371        Inst::new(
2372            "band",
2373            r#"
2374        Bitwise and.
2375        "#,
2376            &formats.binary,
2377        )
2378        .operands_in(vec![Operand::new("x", bits), Operand::new("y", bits)])
2379        .operands_out(vec![Operand::new("a", bits)]),
2380    );
2381
2382    ig.push(
2383        Inst::new(
2384            "bor",
2385            r#"
2386        Bitwise or.
2387        "#,
2388            &formats.binary,
2389        )
2390        .operands_in(vec![Operand::new("x", bits), Operand::new("y", bits)])
2391        .operands_out(vec![Operand::new("a", bits)]),
2392    );
2393
2394    ig.push(
2395        Inst::new(
2396            "bxor",
2397            r#"
2398        Bitwise xor.
2399        "#,
2400            &formats.binary,
2401        )
2402        .operands_in(vec![Operand::new("x", bits), Operand::new("y", bits)])
2403        .operands_out(vec![Operand::new("a", bits)]),
2404    );
2405
2406    ig.push(
2407        Inst::new(
2408            "bnot",
2409            r#"
2410        Bitwise not.
2411        "#,
2412            &formats.unary,
2413        )
2414        .operands_in(vec![Operand::new("x", bits)])
2415        .operands_out(vec![Operand::new("a", bits)]),
2416    );
2417
2418    ig.push(
2419        Inst::new(
2420            "band_not",
2421            r#"
2422        Bitwise and not.
2423
2424        Computes `x & ~y`.
2425        "#,
2426            &formats.binary,
2427        )
2428        .operands_in(vec![Operand::new("x", bits), Operand::new("y", bits)])
2429        .operands_out(vec![Operand::new("a", bits)]),
2430    );
2431
2432    ig.push(
2433        Inst::new(
2434            "bor_not",
2435            r#"
2436        Bitwise or not.
2437
2438        Computes `x | ~y`.
2439        "#,
2440            &formats.binary,
2441        )
2442        .operands_in(vec![Operand::new("x", bits), Operand::new("y", bits)])
2443        .operands_out(vec![Operand::new("a", bits)]),
2444    );
2445
2446    ig.push(
2447        Inst::new(
2448            "bxor_not",
2449            r#"
2450        Bitwise xor not.
2451
2452        Computes `x ^ ~y`.
2453        "#,
2454            &formats.binary,
2455        )
2456        .operands_in(vec![Operand::new("x", bits), Operand::new("y", bits)])
2457        .operands_out(vec![Operand::new("a", bits)]),
2458    );
2459
2460    ig.push(
2461        Inst::new(
2462            "band_imm",
2463            r#"
2464        Bitwise and with immediate.
2465
2466        Same as `band`, but one operand is a zero extended 64 bit immediate constant.
2467
2468        Polymorphic over all scalar integer types, but does not support vector
2469        types.
2470        "#,
2471            &formats.binary_imm64,
2472        )
2473        .operands_in(vec![Operand::new("x", iB), Operand::new("Y", &imm.imm64)])
2474        .operands_out(vec![Operand::new("a", iB)]),
2475    );
2476
2477    ig.push(
2478        Inst::new(
2479            "bor_imm",
2480            r#"
2481        Bitwise or with immediate.
2482
2483        Same as `bor`, but one operand is a zero extended 64 bit immediate constant.
2484
2485        Polymorphic over all scalar integer types, but does not support vector
2486        types.
2487        "#,
2488            &formats.binary_imm64,
2489        )
2490        .operands_in(vec![Operand::new("x", iB), Operand::new("Y", &imm.imm64)])
2491        .operands_out(vec![Operand::new("a", iB)]),
2492    );
2493
2494    ig.push(
2495        Inst::new(
2496            "bxor_imm",
2497            r#"
2498        Bitwise xor with immediate.
2499
2500        Same as `bxor`, but one operand is a zero extended 64 bit immediate constant.
2501
2502        Polymorphic over all scalar integer types, but does not support vector
2503        types.
2504        "#,
2505            &formats.binary_imm64,
2506        )
2507        .operands_in(vec![Operand::new("x", iB), Operand::new("Y", &imm.imm64)])
2508        .operands_out(vec![Operand::new("a", iB)]),
2509    );
2510
2511    ig.push(
2512        Inst::new(
2513            "rotl",
2514            r#"
2515        Rotate left.
2516
2517        Rotate the bits in ``x`` by ``y`` places.
2518        "#,
2519            &formats.binary,
2520        )
2521        .operands_in(vec![
2522            Operand::new("x", Int).with_doc("Scalar or vector value to shift"),
2523            Operand::new("y", iB).with_doc("Number of bits to shift"),
2524        ])
2525        .operands_out(vec![Operand::new("a", Int)]),
2526    );
2527
2528    ig.push(
2529        Inst::new(
2530            "rotr",
2531            r#"
2532        Rotate right.
2533
2534        Rotate the bits in ``x`` by ``y`` places.
2535        "#,
2536            &formats.binary,
2537        )
2538        .operands_in(vec![
2539            Operand::new("x", Int).with_doc("Scalar or vector value to shift"),
2540            Operand::new("y", iB).with_doc("Number of bits to shift"),
2541        ])
2542        .operands_out(vec![Operand::new("a", Int)]),
2543    );
2544
2545    ig.push(
2546        Inst::new(
2547            "rotl_imm",
2548            r#"
2549        Rotate left by immediate.
2550
2551        Same as `rotl`, but one operand is a zero extended 64 bit immediate constant.
2552        "#,
2553            &formats.binary_imm64,
2554        )
2555        .operands_in(vec![
2556            Operand::new("x", Int).with_doc("Scalar or vector value to shift"),
2557            Operand::new("Y", &imm.imm64),
2558        ])
2559        .operands_out(vec![Operand::new("a", Int)]),
2560    );
2561
2562    ig.push(
2563        Inst::new(
2564            "rotr_imm",
2565            r#"
2566        Rotate right by immediate.
2567
2568        Same as `rotr`, but one operand is a zero extended 64 bit immediate constant.
2569        "#,
2570            &formats.binary_imm64,
2571        )
2572        .operands_in(vec![
2573            Operand::new("x", Int).with_doc("Scalar or vector value to shift"),
2574            Operand::new("Y", &imm.imm64),
2575        ])
2576        .operands_out(vec![Operand::new("a", Int)]),
2577    );
2578
2579    ig.push(
2580        Inst::new(
2581            "ishl",
2582            r#"
2583        Integer shift left. Shift the bits in ``x`` towards the MSB by ``y``
2584        places. Shift in zero bits to the LSB.
2585
2586        The shift amount is masked to the size of ``x``.
2587
2588        When shifting a B-bits integer type, this instruction computes:
2589
2590        ```text
2591            s &:= y \pmod B,
2592            a &:= x \cdot 2^s \pmod{2^B}.
2593        ```
2594        "#,
2595            &formats.binary,
2596        )
2597        .operands_in(vec![
2598            Operand::new("x", Int).with_doc("Scalar or vector value to shift"),
2599            Operand::new("y", iB).with_doc("Number of bits to shift"),
2600        ])
2601        .operands_out(vec![Operand::new("a", Int)]),
2602    );
2603
2604    ig.push(
2605        Inst::new(
2606            "ushr",
2607            r#"
2608        Unsigned shift right. Shift bits in ``x`` towards the LSB by ``y``
2609        places, shifting in zero bits to the MSB. Also called a *logical
2610        shift*.
2611
2612        The shift amount is masked to the size of ``x``.
2613
2614        When shifting a B-bits integer type, this instruction computes:
2615
2616        ```text
2617            s &:= y \pmod B,
2618            a &:= \lfloor x \cdot 2^{-s} \rfloor.
2619        ```
2620        "#,
2621            &formats.binary,
2622        )
2623        .operands_in(vec![
2624            Operand::new("x", Int).with_doc("Scalar or vector value to shift"),
2625            Operand::new("y", iB).with_doc("Number of bits to shift"),
2626        ])
2627        .operands_out(vec![Operand::new("a", Int)]),
2628    );
2629
2630    ig.push(
2631        Inst::new(
2632            "sshr",
2633            r#"
2634        Signed shift right. Shift bits in ``x`` towards the LSB by ``y``
2635        places, shifting in sign bits to the MSB. Also called an *arithmetic
2636        shift*.
2637
2638        The shift amount is masked to the size of ``x``.
2639        "#,
2640            &formats.binary,
2641        )
2642        .operands_in(vec![
2643            Operand::new("x", Int).with_doc("Scalar or vector value to shift"),
2644            Operand::new("y", iB).with_doc("Number of bits to shift"),
2645        ])
2646        .operands_out(vec![Operand::new("a", Int)]),
2647    );
2648
2649    ig.push(
2650        Inst::new(
2651            "ishl_imm",
2652            r#"
2653        Integer shift left by immediate.
2654
2655        The shift amount is masked to the size of ``x``.
2656        "#,
2657            &formats.binary_imm64,
2658        )
2659        .operands_in(vec![
2660            Operand::new("x", Int).with_doc("Scalar or vector value to shift"),
2661            Operand::new("Y", &imm.imm64),
2662        ])
2663        .operands_out(vec![Operand::new("a", Int)]),
2664    );
2665
2666    ig.push(
2667        Inst::new(
2668            "ushr_imm",
2669            r#"
2670        Unsigned shift right by immediate.
2671
2672        The shift amount is masked to the size of ``x``.
2673        "#,
2674            &formats.binary_imm64,
2675        )
2676        .operands_in(vec![
2677            Operand::new("x", Int).with_doc("Scalar or vector value to shift"),
2678            Operand::new("Y", &imm.imm64),
2679        ])
2680        .operands_out(vec![Operand::new("a", Int)]),
2681    );
2682
2683    ig.push(
2684        Inst::new(
2685            "sshr_imm",
2686            r#"
2687        Signed shift right by immediate.
2688
2689        The shift amount is masked to the size of ``x``.
2690        "#,
2691            &formats.binary_imm64,
2692        )
2693        .operands_in(vec![
2694            Operand::new("x", Int).with_doc("Scalar or vector value to shift"),
2695            Operand::new("Y", &imm.imm64),
2696        ])
2697        .operands_out(vec![Operand::new("a", Int)]),
2698    );
2699
2700    ig.push(
2701        Inst::new(
2702            "bitrev",
2703            r#"
2704        Reverse the bits of a integer.
2705
2706        Reverses the bits in ``x``.
2707        "#,
2708            &formats.unary,
2709        )
2710        .operands_in(vec![Operand::new("x", iB)])
2711        .operands_out(vec![Operand::new("a", iB)]),
2712    );
2713
2714    ig.push(
2715        Inst::new(
2716            "clz",
2717            r#"
2718        Count leading zero bits.
2719
2720        Starting from the MSB in ``x``, count the number of zero bits before
2721        reaching the first one bit. When ``x`` is zero, returns the size of x
2722        in bits.
2723        "#,
2724            &formats.unary,
2725        )
2726        .operands_in(vec![Operand::new("x", iB)])
2727        .operands_out(vec![Operand::new("a", iB)]),
2728    );
2729
2730    ig.push(
2731        Inst::new(
2732            "cls",
2733            r#"
2734        Count leading sign bits.
2735
2736        Starting from the MSB after the sign bit in ``x``, count the number of
2737        consecutive bits identical to the sign bit. When ``x`` is 0 or -1,
2738        returns one less than the size of x in bits.
2739        "#,
2740            &formats.unary,
2741        )
2742        .operands_in(vec![Operand::new("x", iB)])
2743        .operands_out(vec![Operand::new("a", iB)]),
2744    );
2745
2746    ig.push(
2747        Inst::new(
2748            "ctz",
2749            r#"
2750        Count trailing zeros.
2751
2752        Starting from the LSB in ``x``, count the number of zero bits before
2753        reaching the first one bit. When ``x`` is zero, returns the size of x
2754        in bits.
2755        "#,
2756            &formats.unary,
2757        )
2758        .operands_in(vec![Operand::new("x", iB)])
2759        .operands_out(vec![Operand::new("a", iB)]),
2760    );
2761
2762    ig.push(
2763        Inst::new(
2764            "bswap",
2765            r#"
2766        Reverse the byte order of an integer.
2767
2768        Reverses the bytes in ``x``.
2769        "#,
2770            &formats.unary,
2771        )
2772        .operands_in(vec![Operand::new("x", iSwappable)])
2773        .operands_out(vec![Operand::new("a", iSwappable)]),
2774    );
2775
2776    ig.push(
2777        Inst::new(
2778            "popcnt",
2779            r#"
2780        Population count
2781
2782        Count the number of one bits in ``x``.
2783        "#,
2784            &formats.unary,
2785        )
2786        .operands_in(vec![Operand::new("x", Int)])
2787        .operands_out(vec![Operand::new("a", Int)]),
2788    );
2789
2790    let Float = &TypeVar::new(
2791        "Float",
2792        "A scalar or vector floating point number",
2793        TypeSetBuilder::new()
2794            .floats(Interval::All)
2795            .simd_lanes(Interval::All)
2796            .dynamic_simd_lanes(Interval::All)
2797            .build(),
2798    );
2799
2800    ig.push(
2801        Inst::new(
2802            "fcmp",
2803            r#"
2804        Floating point comparison.
2805
2806        Two IEEE 754-2008 floating point numbers, `x` and `y`, relate to each
2807        other in exactly one of four ways:
2808
2809        ```text
2810        == ==========================================
2811        UN Unordered when one or both numbers is NaN.
2812        EQ When `x = y`. (And `0.0 = -0.0`).
2813        LT When `x < y`.
2814        GT When `x > y`.
2815        == ==========================================
2816        ```
2817
2818        The 14 `floatcc` condition codes each correspond to a subset of
2819        the four relations, except for the empty set which would always be
2820        false, and the full set which would always be true.
2821
2822        The condition codes are divided into 7 'ordered' conditions which don't
2823        include UN, and 7 unordered conditions which all include UN.
2824
2825        ```text
2826        +-------+------------+---------+------------+-------------------------+
2827        |Ordered             |Unordered             |Condition                |
2828        +=======+============+=========+============+=========================+
2829        |ord    |EQ | LT | GT|uno      |UN          |NaNs absent / present.   |
2830        +-------+------------+---------+------------+-------------------------+
2831        |eq     |EQ          |ueq      |UN | EQ     |Equal                    |
2832        +-------+------------+---------+------------+-------------------------+
2833        |one    |LT | GT     |ne       |UN | LT | GT|Not equal                |
2834        +-------+------------+---------+------------+-------------------------+
2835        |lt     |LT          |ult      |UN | LT     |Less than                |
2836        +-------+------------+---------+------------+-------------------------+
2837        |le     |LT | EQ     |ule      |UN | LT | EQ|Less than or equal       |
2838        +-------+------------+---------+------------+-------------------------+
2839        |gt     |GT          |ugt      |UN | GT     |Greater than             |
2840        +-------+------------+---------+------------+-------------------------+
2841        |ge     |GT | EQ     |uge      |UN | GT | EQ|Greater than or equal    |
2842        +-------+------------+---------+------------+-------------------------+
2843        ```
2844
2845        The standard C comparison operators, `<, <=, >, >=`, are all ordered,
2846        so they are false if either operand is NaN. The C equality operator,
2847        `==`, is ordered, and since inequality is defined as the logical
2848        inverse it is *unordered*. They map to the `floatcc` condition
2849        codes as follows:
2850
2851        ```text
2852        ==== ====== ============
2853        C    `Cond` Subset
2854        ==== ====== ============
2855        `==` eq     EQ
2856        `!=` ne     UN | LT | GT
2857        `<`  lt     LT
2858        `<=` le     LT | EQ
2859        `>`  gt     GT
2860        `>=` ge     GT | EQ
2861        ==== ====== ============
2862        ```
2863
2864        This subset of condition codes also corresponds to the WebAssembly
2865        floating point comparisons of the same name.
2866
2867        When this instruction compares floating point vectors, it returns a
2868        vector with the results of lane-wise comparisons.
2869
2870        When comparing scalars, the result is:
2871            - `1` if the condition holds.
2872            - `0` if the condition does not hold.
2873
2874        When comparing vectors, the result is:
2875            - `-1` (i.e. all ones) in each lane where the condition holds.
2876            - `0` in each lane where the condition does not hold.
2877        "#,
2878            &formats.float_compare,
2879        )
2880        .operands_in(vec![
2881            Operand::new("Cond", &imm.floatcc),
2882            Operand::new("x", Float),
2883            Operand::new("y", Float),
2884        ])
2885        .operands_out(vec![Operand::new("a", &Float.as_truthy())]),
2886    );
2887
2888    ig.push(
2889        Inst::new(
2890            "fadd",
2891            r#"
2892        Floating point addition.
2893        "#,
2894            &formats.binary,
2895        )
2896        .operands_in(vec![Operand::new("x", Float), Operand::new("y", Float)])
2897        .operands_out(vec![
2898            Operand::new("a", Float).with_doc("Result of applying operator to each lane"),
2899        ]),
2900    );
2901
2902    ig.push(
2903        Inst::new(
2904            "fsub",
2905            r#"
2906        Floating point subtraction.
2907        "#,
2908            &formats.binary,
2909        )
2910        .operands_in(vec![Operand::new("x", Float), Operand::new("y", Float)])
2911        .operands_out(vec![
2912            Operand::new("a", Float).with_doc("Result of applying operator to each lane"),
2913        ]),
2914    );
2915
2916    ig.push(
2917        Inst::new(
2918            "fmul",
2919            r#"
2920        Floating point multiplication.
2921        "#,
2922            &formats.binary,
2923        )
2924        .operands_in(vec![Operand::new("x", Float), Operand::new("y", Float)])
2925        .operands_out(vec![
2926            Operand::new("a", Float).with_doc("Result of applying operator to each lane"),
2927        ]),
2928    );
2929
2930    ig.push(
2931        Inst::new(
2932            "fdiv",
2933            r#"
2934        Floating point division.
2935
2936        Unlike the integer division instructions ` and
2937        `udiv`, this can't trap. Division by zero is infinity or
2938        NaN, depending on the dividend.
2939        "#,
2940            &formats.binary,
2941        )
2942        .operands_in(vec![Operand::new("x", Float), Operand::new("y", Float)])
2943        .operands_out(vec![
2944            Operand::new("a", Float).with_doc("Result of applying operator to each lane"),
2945        ]),
2946    );
2947
2948    ig.push(
2949        Inst::new(
2950            "sqrt",
2951            r#"
2952        Floating point square root.
2953        "#,
2954            &formats.unary,
2955        )
2956        .operands_in(vec![Operand::new("x", Float)])
2957        .operands_out(vec![
2958            Operand::new("a", Float).with_doc("Result of applying operator to each lane"),
2959        ]),
2960    );
2961
2962    ig.push(
2963        Inst::new(
2964            "fma",
2965            r#"
2966        Floating point fused multiply-and-add.
2967
2968        Computes `a := xy+z` without any intermediate rounding of the
2969        product.
2970        "#,
2971            &formats.ternary,
2972        )
2973        .operands_in(vec![
2974            Operand::new("x", Float),
2975            Operand::new("y", Float),
2976            Operand::new("z", Float),
2977        ])
2978        .operands_out(vec![
2979            Operand::new("a", Float).with_doc("Result of applying operator to each lane"),
2980        ]),
2981    );
2982
2983    ig.push(
2984        Inst::new(
2985            "fneg",
2986            r#"
2987        Floating point negation.
2988
2989        Note that this is a pure bitwise operation.
2990        "#,
2991            &formats.unary,
2992        )
2993        .operands_in(vec![Operand::new("x", Float)])
2994        .operands_out(vec![
2995            Operand::new("a", Float).with_doc("``x`` with its sign bit inverted"),
2996        ]),
2997    );
2998
2999    ig.push(
3000        Inst::new(
3001            "fabs",
3002            r#"
3003        Floating point absolute value.
3004
3005        Note that this is a pure bitwise operation.
3006        "#,
3007            &formats.unary,
3008        )
3009        .operands_in(vec![Operand::new("x", Float)])
3010        .operands_out(vec![
3011            Operand::new("a", Float).with_doc("``x`` with its sign bit cleared"),
3012        ]),
3013    );
3014
3015    ig.push(
3016        Inst::new(
3017            "fcopysign",
3018            r#"
3019        Floating point copy sign.
3020
3021        Note that this is a pure bitwise operation. The sign bit from ``y`` is
3022        copied to the sign bit of ``x``.
3023        "#,
3024            &formats.binary,
3025        )
3026        .operands_in(vec![Operand::new("x", Float), Operand::new("y", Float)])
3027        .operands_out(vec![
3028            Operand::new("a", Float).with_doc("``x`` with its sign bit changed to that of ``y``"),
3029        ]),
3030    );
3031
3032    ig.push(
3033        Inst::new(
3034            "fmin",
3035            r#"
3036        Floating point minimum, propagating NaNs using the WebAssembly rules.
3037
3038        If either operand is NaN, this returns NaN with an unspecified sign. Furthermore, if
3039        each input NaN consists of a mantissa whose most significant bit is 1 and the rest is
3040        0, then the output has the same form. Otherwise, the output mantissa's most significant
3041        bit is 1 and the rest is unspecified.
3042        "#,
3043            &formats.binary,
3044        )
3045        .operands_in(vec![Operand::new("x", Float), Operand::new("y", Float)])
3046        .operands_out(vec![
3047            Operand::new("a", Float).with_doc("The smaller of ``x`` and ``y``"),
3048        ]),
3049    );
3050
3051    ig.push(
3052        Inst::new(
3053            "fmax",
3054            r#"
3055        Floating point maximum, propagating NaNs using the WebAssembly rules.
3056
3057        If either operand is NaN, this returns NaN with an unspecified sign. Furthermore, if
3058        each input NaN consists of a mantissa whose most significant bit is 1 and the rest is
3059        0, then the output has the same form. Otherwise, the output mantissa's most significant
3060        bit is 1 and the rest is unspecified.
3061        "#,
3062            &formats.binary,
3063        )
3064        .operands_in(vec![Operand::new("x", Float), Operand::new("y", Float)])
3065        .operands_out(vec![
3066            Operand::new("a", Float).with_doc("The larger of ``x`` and ``y``"),
3067        ]),
3068    );
3069
3070    ig.push(
3071        Inst::new(
3072            "ceil",
3073            r#"
3074        Round floating point round to integral, towards positive infinity.
3075        "#,
3076            &formats.unary,
3077        )
3078        .operands_in(vec![Operand::new("x", Float)])
3079        .operands_out(vec![
3080            Operand::new("a", Float).with_doc("``x`` rounded to integral value"),
3081        ]),
3082    );
3083
3084    ig.push(
3085        Inst::new(
3086            "floor",
3087            r#"
3088        Round floating point round to integral, towards negative infinity.
3089        "#,
3090            &formats.unary,
3091        )
3092        .operands_in(vec![Operand::new("x", Float)])
3093        .operands_out(vec![
3094            Operand::new("a", Float).with_doc("``x`` rounded to integral value"),
3095        ]),
3096    );
3097
3098    ig.push(
3099        Inst::new(
3100            "trunc",
3101            r#"
3102        Round floating point round to integral, towards zero.
3103        "#,
3104            &formats.unary,
3105        )
3106        .operands_in(vec![Operand::new("x", Float)])
3107        .operands_out(vec![
3108            Operand::new("a", Float).with_doc("``x`` rounded to integral value"),
3109        ]),
3110    );
3111
3112    ig.push(
3113        Inst::new(
3114            "nearest",
3115            r#"
3116        Round floating point round to integral, towards nearest with ties to
3117        even.
3118        "#,
3119            &formats.unary,
3120        )
3121        .operands_in(vec![Operand::new("x", Float)])
3122        .operands_out(vec![
3123            Operand::new("a", Float).with_doc("``x`` rounded to integral value"),
3124        ]),
3125    );
3126
3127    ig.push(
3128        Inst::new(
3129            "bitcast",
3130            r#"
3131        Reinterpret the bits in `x` as a different type.
3132
3133        The input and output types must be storable to memory and of the same
3134        size. A bitcast is equivalent to storing one type and loading the other
3135        type from the same address, both using the specified MemFlags.
3136
3137        Note that this operation only supports the `big` or `little` MemFlags.
3138        The specified byte order only affects the result in the case where
3139        input and output types differ in lane count/size.  In this case, the
3140        operation is only valid if a byte order specifier is provided.
3141        "#,
3142            &formats.load_no_offset,
3143        )
3144        .operands_in(vec![
3145            Operand::new("MemFlags", &imm.memflags),
3146            Operand::new("x", Mem),
3147        ])
3148        .operands_out(vec![
3149            Operand::new("a", MemTo).with_doc("Bits of `x` reinterpreted"),
3150        ]),
3151    );
3152
3153    ig.push(
3154        Inst::new(
3155            "scalar_to_vector",
3156            r#"
3157            Copies a scalar value to a vector value.  The scalar is copied into the
3158            least significant lane of the vector, and all other lanes will be zero.
3159            "#,
3160            &formats.unary,
3161        )
3162        .operands_in(vec![
3163            Operand::new("s", &TxN.lane_of()).with_doc("A scalar value"),
3164        ])
3165        .operands_out(vec![Operand::new("a", TxN).with_doc("A vector value")]),
3166    );
3167
3168    let Truthy = &TypeVar::new(
3169        "Truthy",
3170        "A scalar whose values are truthy",
3171        TypeSetBuilder::new().ints(Interval::All).build(),
3172    );
3173    let IntTo = &TypeVar::new(
3174        "IntTo",
3175        "An integer type",
3176        TypeSetBuilder::new().ints(Interval::All).build(),
3177    );
3178
3179    ig.push(
3180        Inst::new(
3181            "bmask",
3182            r#"
3183        Convert `x` to an integer mask.
3184
3185        Non-zero maps to all 1s and zero maps to all 0s.
3186        "#,
3187            &formats.unary,
3188        )
3189        .operands_in(vec![Operand::new("x", Truthy)])
3190        .operands_out(vec![Operand::new("a", IntTo)]),
3191    );
3192
3193    let Int = &TypeVar::new(
3194        "Int",
3195        "A scalar integer type",
3196        TypeSetBuilder::new().ints(Interval::All).build(),
3197    );
3198
3199    ig.push(
3200        Inst::new(
3201            "ireduce",
3202            r#"
3203        Convert `x` to a smaller integer type by discarding
3204        the most significant bits.
3205
3206        This is the same as reducing modulo `2^n`.
3207        "#,
3208            &formats.unary,
3209        )
3210        .operands_in(vec![
3211            Operand::new("x", &Int.wider())
3212                .with_doc("A scalar integer type, wider than the controlling type"),
3213        ])
3214        .operands_out(vec![Operand::new("a", Int)]),
3215    );
3216
3217    let I16or32or64xN = &TypeVar::new(
3218        "I16or32or64xN",
3219        "A SIMD vector type containing integer lanes 16, 32, or 64 bits wide",
3220        TypeSetBuilder::new()
3221            .ints(16..64)
3222            .simd_lanes(2..8)
3223            .dynamic_simd_lanes(2..8)
3224            .includes_scalars(false)
3225            .build(),
3226    );
3227
3228    ig.push(
3229        Inst::new(
3230            "snarrow",
3231            r#"
3232        Combine `x` and `y` into a vector with twice the lanes but half the integer width while
3233        saturating overflowing values to the signed maximum and minimum.
3234
3235        The lanes will be concatenated after narrowing. For example, when `x` and `y` are `i32x4`
3236        and `x = [x3, x2, x1, x0]` and `y = [y3, y2, y1, y0]`, then after narrowing the value
3237        returned is an `i16x8`: `a = [y3', y2', y1', y0', x3', x2', x1', x0']`.
3238            "#,
3239            &formats.binary,
3240        )
3241        .operands_in(vec![
3242            Operand::new("x", I16or32or64xN),
3243            Operand::new("y", I16or32or64xN),
3244        ])
3245        .operands_out(vec![Operand::new("a", &I16or32or64xN.split_lanes())]),
3246    );
3247
3248    ig.push(
3249        Inst::new(
3250            "unarrow",
3251            r#"
3252        Combine `x` and `y` into a vector with twice the lanes but half the integer width while
3253        saturating overflowing values to the unsigned maximum and minimum.
3254
3255        Note that all input lanes are considered signed: any negative lanes will overflow and be
3256        replaced with the unsigned minimum, `0x00`.
3257
3258        The lanes will be concatenated after narrowing. For example, when `x` and `y` are `i32x4`
3259        and `x = [x3, x2, x1, x0]` and `y = [y3, y2, y1, y0]`, then after narrowing the value
3260        returned is an `i16x8`: `a = [y3', y2', y1', y0', x3', x2', x1', x0']`.
3261            "#,
3262            &formats.binary,
3263        )
3264        .operands_in(vec![
3265            Operand::new("x", I16or32or64xN),
3266            Operand::new("y", I16or32or64xN),
3267        ])
3268        .operands_out(vec![Operand::new("a", &I16or32or64xN.split_lanes())]),
3269    );
3270
3271    ig.push(
3272        Inst::new(
3273            "uunarrow",
3274            r#"
3275        Combine `x` and `y` into a vector with twice the lanes but half the integer width while
3276        saturating overflowing values to the unsigned maximum and minimum.
3277
3278        Note that all input lanes are considered unsigned: any negative values will be interpreted as unsigned, overflowing and being replaced with the unsigned maximum.
3279
3280        The lanes will be concatenated after narrowing. For example, when `x` and `y` are `i32x4`
3281        and `x = [x3, x2, x1, x0]` and `y = [y3, y2, y1, y0]`, then after narrowing the value
3282        returned is an `i16x8`: `a = [y3', y2', y1', y0', x3', x2', x1', x0']`.
3283            "#,
3284            &formats.binary,
3285        )
3286        .operands_in(vec![Operand::new("x", I16or32or64xN), Operand::new("y", I16or32or64xN)])
3287        .operands_out(vec![Operand::new("a", &I16or32or64xN.split_lanes())]),
3288    );
3289
3290    let I8or16or32xN = &TypeVar::new(
3291        "I8or16or32xN",
3292        "A SIMD vector type containing integer lanes 8, 16, or 32 bits wide.",
3293        TypeSetBuilder::new()
3294            .ints(8..32)
3295            .simd_lanes(2..16)
3296            .dynamic_simd_lanes(2..16)
3297            .includes_scalars(false)
3298            .build(),
3299    );
3300
3301    ig.push(
3302        Inst::new(
3303            "swiden_low",
3304            r#"
3305        Widen the low lanes of `x` using signed extension.
3306
3307        This will double the lane width and halve the number of lanes.
3308            "#,
3309            &formats.unary,
3310        )
3311        .operands_in(vec![Operand::new("x", I8or16or32xN)])
3312        .operands_out(vec![Operand::new("a", &I8or16or32xN.merge_lanes())]),
3313    );
3314
3315    ig.push(
3316        Inst::new(
3317            "swiden_high",
3318            r#"
3319        Widen the high lanes of `x` using signed extension.
3320
3321        This will double the lane width and halve the number of lanes.
3322            "#,
3323            &formats.unary,
3324        )
3325        .operands_in(vec![Operand::new("x", I8or16or32xN)])
3326        .operands_out(vec![Operand::new("a", &I8or16or32xN.merge_lanes())]),
3327    );
3328
3329    ig.push(
3330        Inst::new(
3331            "uwiden_low",
3332            r#"
3333        Widen the low lanes of `x` using unsigned extension.
3334
3335        This will double the lane width and halve the number of lanes.
3336            "#,
3337            &formats.unary,
3338        )
3339        .operands_in(vec![Operand::new("x", I8or16or32xN)])
3340        .operands_out(vec![Operand::new("a", &I8or16or32xN.merge_lanes())]),
3341    );
3342
3343    ig.push(
3344        Inst::new(
3345            "uwiden_high",
3346            r#"
3347            Widen the high lanes of `x` using unsigned extension.
3348
3349            This will double the lane width and halve the number of lanes.
3350            "#,
3351            &formats.unary,
3352        )
3353        .operands_in(vec![Operand::new("x", I8or16or32xN)])
3354        .operands_out(vec![Operand::new("a", &I8or16or32xN.merge_lanes())]),
3355    );
3356
3357    ig.push(
3358        Inst::new(
3359            "iadd_pairwise",
3360            r#"
3361        Does lane-wise integer pairwise addition on two operands, putting the
3362        combined results into a single vector result. Here a pair refers to adjacent
3363        lanes in a vector, i.e. i*2 + (i*2+1) for i == num_lanes/2. The first operand
3364        pairwise add results will make up the low half of the resulting vector while
3365        the second operand pairwise add results will make up the upper half of the
3366        resulting vector.
3367            "#,
3368            &formats.binary,
3369        )
3370        .operands_in(vec![
3371            Operand::new("x", I8or16or32xN),
3372            Operand::new("y", I8or16or32xN),
3373        ])
3374        .operands_out(vec![Operand::new("a", I8or16or32xN)]),
3375    );
3376
3377    let I8x16 = &TypeVar::new(
3378        "I8x16",
3379        "A SIMD vector type consisting of 16 lanes of 8-bit integers",
3380        TypeSetBuilder::new()
3381            .ints(8..8)
3382            .simd_lanes(16..16)
3383            .includes_scalars(false)
3384            .build(),
3385    );
3386
3387    ig.push(
3388        Inst::new(
3389            "x86_pmaddubsw",
3390            r#"
3391        An instruction with equivalent semantics to `pmaddubsw` on x86.
3392
3393        This instruction will take signed bytes from the first argument and
3394        multiply them against unsigned bytes in the second argument. Adjacent
3395        pairs are then added, with saturating, to a 16-bit value and are packed
3396        into the result.
3397            "#,
3398            &formats.binary,
3399        )
3400        .operands_in(vec![Operand::new("x", I8x16), Operand::new("y", I8x16)])
3401        .operands_out(vec![Operand::new("a", I16x8)]),
3402    );
3403
3404    ig.push(
3405        Inst::new(
3406            "uextend",
3407            r#"
3408        Convert `x` to a larger integer type by zero-extending.
3409
3410        Each lane in `x` is converted to a larger integer type by adding
3411        zeroes. The result has the same numerical value as `x` when both are
3412        interpreted as unsigned integers.
3413
3414        The result type must have the same number of vector lanes as the input,
3415        and each lane must not have fewer bits that the input lanes. If the
3416        input and output types are the same, this is a no-op.
3417        "#,
3418            &formats.unary,
3419        )
3420        .operands_in(vec![Operand::new("x", &Int.narrower()).with_doc(
3421            "A scalar integer type, narrower than the controlling type",
3422        )])
3423        .operands_out(vec![Operand::new("a", Int)]),
3424    );
3425
3426    ig.push(
3427        Inst::new(
3428            "sextend",
3429            r#"
3430        Convert `x` to a larger integer type by sign-extending.
3431
3432        Each lane in `x` is converted to a larger integer type by replicating
3433        the sign bit. The result has the same numerical value as `x` when both
3434        are interpreted as signed integers.
3435
3436        The result type must have the same number of vector lanes as the input,
3437        and each lane must not have fewer bits that the input lanes. If the
3438        input and output types are the same, this is a no-op.
3439        "#,
3440            &formats.unary,
3441        )
3442        .operands_in(vec![Operand::new("x", &Int.narrower()).with_doc(
3443            "A scalar integer type, narrower than the controlling type",
3444        )])
3445        .operands_out(vec![Operand::new("a", Int)]),
3446    );
3447
3448    let FloatScalar = &TypeVar::new(
3449        "FloatScalar",
3450        "A scalar only floating point number",
3451        TypeSetBuilder::new().floats(Interval::All).build(),
3452    );
3453
3454    ig.push(
3455        Inst::new(
3456            "fpromote",
3457            r#"
3458        Convert `x` to a larger floating point format.
3459
3460        Each lane in `x` is converted to the destination floating point format.
3461        This is an exact operation.
3462
3463        Cranelift currently only supports two floating point formats
3464        - `f32` and `f64`. This may change in the future.
3465
3466        The result type must have the same number of vector lanes as the input,
3467        and the result lanes must not have fewer bits than the input lanes.
3468        "#,
3469            &formats.unary,
3470        )
3471        .operands_in(vec![Operand::new("x", &FloatScalar.narrower()).with_doc(
3472            "A scalar only floating point number, narrower than the controlling type",
3473        )])
3474        .operands_out(vec![Operand::new("a", FloatScalar)]),
3475    );
3476
3477    ig.push(
3478        Inst::new(
3479            "fdemote",
3480            r#"
3481        Convert `x` to a smaller floating point format.
3482
3483        Each lane in `x` is converted to the destination floating point format
3484        by rounding to nearest, ties to even.
3485
3486        Cranelift currently only supports two floating point formats
3487        - `f32` and `f64`. This may change in the future.
3488
3489        The result type must have the same number of vector lanes as the input,
3490        and the result lanes must not have more bits than the input lanes.
3491        "#,
3492            &formats.unary,
3493        )
3494        .operands_in(vec![Operand::new("x", &FloatScalar.wider()).with_doc(
3495            "A scalar only floating point number, wider than the controlling type",
3496        )])
3497        .operands_out(vec![Operand::new("a", FloatScalar)]),
3498    );
3499
3500    let F64x2 = &TypeVar::new(
3501        "F64x2",
3502        "A SIMD vector type consisting of 2 lanes of 64-bit floats",
3503        TypeSetBuilder::new()
3504            .floats(64..64)
3505            .simd_lanes(2..2)
3506            .includes_scalars(false)
3507            .build(),
3508    );
3509    let F32x4 = &TypeVar::new(
3510        "F32x4",
3511        "A SIMD vector type consisting of 4 lanes of 32-bit floats",
3512        TypeSetBuilder::new()
3513            .floats(32..32)
3514            .simd_lanes(4..4)
3515            .includes_scalars(false)
3516            .build(),
3517    );
3518
3519    ig.push(
3520        Inst::new(
3521            "fvdemote",
3522            r#"
3523                Convert `x` to a smaller floating point format.
3524
3525                Each lane in `x` is converted to the destination floating point format
3526                by rounding to nearest, ties to even.
3527
3528                Cranelift currently only supports two floating point formats
3529                - `f32` and `f64`. This may change in the future.
3530
3531                Fvdemote differs from fdemote in that with fvdemote it targets vectors.
3532                Fvdemote is constrained to having the input type being F64x2 and the result
3533                type being F32x4. The result lane that was the upper half of the input lane
3534                is initialized to zero.
3535                "#,
3536            &formats.unary,
3537        )
3538        .operands_in(vec![Operand::new("x", F64x2)])
3539        .operands_out(vec![Operand::new("a", F32x4)]),
3540    );
3541
3542    ig.push(
3543        Inst::new(
3544            "fvpromote_low",
3545            r#"
3546        Converts packed single precision floating point to packed double precision floating point.
3547
3548        Considering only the lower half of the register, the low lanes in `x` are interpreted as
3549        single precision floats that are then converted to a double precision floats.
3550
3551        The result type will have half the number of vector lanes as the input. Fvpromote_low is
3552        constrained to input F32x4 with a result type of F64x2.
3553        "#,
3554            &formats.unary,
3555        )
3556        .operands_in(vec![Operand::new("a", F32x4)])
3557        .operands_out(vec![Operand::new("x", F64x2)]),
3558    );
3559
3560    let IntTo = &TypeVar::new(
3561        "IntTo",
3562        "An scalar only integer type",
3563        TypeSetBuilder::new().ints(Interval::All).build(),
3564    );
3565
3566    ig.push(
3567        Inst::new(
3568            "fcvt_to_uint",
3569            r#"
3570        Converts floating point scalars to unsigned integer.
3571
3572        Only operates on `x` if it is a scalar. If `x` is NaN or if
3573        the unsigned integral value cannot be represented in the result
3574        type, this instruction traps.
3575
3576        "#,
3577            &formats.unary,
3578        )
3579        .operands_in(vec![Operand::new("x", FloatScalar)])
3580        .operands_out(vec![Operand::new("a", IntTo)])
3581        .can_trap()
3582        .side_effects_idempotent(),
3583    );
3584
3585    ig.push(
3586        Inst::new(
3587            "fcvt_to_sint",
3588            r#"
3589        Converts floating point scalars to signed integer.
3590
3591        Only operates on `x` if it is a scalar. If `x` is NaN or if
3592        the unsigned integral value cannot be represented in the result
3593        type, this instruction traps.
3594
3595        "#,
3596            &formats.unary,
3597        )
3598        .operands_in(vec![Operand::new("x", FloatScalar)])
3599        .operands_out(vec![Operand::new("a", IntTo)])
3600        .can_trap()
3601        .side_effects_idempotent(),
3602    );
3603
3604    let IntTo = &TypeVar::new(
3605        "IntTo",
3606        "A larger integer type with the same number of lanes",
3607        TypeSetBuilder::new()
3608            .ints(Interval::All)
3609            .simd_lanes(Interval::All)
3610            .build(),
3611    );
3612
3613    ig.push(
3614        Inst::new(
3615            "fcvt_to_uint_sat",
3616            r#"
3617        Convert floating point to unsigned integer as fcvt_to_uint does, but
3618        saturates the input instead of trapping. NaN and negative values are
3619        converted to 0.
3620        "#,
3621            &formats.unary,
3622        )
3623        .operands_in(vec![Operand::new("x", Float)])
3624        .operands_out(vec![Operand::new("a", IntTo)]),
3625    );
3626
3627    ig.push(
3628        Inst::new(
3629            "fcvt_to_sint_sat",
3630            r#"
3631        Convert floating point to signed integer as fcvt_to_sint does, but
3632        saturates the input instead of trapping. NaN values are converted to 0.
3633        "#,
3634            &formats.unary,
3635        )
3636        .operands_in(vec![Operand::new("x", Float)])
3637        .operands_out(vec![Operand::new("a", IntTo)]),
3638    );
3639
3640    ig.push(
3641        Inst::new(
3642            "x86_cvtt2dq",
3643            r#"
3644        A float-to-integer conversion instruction for vectors-of-floats which
3645        has the same semantics as `cvttp{s,d}2dq` on x86. This specifically
3646        returns `INT_MIN` for NaN or out-of-bounds lanes.
3647        "#,
3648            &formats.unary,
3649        )
3650        .operands_in(vec![Operand::new("x", Float)])
3651        .operands_out(vec![Operand::new("a", IntTo)]),
3652    );
3653
3654    let Int = &TypeVar::new(
3655        "Int",
3656        "A scalar or vector integer type",
3657        TypeSetBuilder::new()
3658            .ints(Interval::All)
3659            .simd_lanes(Interval::All)
3660            .build(),
3661    );
3662
3663    let FloatTo = &TypeVar::new(
3664        "FloatTo",
3665        "A scalar or vector floating point number",
3666        TypeSetBuilder::new()
3667            .floats(Interval::All)
3668            .simd_lanes(Interval::All)
3669            .build(),
3670    );
3671
3672    ig.push(
3673        Inst::new(
3674            "fcvt_from_uint",
3675            r#"
3676        Convert unsigned integer to floating point.
3677
3678        Each lane in `x` is interpreted as an unsigned integer and converted to
3679        floating point using round to nearest, ties to even.
3680
3681        The result type must have the same number of vector lanes as the input.
3682        "#,
3683            &formats.unary,
3684        )
3685        .operands_in(vec![Operand::new("x", Int)])
3686        .operands_out(vec![Operand::new("a", FloatTo)]),
3687    );
3688
3689    ig.push(
3690        Inst::new(
3691            "fcvt_from_sint",
3692            r#"
3693        Convert signed integer to floating point.
3694
3695        Each lane in `x` is interpreted as a signed integer and converted to
3696        floating point using round to nearest, ties to even.
3697
3698        The result type must have the same number of vector lanes as the input.
3699        "#,
3700            &formats.unary,
3701        )
3702        .operands_in(vec![Operand::new("x", Int)])
3703        .operands_out(vec![Operand::new("a", FloatTo)]),
3704    );
3705
3706    let WideInt = &TypeVar::new(
3707        "WideInt",
3708        "An integer type of width `i16` upwards",
3709        TypeSetBuilder::new().ints(16..128).build(),
3710    );
3711
3712    ig.push(
3713        Inst::new(
3714            "isplit",
3715            r#"
3716        Split an integer into low and high parts.
3717
3718        Vectors of integers are split lane-wise, so the results have the same
3719        number of lanes as the input, but the lanes are half the size.
3720
3721        Returns the low half of `x` and the high half of `x` as two independent
3722        values.
3723        "#,
3724            &formats.unary,
3725        )
3726        .operands_in(vec![Operand::new("x", WideInt)])
3727        .operands_out(vec![
3728            Operand::new("lo", &WideInt.half_width()).with_doc("The low bits of `x`"),
3729            Operand::new("hi", &WideInt.half_width()).with_doc("The high bits of `x`"),
3730        ]),
3731    );
3732
3733    ig.push(
3734        Inst::new(
3735            "iconcat",
3736            r#"
3737        Concatenate low and high bits to form a larger integer type.
3738
3739        Vectors of integers are concatenated lane-wise such that the result has
3740        the same number of lanes as the inputs, but the lanes are twice the
3741        size.
3742        "#,
3743            &formats.binary,
3744        )
3745        .operands_in(vec![
3746            Operand::new("lo", NarrowInt),
3747            Operand::new("hi", NarrowInt),
3748        ])
3749        .operands_out(vec![
3750            Operand::new("a", &NarrowInt.double_width())
3751                .with_doc("The concatenation of `lo` and `hi`"),
3752        ]),
3753    );
3754
3755    // Instructions relating to atomic memory accesses and fences
3756    let AtomicMem = &TypeVar::new(
3757        "AtomicMem",
3758        "Any type that can be stored in memory, which can be used in an atomic operation",
3759        TypeSetBuilder::new().ints(8..128).build(),
3760    );
3761
3762    ig.push(
3763        Inst::new(
3764            "atomic_rmw",
3765            r#"
3766        Atomically read-modify-write memory at `p`, with second operand `x`.  The old value is
3767        returned.  `p` has the type of the target word size, and `x` may be any integer type; note
3768        that some targets require specific target features to be enabled in order to support 128-bit
3769        integer atomics.  The type of the returned value is the same as the type of `x`.  This
3770        operation is sequentially consistent and creates happens-before edges that order normal
3771        (non-atomic) loads and stores.
3772        "#,
3773            &formats.atomic_rmw,
3774        )
3775        .operands_in(vec![
3776            Operand::new("MemFlags", &imm.memflags),
3777            Operand::new("AtomicRmwOp", &imm.atomic_rmw_op),
3778            Operand::new("p", iAddr),
3779            Operand::new("x", AtomicMem).with_doc("Value to be atomically stored"),
3780        ])
3781        .operands_out(vec![
3782            Operand::new("a", AtomicMem).with_doc("Value atomically loaded"),
3783        ])
3784        .can_load()
3785        .can_store()
3786        .other_side_effects(),
3787    );
3788
3789    ig.push(
3790        Inst::new(
3791            "atomic_cas",
3792            r#"
3793        Perform an atomic compare-and-swap operation on memory at `p`, with expected value `e`,
3794        storing `x` if the value at `p` equals `e`.  The old value at `p` is returned,
3795        regardless of whether the operation succeeds or fails.  `p` has the type of the target
3796        word size, and `x` and `e` must have the same type and the same size, which may be any
3797        integer type; note that some targets require specific target features to be enabled in order
3798        to support 128-bit integer atomics.  The type of the returned value is the same as the type
3799        of `x` and `e`.  This operation is sequentially consistent and creates happens-before edges
3800        that order normal (non-atomic) loads and stores.
3801        "#,
3802            &formats.atomic_cas,
3803        )
3804        .operands_in(vec![
3805            Operand::new("MemFlags", &imm.memflags),
3806            Operand::new("p", iAddr),
3807            Operand::new("e", AtomicMem).with_doc("Expected value in CAS"),
3808            Operand::new("x", AtomicMem).with_doc("Value to be atomically stored"),
3809        ])
3810        .operands_out(vec![
3811            Operand::new("a", AtomicMem).with_doc("Value atomically loaded"),
3812        ])
3813        .can_load()
3814        .can_store()
3815        .other_side_effects(),
3816    );
3817
3818    ig.push(
3819        Inst::new(
3820            "atomic_load",
3821            r#"
3822        Atomically load from memory at `p`.
3823
3824        This is a polymorphic instruction that can load any value type which has a memory
3825        representation.  It can only be used for integer types; note that some targets require
3826        specific target features to be enabled in order to support 128-bit integer atomics. This
3827        operation is sequentially consistent and creates happens-before edges that order normal
3828        (non-atomic) loads and stores.
3829        "#,
3830            &formats.load_no_offset,
3831        )
3832        .operands_in(vec![
3833            Operand::new("MemFlags", &imm.memflags),
3834            Operand::new("p", iAddr),
3835        ])
3836        .operands_out(vec![
3837            Operand::new("a", AtomicMem).with_doc("Value atomically loaded"),
3838        ])
3839        .can_load()
3840        .other_side_effects(),
3841    );
3842
3843    ig.push(
3844        Inst::new(
3845            "atomic_store",
3846            r#"
3847        Atomically store `x` to memory at `p`.
3848
3849        This is a polymorphic instruction that can store any value type with a memory
3850        representation.  It can only be used for integer types; note that some targets require
3851        specific target features to be enabled in order to support 128-bit integer atomics This
3852        operation is sequentially consistent and creates happens-before edges that order normal
3853        (non-atomic) loads and stores.
3854        "#,
3855            &formats.store_no_offset,
3856        )
3857        .operands_in(vec![
3858            Operand::new("MemFlags", &imm.memflags),
3859            Operand::new("x", AtomicMem).with_doc("Value to be atomically stored"),
3860            Operand::new("p", iAddr),
3861        ])
3862        .can_store()
3863        .other_side_effects(),
3864    );
3865
3866    ig.push(
3867        Inst::new(
3868            "fence",
3869            r#"
3870        A memory fence.  This must provide ordering to ensure that, at a minimum, neither loads
3871        nor stores of any kind may move forwards or backwards across the fence.  This operation
3872        is sequentially consistent.
3873        "#,
3874            &formats.nullary,
3875        )
3876        .other_side_effects(),
3877    );
3878
3879    let TxN = &TypeVar::new(
3880        "TxN",
3881        "A dynamic vector type",
3882        TypeSetBuilder::new()
3883            .ints(Interval::All)
3884            .floats(Interval::All)
3885            .dynamic_simd_lanes(Interval::All)
3886            .build(),
3887    );
3888
3889    ig.push(
3890        Inst::new(
3891            "extract_vector",
3892            r#"
3893        Return a fixed length sub vector, extracted from a dynamic vector.
3894        "#,
3895            &formats.binary_imm8,
3896        )
3897        .operands_in(vec![
3898            Operand::new("x", TxN).with_doc("The dynamic vector to extract from"),
3899            Operand::new("y", &imm.uimm8).with_doc("128-bit vector index"),
3900        ])
3901        .operands_out(vec![
3902            Operand::new("a", &TxN.dynamic_to_vector()).with_doc("New fixed vector"),
3903        ]),
3904    );
3905}