cranelift_codegen_meta/shared/
instructions.rs

1#![expect(non_snake_case, reason = "DSL style here")]
2
3use crate::cdsl::instructions::{
4    AllInstructions, InstructionBuilder as Inst, InstructionGroupBuilder,
5};
6use crate::cdsl::operands::Operand;
7use crate::cdsl::types::{LaneType, ValueType};
8use crate::cdsl::typevar::{Interval, TypeSetBuilder, TypeVar};
9use crate::shared::formats::Formats;
10use crate::shared::types;
11use crate::shared::{entities::EntityRefs, immediates::Immediates};
12
13#[inline(never)]
14fn define_control_flow(
15    ig: &mut InstructionGroupBuilder,
16    formats: &Formats,
17    imm: &Immediates,
18    entities: &EntityRefs,
19) {
20    ig.push(
21        Inst::new(
22            "jump",
23            r#"
24        Jump.
25
26        Unconditionally jump to a basic block, passing the specified
27        block arguments. The number and types of arguments must match the
28        destination block.
29        "#,
30            &formats.jump,
31        )
32        .operands_in(vec![
33            Operand::new("block_call", &entities.block_call)
34                .with_doc("Destination basic block, with its arguments provided"),
35        ])
36        .branches(),
37    );
38
39    let ScalarTruthy = &TypeVar::new(
40        "ScalarTruthy",
41        "A scalar truthy type",
42        TypeSetBuilder::new().ints(Interval::All).build(),
43    );
44
45    ig.push(
46        Inst::new(
47            "brif",
48            r#"
49        Conditional branch when cond is non-zero.
50
51        Take the ``then`` branch when ``c != 0``, and the ``else`` branch otherwise.
52        "#,
53            &formats.brif,
54        )
55        .operands_in(vec![
56            Operand::new("c", ScalarTruthy).with_doc("Controlling value to test"),
57            Operand::new("block_then", &entities.block_then).with_doc("Then block"),
58            Operand::new("block_else", &entities.block_else).with_doc("Else block"),
59        ])
60        .branches(),
61    );
62
63    {
64        let _i32 = &TypeVar::new(
65            "i32",
66            "A 32 bit scalar integer type",
67            TypeSetBuilder::new().ints(32..32).build(),
68        );
69
70        ig.push(
71            Inst::new(
72                "br_table",
73                r#"
74        Indirect branch via jump table.
75
76        Use ``x`` as an unsigned index into the jump table ``JT``. If a jump
77        table entry is found, branch to the corresponding block. If no entry was
78        found or the index is out-of-bounds, branch to the default block of the
79        table.
80
81        Note that this branch instruction can't pass arguments to the targeted
82        blocks. Split critical edges as needed to work around this.
83
84        Do not confuse this with "tables" in WebAssembly. ``br_table`` is for
85        jump tables with destinations within the current function only -- think
86        of a ``match`` in Rust or a ``switch`` in C.  If you want to call a
87        function in a dynamic library, that will typically use
88        ``call_indirect``.
89        "#,
90                &formats.branch_table,
91            )
92            .operands_in(vec![
93                Operand::new("x", _i32).with_doc("i32 index into jump table"),
94                Operand::new("JT", &entities.jump_table),
95            ])
96            .branches(),
97        );
98    }
99
100    let iAddr = &TypeVar::new(
101        "iAddr",
102        "An integer address type",
103        TypeSetBuilder::new().ints(32..64).build(),
104    );
105
106    ig.push(
107        Inst::new(
108            "debugtrap",
109            r#"
110        Encodes an assembly debug trap.
111        "#,
112            &formats.nullary,
113        )
114        .other_side_effects()
115        .can_load()
116        .can_store(),
117    );
118
119    ig.push(
120        Inst::new(
121            "trap",
122            r#"
123        Terminate execution unconditionally.
124        "#,
125            &formats.trap,
126        )
127        .operands_in(vec![Operand::new("code", &imm.trapcode)])
128        .can_trap()
129        .terminates_block(),
130    );
131
132    ig.push(
133        Inst::new(
134            "trapz",
135            r#"
136        Trap when zero.
137
138        if ``c`` is non-zero, execution continues at the following instruction.
139        "#,
140            &formats.cond_trap,
141        )
142        .operands_in(vec![
143            Operand::new("c", ScalarTruthy).with_doc("Controlling value to test"),
144            Operand::new("code", &imm.trapcode),
145        ])
146        .can_trap()
147        // When one `trapz` dominates another `trapz` and they have identical
148        // conditions and trap codes, it is safe to deduplicate them (like GVN,
149        // although there is not actually any value being numbered). Either the
150        // first `trapz` raised a trap and execution halted, or it didn't and
151        // therefore the dominated `trapz` will not raise a trap either.
152        .side_effects_idempotent(),
153    );
154
155    ig.push(
156        Inst::new(
157            "trapnz",
158            r#"
159        Trap when non-zero.
160
161        If ``c`` is zero, execution continues at the following instruction.
162        "#,
163            &formats.cond_trap,
164        )
165        .operands_in(vec![
166            Operand::new("c", ScalarTruthy).with_doc("Controlling value to test"),
167            Operand::new("code", &imm.trapcode),
168        ])
169        .can_trap()
170        // See the above comment for `trapz` and idempotent side effects.
171        .side_effects_idempotent(),
172    );
173
174    ig.push(
175        Inst::new(
176            "return",
177            r#"
178        Return from the function.
179
180        Unconditionally transfer control to the calling function, passing the
181        provided return values. The list of return values must match the
182        function signature's return types.
183        "#,
184            &formats.multiary,
185        )
186        .operands_in(vec![
187            Operand::new("rvals", &entities.varargs).with_doc("return values"),
188        ])
189        .returns(),
190    );
191
192    ig.push(
193        Inst::new(
194            "call",
195            r#"
196        Direct function call.
197
198        Call a function which has been declared in the preamble. The argument
199        types must match the function's signature.
200        "#,
201            &formats.call,
202        )
203        .operands_in(vec![
204            Operand::new("FN", &entities.func_ref)
205                .with_doc("function to call, declared by `function`"),
206            Operand::new("args", &entities.varargs).with_doc("call arguments"),
207        ])
208        .operands_out(vec![
209            Operand::new("rvals", &entities.varargs).with_doc("return values"),
210        ])
211        .call(),
212    );
213
214    ig.push(
215        Inst::new(
216            "call_indirect",
217            r#"
218        Indirect function call.
219
220        Call the function pointed to by `callee` with the given arguments. The
221        called function must match the specified signature.
222
223        Note that this is different from WebAssembly's ``call_indirect``; the
224        callee is a native address, rather than a table index. For WebAssembly,
225        `table_addr` and `load` are used to obtain a native address
226        from a table.
227        "#,
228            &formats.call_indirect,
229        )
230        .operands_in(vec![
231            Operand::new("SIG", &entities.sig_ref).with_doc("function signature"),
232            Operand::new("callee", iAddr).with_doc("address of function to call"),
233            Operand::new("args", &entities.varargs).with_doc("call arguments"),
234        ])
235        .operands_out(vec![
236            Operand::new("rvals", &entities.varargs).with_doc("return values"),
237        ])
238        .call(),
239    );
240
241    ig.push(
242        Inst::new(
243            "return_call",
244            r#"
245        Direct tail call.
246
247        Tail call a function which has been declared in the preamble. The
248        argument types must match the function's signature, the caller and
249        callee calling conventions must be the same, and must be a calling
250        convention that supports tail calls.
251
252        This instruction is a block terminator.
253        "#,
254            &formats.call,
255        )
256        .operands_in(vec![
257            Operand::new("FN", &entities.func_ref)
258                .with_doc("function to call, declared by `function`"),
259            Operand::new("args", &entities.varargs).with_doc("call arguments"),
260        ])
261        .returns()
262        .call(),
263    );
264
265    ig.push(
266        Inst::new(
267            "return_call_indirect",
268            r#"
269        Indirect tail call.
270
271        Call the function pointed to by `callee` with the given arguments. The
272        argument types must match the function's signature, the caller and
273        callee calling conventions must be the same, and must be a calling
274        convention that supports tail calls.
275
276        This instruction is a block terminator.
277
278        Note that this is different from WebAssembly's ``tail_call_indirect``;
279        the callee is a native address, rather than a table index. For
280        WebAssembly, `table_addr` and `load` are used to obtain a native address
281        from a table.
282        "#,
283            &formats.call_indirect,
284        )
285        .operands_in(vec![
286            Operand::new("SIG", &entities.sig_ref).with_doc("function signature"),
287            Operand::new("callee", iAddr).with_doc("address of function to call"),
288            Operand::new("args", &entities.varargs).with_doc("call arguments"),
289        ])
290        .returns()
291        .call(),
292    );
293
294    ig.push(
295        Inst::new(
296            "func_addr",
297            r#"
298        Get the address of a function.
299
300        Compute the absolute address of a function declared in the preamble.
301        The returned address can be used as a ``callee`` argument to
302        `call_indirect`. This is also a method for calling functions that
303        are too far away to be addressable by a direct `call`
304        instruction.
305        "#,
306            &formats.func_addr,
307        )
308        .operands_in(vec![
309            Operand::new("FN", &entities.func_ref)
310                .with_doc("function to call, declared by `function`"),
311        ])
312        .operands_out(vec![Operand::new("addr", iAddr)]),
313    );
314
315    ig.push(
316        Inst::new(
317            "try_call",
318            r#"
319        Call a function, catching the specified exceptions.
320
321        Call the function pointed to by `callee` with the given arguments. On
322        normal return, branch to the first target, with function returns
323        available as `retN` block arguments. On exceptional return,
324        look up the thrown exception tag in the provided exception table;
325        if the tag matches one of the targets, branch to the matching
326        target with the exception payloads available as `exnN` block arguments.
327        If no tag matches, then propagate the exception up the stack.
328
329        It is the Cranelift embedder's responsibility to define the meaning
330        of tags: they are accepted by this instruction and passed through
331        to unwind metadata tables in Cranelift's output. Actual unwinding is
332        outside the purview of the core Cranelift compiler.
333
334        Payload values on exception are passed in fixed register(s) that are
335        defined by the platform and ABI. See the documentation on `CallConv`
336        for details.
337        "#,
338            &formats.try_call,
339        )
340        .operands_in(vec![
341            Operand::new("callee", &entities.func_ref)
342                .with_doc("function to call, declared by `function`"),
343            Operand::new("args", &entities.varargs).with_doc("call arguments"),
344            Operand::new("ET", &entities.exception_table).with_doc("exception table"),
345        ])
346        .call()
347        .branches(),
348    );
349
350    ig.push(
351        Inst::new(
352            "try_call_indirect",
353            r#"
354        Call a function, catching the specified exceptions.
355
356        Call the function pointed to by `callee` with the given arguments. On
357        normal return, branch to the first target, with function returns
358        available as `retN` block arguments. On exceptional return,
359        look up the thrown exception tag in the provided exception table;
360        if the tag matches one of the targets, branch to the matching
361        target with the exception payloads available as `exnN` block arguments.
362        If no tag matches, then propagate the exception up the stack.
363
364        It is the Cranelift embedder's responsibility to define the meaning
365        of tags: they are accepted by this instruction and passed through
366        to unwind metadata tables in Cranelift's output. Actual unwinding is
367        outside the purview of the core Cranelift compiler.
368
369        Payload values on exception are passed in fixed register(s) that are
370        defined by the platform and ABI. See the documentation on `CallConv`
371        for details.
372        "#,
373            &formats.try_call_indirect,
374        )
375        .operands_in(vec![
376            Operand::new("callee", iAddr).with_doc("address of function to call"),
377            Operand::new("args", &entities.varargs).with_doc("call arguments"),
378            Operand::new("ET", &entities.exception_table).with_doc("exception table"),
379        ])
380        .call()
381        .branches(),
382    );
383}
384
385#[inline(never)]
386fn define_simd_lane_access(
387    ig: &mut InstructionGroupBuilder,
388    formats: &Formats,
389    imm: &Immediates,
390    _: &EntityRefs,
391) {
392    let TxN = &TypeVar::new(
393        "TxN",
394        "A SIMD vector type",
395        TypeSetBuilder::new()
396            .ints(Interval::All)
397            .floats(Interval::All)
398            .simd_lanes(Interval::All)
399            .dynamic_simd_lanes(Interval::All)
400            .includes_scalars(false)
401            .build(),
402    );
403
404    ig.push(
405        Inst::new(
406            "splat",
407            r#"
408        Vector splat.
409
410        Return a vector whose lanes are all ``x``.
411        "#,
412            &formats.unary,
413        )
414        .operands_in(vec![
415            Operand::new("x", &TxN.lane_of()).with_doc("Value to splat to all lanes"),
416        ])
417        .operands_out(vec![Operand::new("a", TxN)]),
418    );
419
420    let I8x16 = &TypeVar::new(
421        "I8x16",
422        "A SIMD vector type consisting of 16 lanes of 8-bit integers",
423        TypeSetBuilder::new()
424            .ints(8..8)
425            .simd_lanes(16..16)
426            .includes_scalars(false)
427            .build(),
428    );
429
430    ig.push(
431        Inst::new(
432            "swizzle",
433            r#"
434        Vector swizzle.
435
436        Returns a new vector with byte-width lanes selected from the lanes of the first input
437        vector ``x`` specified in the second input vector ``s``. The indices ``i`` in range
438        ``[0, 15]`` select the ``i``-th element of ``x``. For indices outside of the range the
439        resulting lane is 0. Note that this operates on byte-width lanes.
440        "#,
441            &formats.binary,
442        )
443        .operands_in(vec![
444            Operand::new("x", I8x16).with_doc("Vector to modify by re-arranging lanes"),
445            Operand::new("y", I8x16).with_doc("Mask for re-arranging lanes"),
446        ])
447        .operands_out(vec![Operand::new("a", I8x16)]),
448    );
449
450    ig.push(
451        Inst::new(
452            "x86_pshufb",
453            r#"
454        A vector swizzle lookalike which has the semantics of `pshufb` on x64.
455
456        This instruction will permute the 8-bit lanes of `x` with the indices
457        specified in `y`. Each lane in the mask, `y`, uses the bottom four
458        bits for selecting the lane from `x` unless the most significant bit
459        is set, in which case the lane is zeroed. The output vector will have
460        the following contents when the element of `y` is in these ranges:
461
462        * `[0, 127]` -> `x[y[i] % 16]`
463        * `[128, 255]` -> 0
464        "#,
465            &formats.binary,
466        )
467        .operands_in(vec![
468            Operand::new("x", I8x16).with_doc("Vector to modify by re-arranging lanes"),
469            Operand::new("y", I8x16).with_doc("Mask for re-arranging lanes"),
470        ])
471        .operands_out(vec![Operand::new("a", I8x16)]),
472    );
473
474    ig.push(
475        Inst::new(
476            "insertlane",
477            r#"
478        Insert ``y`` as lane ``Idx`` in x.
479
480        The lane index, ``Idx``, is an immediate value, not an SSA value. It
481        must indicate a valid lane index for the type of ``x``.
482        "#,
483            &formats.ternary_imm8,
484        )
485        .operands_in(vec![
486            Operand::new("x", TxN).with_doc("The vector to modify"),
487            Operand::new("y", &TxN.lane_of()).with_doc("New lane value"),
488            Operand::new("Idx", &imm.uimm8).with_doc("Lane index"),
489        ])
490        .operands_out(vec![Operand::new("a", TxN)]),
491    );
492
493    ig.push(
494        Inst::new(
495            "extractlane",
496            r#"
497        Extract lane ``Idx`` from ``x``.
498
499        The lane index, ``Idx``, is an immediate value, not an SSA value. It
500        must indicate a valid lane index for the type of ``x``. Note that the upper bits of ``a``
501        may or may not be zeroed depending on the ISA but the type system should prevent using
502        ``a`` as anything other than the extracted value.
503        "#,
504            &formats.binary_imm8,
505        )
506        .operands_in(vec![
507            Operand::new("x", TxN),
508            Operand::new("Idx", &imm.uimm8).with_doc("Lane index"),
509        ])
510        .operands_out(vec![Operand::new("a", &TxN.lane_of())]),
511    );
512}
513
514#[inline(never)]
515fn define_simd_arithmetic(
516    ig: &mut InstructionGroupBuilder,
517    formats: &Formats,
518    _: &Immediates,
519    _: &EntityRefs,
520) {
521    let Int = &TypeVar::new(
522        "Int",
523        "A scalar or vector integer type",
524        TypeSetBuilder::new()
525            .ints(Interval::All)
526            .simd_lanes(Interval::All)
527            .build(),
528    );
529
530    ig.push(
531        Inst::new(
532            "smin",
533            r#"
534        Signed integer minimum.
535        "#,
536            &formats.binary,
537        )
538        .operands_in(vec![Operand::new("x", Int), Operand::new("y", Int)])
539        .operands_out(vec![Operand::new("a", Int)]),
540    );
541
542    ig.push(
543        Inst::new(
544            "umin",
545            r#"
546        Unsigned integer minimum.
547        "#,
548            &formats.binary,
549        )
550        .operands_in(vec![Operand::new("x", Int), Operand::new("y", Int)])
551        .operands_out(vec![Operand::new("a", Int)]),
552    );
553
554    ig.push(
555        Inst::new(
556            "smax",
557            r#"
558        Signed integer maximum.
559        "#,
560            &formats.binary,
561        )
562        .operands_in(vec![Operand::new("x", Int), Operand::new("y", Int)])
563        .operands_out(vec![Operand::new("a", Int)]),
564    );
565
566    ig.push(
567        Inst::new(
568            "umax",
569            r#"
570        Unsigned integer maximum.
571        "#,
572            &formats.binary,
573        )
574        .operands_in(vec![Operand::new("x", Int), Operand::new("y", Int)])
575        .operands_out(vec![Operand::new("a", Int)]),
576    );
577
578    let IxN = &TypeVar::new(
579        "IxN",
580        "A SIMD vector type containing integers",
581        TypeSetBuilder::new()
582            .ints(Interval::All)
583            .simd_lanes(Interval::All)
584            .includes_scalars(false)
585            .build(),
586    );
587
588    ig.push(
589        Inst::new(
590            "avg_round",
591            r#"
592        Unsigned average with rounding: `a := (x + y + 1) // 2`
593
594        The addition does not lose any information (such as from overflow).
595        "#,
596            &formats.binary,
597        )
598        .operands_in(vec![Operand::new("x", IxN), Operand::new("y", IxN)])
599        .operands_out(vec![Operand::new("a", IxN)]),
600    );
601
602    ig.push(
603        Inst::new(
604            "uadd_sat",
605            r#"
606        Add with unsigned saturation.
607
608        This is similar to `iadd` but the operands are interpreted as unsigned integers and their
609        summed result, instead of wrapping, will be saturated to the highest unsigned integer for
610        the controlling type (e.g. `0xFF` for i8).
611        "#,
612            &formats.binary,
613        )
614        .operands_in(vec![Operand::new("x", IxN), Operand::new("y", IxN)])
615        .operands_out(vec![Operand::new("a", IxN)]),
616    );
617
618    ig.push(
619        Inst::new(
620            "sadd_sat",
621            r#"
622        Add with signed saturation.
623
624        This is similar to `iadd` but the operands are interpreted as signed integers and their
625        summed result, instead of wrapping, will be saturated to the lowest or highest
626        signed integer for the controlling type (e.g. `0x80` or `0x7F` for i8). For example,
627        since an `sadd_sat.i8` of `0x70` and `0x70` is greater than `0x7F`, the result will be
628        clamped to `0x7F`.
629        "#,
630            &formats.binary,
631        )
632        .operands_in(vec![Operand::new("x", IxN), Operand::new("y", IxN)])
633        .operands_out(vec![Operand::new("a", IxN)]),
634    );
635
636    ig.push(
637        Inst::new(
638            "usub_sat",
639            r#"
640        Subtract with unsigned saturation.
641
642        This is similar to `isub` but the operands are interpreted as unsigned integers and their
643        difference, instead of wrapping, will be saturated to the lowest unsigned integer for
644        the controlling type (e.g. `0x00` for i8).
645        "#,
646            &formats.binary,
647        )
648        .operands_in(vec![Operand::new("x", IxN), Operand::new("y", IxN)])
649        .operands_out(vec![Operand::new("a", IxN)]),
650    );
651
652    ig.push(
653        Inst::new(
654            "ssub_sat",
655            r#"
656        Subtract with signed saturation.
657
658        This is similar to `isub` but the operands are interpreted as signed integers and their
659        difference, instead of wrapping, will be saturated to the lowest or highest
660        signed integer for the controlling type (e.g. `0x80` or `0x7F` for i8).
661        "#,
662            &formats.binary,
663        )
664        .operands_in(vec![Operand::new("x", IxN), Operand::new("y", IxN)])
665        .operands_out(vec![Operand::new("a", IxN)]),
666    );
667}
668
669pub(crate) fn define(
670    all_instructions: &mut AllInstructions,
671    formats: &Formats,
672    imm: &Immediates,
673    entities: &EntityRefs,
674) {
675    let mut ig = InstructionGroupBuilder::new(all_instructions);
676
677    define_control_flow(&mut ig, formats, imm, entities);
678    define_simd_lane_access(&mut ig, formats, imm, entities);
679    define_simd_arithmetic(&mut ig, formats, imm, entities);
680
681    // Operand kind shorthands.
682    let i8: &TypeVar = &ValueType::from(LaneType::from(types::Int::I8)).into();
683    let f16_: &TypeVar = &ValueType::from(LaneType::from(types::Float::F16)).into();
684    let f32_: &TypeVar = &ValueType::from(LaneType::from(types::Float::F32)).into();
685    let f64_: &TypeVar = &ValueType::from(LaneType::from(types::Float::F64)).into();
686    let f128_: &TypeVar = &ValueType::from(LaneType::from(types::Float::F128)).into();
687
688    // Starting definitions.
689    let Int = &TypeVar::new(
690        "Int",
691        "A scalar or vector integer type",
692        TypeSetBuilder::new()
693            .ints(Interval::All)
694            .simd_lanes(Interval::All)
695            .dynamic_simd_lanes(Interval::All)
696            .build(),
697    );
698
699    let NarrowInt = &TypeVar::new(
700        "NarrowInt",
701        "An integer type of width up to `i64`",
702        TypeSetBuilder::new().ints(8..64).build(),
703    );
704
705    let ScalarTruthy = &TypeVar::new(
706        "ScalarTruthy",
707        "A scalar truthy type",
708        TypeSetBuilder::new().ints(Interval::All).build(),
709    );
710
711    let iB = &TypeVar::new(
712        "iB",
713        "A scalar integer type",
714        TypeSetBuilder::new().ints(Interval::All).build(),
715    );
716
717    let iSwappable = &TypeVar::new(
718        "iSwappable",
719        "A multi byte scalar integer type",
720        TypeSetBuilder::new().ints(16..128).build(),
721    );
722
723    let iAddr = &TypeVar::new(
724        "iAddr",
725        "An integer address type",
726        TypeSetBuilder::new().ints(32..64).build(),
727    );
728
729    let TxN = &TypeVar::new(
730        "TxN",
731        "A SIMD vector type",
732        TypeSetBuilder::new()
733            .ints(Interval::All)
734            .floats(Interval::All)
735            .simd_lanes(Interval::All)
736            .includes_scalars(false)
737            .build(),
738    );
739    let Any = &TypeVar::new(
740        "Any",
741        "Any integer, float, or reference scalar or vector type",
742        TypeSetBuilder::new()
743            .ints(Interval::All)
744            .floats(Interval::All)
745            .simd_lanes(Interval::All)
746            .includes_scalars(true)
747            .build(),
748    );
749
750    let Mem = &TypeVar::new(
751        "Mem",
752        "Any type that can be stored in memory",
753        TypeSetBuilder::new()
754            .ints(Interval::All)
755            .floats(Interval::All)
756            .simd_lanes(Interval::All)
757            .dynamic_simd_lanes(Interval::All)
758            .build(),
759    );
760
761    let MemTo = &TypeVar::copy_from(Mem, "MemTo".to_string());
762
763    ig.push(
764        Inst::new(
765            "load",
766            r#"
767        Load from memory at ``p + Offset``.
768
769        This is a polymorphic instruction that can load any value type which
770        has a memory representation.
771        "#,
772            &formats.load,
773        )
774        .operands_in(vec![
775            Operand::new("MemFlags", &imm.memflags),
776            Operand::new("p", iAddr),
777            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
778        ])
779        .operands_out(vec![Operand::new("a", Mem).with_doc("Value loaded")])
780        .can_load(),
781    );
782
783    ig.push(
784        Inst::new(
785            "store",
786            r#"
787        Store ``x`` to memory at ``p + Offset``.
788
789        This is a polymorphic instruction that can store any value type with a
790        memory representation.
791        "#,
792            &formats.store,
793        )
794        .operands_in(vec![
795            Operand::new("MemFlags", &imm.memflags),
796            Operand::new("x", Mem).with_doc("Value to be stored"),
797            Operand::new("p", iAddr),
798            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
799        ])
800        .can_store(),
801    );
802
803    let iExt8 = &TypeVar::new(
804        "iExt8",
805        "An integer type with more than 8 bits",
806        TypeSetBuilder::new().ints(16..64).build(),
807    );
808
809    ig.push(
810        Inst::new(
811            "uload8",
812            r#"
813        Load 8 bits from memory at ``p + Offset`` and zero-extend.
814
815        This is equivalent to ``load.i8`` followed by ``uextend``.
816        "#,
817            &formats.load,
818        )
819        .operands_in(vec![
820            Operand::new("MemFlags", &imm.memflags),
821            Operand::new("p", iAddr),
822            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
823        ])
824        .operands_out(vec![Operand::new("a", iExt8)])
825        .can_load(),
826    );
827
828    ig.push(
829        Inst::new(
830            "sload8",
831            r#"
832        Load 8 bits from memory at ``p + Offset`` and sign-extend.
833
834        This is equivalent to ``load.i8`` followed by ``sextend``.
835        "#,
836            &formats.load,
837        )
838        .operands_in(vec![
839            Operand::new("MemFlags", &imm.memflags),
840            Operand::new("p", iAddr),
841            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
842        ])
843        .operands_out(vec![Operand::new("a", iExt8)])
844        .can_load(),
845    );
846
847    ig.push(
848        Inst::new(
849            "istore8",
850            r#"
851        Store the low 8 bits of ``x`` to memory at ``p + Offset``.
852
853        This is equivalent to ``ireduce.i8`` followed by ``store.i8``.
854        "#,
855            &formats.store,
856        )
857        .operands_in(vec![
858            Operand::new("MemFlags", &imm.memflags),
859            Operand::new("x", iExt8),
860            Operand::new("p", iAddr),
861            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
862        ])
863        .can_store(),
864    );
865
866    let iExt16 = &TypeVar::new(
867        "iExt16",
868        "An integer type with more than 16 bits",
869        TypeSetBuilder::new().ints(32..64).build(),
870    );
871
872    ig.push(
873        Inst::new(
874            "uload16",
875            r#"
876        Load 16 bits from memory at ``p + Offset`` and zero-extend.
877
878        This is equivalent to ``load.i16`` followed by ``uextend``.
879        "#,
880            &formats.load,
881        )
882        .operands_in(vec![
883            Operand::new("MemFlags", &imm.memflags),
884            Operand::new("p", iAddr),
885            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
886        ])
887        .operands_out(vec![Operand::new("a", iExt16)])
888        .can_load(),
889    );
890
891    ig.push(
892        Inst::new(
893            "sload16",
894            r#"
895        Load 16 bits from memory at ``p + Offset`` and sign-extend.
896
897        This is equivalent to ``load.i16`` followed by ``sextend``.
898        "#,
899            &formats.load,
900        )
901        .operands_in(vec![
902            Operand::new("MemFlags", &imm.memflags),
903            Operand::new("p", iAddr),
904            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
905        ])
906        .operands_out(vec![Operand::new("a", iExt16)])
907        .can_load(),
908    );
909
910    ig.push(
911        Inst::new(
912            "istore16",
913            r#"
914        Store the low 16 bits of ``x`` to memory at ``p + Offset``.
915
916        This is equivalent to ``ireduce.i16`` followed by ``store.i16``.
917        "#,
918            &formats.store,
919        )
920        .operands_in(vec![
921            Operand::new("MemFlags", &imm.memflags),
922            Operand::new("x", iExt16),
923            Operand::new("p", iAddr),
924            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
925        ])
926        .can_store(),
927    );
928
929    let iExt32 = &TypeVar::new(
930        "iExt32",
931        "An integer type with more than 32 bits",
932        TypeSetBuilder::new().ints(64..64).build(),
933    );
934
935    ig.push(
936        Inst::new(
937            "uload32",
938            r#"
939        Load 32 bits from memory at ``p + Offset`` and zero-extend.
940
941        This is equivalent to ``load.i32`` followed by ``uextend``.
942        "#,
943            &formats.load,
944        )
945        .operands_in(vec![
946            Operand::new("MemFlags", &imm.memflags),
947            Operand::new("p", iAddr),
948            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
949        ])
950        .operands_out(vec![Operand::new("a", iExt32)])
951        .can_load(),
952    );
953
954    ig.push(
955        Inst::new(
956            "sload32",
957            r#"
958        Load 32 bits from memory at ``p + Offset`` and sign-extend.
959
960        This is equivalent to ``load.i32`` followed by ``sextend``.
961        "#,
962            &formats.load,
963        )
964        .operands_in(vec![
965            Operand::new("MemFlags", &imm.memflags),
966            Operand::new("p", iAddr),
967            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
968        ])
969        .operands_out(vec![Operand::new("a", iExt32)])
970        .can_load(),
971    );
972
973    ig.push(
974        Inst::new(
975            "istore32",
976            r#"
977        Store the low 32 bits of ``x`` to memory at ``p + Offset``.
978
979        This is equivalent to ``ireduce.i32`` followed by ``store.i32``.
980        "#,
981            &formats.store,
982        )
983        .operands_in(vec![
984            Operand::new("MemFlags", &imm.memflags),
985            Operand::new("x", iExt32),
986            Operand::new("p", iAddr),
987            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
988        ])
989        .can_store(),
990    );
991    ig.push(
992        Inst::new(
993            "stack_switch",
994            r#"
995        Suspends execution of the current stack and resumes execution of another
996        one.
997
998        The target stack to switch to is identified by the data stored at
999        ``load_context_ptr``. Before switching, this instruction stores
1000        analogous information about the
1001        current (i.e., original) stack at ``store_context_ptr``, to
1002        enabled switching back to the original stack at a later point.
1003
1004        The size, alignment and layout of the information stored at
1005        ``load_context_ptr`` and ``store_context_ptr`` is platform-dependent.
1006        The instruction assumes that ``load_context_ptr`` and
1007        ``store_context_ptr`` are valid pointers to memory with said layout and
1008        alignment, and does not perform any checks on these pointers or the data
1009        stored there.
1010
1011        The instruction is experimental and only supported on x64 Linux at the
1012        moment.
1013
1014        When switching from a stack A to a stack B, one of the following cases
1015        must apply:
1016        1. Stack B was previously suspended using a ``stack_switch`` instruction.
1017        2. Stack B is a newly initialized stack. The necessary initialization is
1018        platform-dependent and will generally involve running some kind of
1019        trampoline to start execution of a function on the new stack.
1020
1021        In both cases, the ``in_payload`` argument of the ``stack_switch``
1022        instruction executed on A is passed to stack B. In the first case above,
1023        it will be the result value of the earlier ``stack_switch`` instruction
1024        executed on stack B. In the second case, the value will be accessible to
1025        the trampoline in a platform-dependent register.
1026
1027        The pointers ``load_context_ptr`` and ``store_context_ptr`` are allowed
1028        to be equal; the instruction ensures that all data is loaded from the
1029        former before writing to the latter.
1030
1031        Stack switching is one-shot in the sense that each ``stack_switch``
1032        operation effectively consumes the context identified by
1033        ``load_context_ptr``. In other words, performing two ``stack_switches``
1034        using the same ``load_context_ptr`` causes undefined behavior, unless
1035        the context at ``load_context_ptr`` is overwritten by another
1036        `stack_switch` in between.
1037        "#,
1038            &formats.ternary,
1039        )
1040        .operands_in(vec![
1041            Operand::new("store_context_ptr", iAddr),
1042            Operand::new("load_context_ptr", iAddr),
1043            Operand::new("in_payload0", iAddr),
1044        ])
1045        .operands_out(vec![Operand::new("out_payload0", iAddr)])
1046        .other_side_effects()
1047        .can_load()
1048        .can_store()
1049        .call(),
1050    );
1051
1052    let I16x8 = &TypeVar::new(
1053        "I16x8",
1054        "A SIMD vector with exactly 8 lanes of 16-bit values",
1055        TypeSetBuilder::new()
1056            .ints(16..16)
1057            .simd_lanes(8..8)
1058            .includes_scalars(false)
1059            .build(),
1060    );
1061
1062    ig.push(
1063        Inst::new(
1064            "uload8x8",
1065            r#"
1066        Load an 8x8 vector (64 bits) from memory at ``p + Offset`` and zero-extend into an i16x8
1067        vector.
1068        "#,
1069            &formats.load,
1070        )
1071        .operands_in(vec![
1072            Operand::new("MemFlags", &imm.memflags),
1073            Operand::new("p", iAddr),
1074            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
1075        ])
1076        .operands_out(vec![Operand::new("a", I16x8).with_doc("Value loaded")])
1077        .can_load(),
1078    );
1079
1080    ig.push(
1081        Inst::new(
1082            "sload8x8",
1083            r#"
1084        Load an 8x8 vector (64 bits) from memory at ``p + Offset`` and sign-extend into an i16x8
1085        vector.
1086        "#,
1087            &formats.load,
1088        )
1089        .operands_in(vec![
1090            Operand::new("MemFlags", &imm.memflags),
1091            Operand::new("p", iAddr),
1092            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
1093        ])
1094        .operands_out(vec![Operand::new("a", I16x8).with_doc("Value loaded")])
1095        .can_load(),
1096    );
1097
1098    let I32x4 = &TypeVar::new(
1099        "I32x4",
1100        "A SIMD vector with exactly 4 lanes of 32-bit values",
1101        TypeSetBuilder::new()
1102            .ints(32..32)
1103            .simd_lanes(4..4)
1104            .includes_scalars(false)
1105            .build(),
1106    );
1107
1108    ig.push(
1109        Inst::new(
1110            "uload16x4",
1111            r#"
1112        Load a 16x4 vector (64 bits) from memory at ``p + Offset`` and zero-extend into an i32x4
1113        vector.
1114        "#,
1115            &formats.load,
1116        )
1117        .operands_in(vec![
1118            Operand::new("MemFlags", &imm.memflags),
1119            Operand::new("p", iAddr),
1120            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
1121        ])
1122        .operands_out(vec![Operand::new("a", I32x4).with_doc("Value loaded")])
1123        .can_load(),
1124    );
1125
1126    ig.push(
1127        Inst::new(
1128            "sload16x4",
1129            r#"
1130        Load a 16x4 vector (64 bits) from memory at ``p + Offset`` and sign-extend into an i32x4
1131        vector.
1132        "#,
1133            &formats.load,
1134        )
1135        .operands_in(vec![
1136            Operand::new("MemFlags", &imm.memflags),
1137            Operand::new("p", iAddr),
1138            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
1139        ])
1140        .operands_out(vec![Operand::new("a", I32x4).with_doc("Value loaded")])
1141        .can_load(),
1142    );
1143
1144    let I64x2 = &TypeVar::new(
1145        "I64x2",
1146        "A SIMD vector with exactly 2 lanes of 64-bit values",
1147        TypeSetBuilder::new()
1148            .ints(64..64)
1149            .simd_lanes(2..2)
1150            .includes_scalars(false)
1151            .build(),
1152    );
1153
1154    ig.push(
1155        Inst::new(
1156            "uload32x2",
1157            r#"
1158        Load an 32x2 vector (64 bits) from memory at ``p + Offset`` and zero-extend into an i64x2
1159        vector.
1160        "#,
1161            &formats.load,
1162        )
1163        .operands_in(vec![
1164            Operand::new("MemFlags", &imm.memflags),
1165            Operand::new("p", iAddr),
1166            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
1167        ])
1168        .operands_out(vec![Operand::new("a", I64x2).with_doc("Value loaded")])
1169        .can_load(),
1170    );
1171
1172    ig.push(
1173        Inst::new(
1174            "sload32x2",
1175            r#"
1176        Load a 32x2 vector (64 bits) from memory at ``p + Offset`` and sign-extend into an i64x2
1177        vector.
1178        "#,
1179            &formats.load,
1180        )
1181        .operands_in(vec![
1182            Operand::new("MemFlags", &imm.memflags),
1183            Operand::new("p", iAddr),
1184            Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
1185        ])
1186        .operands_out(vec![Operand::new("a", I64x2).with_doc("Value loaded")])
1187        .can_load(),
1188    );
1189
1190    ig.push(
1191        Inst::new(
1192            "stack_load",
1193            r#"
1194        Load a value from a stack slot at the constant offset.
1195
1196        This is a polymorphic instruction that can load any value type which
1197        has a memory representation.
1198
1199        The offset is an immediate constant, not an SSA value. The memory
1200        access cannot go out of bounds, i.e.
1201        `sizeof(a) + Offset <= sizeof(SS)`.
1202        "#,
1203            &formats.stack_load,
1204        )
1205        .operands_in(vec![
1206            Operand::new("SS", &entities.stack_slot),
1207            Operand::new("Offset", &imm.offset32).with_doc("In-bounds offset into stack slot"),
1208        ])
1209        .operands_out(vec![Operand::new("a", Mem).with_doc("Value loaded")])
1210        .can_load(),
1211    );
1212
1213    ig.push(
1214        Inst::new(
1215            "stack_store",
1216            r#"
1217        Store a value to a stack slot at a constant offset.
1218
1219        This is a polymorphic instruction that can store any value type with a
1220        memory representation.
1221
1222        The offset is an immediate constant, not an SSA value. The memory
1223        access cannot go out of bounds, i.e.
1224        `sizeof(a) + Offset <= sizeof(SS)`.
1225        "#,
1226            &formats.stack_store,
1227        )
1228        .operands_in(vec![
1229            Operand::new("x", Mem).with_doc("Value to be stored"),
1230            Operand::new("SS", &entities.stack_slot),
1231            Operand::new("Offset", &imm.offset32).with_doc("In-bounds offset into stack slot"),
1232        ])
1233        .can_store(),
1234    );
1235
1236    ig.push(
1237        Inst::new(
1238            "stack_addr",
1239            r#"
1240        Get the address of a stack slot.
1241
1242        Compute the absolute address of a byte in a stack slot. The offset must
1243        refer to a byte inside the stack slot:
1244        `0 <= Offset < sizeof(SS)`.
1245        "#,
1246            &formats.stack_load,
1247        )
1248        .operands_in(vec![
1249            Operand::new("SS", &entities.stack_slot),
1250            Operand::new("Offset", &imm.offset32).with_doc("In-bounds offset into stack slot"),
1251        ])
1252        .operands_out(vec![Operand::new("addr", iAddr)]),
1253    );
1254
1255    ig.push(
1256        Inst::new(
1257            "dynamic_stack_load",
1258            r#"
1259        Load a value from a dynamic stack slot.
1260
1261        This is a polymorphic instruction that can load any value type which
1262        has a memory representation.
1263        "#,
1264            &formats.dynamic_stack_load,
1265        )
1266        .operands_in(vec![Operand::new("DSS", &entities.dynamic_stack_slot)])
1267        .operands_out(vec![Operand::new("a", Mem).with_doc("Value loaded")])
1268        .can_load(),
1269    );
1270
1271    ig.push(
1272        Inst::new(
1273            "dynamic_stack_store",
1274            r#"
1275        Store a value to a dynamic stack slot.
1276
1277        This is a polymorphic instruction that can store any dynamic value type with a
1278        memory representation.
1279        "#,
1280            &formats.dynamic_stack_store,
1281        )
1282        .operands_in(vec![
1283            Operand::new("x", Mem).with_doc("Value to be stored"),
1284            Operand::new("DSS", &entities.dynamic_stack_slot),
1285        ])
1286        .can_store(),
1287    );
1288
1289    ig.push(
1290        Inst::new(
1291            "dynamic_stack_addr",
1292            r#"
1293        Get the address of a dynamic stack slot.
1294
1295        Compute the absolute address of the first byte of a dynamic stack slot.
1296        "#,
1297            &formats.dynamic_stack_load,
1298        )
1299        .operands_in(vec![Operand::new("DSS", &entities.dynamic_stack_slot)])
1300        .operands_out(vec![Operand::new("addr", iAddr)]),
1301    );
1302
1303    ig.push(
1304        Inst::new(
1305            "global_value",
1306            r#"
1307        Compute the value of global GV.
1308        "#,
1309            &formats.unary_global_value,
1310        )
1311        .operands_in(vec![Operand::new("GV", &entities.global_value)])
1312        .operands_out(vec![Operand::new("a", Mem).with_doc("Value loaded")]),
1313    );
1314
1315    ig.push(
1316        Inst::new(
1317            "symbol_value",
1318            r#"
1319        Compute the value of global GV, which is a symbolic value.
1320        "#,
1321            &formats.unary_global_value,
1322        )
1323        .operands_in(vec![Operand::new("GV", &entities.global_value)])
1324        .operands_out(vec![Operand::new("a", Mem).with_doc("Value loaded")]),
1325    );
1326
1327    ig.push(
1328        Inst::new(
1329            "tls_value",
1330            r#"
1331        Compute the value of global GV, which is a TLS (thread local storage) value.
1332        "#,
1333            &formats.unary_global_value,
1334        )
1335        .operands_in(vec![Operand::new("GV", &entities.global_value)])
1336        .operands_out(vec![Operand::new("a", Mem).with_doc("Value loaded")]),
1337    );
1338
1339    // Note this instruction is marked as having other side-effects, so GVN won't try to hoist it,
1340    // which would result in it being subject to spilling. While not hoisting would generally hurt
1341    // performance, since a computed value used many times may need to be regenerated before each
1342    // use, it is not the case here: this instruction doesn't generate any code.  That's because,
1343    // by definition the pinned register is never used by the register allocator, but is written to
1344    // and read explicitly and exclusively by set_pinned_reg and get_pinned_reg.
1345    ig.push(
1346        Inst::new(
1347            "get_pinned_reg",
1348            r#"
1349            Gets the content of the pinned register, when it's enabled.
1350        "#,
1351            &formats.nullary,
1352        )
1353        .operands_out(vec![Operand::new("addr", iAddr)])
1354        .other_side_effects(),
1355    );
1356
1357    ig.push(
1358        Inst::new(
1359            "set_pinned_reg",
1360            r#"
1361        Sets the content of the pinned register, when it's enabled.
1362        "#,
1363            &formats.unary,
1364        )
1365        .operands_in(vec![Operand::new("addr", iAddr)])
1366        .other_side_effects(),
1367    );
1368
1369    ig.push(
1370        Inst::new(
1371            "get_frame_pointer",
1372            r#"
1373        Get the address in the frame pointer register.
1374
1375        Usage of this instruction requires setting `preserve_frame_pointers` to `true`.
1376        "#,
1377            &formats.nullary,
1378        )
1379        .operands_out(vec![Operand::new("addr", iAddr)]),
1380    );
1381
1382    ig.push(
1383        Inst::new(
1384            "get_stack_pointer",
1385            r#"
1386        Get the address in the stack pointer register.
1387        "#,
1388            &formats.nullary,
1389        )
1390        .operands_out(vec![Operand::new("addr", iAddr)]),
1391    );
1392
1393    ig.push(
1394        Inst::new(
1395            "get_return_address",
1396            r#"
1397        Get the PC where this function will transfer control to when it returns.
1398
1399        Usage of this instruction requires setting `preserve_frame_pointers` to `true`.
1400        "#,
1401            &formats.nullary,
1402        )
1403        .operands_out(vec![Operand::new("addr", iAddr)]),
1404    );
1405
1406    ig.push(
1407        Inst::new(
1408            "iconst",
1409            r#"
1410        Integer constant.
1411
1412        Create a scalar integer SSA value with an immediate constant value, or
1413        an integer vector where all the lanes have the same value.
1414        "#,
1415            &formats.unary_imm,
1416        )
1417        .operands_in(vec![Operand::new("N", &imm.imm64)])
1418        .operands_out(vec![
1419            Operand::new("a", NarrowInt).with_doc("A constant integer scalar or vector value"),
1420        ]),
1421    );
1422
1423    ig.push(
1424        Inst::new(
1425            "f16const",
1426            r#"
1427        Floating point constant.
1428
1429        Create a `f16` SSA value with an immediate constant value.
1430        "#,
1431            &formats.unary_ieee16,
1432        )
1433        .operands_in(vec![Operand::new("N", &imm.ieee16)])
1434        .operands_out(vec![
1435            Operand::new("a", f16_).with_doc("A constant f16 scalar value"),
1436        ]),
1437    );
1438
1439    ig.push(
1440        Inst::new(
1441            "f32const",
1442            r#"
1443        Floating point constant.
1444
1445        Create a `f32` SSA value with an immediate constant value.
1446        "#,
1447            &formats.unary_ieee32,
1448        )
1449        .operands_in(vec![Operand::new("N", &imm.ieee32)])
1450        .operands_out(vec![
1451            Operand::new("a", f32_).with_doc("A constant f32 scalar value"),
1452        ]),
1453    );
1454
1455    ig.push(
1456        Inst::new(
1457            "f64const",
1458            r#"
1459        Floating point constant.
1460
1461        Create a `f64` SSA value with an immediate constant value.
1462        "#,
1463            &formats.unary_ieee64,
1464        )
1465        .operands_in(vec![Operand::new("N", &imm.ieee64)])
1466        .operands_out(vec![
1467            Operand::new("a", f64_).with_doc("A constant f64 scalar value"),
1468        ]),
1469    );
1470
1471    ig.push(
1472        Inst::new(
1473            "f128const",
1474            r#"
1475        Floating point constant.
1476
1477        Create a `f128` SSA value with an immediate constant value.
1478        "#,
1479            &formats.unary_const,
1480        )
1481        .operands_in(vec![Operand::new("N", &imm.pool_constant)])
1482        .operands_out(vec![
1483            Operand::new("a", f128_).with_doc("A constant f128 scalar value"),
1484        ]),
1485    );
1486
1487    ig.push(
1488        Inst::new(
1489            "vconst",
1490            r#"
1491        SIMD vector constant.
1492
1493        Construct a vector with the given immediate bytes.
1494        "#,
1495            &formats.unary_const,
1496        )
1497        .operands_in(vec![
1498            Operand::new("N", &imm.pool_constant)
1499                .with_doc("The 16 immediate bytes of a 128-bit vector"),
1500        ])
1501        .operands_out(vec![
1502            Operand::new("a", TxN).with_doc("A constant vector value"),
1503        ]),
1504    );
1505
1506    let Tx16 = &TypeVar::new(
1507        "Tx16",
1508        "A SIMD vector with exactly 16 lanes of 8-bit values; eventually this may support other \
1509         lane counts and widths",
1510        TypeSetBuilder::new()
1511            .ints(8..8)
1512            .simd_lanes(16..16)
1513            .includes_scalars(false)
1514            .build(),
1515    );
1516
1517    ig.push(
1518        Inst::new(
1519            "shuffle",
1520            r#"
1521        SIMD vector shuffle.
1522
1523        Shuffle two vectors using the given immediate bytes. For each of the 16 bytes of the
1524        immediate, a value i of 0-15 selects the i-th element of the first vector and a value i of
1525        16-31 selects the (i-16)th element of the second vector. Immediate values outside of the
1526        0-31 range are not valid.
1527        "#,
1528            &formats.shuffle,
1529        )
1530        .operands_in(vec![
1531            Operand::new("a", Tx16).with_doc("A vector value"),
1532            Operand::new("b", Tx16).with_doc("A vector value"),
1533            Operand::new("mask", &imm.uimm128)
1534                .with_doc("The 16 immediate bytes used for selecting the elements to shuffle"),
1535        ])
1536        .operands_out(vec![Operand::new("a", Tx16).with_doc("A vector value")]),
1537    );
1538
1539    ig.push(Inst::new(
1540        "nop",
1541        r#"
1542        Just a dummy instruction.
1543
1544        Note: this doesn't compile to a machine code nop.
1545        "#,
1546        &formats.nullary,
1547    ));
1548
1549    ig.push(
1550        Inst::new(
1551            "select",
1552            r#"
1553        Conditional select.
1554
1555        This instruction selects whole values. Use `bitselect` to choose each
1556        bit according to a mask.
1557        "#,
1558            &formats.ternary,
1559        )
1560        .operands_in(vec![
1561            Operand::new("c", ScalarTruthy).with_doc("Controlling value to test"),
1562            Operand::new("x", Any).with_doc("Value to use when `c` is true"),
1563            Operand::new("y", Any).with_doc("Value to use when `c` is false"),
1564        ])
1565        .operands_out(vec![Operand::new("a", Any)]),
1566    );
1567
1568    ig.push(
1569        Inst::new(
1570            "select_spectre_guard",
1571            r#"
1572            Conditional select intended for Spectre guards.
1573
1574            This operation is semantically equivalent to a select instruction.
1575            However, this instruction prohibits all speculation on the
1576            controlling value when determining which input to use as the result.
1577            As such, it is suitable for use in Spectre guards.
1578
1579            For example, on a target which may speculatively execute branches,
1580            the lowering of this instruction is guaranteed to not conditionally
1581            branch. Instead it will typically lower to a conditional move
1582            instruction. (No Spectre-vulnerable processors are known to perform
1583            value speculation on conditional move instructions.)
1584
1585            Ensure that the instruction you're trying to protect from Spectre
1586            attacks has a data dependency on the result of this instruction.
1587            That prevents an out-of-order CPU from evaluating that instruction
1588            until the result of this one is known, which in turn will be blocked
1589            until the controlling value is known.
1590
1591            Typical usage is to use a bounds-check as the controlling value,
1592            and select between either a null pointer if the bounds-check
1593            fails, or an in-bounds address otherwise, so that dereferencing
1594            the resulting address with a load or store instruction will trap if
1595            the bounds-check failed. When this instruction is used in this way,
1596            any microarchitectural side effects of the memory access will only
1597            occur after the bounds-check finishes, which ensures that no Spectre
1598            vulnerability will exist.
1599
1600            Optimization opportunities for this instruction are limited compared
1601            to a normal select instruction, but it is allowed to be replaced
1602            by other values which are functionally equivalent as long as doing
1603            so does not introduce any new opportunities to speculate on the
1604            controlling value.
1605            "#,
1606            &formats.ternary,
1607        )
1608        .operands_in(vec![
1609            Operand::new("c", ScalarTruthy).with_doc("Controlling value to test"),
1610            Operand::new("x", Any).with_doc("Value to use when `c` is true"),
1611            Operand::new("y", Any).with_doc("Value to use when `c` is false"),
1612        ])
1613        .operands_out(vec![Operand::new("a", Any)]),
1614    );
1615
1616    ig.push(
1617        Inst::new(
1618            "bitselect",
1619            r#"
1620        Conditional select of bits.
1621
1622        For each bit in `c`, this instruction selects the corresponding bit from `x` if the bit
1623        in `x` is 1 and the corresponding bit from `y` if the bit in `c` is 0. See also:
1624        `select`.
1625        "#,
1626            &formats.ternary,
1627        )
1628        .operands_in(vec![
1629            Operand::new("c", Any).with_doc("Controlling value to test"),
1630            Operand::new("x", Any).with_doc("Value to use when `c` is true"),
1631            Operand::new("y", Any).with_doc("Value to use when `c` is false"),
1632        ])
1633        .operands_out(vec![Operand::new("a", Any)]),
1634    );
1635
1636    ig.push(
1637        Inst::new(
1638            "x86_blendv",
1639            r#"
1640        A bitselect-lookalike instruction except with the semantics of
1641        `blendv`-related instructions on x86.
1642
1643        This instruction will use the top bit of each lane in `c`, the condition
1644        mask. If the bit is 1 then the corresponding lane from `x` is chosen.
1645        Otherwise the corresponding lane from `y` is chosen.
1646
1647            "#,
1648            &formats.ternary,
1649        )
1650        .operands_in(vec![
1651            Operand::new("c", Any).with_doc("Controlling value to test"),
1652            Operand::new("x", Any).with_doc("Value to use when `c` is true"),
1653            Operand::new("y", Any).with_doc("Value to use when `c` is false"),
1654        ])
1655        .operands_out(vec![Operand::new("a", Any)]),
1656    );
1657
1658    ig.push(
1659        Inst::new(
1660            "vany_true",
1661            r#"
1662        Reduce a vector to a scalar boolean.
1663
1664        Return a scalar boolean true if any lane in ``a`` is non-zero, false otherwise.
1665        "#,
1666            &formats.unary,
1667        )
1668        .operands_in(vec![Operand::new("a", TxN)])
1669        .operands_out(vec![Operand::new("s", i8)]),
1670    );
1671
1672    ig.push(
1673        Inst::new(
1674            "vall_true",
1675            r#"
1676        Reduce a vector to a scalar boolean.
1677
1678        Return a scalar boolean true if all lanes in ``i`` are non-zero, false otherwise.
1679        "#,
1680            &formats.unary,
1681        )
1682        .operands_in(vec![Operand::new("a", TxN)])
1683        .operands_out(vec![Operand::new("s", i8)]),
1684    );
1685
1686    ig.push(
1687        Inst::new(
1688            "vhigh_bits",
1689            r#"
1690        Reduce a vector to a scalar integer.
1691
1692        Return a scalar integer, consisting of the concatenation of the most significant bit
1693        of each lane of ``a``.
1694        "#,
1695            &formats.unary,
1696        )
1697        .operands_in(vec![Operand::new("a", TxN)])
1698        .operands_out(vec![Operand::new("x", NarrowInt)]),
1699    );
1700
1701    ig.push(
1702        Inst::new(
1703            "icmp",
1704            r#"
1705        Integer comparison.
1706
1707        The condition code determines if the operands are interpreted as signed
1708        or unsigned integers.
1709
1710        | Signed | Unsigned | Condition             |
1711        |--------|----------|-----------------------|
1712        | eq     | eq       | Equal                 |
1713        | ne     | ne       | Not equal             |
1714        | slt    | ult      | Less than             |
1715        | sge    | uge      | Greater than or equal |
1716        | sgt    | ugt      | Greater than          |
1717        | sle    | ule      | Less than or equal    |
1718
1719        When this instruction compares integer vectors, it returns a vector of
1720        lane-wise comparisons.
1721
1722        When comparing scalars, the result is:
1723            - `1` if the condition holds.
1724            - `0` if the condition does not hold.
1725
1726        When comparing vectors, the result is:
1727            - `-1` (i.e. all ones) in each lane where the condition holds.
1728            - `0` in each lane where the condition does not hold.
1729        "#,
1730            &formats.int_compare,
1731        )
1732        .operands_in(vec![
1733            Operand::new("Cond", &imm.intcc),
1734            Operand::new("x", Int),
1735            Operand::new("y", Int),
1736        ])
1737        .operands_out(vec![Operand::new("a", &Int.as_truthy())]),
1738    );
1739
1740    ig.push(
1741        Inst::new(
1742            "icmp_imm",
1743            r#"
1744        Compare scalar integer to a constant.
1745
1746        This is the same as the `icmp` instruction, except one operand is
1747        a sign extended 64 bit immediate constant.
1748
1749        This instruction can only compare scalars. Use `icmp` for
1750        lane-wise vector comparisons.
1751        "#,
1752            &formats.int_compare_imm,
1753        )
1754        .operands_in(vec![
1755            Operand::new("Cond", &imm.intcc),
1756            Operand::new("x", iB),
1757            Operand::new("Y", &imm.imm64),
1758        ])
1759        .operands_out(vec![Operand::new("a", i8)]),
1760    );
1761
1762    ig.push(
1763        Inst::new(
1764            "iadd",
1765            r#"
1766        Wrapping integer addition: `a := x + y \pmod{2^B}`.
1767
1768        This instruction does not depend on the signed/unsigned interpretation
1769        of the operands.
1770        "#,
1771            &formats.binary,
1772        )
1773        .operands_in(vec![Operand::new("x", Int), Operand::new("y", Int)])
1774        .operands_out(vec![Operand::new("a", Int)]),
1775    );
1776
1777    ig.push(
1778        Inst::new(
1779            "isub",
1780            r#"
1781        Wrapping integer subtraction: `a := x - y \pmod{2^B}`.
1782
1783        This instruction does not depend on the signed/unsigned interpretation
1784        of the operands.
1785        "#,
1786            &formats.binary,
1787        )
1788        .operands_in(vec![Operand::new("x", Int), Operand::new("y", Int)])
1789        .operands_out(vec![Operand::new("a", Int)]),
1790    );
1791
1792    ig.push(
1793        Inst::new(
1794            "ineg",
1795            r#"
1796        Integer negation: `a := -x \pmod{2^B}`.
1797        "#,
1798            &formats.unary,
1799        )
1800        .operands_in(vec![Operand::new("x", Int)])
1801        .operands_out(vec![Operand::new("a", Int)]),
1802    );
1803
1804    ig.push(
1805        Inst::new(
1806            "iabs",
1807            r#"
1808        Integer absolute value with wrapping: `a := |x|`.
1809        "#,
1810            &formats.unary,
1811        )
1812        .operands_in(vec![Operand::new("x", Int)])
1813        .operands_out(vec![Operand::new("a", Int)]),
1814    );
1815
1816    ig.push(
1817        Inst::new(
1818            "imul",
1819            r#"
1820        Wrapping integer multiplication: `a := x y \pmod{2^B}`.
1821
1822        This instruction does not depend on the signed/unsigned interpretation
1823        of the operands.
1824
1825        Polymorphic over all integer types (vector and scalar).
1826        "#,
1827            &formats.binary,
1828        )
1829        .operands_in(vec![Operand::new("x", Int), Operand::new("y", Int)])
1830        .operands_out(vec![Operand::new("a", Int)]),
1831    );
1832
1833    ig.push(
1834        Inst::new(
1835            "umulhi",
1836            r#"
1837        Unsigned integer multiplication, producing the high half of a
1838        double-length result.
1839
1840        Polymorphic over all integer types (vector and scalar).
1841        "#,
1842            &formats.binary,
1843        )
1844        .operands_in(vec![Operand::new("x", Int), Operand::new("y", Int)])
1845        .operands_out(vec![Operand::new("a", Int)]),
1846    );
1847
1848    ig.push(
1849        Inst::new(
1850            "smulhi",
1851            r#"
1852        Signed integer multiplication, producing the high half of a
1853        double-length result.
1854
1855        Polymorphic over all integer types (vector and scalar).
1856        "#,
1857            &formats.binary,
1858        )
1859        .operands_in(vec![Operand::new("x", Int), Operand::new("y", Int)])
1860        .operands_out(vec![Operand::new("a", Int)]),
1861    );
1862
1863    let I16or32 = &TypeVar::new(
1864        "I16or32",
1865        "A vector integer type with 16- or 32-bit numbers",
1866        TypeSetBuilder::new().ints(16..32).simd_lanes(4..8).build(),
1867    );
1868
1869    ig.push(
1870        Inst::new(
1871            "sqmul_round_sat",
1872            r#"
1873        Fixed-point multiplication of numbers in the QN format, where N + 1
1874        is the number bitwidth:
1875        `a := signed_saturate((x * y + (1 << (Q - 1))) >> Q)`
1876
1877        Polymorphic over all integer vector types with 16- or 32-bit numbers.
1878        "#,
1879            &formats.binary,
1880        )
1881        .operands_in(vec![Operand::new("x", I16or32), Operand::new("y", I16or32)])
1882        .operands_out(vec![Operand::new("a", I16or32)]),
1883    );
1884
1885    ig.push(
1886        Inst::new(
1887            "x86_pmulhrsw",
1888            r#"
1889        A similar instruction to `sqmul_round_sat` except with the semantics
1890        of x86's `pmulhrsw` instruction.
1891
1892        This is the same as `sqmul_round_sat` except when both input lanes are
1893        `i16::MIN`.
1894        "#,
1895            &formats.binary,
1896        )
1897        .operands_in(vec![Operand::new("x", I16or32), Operand::new("y", I16or32)])
1898        .operands_out(vec![Operand::new("a", I16or32)]),
1899    );
1900
1901    // Integer division and remainder are scalar-only; most
1902    // hardware does not directly support vector integer division.
1903
1904    ig.push(
1905        Inst::new(
1906            "udiv",
1907            r#"
1908        Unsigned integer division: `a := \lfloor {x \over y} \rfloor`.
1909
1910        This operation traps if the divisor is zero.
1911        "#,
1912            &formats.binary,
1913        )
1914        .operands_in(vec![Operand::new("x", iB), Operand::new("y", iB)])
1915        .operands_out(vec![Operand::new("a", iB)])
1916        .can_trap()
1917        .side_effects_idempotent(),
1918    );
1919
1920    ig.push(
1921        Inst::new(
1922            "sdiv",
1923            r#"
1924        Signed integer division rounded toward zero: `a := sign(xy)
1925        \lfloor {|x| \over |y|}\rfloor`.
1926
1927        This operation traps if the divisor is zero, or if the result is not
1928        representable in `B` bits two's complement. This only happens
1929        when `x = -2^{B-1}, y = -1`.
1930        "#,
1931            &formats.binary,
1932        )
1933        .operands_in(vec![Operand::new("x", iB), Operand::new("y", iB)])
1934        .operands_out(vec![Operand::new("a", iB)])
1935        .can_trap()
1936        .side_effects_idempotent(),
1937    );
1938
1939    ig.push(
1940        Inst::new(
1941            "urem",
1942            r#"
1943        Unsigned integer remainder.
1944
1945        This operation traps if the divisor is zero.
1946        "#,
1947            &formats.binary,
1948        )
1949        .operands_in(vec![Operand::new("x", iB), Operand::new("y", iB)])
1950        .operands_out(vec![Operand::new("a", iB)])
1951        .can_trap()
1952        .side_effects_idempotent(),
1953    );
1954
1955    ig.push(
1956        Inst::new(
1957            "srem",
1958            r#"
1959        Signed integer remainder. The result has the sign of the dividend.
1960
1961        This operation traps if the divisor is zero.
1962        "#,
1963            &formats.binary,
1964        )
1965        .operands_in(vec![Operand::new("x", iB), Operand::new("y", iB)])
1966        .operands_out(vec![Operand::new("a", iB)])
1967        .can_trap()
1968        .side_effects_idempotent(),
1969    );
1970
1971    ig.push(
1972        Inst::new(
1973            "iadd_imm",
1974            r#"
1975        Add immediate integer.
1976
1977        Same as `iadd`, but one operand is a sign extended 64 bit immediate constant.
1978
1979        Polymorphic over all scalar integer types, but does not support vector
1980        types.
1981        "#,
1982            &formats.binary_imm64,
1983        )
1984        .operands_in(vec![Operand::new("x", iB), Operand::new("Y", &imm.imm64)])
1985        .operands_out(vec![Operand::new("a", iB)]),
1986    );
1987
1988    ig.push(
1989        Inst::new(
1990            "imul_imm",
1991            r#"
1992        Integer multiplication by immediate constant.
1993
1994        Same as `imul`, but one operand is a sign extended 64 bit immediate constant.
1995
1996        Polymorphic over all scalar integer types, but does not support vector
1997        types.
1998        "#,
1999            &formats.binary_imm64,
2000        )
2001        .operands_in(vec![Operand::new("x", iB), Operand::new("Y", &imm.imm64)])
2002        .operands_out(vec![Operand::new("a", iB)]),
2003    );
2004
2005    ig.push(
2006        Inst::new(
2007            "udiv_imm",
2008            r#"
2009        Unsigned integer division by an immediate constant.
2010
2011        Same as `udiv`, but one operand is a zero extended 64 bit immediate constant.
2012
2013        This operation traps if the divisor is zero.
2014        "#,
2015            &formats.binary_imm64,
2016        )
2017        .operands_in(vec![Operand::new("x", iB), Operand::new("Y", &imm.imm64)])
2018        .operands_out(vec![Operand::new("a", iB)]),
2019    );
2020
2021    ig.push(
2022        Inst::new(
2023            "sdiv_imm",
2024            r#"
2025        Signed integer division by an immediate constant.
2026
2027        Same as `sdiv`, but one operand is a sign extended 64 bit immediate constant.
2028
2029        This operation traps if the divisor is zero, or if the result is not
2030        representable in `B` bits two's complement. This only happens
2031        when `x = -2^{B-1}, Y = -1`.
2032        "#,
2033            &formats.binary_imm64,
2034        )
2035        .operands_in(vec![Operand::new("x", iB), Operand::new("Y", &imm.imm64)])
2036        .operands_out(vec![Operand::new("a", iB)]),
2037    );
2038
2039    ig.push(
2040        Inst::new(
2041            "urem_imm",
2042            r#"
2043        Unsigned integer remainder with immediate divisor.
2044
2045        Same as `urem`, but one operand is a zero extended 64 bit immediate constant.
2046
2047        This operation traps if the divisor is zero.
2048        "#,
2049            &formats.binary_imm64,
2050        )
2051        .operands_in(vec![Operand::new("x", iB), Operand::new("Y", &imm.imm64)])
2052        .operands_out(vec![Operand::new("a", iB)]),
2053    );
2054
2055    ig.push(
2056        Inst::new(
2057            "srem_imm",
2058            r#"
2059        Signed integer remainder with immediate divisor.
2060
2061        Same as `srem`, but one operand is a sign extended 64 bit immediate constant.
2062
2063        This operation traps if the divisor is zero.
2064        "#,
2065            &formats.binary_imm64,
2066        )
2067        .operands_in(vec![Operand::new("x", iB), Operand::new("Y", &imm.imm64)])
2068        .operands_out(vec![Operand::new("a", iB)]),
2069    );
2070
2071    ig.push(
2072        Inst::new(
2073            "irsub_imm",
2074            r#"
2075        Immediate reverse wrapping subtraction: `a := Y - x \pmod{2^B}`.
2076
2077        The immediate operand is a sign extended 64 bit constant.
2078
2079        Also works as integer negation when `Y = 0`. Use `iadd_imm`
2080        with a negative immediate operand for the reverse immediate
2081        subtraction.
2082
2083        Polymorphic over all scalar integer types, but does not support vector
2084        types.
2085        "#,
2086            &formats.binary_imm64,
2087        )
2088        .operands_in(vec![Operand::new("x", iB), Operand::new("Y", &imm.imm64)])
2089        .operands_out(vec![Operand::new("a", iB)]),
2090    );
2091
2092    ig.push(
2093        Inst::new(
2094            "sadd_overflow_cin",
2095            r#"
2096        Add signed integers with carry in and overflow out.
2097
2098        Same as `sadd_overflow` with an additional carry input. The `c_in` type
2099        is interpreted as 1 if it's nonzero or 0 if it's zero.
2100        "#,
2101            &formats.ternary,
2102        )
2103        .operands_in(vec![
2104            Operand::new("x", iB),
2105            Operand::new("y", iB),
2106            Operand::new("c_in", i8).with_doc("Input carry flag"),
2107        ])
2108        .operands_out(vec![
2109            Operand::new("a", iB),
2110            Operand::new("c_out", i8).with_doc("Output carry flag"),
2111        ]),
2112    );
2113
2114    ig.push(
2115        Inst::new(
2116            "uadd_overflow_cin",
2117            r#"
2118        Add unsigned integers with carry in and overflow out.
2119
2120        Same as `uadd_overflow` with an additional carry input. The `c_in` type
2121        is interpreted as 1 if it's nonzero or 0 if it's zero.
2122        "#,
2123            &formats.ternary,
2124        )
2125        .operands_in(vec![
2126            Operand::new("x", iB),
2127            Operand::new("y", iB),
2128            Operand::new("c_in", i8).with_doc("Input carry flag"),
2129        ])
2130        .operands_out(vec![
2131            Operand::new("a", iB),
2132            Operand::new("c_out", i8).with_doc("Output carry flag"),
2133        ]),
2134    );
2135
2136    {
2137        let of_out = Operand::new("of", i8).with_doc("Overflow flag");
2138        ig.push(
2139            Inst::new(
2140                "uadd_overflow",
2141                r#"
2142            Add integers unsigned with overflow out.
2143            ``of`` is set when the addition overflowed.
2144            ```text
2145                a &= x + y \pmod 2^B \\
2146                of &= x+y >= 2^B
2147            ```
2148            Polymorphic over all scalar integer types, but does not support vector
2149            types.
2150            "#,
2151                &formats.binary,
2152            )
2153            .operands_in(vec![Operand::new("x", iB), Operand::new("y", iB)])
2154            .operands_out(vec![Operand::new("a", iB), of_out.clone()]),
2155        );
2156
2157        ig.push(
2158            Inst::new(
2159                "sadd_overflow",
2160                r#"
2161            Add integers signed with overflow out.
2162            ``of`` is set when the addition over- or underflowed.
2163            Polymorphic over all scalar integer types, but does not support vector
2164            types.
2165            "#,
2166                &formats.binary,
2167            )
2168            .operands_in(vec![Operand::new("x", iB), Operand::new("y", iB)])
2169            .operands_out(vec![Operand::new("a", iB), of_out.clone()]),
2170        );
2171
2172        ig.push(
2173            Inst::new(
2174                "usub_overflow",
2175                r#"
2176            Subtract integers unsigned with overflow out.
2177            ``of`` is set when the subtraction underflowed.
2178            ```text
2179                a &= x - y \pmod 2^B \\
2180                of &= x - y < 0
2181            ```
2182            Polymorphic over all scalar integer types, but does not support vector
2183            types.
2184            "#,
2185                &formats.binary,
2186            )
2187            .operands_in(vec![Operand::new("x", iB), Operand::new("y", iB)])
2188            .operands_out(vec![Operand::new("a", iB), of_out.clone()]),
2189        );
2190
2191        ig.push(
2192            Inst::new(
2193                "ssub_overflow",
2194                r#"
2195            Subtract integers signed with overflow out.
2196            ``of`` is set when the subtraction over- or underflowed.
2197            Polymorphic over all scalar integer types, but does not support vector
2198            types.
2199            "#,
2200                &formats.binary,
2201            )
2202            .operands_in(vec![Operand::new("x", iB), Operand::new("y", iB)])
2203            .operands_out(vec![Operand::new("a", iB), of_out.clone()]),
2204        );
2205
2206        {
2207            let NarrowScalar = &TypeVar::new(
2208                "NarrowScalar",
2209                "A scalar integer type up to 64 bits",
2210                TypeSetBuilder::new().ints(8..64).build(),
2211            );
2212
2213            ig.push(
2214                Inst::new(
2215                    "umul_overflow",
2216                    r#"
2217                Multiply integers unsigned with overflow out.
2218                ``of`` is set when the multiplication overflowed.
2219                ```text
2220                    a &= x * y \pmod 2^B \\
2221                    of &= x * y > 2^B
2222                ```
2223                Polymorphic over all scalar integer types except i128, but does not support vector
2224                types.
2225                "#,
2226                    &formats.binary,
2227                )
2228                .operands_in(vec![
2229                    Operand::new("x", NarrowScalar),
2230                    Operand::new("y", NarrowScalar),
2231                ])
2232                .operands_out(vec![Operand::new("a", NarrowScalar), of_out.clone()]),
2233            );
2234
2235            ig.push(
2236                Inst::new(
2237                    "smul_overflow",
2238                    r#"
2239                Multiply integers signed with overflow out.
2240                ``of`` is set when the multiplication over- or underflowed.
2241                Polymorphic over all scalar integer types except i128, but does not support vector
2242                types.
2243                "#,
2244                    &formats.binary,
2245                )
2246                .operands_in(vec![
2247                    Operand::new("x", NarrowScalar),
2248                    Operand::new("y", NarrowScalar),
2249                ])
2250                .operands_out(vec![Operand::new("a", NarrowScalar), of_out.clone()]),
2251            );
2252        }
2253    }
2254
2255    let i32_64 = &TypeVar::new(
2256        "i32_64",
2257        "A 32 or 64-bit scalar integer type",
2258        TypeSetBuilder::new().ints(32..64).build(),
2259    );
2260
2261    ig.push(
2262        Inst::new(
2263            "uadd_overflow_trap",
2264            r#"
2265        Unsigned addition of x and y, trapping if the result overflows.
2266
2267        Accepts 32 or 64-bit integers, and does not support vector types.
2268        "#,
2269            &formats.int_add_trap,
2270        )
2271        .operands_in(vec![
2272            Operand::new("x", i32_64),
2273            Operand::new("y", i32_64),
2274            Operand::new("code", &imm.trapcode),
2275        ])
2276        .operands_out(vec![Operand::new("a", i32_64)])
2277        .can_trap()
2278        .side_effects_idempotent(),
2279    );
2280
2281    ig.push(
2282        Inst::new(
2283            "ssub_overflow_bin",
2284            r#"
2285        Subtract signed integers with borrow in and overflow out.
2286
2287        Same as `ssub_overflow` with an additional borrow input. The `b_in` type
2288        is interpreted as 1 if it's nonzero or 0 if it's zero. The computation
2289        performed here is `x - (y + (b_in != 0))`.
2290        "#,
2291            &formats.ternary,
2292        )
2293        .operands_in(vec![
2294            Operand::new("x", iB),
2295            Operand::new("y", iB),
2296            Operand::new("b_in", i8).with_doc("Input borrow flag"),
2297        ])
2298        .operands_out(vec![
2299            Operand::new("a", iB),
2300            Operand::new("b_out", i8).with_doc("Output borrow flag"),
2301        ]),
2302    );
2303
2304    ig.push(
2305        Inst::new(
2306            "usub_overflow_bin",
2307            r#"
2308        Subtract unsigned integers with borrow in and overflow out.
2309
2310        Same as `usub_overflow` with an additional borrow input. The `b_in` type
2311        is interpreted as 1 if it's nonzero or 0 if it's zero. The computation
2312        performed here is `x - (y + (b_in != 0))`.
2313        "#,
2314            &formats.ternary,
2315        )
2316        .operands_in(vec![
2317            Operand::new("x", iB),
2318            Operand::new("y", iB),
2319            Operand::new("b_in", i8).with_doc("Input borrow flag"),
2320        ])
2321        .operands_out(vec![
2322            Operand::new("a", iB),
2323            Operand::new("b_out", i8).with_doc("Output borrow flag"),
2324        ]),
2325    );
2326
2327    let bits = &TypeVar::new(
2328        "bits",
2329        "Any integer, float, or vector type",
2330        TypeSetBuilder::new()
2331            .ints(Interval::All)
2332            .floats(Interval::All)
2333            .simd_lanes(Interval::All)
2334            .includes_scalars(true)
2335            .build(),
2336    );
2337
2338    ig.push(
2339        Inst::new(
2340            "band",
2341            r#"
2342        Bitwise and.
2343        "#,
2344            &formats.binary,
2345        )
2346        .operands_in(vec![Operand::new("x", bits), Operand::new("y", bits)])
2347        .operands_out(vec![Operand::new("a", bits)]),
2348    );
2349
2350    ig.push(
2351        Inst::new(
2352            "bor",
2353            r#"
2354        Bitwise or.
2355        "#,
2356            &formats.binary,
2357        )
2358        .operands_in(vec![Operand::new("x", bits), Operand::new("y", bits)])
2359        .operands_out(vec![Operand::new("a", bits)]),
2360    );
2361
2362    ig.push(
2363        Inst::new(
2364            "bxor",
2365            r#"
2366        Bitwise xor.
2367        "#,
2368            &formats.binary,
2369        )
2370        .operands_in(vec![Operand::new("x", bits), Operand::new("y", bits)])
2371        .operands_out(vec![Operand::new("a", bits)]),
2372    );
2373
2374    ig.push(
2375        Inst::new(
2376            "bnot",
2377            r#"
2378        Bitwise not.
2379        "#,
2380            &formats.unary,
2381        )
2382        .operands_in(vec![Operand::new("x", bits)])
2383        .operands_out(vec![Operand::new("a", bits)]),
2384    );
2385
2386    ig.push(
2387        Inst::new(
2388            "band_not",
2389            r#"
2390        Bitwise and not.
2391
2392        Computes `x & ~y`.
2393        "#,
2394            &formats.binary,
2395        )
2396        .operands_in(vec![Operand::new("x", bits), Operand::new("y", bits)])
2397        .operands_out(vec![Operand::new("a", bits)]),
2398    );
2399
2400    ig.push(
2401        Inst::new(
2402            "bor_not",
2403            r#"
2404        Bitwise or not.
2405
2406        Computes `x | ~y`.
2407        "#,
2408            &formats.binary,
2409        )
2410        .operands_in(vec![Operand::new("x", bits), Operand::new("y", bits)])
2411        .operands_out(vec![Operand::new("a", bits)]),
2412    );
2413
2414    ig.push(
2415        Inst::new(
2416            "bxor_not",
2417            r#"
2418        Bitwise xor not.
2419
2420        Computes `x ^ ~y`.
2421        "#,
2422            &formats.binary,
2423        )
2424        .operands_in(vec![Operand::new("x", bits), Operand::new("y", bits)])
2425        .operands_out(vec![Operand::new("a", bits)]),
2426    );
2427
2428    ig.push(
2429        Inst::new(
2430            "band_imm",
2431            r#"
2432        Bitwise and with immediate.
2433
2434        Same as `band`, but one operand is a zero extended 64 bit immediate constant.
2435
2436        Polymorphic over all scalar integer types, but does not support vector
2437        types.
2438        "#,
2439            &formats.binary_imm64,
2440        )
2441        .operands_in(vec![Operand::new("x", iB), Operand::new("Y", &imm.imm64)])
2442        .operands_out(vec![Operand::new("a", iB)]),
2443    );
2444
2445    ig.push(
2446        Inst::new(
2447            "bor_imm",
2448            r#"
2449        Bitwise or with immediate.
2450
2451        Same as `bor`, but one operand is a zero extended 64 bit immediate constant.
2452
2453        Polymorphic over all scalar integer types, but does not support vector
2454        types.
2455        "#,
2456            &formats.binary_imm64,
2457        )
2458        .operands_in(vec![Operand::new("x", iB), Operand::new("Y", &imm.imm64)])
2459        .operands_out(vec![Operand::new("a", iB)]),
2460    );
2461
2462    ig.push(
2463        Inst::new(
2464            "bxor_imm",
2465            r#"
2466        Bitwise xor with immediate.
2467
2468        Same as `bxor`, but one operand is a zero extended 64 bit immediate constant.
2469
2470        Polymorphic over all scalar integer types, but does not support vector
2471        types.
2472        "#,
2473            &formats.binary_imm64,
2474        )
2475        .operands_in(vec![Operand::new("x", iB), Operand::new("Y", &imm.imm64)])
2476        .operands_out(vec![Operand::new("a", iB)]),
2477    );
2478
2479    ig.push(
2480        Inst::new(
2481            "rotl",
2482            r#"
2483        Rotate left.
2484
2485        Rotate the bits in ``x`` by ``y`` places.
2486        "#,
2487            &formats.binary,
2488        )
2489        .operands_in(vec![
2490            Operand::new("x", Int).with_doc("Scalar or vector value to shift"),
2491            Operand::new("y", iB).with_doc("Number of bits to shift"),
2492        ])
2493        .operands_out(vec![Operand::new("a", Int)]),
2494    );
2495
2496    ig.push(
2497        Inst::new(
2498            "rotr",
2499            r#"
2500        Rotate right.
2501
2502        Rotate the bits in ``x`` by ``y`` places.
2503        "#,
2504            &formats.binary,
2505        )
2506        .operands_in(vec![
2507            Operand::new("x", Int).with_doc("Scalar or vector value to shift"),
2508            Operand::new("y", iB).with_doc("Number of bits to shift"),
2509        ])
2510        .operands_out(vec![Operand::new("a", Int)]),
2511    );
2512
2513    ig.push(
2514        Inst::new(
2515            "rotl_imm",
2516            r#"
2517        Rotate left by immediate.
2518
2519        Same as `rotl`, but one operand is a zero extended 64 bit immediate constant.
2520        "#,
2521            &formats.binary_imm64,
2522        )
2523        .operands_in(vec![
2524            Operand::new("x", Int).with_doc("Scalar or vector value to shift"),
2525            Operand::new("Y", &imm.imm64),
2526        ])
2527        .operands_out(vec![Operand::new("a", Int)]),
2528    );
2529
2530    ig.push(
2531        Inst::new(
2532            "rotr_imm",
2533            r#"
2534        Rotate right by immediate.
2535
2536        Same as `rotr`, but one operand is a zero extended 64 bit immediate constant.
2537        "#,
2538            &formats.binary_imm64,
2539        )
2540        .operands_in(vec![
2541            Operand::new("x", Int).with_doc("Scalar or vector value to shift"),
2542            Operand::new("Y", &imm.imm64),
2543        ])
2544        .operands_out(vec![Operand::new("a", Int)]),
2545    );
2546
2547    ig.push(
2548        Inst::new(
2549            "ishl",
2550            r#"
2551        Integer shift left. Shift the bits in ``x`` towards the MSB by ``y``
2552        places. Shift in zero bits to the LSB.
2553
2554        The shift amount is masked to the size of ``x``.
2555
2556        When shifting a B-bits integer type, this instruction computes:
2557
2558        ```text
2559            s &:= y \pmod B,
2560            a &:= x \cdot 2^s \pmod{2^B}.
2561        ```
2562        "#,
2563            &formats.binary,
2564        )
2565        .operands_in(vec![
2566            Operand::new("x", Int).with_doc("Scalar or vector value to shift"),
2567            Operand::new("y", iB).with_doc("Number of bits to shift"),
2568        ])
2569        .operands_out(vec![Operand::new("a", Int)]),
2570    );
2571
2572    ig.push(
2573        Inst::new(
2574            "ushr",
2575            r#"
2576        Unsigned shift right. Shift bits in ``x`` towards the LSB by ``y``
2577        places, shifting in zero bits to the MSB. Also called a *logical
2578        shift*.
2579
2580        The shift amount is masked to the size of ``x``.
2581
2582        When shifting a B-bits integer type, this instruction computes:
2583
2584        ```text
2585            s &:= y \pmod B,
2586            a &:= \lfloor x \cdot 2^{-s} \rfloor.
2587        ```
2588        "#,
2589            &formats.binary,
2590        )
2591        .operands_in(vec![
2592            Operand::new("x", Int).with_doc("Scalar or vector value to shift"),
2593            Operand::new("y", iB).with_doc("Number of bits to shift"),
2594        ])
2595        .operands_out(vec![Operand::new("a", Int)]),
2596    );
2597
2598    ig.push(
2599        Inst::new(
2600            "sshr",
2601            r#"
2602        Signed shift right. Shift bits in ``x`` towards the LSB by ``y``
2603        places, shifting in sign bits to the MSB. Also called an *arithmetic
2604        shift*.
2605
2606        The shift amount is masked to the size of ``x``.
2607        "#,
2608            &formats.binary,
2609        )
2610        .operands_in(vec![
2611            Operand::new("x", Int).with_doc("Scalar or vector value to shift"),
2612            Operand::new("y", iB).with_doc("Number of bits to shift"),
2613        ])
2614        .operands_out(vec![Operand::new("a", Int)]),
2615    );
2616
2617    ig.push(
2618        Inst::new(
2619            "ishl_imm",
2620            r#"
2621        Integer shift left by immediate.
2622
2623        The shift amount is masked to the size of ``x``.
2624        "#,
2625            &formats.binary_imm64,
2626        )
2627        .operands_in(vec![
2628            Operand::new("x", Int).with_doc("Scalar or vector value to shift"),
2629            Operand::new("Y", &imm.imm64),
2630        ])
2631        .operands_out(vec![Operand::new("a", Int)]),
2632    );
2633
2634    ig.push(
2635        Inst::new(
2636            "ushr_imm",
2637            r#"
2638        Unsigned shift right by immediate.
2639
2640        The shift amount is masked to the size of ``x``.
2641        "#,
2642            &formats.binary_imm64,
2643        )
2644        .operands_in(vec![
2645            Operand::new("x", Int).with_doc("Scalar or vector value to shift"),
2646            Operand::new("Y", &imm.imm64),
2647        ])
2648        .operands_out(vec![Operand::new("a", Int)]),
2649    );
2650
2651    ig.push(
2652        Inst::new(
2653            "sshr_imm",
2654            r#"
2655        Signed shift right by immediate.
2656
2657        The shift amount is masked to the size of ``x``.
2658        "#,
2659            &formats.binary_imm64,
2660        )
2661        .operands_in(vec![
2662            Operand::new("x", Int).with_doc("Scalar or vector value to shift"),
2663            Operand::new("Y", &imm.imm64),
2664        ])
2665        .operands_out(vec![Operand::new("a", Int)]),
2666    );
2667
2668    ig.push(
2669        Inst::new(
2670            "bitrev",
2671            r#"
2672        Reverse the bits of a integer.
2673
2674        Reverses the bits in ``x``.
2675        "#,
2676            &formats.unary,
2677        )
2678        .operands_in(vec![Operand::new("x", iB)])
2679        .operands_out(vec![Operand::new("a", iB)]),
2680    );
2681
2682    ig.push(
2683        Inst::new(
2684            "clz",
2685            r#"
2686        Count leading zero bits.
2687
2688        Starting from the MSB in ``x``, count the number of zero bits before
2689        reaching the first one bit. When ``x`` is zero, returns the size of x
2690        in bits.
2691        "#,
2692            &formats.unary,
2693        )
2694        .operands_in(vec![Operand::new("x", iB)])
2695        .operands_out(vec![Operand::new("a", iB)]),
2696    );
2697
2698    ig.push(
2699        Inst::new(
2700            "cls",
2701            r#"
2702        Count leading sign bits.
2703
2704        Starting from the MSB after the sign bit in ``x``, count the number of
2705        consecutive bits identical to the sign bit. When ``x`` is 0 or -1,
2706        returns one less than the size of x in bits.
2707        "#,
2708            &formats.unary,
2709        )
2710        .operands_in(vec![Operand::new("x", iB)])
2711        .operands_out(vec![Operand::new("a", iB)]),
2712    );
2713
2714    ig.push(
2715        Inst::new(
2716            "ctz",
2717            r#"
2718        Count trailing zeros.
2719
2720        Starting from the LSB in ``x``, count the number of zero bits before
2721        reaching the first one bit. When ``x`` is zero, returns the size of x
2722        in bits.
2723        "#,
2724            &formats.unary,
2725        )
2726        .operands_in(vec![Operand::new("x", iB)])
2727        .operands_out(vec![Operand::new("a", iB)]),
2728    );
2729
2730    ig.push(
2731        Inst::new(
2732            "bswap",
2733            r#"
2734        Reverse the byte order of an integer.
2735
2736        Reverses the bytes in ``x``.
2737        "#,
2738            &formats.unary,
2739        )
2740        .operands_in(vec![Operand::new("x", iSwappable)])
2741        .operands_out(vec![Operand::new("a", iSwappable)]),
2742    );
2743
2744    ig.push(
2745        Inst::new(
2746            "popcnt",
2747            r#"
2748        Population count
2749
2750        Count the number of one bits in ``x``.
2751        "#,
2752            &formats.unary,
2753        )
2754        .operands_in(vec![Operand::new("x", Int)])
2755        .operands_out(vec![Operand::new("a", Int)]),
2756    );
2757
2758    let Float = &TypeVar::new(
2759        "Float",
2760        "A scalar or vector floating point number",
2761        TypeSetBuilder::new()
2762            .floats(Interval::All)
2763            .simd_lanes(Interval::All)
2764            .dynamic_simd_lanes(Interval::All)
2765            .build(),
2766    );
2767
2768    ig.push(
2769        Inst::new(
2770            "fcmp",
2771            r#"
2772        Floating point comparison.
2773
2774        Two IEEE 754-2008 floating point numbers, `x` and `y`, relate to each
2775        other in exactly one of four ways:
2776
2777        ```text
2778        == ==========================================
2779        UN Unordered when one or both numbers is NaN.
2780        EQ When `x = y`. (And `0.0 = -0.0`).
2781        LT When `x < y`.
2782        GT When `x > y`.
2783        == ==========================================
2784        ```
2785
2786        The 14 `floatcc` condition codes each correspond to a subset of
2787        the four relations, except for the empty set which would always be
2788        false, and the full set which would always be true.
2789
2790        The condition codes are divided into 7 'ordered' conditions which don't
2791        include UN, and 7 unordered conditions which all include UN.
2792
2793        ```text
2794        +-------+------------+---------+------------+-------------------------+
2795        |Ordered             |Unordered             |Condition                |
2796        +=======+============+=========+============+=========================+
2797        |ord    |EQ | LT | GT|uno      |UN          |NaNs absent / present.   |
2798        +-------+------------+---------+------------+-------------------------+
2799        |eq     |EQ          |ueq      |UN | EQ     |Equal                    |
2800        +-------+------------+---------+------------+-------------------------+
2801        |one    |LT | GT     |ne       |UN | LT | GT|Not equal                |
2802        +-------+------------+---------+------------+-------------------------+
2803        |lt     |LT          |ult      |UN | LT     |Less than                |
2804        +-------+------------+---------+------------+-------------------------+
2805        |le     |LT | EQ     |ule      |UN | LT | EQ|Less than or equal       |
2806        +-------+------------+---------+------------+-------------------------+
2807        |gt     |GT          |ugt      |UN | GT     |Greater than             |
2808        +-------+------------+---------+------------+-------------------------+
2809        |ge     |GT | EQ     |uge      |UN | GT | EQ|Greater than or equal    |
2810        +-------+------------+---------+------------+-------------------------+
2811        ```
2812
2813        The standard C comparison operators, `<, <=, >, >=`, are all ordered,
2814        so they are false if either operand is NaN. The C equality operator,
2815        `==`, is ordered, and since inequality is defined as the logical
2816        inverse it is *unordered*. They map to the `floatcc` condition
2817        codes as follows:
2818
2819        ```text
2820        ==== ====== ============
2821        C    `Cond` Subset
2822        ==== ====== ============
2823        `==` eq     EQ
2824        `!=` ne     UN | LT | GT
2825        `<`  lt     LT
2826        `<=` le     LT | EQ
2827        `>`  gt     GT
2828        `>=` ge     GT | EQ
2829        ==== ====== ============
2830        ```
2831
2832        This subset of condition codes also corresponds to the WebAssembly
2833        floating point comparisons of the same name.
2834
2835        When this instruction compares floating point vectors, it returns a
2836        vector with the results of lane-wise comparisons.
2837
2838        When comparing scalars, the result is:
2839            - `1` if the condition holds.
2840            - `0` if the condition does not hold.
2841
2842        When comparing vectors, the result is:
2843            - `-1` (i.e. all ones) in each lane where the condition holds.
2844            - `0` in each lane where the condition does not hold.
2845        "#,
2846            &formats.float_compare,
2847        )
2848        .operands_in(vec![
2849            Operand::new("Cond", &imm.floatcc),
2850            Operand::new("x", Float),
2851            Operand::new("y", Float),
2852        ])
2853        .operands_out(vec![Operand::new("a", &Float.as_truthy())]),
2854    );
2855
2856    ig.push(
2857        Inst::new(
2858            "fadd",
2859            r#"
2860        Floating point addition.
2861        "#,
2862            &formats.binary,
2863        )
2864        .operands_in(vec![Operand::new("x", Float), Operand::new("y", Float)])
2865        .operands_out(vec![
2866            Operand::new("a", Float).with_doc("Result of applying operator to each lane"),
2867        ]),
2868    );
2869
2870    ig.push(
2871        Inst::new(
2872            "fsub",
2873            r#"
2874        Floating point subtraction.
2875        "#,
2876            &formats.binary,
2877        )
2878        .operands_in(vec![Operand::new("x", Float), Operand::new("y", Float)])
2879        .operands_out(vec![
2880            Operand::new("a", Float).with_doc("Result of applying operator to each lane"),
2881        ]),
2882    );
2883
2884    ig.push(
2885        Inst::new(
2886            "fmul",
2887            r#"
2888        Floating point multiplication.
2889        "#,
2890            &formats.binary,
2891        )
2892        .operands_in(vec![Operand::new("x", Float), Operand::new("y", Float)])
2893        .operands_out(vec![
2894            Operand::new("a", Float).with_doc("Result of applying operator to each lane"),
2895        ]),
2896    );
2897
2898    ig.push(
2899        Inst::new(
2900            "fdiv",
2901            r#"
2902        Floating point division.
2903
2904        Unlike the integer division instructions ` and
2905        `udiv`, this can't trap. Division by zero is infinity or
2906        NaN, depending on the dividend.
2907        "#,
2908            &formats.binary,
2909        )
2910        .operands_in(vec![Operand::new("x", Float), Operand::new("y", Float)])
2911        .operands_out(vec![
2912            Operand::new("a", Float).with_doc("Result of applying operator to each lane"),
2913        ]),
2914    );
2915
2916    ig.push(
2917        Inst::new(
2918            "sqrt",
2919            r#"
2920        Floating point square root.
2921        "#,
2922            &formats.unary,
2923        )
2924        .operands_in(vec![Operand::new("x", Float)])
2925        .operands_out(vec![
2926            Operand::new("a", Float).with_doc("Result of applying operator to each lane"),
2927        ]),
2928    );
2929
2930    ig.push(
2931        Inst::new(
2932            "fma",
2933            r#"
2934        Floating point fused multiply-and-add.
2935
2936        Computes `a := xy+z` without any intermediate rounding of the
2937        product.
2938        "#,
2939            &formats.ternary,
2940        )
2941        .operands_in(vec![
2942            Operand::new("x", Float),
2943            Operand::new("y", Float),
2944            Operand::new("z", Float),
2945        ])
2946        .operands_out(vec![
2947            Operand::new("a", Float).with_doc("Result of applying operator to each lane"),
2948        ]),
2949    );
2950
2951    ig.push(
2952        Inst::new(
2953            "fneg",
2954            r#"
2955        Floating point negation.
2956
2957        Note that this is a pure bitwise operation.
2958        "#,
2959            &formats.unary,
2960        )
2961        .operands_in(vec![Operand::new("x", Float)])
2962        .operands_out(vec![
2963            Operand::new("a", Float).with_doc("``x`` with its sign bit inverted"),
2964        ]),
2965    );
2966
2967    ig.push(
2968        Inst::new(
2969            "fabs",
2970            r#"
2971        Floating point absolute value.
2972
2973        Note that this is a pure bitwise operation.
2974        "#,
2975            &formats.unary,
2976        )
2977        .operands_in(vec![Operand::new("x", Float)])
2978        .operands_out(vec![
2979            Operand::new("a", Float).with_doc("``x`` with its sign bit cleared"),
2980        ]),
2981    );
2982
2983    ig.push(
2984        Inst::new(
2985            "fcopysign",
2986            r#"
2987        Floating point copy sign.
2988
2989        Note that this is a pure bitwise operation. The sign bit from ``y`` is
2990        copied to the sign bit of ``x``.
2991        "#,
2992            &formats.binary,
2993        )
2994        .operands_in(vec![Operand::new("x", Float), Operand::new("y", Float)])
2995        .operands_out(vec![
2996            Operand::new("a", Float).with_doc("``x`` with its sign bit changed to that of ``y``"),
2997        ]),
2998    );
2999
3000    ig.push(
3001        Inst::new(
3002            "fmin",
3003            r#"
3004        Floating point minimum, propagating NaNs using the WebAssembly rules.
3005
3006        If either operand is NaN, this returns NaN with an unspecified sign. Furthermore, if
3007        each input NaN consists of a mantissa whose most significant bit is 1 and the rest is
3008        0, then the output has the same form. Otherwise, the output mantissa's most significant
3009        bit is 1 and the rest is unspecified.
3010        "#,
3011            &formats.binary,
3012        )
3013        .operands_in(vec![Operand::new("x", Float), Operand::new("y", Float)])
3014        .operands_out(vec![
3015            Operand::new("a", Float).with_doc("The smaller of ``x`` and ``y``"),
3016        ]),
3017    );
3018
3019    ig.push(
3020        Inst::new(
3021            "fmax",
3022            r#"
3023        Floating point maximum, propagating NaNs using the WebAssembly rules.
3024
3025        If either operand is NaN, this returns NaN with an unspecified sign. Furthermore, if
3026        each input NaN consists of a mantissa whose most significant bit is 1 and the rest is
3027        0, then the output has the same form. Otherwise, the output mantissa's most significant
3028        bit is 1 and the rest is unspecified.
3029        "#,
3030            &formats.binary,
3031        )
3032        .operands_in(vec![Operand::new("x", Float), Operand::new("y", Float)])
3033        .operands_out(vec![
3034            Operand::new("a", Float).with_doc("The larger of ``x`` and ``y``"),
3035        ]),
3036    );
3037
3038    ig.push(
3039        Inst::new(
3040            "ceil",
3041            r#"
3042        Round floating point round to integral, towards positive infinity.
3043        "#,
3044            &formats.unary,
3045        )
3046        .operands_in(vec![Operand::new("x", Float)])
3047        .operands_out(vec![
3048            Operand::new("a", Float).with_doc("``x`` rounded to integral value"),
3049        ]),
3050    );
3051
3052    ig.push(
3053        Inst::new(
3054            "floor",
3055            r#"
3056        Round floating point round to integral, towards negative infinity.
3057        "#,
3058            &formats.unary,
3059        )
3060        .operands_in(vec![Operand::new("x", Float)])
3061        .operands_out(vec![
3062            Operand::new("a", Float).with_doc("``x`` rounded to integral value"),
3063        ]),
3064    );
3065
3066    ig.push(
3067        Inst::new(
3068            "trunc",
3069            r#"
3070        Round floating point round to integral, towards zero.
3071        "#,
3072            &formats.unary,
3073        )
3074        .operands_in(vec![Operand::new("x", Float)])
3075        .operands_out(vec![
3076            Operand::new("a", Float).with_doc("``x`` rounded to integral value"),
3077        ]),
3078    );
3079
3080    ig.push(
3081        Inst::new(
3082            "nearest",
3083            r#"
3084        Round floating point round to integral, towards nearest with ties to
3085        even.
3086        "#,
3087            &formats.unary,
3088        )
3089        .operands_in(vec![Operand::new("x", Float)])
3090        .operands_out(vec![
3091            Operand::new("a", Float).with_doc("``x`` rounded to integral value"),
3092        ]),
3093    );
3094
3095    ig.push(
3096        Inst::new(
3097            "bitcast",
3098            r#"
3099        Reinterpret the bits in `x` as a different type.
3100
3101        The input and output types must be storable to memory and of the same
3102        size. A bitcast is equivalent to storing one type and loading the other
3103        type from the same address, both using the specified MemFlags.
3104
3105        Note that this operation only supports the `big` or `little` MemFlags.
3106        The specified byte order only affects the result in the case where
3107        input and output types differ in lane count/size.  In this case, the
3108        operation is only valid if a byte order specifier is provided.
3109        "#,
3110            &formats.load_no_offset,
3111        )
3112        .operands_in(vec![
3113            Operand::new("MemFlags", &imm.memflags),
3114            Operand::new("x", Mem),
3115        ])
3116        .operands_out(vec![
3117            Operand::new("a", MemTo).with_doc("Bits of `x` reinterpreted"),
3118        ]),
3119    );
3120
3121    ig.push(
3122        Inst::new(
3123            "scalar_to_vector",
3124            r#"
3125            Copies a scalar value to a vector value.  The scalar is copied into the
3126            least significant lane of the vector, and all other lanes will be zero.
3127            "#,
3128            &formats.unary,
3129        )
3130        .operands_in(vec![
3131            Operand::new("s", &TxN.lane_of()).with_doc("A scalar value"),
3132        ])
3133        .operands_out(vec![Operand::new("a", TxN).with_doc("A vector value")]),
3134    );
3135
3136    let Truthy = &TypeVar::new(
3137        "Truthy",
3138        "A scalar whose values are truthy",
3139        TypeSetBuilder::new().ints(Interval::All).build(),
3140    );
3141    let IntTo = &TypeVar::new(
3142        "IntTo",
3143        "An integer type",
3144        TypeSetBuilder::new().ints(Interval::All).build(),
3145    );
3146
3147    ig.push(
3148        Inst::new(
3149            "bmask",
3150            r#"
3151        Convert `x` to an integer mask.
3152
3153        Non-zero maps to all 1s and zero maps to all 0s.
3154        "#,
3155            &formats.unary,
3156        )
3157        .operands_in(vec![Operand::new("x", Truthy)])
3158        .operands_out(vec![Operand::new("a", IntTo)]),
3159    );
3160
3161    let Int = &TypeVar::new(
3162        "Int",
3163        "A scalar integer type",
3164        TypeSetBuilder::new().ints(Interval::All).build(),
3165    );
3166
3167    ig.push(
3168        Inst::new(
3169            "ireduce",
3170            r#"
3171        Convert `x` to a smaller integer type by discarding
3172        the most significant bits.
3173
3174        This is the same as reducing modulo `2^n`.
3175        "#,
3176            &formats.unary,
3177        )
3178        .operands_in(vec![
3179            Operand::new("x", &Int.wider())
3180                .with_doc("A scalar integer type, wider than the controlling type"),
3181        ])
3182        .operands_out(vec![Operand::new("a", Int)]),
3183    );
3184
3185    let I16or32or64xN = &TypeVar::new(
3186        "I16or32or64xN",
3187        "A SIMD vector type containing integer lanes 16, 32, or 64 bits wide",
3188        TypeSetBuilder::new()
3189            .ints(16..64)
3190            .simd_lanes(2..8)
3191            .dynamic_simd_lanes(2..8)
3192            .includes_scalars(false)
3193            .build(),
3194    );
3195
3196    ig.push(
3197        Inst::new(
3198            "snarrow",
3199            r#"
3200        Combine `x` and `y` into a vector with twice the lanes but half the integer width while
3201        saturating overflowing values to the signed maximum and minimum.
3202
3203        The lanes will be concatenated after narrowing. For example, when `x` and `y` are `i32x4`
3204        and `x = [x3, x2, x1, x0]` and `y = [y3, y2, y1, y0]`, then after narrowing the value
3205        returned is an `i16x8`: `a = [y3', y2', y1', y0', x3', x2', x1', x0']`.
3206            "#,
3207            &formats.binary,
3208        )
3209        .operands_in(vec![
3210            Operand::new("x", I16or32or64xN),
3211            Operand::new("y", I16or32or64xN),
3212        ])
3213        .operands_out(vec![Operand::new("a", &I16or32or64xN.split_lanes())]),
3214    );
3215
3216    ig.push(
3217        Inst::new(
3218            "unarrow",
3219            r#"
3220        Combine `x` and `y` into a vector with twice the lanes but half the integer width while
3221        saturating overflowing values to the unsigned maximum and minimum.
3222
3223        Note that all input lanes are considered signed: any negative lanes will overflow and be
3224        replaced with the unsigned minimum, `0x00`.
3225
3226        The lanes will be concatenated after narrowing. For example, when `x` and `y` are `i32x4`
3227        and `x = [x3, x2, x1, x0]` and `y = [y3, y2, y1, y0]`, then after narrowing the value
3228        returned is an `i16x8`: `a = [y3', y2', y1', y0', x3', x2', x1', x0']`.
3229            "#,
3230            &formats.binary,
3231        )
3232        .operands_in(vec![
3233            Operand::new("x", I16or32or64xN),
3234            Operand::new("y", I16or32or64xN),
3235        ])
3236        .operands_out(vec![Operand::new("a", &I16or32or64xN.split_lanes())]),
3237    );
3238
3239    ig.push(
3240        Inst::new(
3241            "uunarrow",
3242            r#"
3243        Combine `x` and `y` into a vector with twice the lanes but half the integer width while
3244        saturating overflowing values to the unsigned maximum and minimum.
3245
3246        Note that all input lanes are considered unsigned: any negative values will be interpreted as unsigned, overflowing and being replaced with the unsigned maximum.
3247
3248        The lanes will be concatenated after narrowing. For example, when `x` and `y` are `i32x4`
3249        and `x = [x3, x2, x1, x0]` and `y = [y3, y2, y1, y0]`, then after narrowing the value
3250        returned is an `i16x8`: `a = [y3', y2', y1', y0', x3', x2', x1', x0']`.
3251            "#,
3252            &formats.binary,
3253        )
3254        .operands_in(vec![Operand::new("x", I16or32or64xN), Operand::new("y", I16or32or64xN)])
3255        .operands_out(vec![Operand::new("a", &I16or32or64xN.split_lanes())]),
3256    );
3257
3258    let I8or16or32xN = &TypeVar::new(
3259        "I8or16or32xN",
3260        "A SIMD vector type containing integer lanes 8, 16, or 32 bits wide.",
3261        TypeSetBuilder::new()
3262            .ints(8..32)
3263            .simd_lanes(2..16)
3264            .dynamic_simd_lanes(2..16)
3265            .includes_scalars(false)
3266            .build(),
3267    );
3268
3269    ig.push(
3270        Inst::new(
3271            "swiden_low",
3272            r#"
3273        Widen the low lanes of `x` using signed extension.
3274
3275        This will double the lane width and halve the number of lanes.
3276            "#,
3277            &formats.unary,
3278        )
3279        .operands_in(vec![Operand::new("x", I8or16or32xN)])
3280        .operands_out(vec![Operand::new("a", &I8or16or32xN.merge_lanes())]),
3281    );
3282
3283    ig.push(
3284        Inst::new(
3285            "swiden_high",
3286            r#"
3287        Widen the high lanes of `x` using signed extension.
3288
3289        This will double the lane width and halve the number of lanes.
3290            "#,
3291            &formats.unary,
3292        )
3293        .operands_in(vec![Operand::new("x", I8or16or32xN)])
3294        .operands_out(vec![Operand::new("a", &I8or16or32xN.merge_lanes())]),
3295    );
3296
3297    ig.push(
3298        Inst::new(
3299            "uwiden_low",
3300            r#"
3301        Widen the low lanes of `x` using unsigned extension.
3302
3303        This will double the lane width and halve the number of lanes.
3304            "#,
3305            &formats.unary,
3306        )
3307        .operands_in(vec![Operand::new("x", I8or16or32xN)])
3308        .operands_out(vec![Operand::new("a", &I8or16or32xN.merge_lanes())]),
3309    );
3310
3311    ig.push(
3312        Inst::new(
3313            "uwiden_high",
3314            r#"
3315            Widen the high lanes of `x` using unsigned extension.
3316
3317            This will double the lane width and halve the number of lanes.
3318            "#,
3319            &formats.unary,
3320        )
3321        .operands_in(vec![Operand::new("x", I8or16or32xN)])
3322        .operands_out(vec![Operand::new("a", &I8or16or32xN.merge_lanes())]),
3323    );
3324
3325    ig.push(
3326        Inst::new(
3327            "iadd_pairwise",
3328            r#"
3329        Does lane-wise integer pairwise addition on two operands, putting the
3330        combined results into a single vector result. Here a pair refers to adjacent
3331        lanes in a vector, i.e. i*2 + (i*2+1) for i == num_lanes/2. The first operand
3332        pairwise add results will make up the low half of the resulting vector while
3333        the second operand pairwise add results will make up the upper half of the
3334        resulting vector.
3335            "#,
3336            &formats.binary,
3337        )
3338        .operands_in(vec![
3339            Operand::new("x", I8or16or32xN),
3340            Operand::new("y", I8or16or32xN),
3341        ])
3342        .operands_out(vec![Operand::new("a", I8or16or32xN)]),
3343    );
3344
3345    let I8x16 = &TypeVar::new(
3346        "I8x16",
3347        "A SIMD vector type consisting of 16 lanes of 8-bit integers",
3348        TypeSetBuilder::new()
3349            .ints(8..8)
3350            .simd_lanes(16..16)
3351            .includes_scalars(false)
3352            .build(),
3353    );
3354
3355    ig.push(
3356        Inst::new(
3357            "x86_pmaddubsw",
3358            r#"
3359        An instruction with equivalent semantics to `pmaddubsw` on x86.
3360
3361        This instruction will take signed bytes from the first argument and
3362        multiply them against unsigned bytes in the second argument. Adjacent
3363        pairs are then added, with saturating, to a 16-bit value and are packed
3364        into the result.
3365            "#,
3366            &formats.binary,
3367        )
3368        .operands_in(vec![Operand::new("x", I8x16), Operand::new("y", I8x16)])
3369        .operands_out(vec![Operand::new("a", I16x8)]),
3370    );
3371
3372    ig.push(
3373        Inst::new(
3374            "uextend",
3375            r#"
3376        Convert `x` to a larger integer type by zero-extending.
3377
3378        Each lane in `x` is converted to a larger integer type by adding
3379        zeroes. The result has the same numerical value as `x` when both are
3380        interpreted as unsigned integers.
3381
3382        The result type must have the same number of vector lanes as the input,
3383        and each lane must not have fewer bits that the input lanes. If the
3384        input and output types are the same, this is a no-op.
3385        "#,
3386            &formats.unary,
3387        )
3388        .operands_in(vec![Operand::new("x", &Int.narrower()).with_doc(
3389            "A scalar integer type, narrower than the controlling type",
3390        )])
3391        .operands_out(vec![Operand::new("a", Int)]),
3392    );
3393
3394    ig.push(
3395        Inst::new(
3396            "sextend",
3397            r#"
3398        Convert `x` to a larger integer type by sign-extending.
3399
3400        Each lane in `x` is converted to a larger integer type by replicating
3401        the sign bit. The result has the same numerical value as `x` when both
3402        are interpreted as signed integers.
3403
3404        The result type must have the same number of vector lanes as the input,
3405        and each lane must not have fewer bits that the input lanes. If the
3406        input and output types are the same, this is a no-op.
3407        "#,
3408            &formats.unary,
3409        )
3410        .operands_in(vec![Operand::new("x", &Int.narrower()).with_doc(
3411            "A scalar integer type, narrower than the controlling type",
3412        )])
3413        .operands_out(vec![Operand::new("a", Int)]),
3414    );
3415
3416    let FloatScalar = &TypeVar::new(
3417        "FloatScalar",
3418        "A scalar only floating point number",
3419        TypeSetBuilder::new().floats(Interval::All).build(),
3420    );
3421
3422    ig.push(
3423        Inst::new(
3424            "fpromote",
3425            r#"
3426        Convert `x` to a larger floating point format.
3427
3428        Each lane in `x` is converted to the destination floating point format.
3429        This is an exact operation.
3430
3431        Cranelift currently only supports two floating point formats
3432        - `f32` and `f64`. This may change in the future.
3433
3434        The result type must have the same number of vector lanes as the input,
3435        and the result lanes must not have fewer bits than the input lanes.
3436        "#,
3437            &formats.unary,
3438        )
3439        .operands_in(vec![Operand::new("x", &FloatScalar.narrower()).with_doc(
3440            "A scalar only floating point number, narrower than the controlling type",
3441        )])
3442        .operands_out(vec![Operand::new("a", FloatScalar)]),
3443    );
3444
3445    ig.push(
3446        Inst::new(
3447            "fdemote",
3448            r#"
3449        Convert `x` to a smaller floating point format.
3450
3451        Each lane in `x` is converted to the destination floating point format
3452        by rounding to nearest, ties to even.
3453
3454        Cranelift currently only supports two floating point formats
3455        - `f32` and `f64`. This may change in the future.
3456
3457        The result type must have the same number of vector lanes as the input,
3458        and the result lanes must not have more bits than the input lanes.
3459        "#,
3460            &formats.unary,
3461        )
3462        .operands_in(vec![Operand::new("x", &FloatScalar.wider()).with_doc(
3463            "A scalar only floating point number, wider than the controlling type",
3464        )])
3465        .operands_out(vec![Operand::new("a", FloatScalar)]),
3466    );
3467
3468    let F64x2 = &TypeVar::new(
3469        "F64x2",
3470        "A SIMD vector type consisting of 2 lanes of 64-bit floats",
3471        TypeSetBuilder::new()
3472            .floats(64..64)
3473            .simd_lanes(2..2)
3474            .includes_scalars(false)
3475            .build(),
3476    );
3477    let F32x4 = &TypeVar::new(
3478        "F32x4",
3479        "A SIMD vector type consisting of 4 lanes of 32-bit floats",
3480        TypeSetBuilder::new()
3481            .floats(32..32)
3482            .simd_lanes(4..4)
3483            .includes_scalars(false)
3484            .build(),
3485    );
3486
3487    ig.push(
3488        Inst::new(
3489            "fvdemote",
3490            r#"
3491                Convert `x` to a smaller floating point format.
3492
3493                Each lane in `x` is converted to the destination floating point format
3494                by rounding to nearest, ties to even.
3495
3496                Cranelift currently only supports two floating point formats
3497                - `f32` and `f64`. This may change in the future.
3498
3499                Fvdemote differs from fdemote in that with fvdemote it targets vectors.
3500                Fvdemote is constrained to having the input type being F64x2 and the result
3501                type being F32x4. The result lane that was the upper half of the input lane
3502                is initialized to zero.
3503                "#,
3504            &formats.unary,
3505        )
3506        .operands_in(vec![Operand::new("x", F64x2)])
3507        .operands_out(vec![Operand::new("a", F32x4)]),
3508    );
3509
3510    ig.push(
3511        Inst::new(
3512            "fvpromote_low",
3513            r#"
3514        Converts packed single precision floating point to packed double precision floating point.
3515
3516        Considering only the lower half of the register, the low lanes in `x` are interpreted as
3517        single precision floats that are then converted to a double precision floats.
3518
3519        The result type will have half the number of vector lanes as the input. Fvpromote_low is
3520        constrained to input F32x4 with a result type of F64x2.
3521        "#,
3522            &formats.unary,
3523        )
3524        .operands_in(vec![Operand::new("a", F32x4)])
3525        .operands_out(vec![Operand::new("x", F64x2)]),
3526    );
3527
3528    let IntTo = &TypeVar::new(
3529        "IntTo",
3530        "An scalar only integer type",
3531        TypeSetBuilder::new().ints(Interval::All).build(),
3532    );
3533
3534    ig.push(
3535        Inst::new(
3536            "fcvt_to_uint",
3537            r#"
3538        Converts floating point scalars to unsigned integer.
3539
3540        Only operates on `x` if it is a scalar. If `x` is NaN or if
3541        the unsigned integral value cannot be represented in the result
3542        type, this instruction traps.
3543
3544        "#,
3545            &formats.unary,
3546        )
3547        .operands_in(vec![Operand::new("x", FloatScalar)])
3548        .operands_out(vec![Operand::new("a", IntTo)])
3549        .can_trap()
3550        .side_effects_idempotent(),
3551    );
3552
3553    ig.push(
3554        Inst::new(
3555            "fcvt_to_sint",
3556            r#"
3557        Converts floating point scalars to signed integer.
3558
3559        Only operates on `x` if it is a scalar. If `x` is NaN or if
3560        the unsigned integral value cannot be represented in the result
3561        type, this instruction traps.
3562
3563        "#,
3564            &formats.unary,
3565        )
3566        .operands_in(vec![Operand::new("x", FloatScalar)])
3567        .operands_out(vec![Operand::new("a", IntTo)])
3568        .can_trap()
3569        .side_effects_idempotent(),
3570    );
3571
3572    let IntTo = &TypeVar::new(
3573        "IntTo",
3574        "A larger integer type with the same number of lanes",
3575        TypeSetBuilder::new()
3576            .ints(Interval::All)
3577            .simd_lanes(Interval::All)
3578            .build(),
3579    );
3580
3581    ig.push(
3582        Inst::new(
3583            "fcvt_to_uint_sat",
3584            r#"
3585        Convert floating point to unsigned integer as fcvt_to_uint does, but
3586        saturates the input instead of trapping. NaN and negative values are
3587        converted to 0.
3588        "#,
3589            &formats.unary,
3590        )
3591        .operands_in(vec![Operand::new("x", Float)])
3592        .operands_out(vec![Operand::new("a", IntTo)]),
3593    );
3594
3595    ig.push(
3596        Inst::new(
3597            "fcvt_to_sint_sat",
3598            r#"
3599        Convert floating point to signed integer as fcvt_to_sint does, but
3600        saturates the input instead of trapping. NaN values are converted to 0.
3601        "#,
3602            &formats.unary,
3603        )
3604        .operands_in(vec![Operand::new("x", Float)])
3605        .operands_out(vec![Operand::new("a", IntTo)]),
3606    );
3607
3608    ig.push(
3609        Inst::new(
3610            "x86_cvtt2dq",
3611            r#"
3612        A float-to-integer conversion instruction for vectors-of-floats which
3613        has the same semantics as `cvttp{s,d}2dq` on x86. This specifically
3614        returns `INT_MIN` for NaN or out-of-bounds lanes.
3615        "#,
3616            &formats.unary,
3617        )
3618        .operands_in(vec![Operand::new("x", Float)])
3619        .operands_out(vec![Operand::new("a", IntTo)]),
3620    );
3621
3622    let Int = &TypeVar::new(
3623        "Int",
3624        "A scalar or vector integer type",
3625        TypeSetBuilder::new()
3626            .ints(Interval::All)
3627            .simd_lanes(Interval::All)
3628            .build(),
3629    );
3630
3631    let FloatTo = &TypeVar::new(
3632        "FloatTo",
3633        "A scalar or vector floating point number",
3634        TypeSetBuilder::new()
3635            .floats(Interval::All)
3636            .simd_lanes(Interval::All)
3637            .build(),
3638    );
3639
3640    ig.push(
3641        Inst::new(
3642            "fcvt_from_uint",
3643            r#"
3644        Convert unsigned integer to floating point.
3645
3646        Each lane in `x` is interpreted as an unsigned integer and converted to
3647        floating point using round to nearest, ties to even.
3648
3649        The result type must have the same number of vector lanes as the input.
3650        "#,
3651            &formats.unary,
3652        )
3653        .operands_in(vec![Operand::new("x", Int)])
3654        .operands_out(vec![Operand::new("a", FloatTo)]),
3655    );
3656
3657    ig.push(
3658        Inst::new(
3659            "fcvt_from_sint",
3660            r#"
3661        Convert signed integer to floating point.
3662
3663        Each lane in `x` is interpreted as a signed integer and converted to
3664        floating point using round to nearest, ties to even.
3665
3666        The result type must have the same number of vector lanes as the input.
3667        "#,
3668            &formats.unary,
3669        )
3670        .operands_in(vec![Operand::new("x", Int)])
3671        .operands_out(vec![Operand::new("a", FloatTo)]),
3672    );
3673
3674    let WideInt = &TypeVar::new(
3675        "WideInt",
3676        "An integer type of width `i16` upwards",
3677        TypeSetBuilder::new().ints(16..128).build(),
3678    );
3679
3680    ig.push(
3681        Inst::new(
3682            "isplit",
3683            r#"
3684        Split an integer into low and high parts.
3685
3686        Vectors of integers are split lane-wise, so the results have the same
3687        number of lanes as the input, but the lanes are half the size.
3688
3689        Returns the low half of `x` and the high half of `x` as two independent
3690        values.
3691        "#,
3692            &formats.unary,
3693        )
3694        .operands_in(vec![Operand::new("x", WideInt)])
3695        .operands_out(vec![
3696            Operand::new("lo", &WideInt.half_width()).with_doc("The low bits of `x`"),
3697            Operand::new("hi", &WideInt.half_width()).with_doc("The high bits of `x`"),
3698        ]),
3699    );
3700
3701    ig.push(
3702        Inst::new(
3703            "iconcat",
3704            r#"
3705        Concatenate low and high bits to form a larger integer type.
3706
3707        Vectors of integers are concatenated lane-wise such that the result has
3708        the same number of lanes as the inputs, but the lanes are twice the
3709        size.
3710        "#,
3711            &formats.binary,
3712        )
3713        .operands_in(vec![
3714            Operand::new("lo", NarrowInt),
3715            Operand::new("hi", NarrowInt),
3716        ])
3717        .operands_out(vec![
3718            Operand::new("a", &NarrowInt.double_width())
3719                .with_doc("The concatenation of `lo` and `hi`"),
3720        ]),
3721    );
3722
3723    // Instructions relating to atomic memory accesses and fences
3724    let AtomicMem = &TypeVar::new(
3725        "AtomicMem",
3726        "Any type that can be stored in memory, which can be used in an atomic operation",
3727        TypeSetBuilder::new().ints(8..128).build(),
3728    );
3729
3730    ig.push(
3731        Inst::new(
3732            "atomic_rmw",
3733            r#"
3734        Atomically read-modify-write memory at `p`, with second operand `x`.  The old value is
3735        returned.  `p` has the type of the target word size, and `x` may be any integer type; note
3736        that some targets require specific target features to be enabled in order to support 128-bit
3737        integer atomics.  The type of the returned value is the same as the type of `x`.  This
3738        operation is sequentially consistent and creates happens-before edges that order normal
3739        (non-atomic) loads and stores.
3740        "#,
3741            &formats.atomic_rmw,
3742        )
3743        .operands_in(vec![
3744            Operand::new("MemFlags", &imm.memflags),
3745            Operand::new("AtomicRmwOp", &imm.atomic_rmw_op),
3746            Operand::new("p", iAddr),
3747            Operand::new("x", AtomicMem).with_doc("Value to be atomically stored"),
3748        ])
3749        .operands_out(vec![
3750            Operand::new("a", AtomicMem).with_doc("Value atomically loaded"),
3751        ])
3752        .can_load()
3753        .can_store()
3754        .other_side_effects(),
3755    );
3756
3757    ig.push(
3758        Inst::new(
3759            "atomic_cas",
3760            r#"
3761        Perform an atomic compare-and-swap operation on memory at `p`, with expected value `e`,
3762        storing `x` if the value at `p` equals `e`.  The old value at `p` is returned,
3763        regardless of whether the operation succeeds or fails.  `p` has the type of the target
3764        word size, and `x` and `e` must have the same type and the same size, which may be any
3765        integer type; note that some targets require specific target features to be enabled in order
3766        to support 128-bit integer atomics.  The type of the returned value is the same as the type
3767        of `x` and `e`.  This operation is sequentially consistent and creates happens-before edges
3768        that order normal (non-atomic) loads and stores.
3769        "#,
3770            &formats.atomic_cas,
3771        )
3772        .operands_in(vec![
3773            Operand::new("MemFlags", &imm.memflags),
3774            Operand::new("p", iAddr),
3775            Operand::new("e", AtomicMem).with_doc("Expected value in CAS"),
3776            Operand::new("x", AtomicMem).with_doc("Value to be atomically stored"),
3777        ])
3778        .operands_out(vec![
3779            Operand::new("a", AtomicMem).with_doc("Value atomically loaded"),
3780        ])
3781        .can_load()
3782        .can_store()
3783        .other_side_effects(),
3784    );
3785
3786    ig.push(
3787        Inst::new(
3788            "atomic_load",
3789            r#"
3790        Atomically load from memory at `p`.
3791
3792        This is a polymorphic instruction that can load any value type which has a memory
3793        representation.  It can only be used for integer types; note that some targets require
3794        specific target features to be enabled in order to support 128-bit integer atomics. This
3795        operation is sequentially consistent and creates happens-before edges that order normal
3796        (non-atomic) loads and stores.
3797        "#,
3798            &formats.load_no_offset,
3799        )
3800        .operands_in(vec![
3801            Operand::new("MemFlags", &imm.memflags),
3802            Operand::new("p", iAddr),
3803        ])
3804        .operands_out(vec![
3805            Operand::new("a", AtomicMem).with_doc("Value atomically loaded"),
3806        ])
3807        .can_load()
3808        .other_side_effects(),
3809    );
3810
3811    ig.push(
3812        Inst::new(
3813            "atomic_store",
3814            r#"
3815        Atomically store `x` to memory at `p`.
3816
3817        This is a polymorphic instruction that can store any value type with a memory
3818        representation.  It can only be used for integer types; note that some targets require
3819        specific target features to be enabled in order to support 128-bit integer atomics This
3820        operation is sequentially consistent and creates happens-before edges that order normal
3821        (non-atomic) loads and stores.
3822        "#,
3823            &formats.store_no_offset,
3824        )
3825        .operands_in(vec![
3826            Operand::new("MemFlags", &imm.memflags),
3827            Operand::new("x", AtomicMem).with_doc("Value to be atomically stored"),
3828            Operand::new("p", iAddr),
3829        ])
3830        .can_store()
3831        .other_side_effects(),
3832    );
3833
3834    ig.push(
3835        Inst::new(
3836            "fence",
3837            r#"
3838        A memory fence.  This must provide ordering to ensure that, at a minimum, neither loads
3839        nor stores of any kind may move forwards or backwards across the fence.  This operation
3840        is sequentially consistent.
3841        "#,
3842            &formats.nullary,
3843        )
3844        .other_side_effects(),
3845    );
3846
3847    let TxN = &TypeVar::new(
3848        "TxN",
3849        "A dynamic vector type",
3850        TypeSetBuilder::new()
3851            .ints(Interval::All)
3852            .floats(Interval::All)
3853            .dynamic_simd_lanes(Interval::All)
3854            .build(),
3855    );
3856
3857    ig.push(
3858        Inst::new(
3859            "extract_vector",
3860            r#"
3861        Return a fixed length sub vector, extracted from a dynamic vector.
3862        "#,
3863            &formats.binary_imm8,
3864        )
3865        .operands_in(vec![
3866            Operand::new("x", TxN).with_doc("The dynamic vector to extract from"),
3867            Operand::new("y", &imm.uimm8).with_doc("128-bit vector index"),
3868        ])
3869        .operands_out(vec![
3870            Operand::new("a", &TxN.dynamic_to_vector()).with_doc("New fixed vector"),
3871        ]),
3872    );
3873}