Merging upstream version 1.75.0+dfsg1.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 189 insertions, 249 deletions

diff --git a/compiler/rustc_codegen_cranelift/src/inline_asm.rs b/compiler/rustc_codegen_cranelift/src/inline_asm.rs
index 50bbf8105..ce0eecca8 100644
--- a/compiler/rustc_codegen_cranelift/src/inline_asm.rs
+++ b/compiler/rustc_codegen_cranelift/src/inline_asm.rs
@@ -1,18 +1,19 @@
 //! Codegen of `asm!` invocations.
 
-use crate::prelude::*;
-
 use std::fmt::Write;
 
 use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_middle::mir::InlineAsmOperand;
 use rustc_span::sym;
 use rustc_target::asm::*;
+use target_lexicon::BinaryFormat;
+
+use crate::prelude::*;
 
 enum CInlineAsmOperand<'tcx> {
     In {
         reg: InlineAsmRegOrRegClass,
-        value: CValue<'tcx>,
+        value: Value,
     },
     Out {
         reg: InlineAsmRegOrRegClass,
@@ -22,7 +23,7 @@ enum CInlineAsmOperand<'tcx> {
     InOut {
         reg: InlineAsmRegOrRegClass,
         _late: bool,
-        in_value: CValue<'tcx>,
+        in_value: Value,
         out_place: Option<CPlace<'tcx>>,
     },
     Const {
@@ -43,191 +44,23 @@ pub(crate) fn codegen_inline_asm<'tcx>(
 ) {
     // FIXME add .eh_frame unwind info directives
 
-    if !template.is_empty() {
-        // Used by panic_abort
-        if template[0] == InlineAsmTemplatePiece::String("int $$0x29".to_string()) {
-            fx.bcx.ins().trap(TrapCode::User(1));
-            return;
-        }
-
-        // Used by stdarch
-        if template[0] == InlineAsmTemplatePiece::String("mov ".to_string())
-            && matches!(
-                template[1],
-                InlineAsmTemplatePiece::Placeholder {
-                    operand_idx: 0,
-                    modifier: Some('r'),
-                    span: _
-                }
-            )
-            && template[2] == InlineAsmTemplatePiece::String(", rbx".to_string())
-            && template[3] == InlineAsmTemplatePiece::String("\n".to_string())
-            && template[4] == InlineAsmTemplatePiece::String("cpuid".to_string())
-            && template[5] == InlineAsmTemplatePiece::String("\n".to_string())
-            && template[6] == InlineAsmTemplatePiece::String("xchg ".to_string())
-            && matches!(
-                template[7],
-                InlineAsmTemplatePiece::Placeholder {
-                    operand_idx: 0,
-                    modifier: Some('r'),
-                    span: _
-                }
-            )
-            && template[8] == InlineAsmTemplatePiece::String(", rbx".to_string())
-        {
-            assert_eq!(operands.len(), 4);
-            let (leaf, eax_place) = match operands[1] {
-                InlineAsmOperand::InOut {
-                    reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)),
-                    late: _,
-                    ref in_value,
-                    out_place: Some(out_place),
-                } => (
-                    crate::base::codegen_operand(fx, in_value).load_scalar(fx),
-                    crate::base::codegen_place(fx, out_place),
-                ),
-                _ => unreachable!(),
-            };
-            let ebx_place = match operands[0] {
-                InlineAsmOperand::Out {
-                    reg:
-                        InlineAsmRegOrRegClass::RegClass(InlineAsmRegClass::X86(
-                            X86InlineAsmRegClass::reg,
-                        )),
-                    late: _,
-                    place: Some(place),
-                } => crate::base::codegen_place(fx, place),
-                _ => unreachable!(),
-            };
-            let (sub_leaf, ecx_place) = match operands[2] {
-                InlineAsmOperand::InOut {
-                    reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx)),
-                    late: _,
-                    ref in_value,
-                    out_place: Some(out_place),
-                } => (
-                    crate::base::codegen_operand(fx, in_value).load_scalar(fx),
-                    crate::base::codegen_place(fx, out_place),
-                ),
-                _ => unreachable!(),
-            };
-            let edx_place = match operands[3] {
-                InlineAsmOperand::Out {
-                    reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)),
-                    late: _,
-                    place: Some(place),
-                } => crate::base::codegen_place(fx, place),
-                _ => unreachable!(),
-            };
-
-            let (eax, ebx, ecx, edx) = crate::intrinsics::codegen_cpuid_call(fx, leaf, sub_leaf);
-
-            eax_place.write_cvalue(fx, CValue::by_val(eax, fx.layout_of(fx.tcx.types.u32)));
-            ebx_place.write_cvalue(fx, CValue::by_val(ebx, fx.layout_of(fx.tcx.types.u32)));
-            ecx_place.write_cvalue(fx, CValue::by_val(ecx, fx.layout_of(fx.tcx.types.u32)));
-            edx_place.write_cvalue(fx, CValue::by_val(edx, fx.layout_of(fx.tcx.types.u32)));
-            let destination_block = fx.get_block(destination.unwrap());
-            fx.bcx.ins().jump(destination_block, &[]);
-            return;
-        }
-
-        // Used by compiler-builtins
-        if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") {
-            // ___chkstk, ___chkstk_ms and __alloca are only used on Windows
-            crate::trap::trap_unimplemented(fx, "Stack probes are not supported");
-            return;
-        } else if fx.tcx.symbol_name(fx.instance).name == "__alloca" {
-            crate::trap::trap_unimplemented(fx, "Alloca is not supported");
-            return;
-        }
-
-        // Used by measureme
-        if template[0] == InlineAsmTemplatePiece::String("xor %eax, %eax".to_string())
-            && template[1] == InlineAsmTemplatePiece::String("\n".to_string())
-            && template[2] == InlineAsmTemplatePiece::String("mov %rbx, ".to_string())
-            && matches!(
-                template[3],
-                InlineAsmTemplatePiece::Placeholder {
-                    operand_idx: 0,
-                    modifier: Some('r'),
-                    span: _
-                }
-            )
-            && template[4] == InlineAsmTemplatePiece::String("\n".to_string())
-            && template[5] == InlineAsmTemplatePiece::String("cpuid".to_string())
-            && template[6] == InlineAsmTemplatePiece::String("\n".to_string())
-            && template[7] == InlineAsmTemplatePiece::String("mov ".to_string())
-            && matches!(
-                template[8],
-                InlineAsmTemplatePiece::Placeholder {
-                    operand_idx: 0,
-                    modifier: Some('r'),
-                    span: _
-                }
-            )
-            && template[9] == InlineAsmTemplatePiece::String(", %rbx".to_string())
-        {
-            let destination_block = fx.get_block(destination.unwrap());
-            fx.bcx.ins().jump(destination_block, &[]);
-            return;
-        } else if template[0] == InlineAsmTemplatePiece::String("rdpmc".to_string()) {
-            // Return zero dummy values for all performance counters
-            match operands[0] {
-                InlineAsmOperand::In {
-                    reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx)),
-                    value: _,
-                } => {}
-                _ => unreachable!(),
-            };
-            let lo = match operands[1] {
-                InlineAsmOperand::Out {
-                    reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)),
-                    late: true,
-                    place: Some(place),
-                } => crate::base::codegen_place(fx, place),
-                _ => unreachable!(),
-            };
-            let hi = match operands[2] {
-                InlineAsmOperand::Out {
-                    reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)),
-                    late: true,
-                    place: Some(place),
-                } => crate::base::codegen_place(fx, place),
-                _ => unreachable!(),
-            };
-
-            let u32_layout = fx.layout_of(fx.tcx.types.u32);
-            let zero = fx.bcx.ins().iconst(types::I32, 0);
-            lo.write_cvalue(fx, CValue::by_val(zero, u32_layout));
-            hi.write_cvalue(fx, CValue::by_val(zero, u32_layout));
-
-            let destination_block = fx.get_block(destination.unwrap());
-            fx.bcx.ins().jump(destination_block, &[]);
-            return;
-        } else if template[0] == InlineAsmTemplatePiece::String("lock xadd ".to_string())
-            && matches!(
-                template[1],
-                InlineAsmTemplatePiece::Placeholder { operand_idx: 1, modifier: None, span: _ }
-            )
-            && template[2] == InlineAsmTemplatePiece::String(", (".to_string())
-            && matches!(
-                template[3],
-                InlineAsmTemplatePiece::Placeholder { operand_idx: 0, modifier: None, span: _ }
-            )
-            && template[4] == InlineAsmTemplatePiece::String(")".to_string())
-        {
-            let destination_block = fx.get_block(destination.unwrap());
-            fx.bcx.ins().jump(destination_block, &[]);
-            return;
-        }
+    // Used by panic_abort on Windows, but uses a syntax which only happens to work with
+    // asm!() by accident and breaks with the GNU assembler as well as global_asm!() for
+    // the LLVM backend.
+    if template.len() == 1
+        && template[0] == InlineAsmTemplatePiece::String("int $$0x29".to_string())
+    {
+        fx.bcx.ins().trap(TrapCode::User(1));
+        return;
     }
 
     let operands = operands
         .into_iter()
         .map(|operand| match *operand {
-            InlineAsmOperand::In { reg, ref value } => {
-                CInlineAsmOperand::In { reg, value: crate::base::codegen_operand(fx, value) }
-            }
+            InlineAsmOperand::In { reg, ref value } => CInlineAsmOperand::In {
+                reg,
+                value: crate::base::codegen_operand(fx, value).load_scalar(fx),
+            },
             InlineAsmOperand::Out { reg, late, ref place } => CInlineAsmOperand::Out {
                 reg,
                 late,
@@ -237,7 +70,7 @@ pub(crate) fn codegen_inline_asm<'tcx>(
                 CInlineAsmOperand::InOut {
                     reg,
                     _late: late,
-                    in_value: crate::base::codegen_operand(fx, in_value),
+                    in_value: crate::base::codegen_operand(fx, in_value).load_scalar(fx),
                     out_place: out_place.map(|place| crate::base::codegen_place(fx, place)),
                 }
             }
@@ -252,6 +85,12 @@ pub(crate) fn codegen_inline_asm<'tcx>(
                 CInlineAsmOperand::Const { value }
             }
             InlineAsmOperand::SymFn { ref value } => {
+                if cfg!(not(feature = "inline_asm_sym")) {
+                    fx.tcx
+                        .sess
+                        .span_err(span, "asm! and global_asm! sym operands are not yet supported");
+                }
+
                 let const_ = fx.monomorphize(value.const_);
                 if let ty::FnDef(def_id, args) = *const_.ty().kind() {
                     let instance = ty::Instance::resolve_for_fn_ptr(
@@ -329,7 +168,7 @@ pub(crate) fn codegen_inline_asm<'tcx>(
     for (i, operand) in operands.iter().enumerate() {
         match operand {
             CInlineAsmOperand::In { reg: _, value } => {
-                inputs.push((asm_gen.stack_slots_input[i].unwrap(), value.load_scalar(fx)));
+                inputs.push((asm_gen.stack_slots_input[i].unwrap(), *value));
             }
             CInlineAsmOperand::Out { reg: _, late: _, place } => {
                 if let Some(place) = place {
@@ -337,7 +176,7 @@ pub(crate) fn codegen_inline_asm<'tcx>(
                 }
             }
             CInlineAsmOperand::InOut { reg: _, _late: _, in_value, out_place } => {
-                inputs.push((asm_gen.stack_slots_input[i].unwrap(), in_value.load_scalar(fx)));
+                inputs.push((asm_gen.stack_slots_input[i].unwrap(), *in_value));
                 if let Some(out_place) = out_place {
                     outputs.push((asm_gen.stack_slots_output[i].unwrap(), *out_place));
                 }
@@ -589,11 +428,29 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
     }
 
     fn generate_asm_wrapper(&self, asm_name: &str) -> String {
+        let binary_format = crate::target_triple(self.tcx.sess).binary_format;
+
         let mut generated_asm = String::new();
-        writeln!(generated_asm, ".globl {}", asm_name).unwrap();
-        writeln!(generated_asm, ".type {},@function", asm_name).unwrap();
-        writeln!(generated_asm, ".section .text.{},\"ax\",@progbits", asm_name).unwrap();
-        writeln!(generated_asm, "{}:", asm_name).unwrap();
+        match binary_format {
+            BinaryFormat::Elf => {
+                writeln!(generated_asm, ".globl {}", asm_name).unwrap();
+                writeln!(generated_asm, ".type {},@function", asm_name).unwrap();
+                writeln!(generated_asm, ".section .text.{},\"ax\",@progbits", asm_name).unwrap();
+                writeln!(generated_asm, "{}:", asm_name).unwrap();
+            }
+            BinaryFormat::Macho => {
+                writeln!(generated_asm, ".globl _{}", asm_name).unwrap();
+                writeln!(generated_asm, "_{}:", asm_name).unwrap();
+            }
+            BinaryFormat::Coff => {
+                writeln!(generated_asm, ".globl {}", asm_name).unwrap();
+                writeln!(generated_asm, "{}:", asm_name).unwrap();
+            }
+            _ => self
+                .tcx
+                .sess
+                .fatal(format!("Unsupported binary format for inline asm: {binary_format:?}")),
+        }
 
         let is_x86 = matches!(self.arch, InlineAsmArch::X86 | InlineAsmArch::X86_64);
 
@@ -690,8 +547,19 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
         if is_x86 {
             generated_asm.push_str(".att_syntax\n");
         }
-        writeln!(generated_asm, ".size {name}, .-{name}", name = asm_name).unwrap();
-        generated_asm.push_str(".text\n");
+
+        match binary_format {
+            BinaryFormat::Elf => {
+                writeln!(generated_asm, ".size {name}, .-{name}", name = asm_name).unwrap();
+                generated_asm.push_str(".text\n");
+            }
+            BinaryFormat::Macho | BinaryFormat::Coff => {}
+            _ => self
+                .tcx
+                .sess
+                .fatal(format!("Unsupported binary format for inline asm: {binary_format:?}")),
+        }
+
         generated_asm.push_str("\n\n");
 
         generated_asm
@@ -699,25 +567,26 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
 
     fn prologue(generated_asm: &mut String, arch: InlineAsmArch) {
         match arch {
-            InlineAsmArch::X86 => {
-                generated_asm.push_str("    push ebp\n");
-                generated_asm.push_str("    mov ebp,[esp+8]\n");
-            }
             InlineAsmArch::X86_64 => {
                 generated_asm.push_str("    push rbp\n");
-                generated_asm.push_str("    mov rbp,rdi\n");
-            }
-            InlineAsmArch::RiscV32 => {
-                generated_asm.push_str("    addi sp, sp, -8\n");
-                generated_asm.push_str("    sw ra, 4(sp)\n");
-                generated_asm.push_str("    sw s0, 0(sp)\n");
-                generated_asm.push_str("    mv s0, a0\n");
+                generated_asm.push_str("    mov rbp,rsp\n");
+                generated_asm.push_str("    push rbx\n"); // rbx is callee saved
+                // rbx is reserved by LLVM for the "base pointer", so rustc doesn't allow using it
+                generated_asm.push_str("    mov rbx,rdi\n");
+            }
+            InlineAsmArch::AArch64 => {
+                generated_asm.push_str("    stp fp, lr, [sp, #-32]!\n");
+                generated_asm.push_str("    mov fp, sp\n");
+                generated_asm.push_str("    str x19, [sp, #24]\n"); // x19 is callee saved
+                // x19 is reserved by LLVM for the "base pointer", so rustc doesn't allow using it
+                generated_asm.push_str("    mov x19, x0\n");
             }
             InlineAsmArch::RiscV64 => {
                 generated_asm.push_str("    addi sp, sp, -16\n");
                 generated_asm.push_str("    sd ra, 8(sp)\n");
-                generated_asm.push_str("    sd s0, 0(sp)\n");
-                generated_asm.push_str("    mv s0, a0\n");
+                generated_asm.push_str("    sd s1, 0(sp)\n"); // s1 is callee saved
+                // s1/x9 is reserved by LLVM for the "base pointer", so rustc doesn't allow using it
+                generated_asm.push_str("    mv s1, a0\n");
             }
             _ => unimplemented!("prologue for {:?}", arch),
         }
@@ -725,22 +594,18 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
 
     fn epilogue(generated_asm: &mut String, arch: InlineAsmArch) {
         match arch {
-            InlineAsmArch::X86 => {
-                generated_asm.push_str("    pop ebp\n");
-                generated_asm.push_str("    ret\n");
-            }
             InlineAsmArch::X86_64 => {
+                generated_asm.push_str("    pop rbx\n");
                 generated_asm.push_str("    pop rbp\n");
                 generated_asm.push_str("    ret\n");
             }
-            InlineAsmArch::RiscV32 => {
-                generated_asm.push_str("    lw s0, 0(sp)\n");
-                generated_asm.push_str("    lw ra, 4(sp)\n");
-                generated_asm.push_str("    addi sp, sp, 8\n");
+            InlineAsmArch::AArch64 => {
+                generated_asm.push_str("    ldr x19, [sp, #24]\n");
+                generated_asm.push_str("    ldp fp, lr, [sp], #32\n");
                 generated_asm.push_str("    ret\n");
             }
             InlineAsmArch::RiscV64 => {
-                generated_asm.push_str("    ld s0, 0(sp)\n");
+                generated_asm.push_str("    ld s1, 0(sp)\n");
                 generated_asm.push_str("    ld ra, 8(sp)\n");
                 generated_asm.push_str("    addi sp, sp, 16\n");
                 generated_asm.push_str("    ret\n");
@@ -751,10 +616,13 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
 
     fn epilogue_noreturn(generated_asm: &mut String, arch: InlineAsmArch) {
         match arch {
-            InlineAsmArch::X86 | InlineAsmArch::X86_64 => {
+            InlineAsmArch::X86_64 => {
                 generated_asm.push_str("    ud2\n");
             }
-            InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => {
+            InlineAsmArch::AArch64 => {
+                generated_asm.push_str("    brk     #0x1\n");
+            }
+            InlineAsmArch::RiscV64 => {
                 generated_asm.push_str("    ebreak\n");
             }
             _ => unimplemented!("epilogue_noreturn for {:?}", arch),
@@ -768,25 +636,20 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
         offset: Size,
     ) {
         match arch {
-            InlineAsmArch::X86 => {
-                write!(generated_asm, "    mov [ebp+0x{:x}], ", offset.bytes()).unwrap();
-                reg.emit(generated_asm, InlineAsmArch::X86, None).unwrap();
-                generated_asm.push('\n');
-            }
             InlineAsmArch::X86_64 => {
-                write!(generated_asm, "    mov [rbp+0x{:x}], ", offset.bytes()).unwrap();
+                write!(generated_asm, "    mov [rbx+0x{:x}], ", offset.bytes()).unwrap();
                 reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
                 generated_asm.push('\n');
             }
-            InlineAsmArch::RiscV32 => {
-                generated_asm.push_str("    sw ");
-                reg.emit(generated_asm, InlineAsmArch::RiscV32, None).unwrap();
-                writeln!(generated_asm, ", 0x{:x}(s0)", offset.bytes()).unwrap();
+            InlineAsmArch::AArch64 => {
+                generated_asm.push_str("    str ");
+                reg.emit(generated_asm, InlineAsmArch::AArch64, None).unwrap();
+                writeln!(generated_asm, ", [x19, 0x{:x}]", offset.bytes()).unwrap();
             }
             InlineAsmArch::RiscV64 => {
                 generated_asm.push_str("    sd ");
                 reg.emit(generated_asm, InlineAsmArch::RiscV64, None).unwrap();
-                writeln!(generated_asm, ", 0x{:x}(s0)", offset.bytes()).unwrap();
+                writeln!(generated_asm, ", 0x{:x}(s1)", offset.bytes()).unwrap();
             }
             _ => unimplemented!("save_register for {:?}", arch),
         }
@@ -799,25 +662,20 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
         offset: Size,
     ) {
         match arch {
-            InlineAsmArch::X86 => {
-                generated_asm.push_str("    mov ");
-                reg.emit(generated_asm, InlineAsmArch::X86, None).unwrap();
-                writeln!(generated_asm, ", [ebp+0x{:x}]", offset.bytes()).unwrap();
-            }
             InlineAsmArch::X86_64 => {
                 generated_asm.push_str("    mov ");
                 reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
-                writeln!(generated_asm, ", [rbp+0x{:x}]", offset.bytes()).unwrap();
+                writeln!(generated_asm, ", [rbx+0x{:x}]", offset.bytes()).unwrap();
             }
-            InlineAsmArch::RiscV32 => {
-                generated_asm.push_str("    lw ");
-                reg.emit(generated_asm, InlineAsmArch::RiscV32, None).unwrap();
-                writeln!(generated_asm, ", 0x{:x}(s0)", offset.bytes()).unwrap();
+            InlineAsmArch::AArch64 => {
+                generated_asm.push_str("    ldr ");
+                reg.emit(generated_asm, InlineAsmArch::AArch64, None).unwrap();
+                writeln!(generated_asm, ", [x19, 0x{:x}]", offset.bytes()).unwrap();
             }
             InlineAsmArch::RiscV64 => {
                 generated_asm.push_str("    ld ");
                 reg.emit(generated_asm, InlineAsmArch::RiscV64, None).unwrap();
-                writeln!(generated_asm, ", 0x{:x}(s0)", offset.bytes()).unwrap();
+                writeln!(generated_asm, ", 0x{:x}(s1)", offset.bytes()).unwrap();
             }
             _ => unimplemented!("restore_register for {:?}", arch),
         }
@@ -831,13 +689,7 @@ fn call_inline_asm<'tcx>(
     inputs: Vec<(Size, Value)>,
     outputs: Vec<(Size, CPlace<'tcx>)>,
 ) {
-    let stack_slot = fx.bcx.func.create_sized_stack_slot(StackSlotData {
-        kind: StackSlotKind::ExplicitSlot,
-        size: u32::try_from(slot_size.bytes()).unwrap(),
-    });
-    if fx.clif_comments.enabled() {
-        fx.add_comment(stack_slot, "inline asm scratch slot");
-    }
+    let stack_slot = fx.create_stack_slot(u32::try_from(slot_size.bytes()).unwrap(), 16);
 
     let inline_asm_func = fx
         .module
@@ -857,15 +709,103 @@ fn call_inline_asm<'tcx>(
     }
 
     for (offset, value) in inputs {
-        fx.bcx.ins().stack_store(value, stack_slot, i32::try_from(offset.bytes()).unwrap());
+        stack_slot.offset(fx, i32::try_from(offset.bytes()).unwrap().into()).store(
+            fx,
+            value,
+            MemFlags::trusted(),
+        );
     }
 
-    let stack_slot_addr = fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0);
+    let stack_slot_addr = stack_slot.get_addr(fx);
     fx.bcx.ins().call(inline_asm_func, &[stack_slot_addr]);
 
     for (offset, place) in outputs {
         let ty = fx.clif_type(place.layout().ty).unwrap();
-        let value = fx.bcx.ins().stack_load(ty, stack_slot, i32::try_from(offset.bytes()).unwrap());
+        let value = stack_slot.offset(fx, i32::try_from(offset.bytes()).unwrap().into()).load(
+            fx,
+            ty,
+            MemFlags::trusted(),
+        );
         place.write_cvalue(fx, CValue::by_val(value, place.layout()));
     }
 }
+
+pub(crate) fn codegen_xgetbv<'tcx>(
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
+    xcr_no: Value,
+    ret: CPlace<'tcx>,
+) {
+    // FIXME add .eh_frame unwind info directives
+
+    let operands = vec![
+        CInlineAsmOperand::In {
+            reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx)),
+            value: xcr_no,
+        },
+        CInlineAsmOperand::Out {
+            reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)),
+            late: true,
+            place: Some(ret),
+        },
+        CInlineAsmOperand::Out {
+            reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)),
+            late: true,
+            place: None,
+        },
+    ];
+    let options = InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM;
+
+    let mut inputs = Vec::new();
+    let mut outputs = Vec::new();
+
+    let mut asm_gen = InlineAssemblyGenerator {
+        tcx: fx.tcx,
+        arch: fx.tcx.sess.asm_arch.unwrap(),
+        enclosing_def_id: fx.instance.def_id(),
+        template: &[InlineAsmTemplatePiece::String(
+            "
+            xgetbv
+            // out = rdx << 32 | rax
+            shl rdx, 32
+            or rax, rdx
+            "
+            .to_string(),
+        )],
+        operands: &operands,
+        options,
+        registers: Vec::new(),
+        stack_slots_clobber: Vec::new(),
+        stack_slots_input: Vec::new(),
+        stack_slots_output: Vec::new(),
+        stack_slot_size: Size::from_bytes(0),
+    };
+    asm_gen.allocate_registers();
+    asm_gen.allocate_stack_slots();
+
+    let inline_asm_index = fx.cx.inline_asm_index.get();
+    fx.cx.inline_asm_index.set(inline_asm_index + 1);
+    let asm_name = format!(
+        "__inline_asm_{}_n{}",
+        fx.cx.cgu_name.as_str().replace('.', "__").replace('-', "_"),
+        inline_asm_index
+    );
+
+    let generated_asm = asm_gen.generate_asm_wrapper(&asm_name);
+    fx.cx.global_asm.push_str(&generated_asm);
+
+    for (i, operand) in operands.iter().enumerate() {
+        match operand {
+            CInlineAsmOperand::In { reg: _, value } => {
+                inputs.push((asm_gen.stack_slots_input[i].unwrap(), *value));
+            }
+            CInlineAsmOperand::Out { reg: _, late: _, place } => {
+                if let Some(place) = place {
+                    outputs.push((asm_gen.stack_slots_output[i].unwrap(), *place));
+                }
+            }
+            _ => unreachable!(),
+        }
+    }
+
+    call_inline_asm(fx, &asm_name, asm_gen.stack_slot_size, inputs, outputs);
+}