PVE 开启硬件显卡直通功能

首先编辑 GRUB 配置文件：

root@pve:~# vim /etc/default/grub
root@pve:~#
root@pve:~# cat /etc/default/grub
# If you change this file, run 'update-grub' afterwards to update
# /boot/grub/grub.cfg.
# For full documentation of the options in this file, see:
#   info -f grub -n 'Simple configuration'


GRUB_DEFAULT=0
GRUB_TIMEOUT=5
GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian`
GRUB_CMDLINE_LINUX_DEFAULT="quiet intel_iommu=on video=efifb:off"
GRUB_CMDLINE_LINUX=""


# Uncomment to enable BadRAM filtering, modify to suit your needs
# This works with Linux (no patch required) and with any kernel that obtains
# the memory map information from GRUB (GNU Mach, kernel of FreeBSD ...)
#GRUB_BADRAM="0x01234567,0xfefefefe,0x89abcdef,0xefefefef"


# Uncomment to disable graphical terminal (grub-pc only)
#GRUB_TERMINAL=console


# The resolution used on graphical terminal
# note that you can use only modes which your graphic card supports via VBE
# you can see them in real GRUB with the command `vbeinfo'
#GRUB_GFXMODE=640x480


# Uncomment if you don't want GRUB to pass "root=UUID=xxx" parameter to Linux
#GRUB_DISABLE_LINUX_UUID=true


# Uncomment to disable generation of recovery mode menu entries
#GRUB_DISABLE_RECOVERY="true"


# Uncomment to get a beep at grub start
#GRUB_INIT_TUNE="480 440 1"
root@pve:~#


开启IOMMU支持：


GRUB_CMDLINE_LINUX_DEFAULT="quiet intel_iommu=on video=efifb:off"
如果是AMD的CPU：


GRUB_CMDLINE_LINUX_DEFAULT="quiet amd_iommu=on video=efifb:off"

更新 GRUB：

root@pve:~# update-grub
Generating grub configuration file ...
Found linux image: /boot/vmlinuz-5.11.22-5-pve
Found initrd image: /boot/initrd.img-5.11.22-5-pve
Found linux image: /boot/vmlinuz-5.11.22-4-pve
Found initrd image: /boot/initrd.img-5.11.22-4-pve
Found memtest86+ image: /boot/memtest86+.bin
Found memtest86+ multiboot image: /boot/memtest86+_multiboot.bin
done
root@pve:~#

添加所需的系统模块（驱动）：

root@pve:~#
root@pve:~# echo "vfio" >> /etc/modules
root@pve:~# echo "vfio_iommu_type1" >> /etc/modules
root@pve:~# echo "vfio_pci" >> /etc/modules
root@pve:~# echo "vfio_virqfd" >> /etc/modules
root@pve:~#
root@pve:~# cat /etc/modules
# /etc/modules: kernel modules to load at boot time.
#
# This file contains the names of kernel modules that should be loaded
# at boot time, one per line. Lines beginning with "#" are ignored.



# Generated by sensors-detect on Fri Sep 24 17:22:44 2021
# Chip drivers
coretemp
vfio
vfio_iommu_type1
vfio_pci
vfio_virqfd

接着添加模块（驱动）黑名单，即让 GPU 设备在下次系统启动之后不使用这些驱动，把设备腾出来给 vfio 驱动用：

Intel 核显：

echo "blacklist snd_hda_intel" >> /etc/modprobe.d/pve-blacklist.conf
echo "blacklist snd_hda_codec_hdmi" >> /etc/modprobe.d/pve-blacklist.conf
echo "blacklist i915" >> /etc/modprobe.d/pve-blacklist.conf

N 卡 / A 卡：

echo "blacklist nouveau" >> /etc/modprobe.d/pve-blacklist.conf
echo "blacklist radeon" >> /etc/modprobe.d/pve-blacklist.conf

如果是 N 卡还需要加入下面的配置到 kvm.conf（据老外说是避免一些莫名其妙的错误）：

echo "options kvm ignore_msrs=1" > /etc/modprobe.d/kvm.conf

root@pve:~# echo "blacklist nouveau" >> /etc/modprobe.d/pve-blacklist.conf
root@pve:~# echo "blacklist radeon" >> /etc/modprobe.d/pve-blacklist.conf
root@pve:~#
root@pve:~# cat /etc/modprobe.d/pve-blacklist.conf
# This file contains a list of modules which are not supported by Proxmox VE


# nidiafb see bugreport https://bugzilla.proxmox.com/show_bug.cgi?id=701
blacklist nvidiafb
blacklist nouveau
blacklist radeon
root@pve:~#
root@pve:~# echo "options kvm ignore_msrs=1" > /etc/modprobe.d/kvm.conf
root@pve:~#
root@pve:~# cat /etc/modprobe.d/kvm.conf
options kvm ignore_msrs=1

更新内核：

root@pve:~# update-initramfs -u
update-initramfs: Generating /boot/initrd.img-5.11.22-5-pve
Running hook script 'zz-proxmox-boot'..
Re-executing '/etc/kernel/postinst.d/zz-proxmox-boot' in new private mount namespace..
No /etc/kernel/proxmox-boot-uuids found, skipping ESP sync.
root@pve:~#

重启机器：

root@pve:~# reboot

重启上来之后检查模块是否正常加载：

root@pve:~# lsmod | grep vfio
vfio_pci               57344  1
vfio_virqfd            16384  1 vfio_pci
irqbypass              16384  11 vfio_pci,kvm
vfio_iommu_type1       36864  1
vfio                   36864  5 vfio_iommu_type1,vfio_pci
root@pve:~#

查看显卡

root@pve:~# lspci -nn | grep NV
86:00.0 3D controller [0302]: NVIDIA Corporation TU104GL [Tesla T4] [10de:1eb8] (rev a1)
root@pve:~#
root@pve:~#

查看显卡 ID，写入到配置中

root@pve:~# lspci -n -s 86:00
86:00.0 0302: 10de:1eb8 (rev a1)
root@pve:~#
root@pve:~#
root@pve:~# echo "options vfio-pci ids=10de:1eb8" > /etc/modprobe.d/vfio.conf
root@pve:~#