自动化kolla-ansible部署openstack+GPU透传方法

admin

1. CentOS7.x-8.x系列为虚拟机配置GPU直通
! `& t. W# c6 G" X, D% B) V复制代码
1. 编辑文件vim  /etc/modules， 添加以下内容：
" F9 ~# Y% p/ D, wpci_stub
vfio
( z/ u0 P) |6 C5 d* d: Gvfio_iommu_type1
* f/ f% [8 ~, W; S" Q1 Vvfio_pci
kvm
kvm_intel
* N7 P. l! E% |
  q# C( v7 V" d% l0 i2. 在KVM主机上启用IOMMU
#对于Intel芯片：
6 Q2 U3 p- S% w4 pGRUB_CMDLINE_LINUX_DEFAULT="intel_iommu=on"
2 S: q/ Q! F' F/ N6 \! y0 {% @#对于AMD芯片：
- X) Y/ L! t( G% p5 f# P! @# d8 fGRUB_CMDLINE_LINUX_DEFAULT="iommu=pt iommu=1"
! Z" H% O2 N$ c6 P4 @' t9 X
+ N8 U4 V* [/ |) q: |/ u0 uvim /etc/default/grub
$ N" r# Q3 ^2 z0 p. m. K* a/ @
- s! B9 U' ^1 s( yGRUB_TIMEOUT=5
GRUB_DISTRIBUTOR="$(sed 's, release .*$,,g' /etc/system-release)"
GRUB_DEFAULT=saved
4 c+ C7 D& y- v) o! F6 WGRUB_DISABLE_SUBMENU=true
GRUB_TERMINAL_OUTPUT="console"
5 u3 t( }) I. K% `9 z0 |GRUB_CMDLINE_LINUX="crashkernel=auto rhgb quiet intel_iommu=on"
GRUB_DISABLE_RECOVERY="true"

0 A6 J/ ]- W, E/ o" T. O3 Y/ g+ _   3.  重新生成grub
   EFI
   grub2-mkconfig -o /boot/efi/EFI/centos/grub.cfg
) M* Y6 {- r5 Z3 Z; M) {" z/ \- \   非EFI
  A& u3 u1 h) L- c( Z( F/ L& }   grub2-mkconfig -o /boot/grub2/grub.cfg
4.  将下列内容加入到blacklist中以避免被宿主机占用，编辑文件
vim  /etc/modprobe.d/blacklist.conf
blacklist snd_hda_intel
blacklist amd76x_edac
7 r( K6 Q9 w2 G$ U1 Kblacklist vga16fb
4 K9 j* V% f# M. c7 s3 O# Ublacklist nouveau
8 u5 E, U: p5 \blacklist rivafb
blacklist nvidiafb
; x1 D$ m! k& |; U# Bblacklist rivatv
blacklist nvidia

5.  查找显卡的Product ID 以及 Vendor ID：
yum install pciutils -y
lspci -nn | grep NVIDIA
如下：
[root@stein-a ~]#
0 |: N! J% W( l: T7 k03:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP104GL [Quadro P4000] [10de:1bb1] (rev a1)
03:00.1 Audio device [0403]: NVIDIA Corporation GP104 High Definition Audio Controller [10de:10f0] (rev a1)
* O8 ]4 u, ~# b
6.  编辑
* s  n( v! a8 M4 A, u& Fvim /etc/modprobe.d/vfio.conf
3 ~0 }+ Y  L% b2 k( e3 H# create new: for [ids=***], specify [vendor-ID:device-ID]
+ Y" p4 E! B  `6 S  xoptions vfio-pci ids=10de:1bb1,10de:10f0

7.  写入到系统启动项
echo 'vfio-pci' > /etc/modules-load.d/vfio-pci.conf
) N8 P" {8 D* O( W6 \5 w
8.  重新生成initramfs
mv /boot/initramfs-$(uname -r).img /boot/initramfs-$(uname -r).img.bak
8 d/ x. _2 v1 ^" T, e# G/ cdracut -v /boot/initramfs-$(uname -r).img $(uname -r)
$ l. [6 A' c- O
- {8 b* M/ ]5 c) f: g# P* }9.  重启系统
0 B& ^) \& `1 J! v$ X* q) Hreboot
) i2 k4 x# i" y  z# j
10. 验证
3 J1 J; b0 U2 M4 Slspci -nnk -d 10de:1bb1
) a7 K) W$ Z+ ^dmesg | grep -i vfio
: t& a8 ^# N5 _: n0 m$ ?0 z: h[root@stein-a ~]# lspci -nnk -d 10de:1bb1
* Y6 f5 ]* V7 {" I8 p( e7 u3 l03:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP104GL [Quadro P4000] [10de:1bb1] (rev a1)
        Subsystem: NVIDIA Corporation Device [10de:11a3]
        Kernel driver in use: vfio-pci
0 _* I/ y) N, ]$ Y+ b: k        Kernel modules: nouveau
. i3 P" @. Q+ a, y, p8 Y* _* e[root@stein-a ~]# dmesg | grep -i vfio
[    2.503115] VFIO - User Level meta-driver version: 0.3
[    2.515645] vfio_pci: add [10de:1bb1[ffff:ffff]] class 0x000000/00000000
[    2.515752] vfio_pci: add [10de:10f0[ffff:ffff]] class 0x000000/00000000
: {+ H' l0 i/ K. e  s' \. Q3 J1 \[root@stein-a ~]#
% I/ }$ |" f6 {5 f0 |" z4 n% E复制代码

% H) d4 F3 R1 @* [2. Ubuntu18.04系列为虚拟机配置GPU直通
复制代码
1. 编辑文件vim  /etc/modules， 添加以下内容：
pci_stub
& K# F6 L9 g5 O8 \  F" I2 lvfio
vfio_iommu_type1
. Y+ s' ]- U+ Fvfio_pci
7 x: o  G3 T+ |7 |: {! y6 Ikvm
/ w7 X' h3 O- ]* F4 C+ J3 ikvm_intel
3 ^# S; s$ [$ b. \
2. 在KVM主机上启用IOMMU
& V; U# x2 ^. t; U5 t5 O#对于Intel芯片：
( T* F5 L+ O, Z7 e  QGRUB_CMDLINE_LINUX_DEFAULT="quiet splash intel_iommu=on"
2 D5 }$ L- B3 J2 a4 n8 O3 e& U1 W#对于AMD芯片：
GRUB_CMDLINE_LINUX_DEFAULT="iommu=pt iommu=1"
  D: {# m# v4 v4 h' I
9 x5 l8 b+ Q* U  x7 Cvim /etc/default/grub

" M. P9 k* h8 G7 F: Q& eGRUB_DEFAULT=0
9 v6 Q2 C9 S. _" @7 d8 oGRUB_TIMEOUT_STYLE=hidden
+ ]: P$ t" v" U0 @GRUB_TIMEOUT=0
: c" ~/ Y9 |% mGRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian`
GRUB_CMDLINE_LINUX_DEFAULT="quiet splash intel_iommu=on"
GRUB_CMDLINE_LINUX=""

   3.  重新生成grub
   EFI
, G  Z* a3 J; l5 y$ B   grub2-mkconfig -o /boot/efi/EFI/centos/grub.cfg
   非EFI
   grub2-mkconfig -o /boot/grub2/grub.cfg
+ z0 r9 M) O9 i, ^4 R- U4 ^* s9 K4.  将下列内容加入到blacklist中以避免被宿主机占用，编辑文件
, J' U3 G# ]+ H+ P# q9 Lvim  /etc/modprobe.d/blacklist.conf
# J4 y1 W; o* \% rblacklist snd_hda_intel
blacklist amd76x_edac
blacklist vga16fb
+ Q- @! {1 z4 Yblacklist nouveau
blacklist rivafb
/ B7 Z1 D5 U/ ^blacklist nvidiafb
blacklist rivatv
blacklist nvidia

5.  查找显卡的Product ID 以及 Vendor ID：
% U, _1 [  s# c0 k: Eapt install pciutils -y
7 t/ e$ y" i$ ^; V4 Slspci -nn | grep NVIDIA
如下：
[root@stein-a ~]# lspci -nn | grep NVIDIA
& o" X6 a/ t( U. G, r03:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP104GL [Quadro P4000] [10de:1bb1] (rev a1)
$ L0 A1 p  T4 z- I03:00.1 Audio device [0403]: NVIDIA Corporation GP104 High Definition Audio Controller [10de:10f0] (rev a1)

6.  编辑
" X, u' n- A- E1 _vim /etc/modprobe.d/vfio.conf
# create new: for [ids=***], specify [vendor-ID:device-ID]
' p6 ~% m1 l+ G. V' Doptions vfio-pci ids=10de:1bb1,10de:10f0
6 ]+ x2 y/ l% w- `# e$ q( [
7.  写入到系统启动项
5 p0 j* i% B6 G! m2 zecho 'vfio-pci' > /etc/modules-load.d/vfio-pci.conf
6 d* [$ k0 k5 W7 q) J/ D
0 u3 t! z3 `2 \; X. T$ I8.  重新生成initramfs
dracut -v /boot/initramfs-$(uname -r).img $(uname -r)

9.  重启系统
$ A" ?% u8 S. s8 Ereboot
6 E( f6 \2 @4 @2 f5 Q
& H" Q0 P7 i& ^4 j, K10. 验证
lspci -nnk -d 10de:1bb1
dmesg | grep -i vfio
" `9 v" X# N- @2 i6 Oroot@kvm:~# lspci -nnk -d 10de:1bb1
dmesg | grep -i vfio
03:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP104GL [Quadro P4000] [10de:1bb1] (rev a1)
        Subsystem: NVIDIA Corporation GP104GL [Quadro P4000] [10de:11a3]
# n3 ]3 ^7 }6 K: C$ c  Y1 ~$ g# m        Kernel driver in use: vfio-pci
8 Z% }! x5 L# g' z$ Q2 P, Q7 v        Kernel modules: nvidiafb, nouveau
7 U; h, X, }/ M" J) ~root@kvm:~# dmesg | grep -i vfio
[    3.838714] VFIO - User Level meta-driver version: 0.3
( v# Z  e: O% ]4 b) _7 ?- Y[    3.846238] vfio-pci 0000:03:00.0: vgaarb: changed VGA decodes: olddecodes=io+mem,decodes=io+mem:owns=none
[    3.866370] vfio_pci: add [10de:1bb1[ffffffff:ffffffff]] class 0x000000/00000000
[    3.886375] vfio_pci: add [10de:10f0[ffffffff:ffffffff]] class 0x000000/00000000
复制代码

: Q( b$ V+ F, F/ I5 j复制代码
#如果你单机部署的，在单机下配置。
#如果你是高可用部署的，在三台控制节点配置
- k1 O$ s$ U- G1. 添加pci
vim /etc/kolla/config/nova/nova-compute.conf
[libvirt]
$ {! J4 t; [6 Ninject_password=true
' Z9 u  B8 W% v; @cpu_mode=host-passthrough
5 b- o! x* U; }! Uvirt_type = kvm
) ?7 {3 d  X2 Q0 R* Q[pci]
passthrough_whitelist: { "vendor_id": "10de", "product_id": "1bb1" }
, s8 ]/ _5 H( Y) Z( H2 R
2. 修改nova.conf
$ O4 b- C7 N4 O6 b, W5 b3 f. ovim /etc/kolla/config/nova.conf
* f* A- `$ N+ d; K* [8 U. N" ^4 t[DEFAULT]
service_down_time = 120
cpu_allocation_ratio = 4.0   
disk_allocation_ratio=1.0
ram_allocation_ratio = 1.0   
0 M, E! a2 F3 E( I6 t8 z; D0 I& `reserved_host_disk_mb = 4096
reserved_host_memory_mb = 4096
allow_resize_to_same_host = True   
7 y2 V9 ?6 j$ L1 {' yremove_unused_base_images = False
image_cache_manager_interval = 0
; {6 F5 [* B9 \8 Z5 tresume_guests_state_on_host_boot = True

[PCI]
) r, t3 }: G) |2 C) Aalias: { "vendor_id":"10de", "product_id":"1bb1", "device_type":"type-PCI", "name":"quadro-p4000" }
  m5 N/ {! K; D( ~1 M[filter_scheduler]
* K( a3 b4 D7 ~& y5 x1 E. ]. r- Denabled_filters = RetryFilter, AvailabilityZoneFilter, ComputeFilter, ComputeCapabilitiesFilter, ImagePropertiesFilter, ServerGroupAntiAffinityFilter, ServerGroupAffinityFilter, PciPassthroughFilter
8 `5 s! I. |6 ]6 M7 @' uavailable_filters = nova.scheduler.filters.all_filters

3. GPU 类型实例创建  
openstack flavor create --vcpus 4 --ram 8192 --disk 30 --property "pci_passthrough:alias"="quadro-p400:1" g1.4c.8m.p400
复制代码

3. CentOS7.x系列 安装显卡驱动
复制代码
1.  查看是否含有英伟达显卡
lspci | grep -i NVIDIA
#下面说明有1块英伟达的显卡
[root@train-all ~]#  lspci | grep -i NVIDIA
& O' K( M/ |) S' {) d) p04:00.0 VGA compatible controller: NVIDIA Corporation GP104GL [Quadro P4000] (rev a1)
1 Q9 M, g: B8 S3 C, i$ A: b04:00.1 Audio device: NVIDIA Corporation GP104 High Definition Audio Controller (rev a1)
[root@train-all ~]#
% _9 c4 |6 a% H& S, J/ H
/ v/ ?% r/ u" z8 N6 ?  R2.  添加ELRepo源
$ \) x1 l) v% F. T; Nrpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org

3.  安装ELRepo
rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-2.el7.elrepo.noarch.rpm
; I  G' h5 q4 O2 Q: o' D4 o
4.  安装nvidia-detect
/ J* y. t9 F* ~6 X* W' s1 y* {/ `7 Nyum install nvidia-detect -y

5.  运行nvidia-detect
nvidia-detect -v
# x( R& ^: k" l) x; T9 \
( o9 v- L3 H) B  y0 R6.  查找驱动程序
; H' \" i3 `1 Q* m$ V, T) L& tyum search kmod-nvidia

7.  安装驱动程序
7 X, R, V5 r( N0 [% p- G- wyum install kmod-nvidia.x86_64 -y

8.  查看禁用Nouveau
5 m5 f2 j2 u  Nlsmod | grep nouveau
6 h: O' h. v( S1 T; Y6 z#若没有输出 则说明禁用成功，否则执行下面的命令
# x( e/ Y5 T: O. x* }8 A
9.  在/etc/modprobe.d/blacklist-nouveau.conf中创建一个文件，其内容如下：
vi /etc/modprobe.d/blacklist-nouveau.conf
添加
blacklist nouveau
options nouveau modeset=0
$ Z, n6 K3 [/ I  e
10. 重新生成内核initramfs
dracut --force

11.  重启系统
, Z! ^( v" w" y2 D$ B( w1 areboot

" Q) @1 g/ v# ~8 Z6 U( N12.  测试
! \# G) j& R/ w$ z2 ^7 c2 q- tnvidia-smi