自动化kolla-ansible部署openstack+GPU透传方法

admin

1. CentOS7.x-8.x系列为虚拟机配置GPU直通
复制代码
1. 编辑文件vim  /etc/modules， 添加以下内容：
pci_stub
) \& S0 n6 l0 X* g) i8 \5 Qvfio
vfio_iommu_type1
vfio_pci
kvm
: T& e% _, w; R& _4 Ekvm_intel
' P5 c( N  p3 ]6 c: u
2. 在KVM主机上启用IOMMU
7 v9 X, V/ r+ m: c#对于Intel芯片：
GRUB_CMDLINE_LINUX_DEFAULT="intel_iommu=on"
$ s3 Y: {" B% L# r- U#对于AMD芯片：
9 ]. M; X2 {1 |/ I; GGRUB_CMDLINE_LINUX_DEFAULT="iommu=pt iommu=1"

vim /etc/default/grub

GRUB_TIMEOUT=5
  s, b) M. R% W9 \' VGRUB_DISTRIBUTOR="$(sed 's, release .*$,,g' /etc/system-release)"
: k. H6 X& E' p/ IGRUB_DEFAULT=saved
% g) e! \. O2 F/ D7 e8 R( {( aGRUB_DISABLE_SUBMENU=true
$ q  }( M* h1 F* j; h( BGRUB_TERMINAL_OUTPUT="console"
GRUB_CMDLINE_LINUX="crashkernel=auto rhgb quiet intel_iommu=on"
GRUB_DISABLE_RECOVERY="true"
# [$ b8 K+ d. i8 n
. j7 ~/ p/ f* v4 T6 f   3.  重新生成grub
& n# j+ O* h, ?9 y' c   EFI
3 B0 `) h( K/ D8 B* p# b   grub2-mkconfig -o /boot/efi/EFI/centos/grub.cfg
& ]5 N  J0 z2 Z9 s/ Q4 ~* ]- p$ _. `   非EFI
! Z& \) I3 n' Q; e8 ^   grub2-mkconfig -o /boot/grub2/grub.cfg
& N) l& u" R( i/ T0 B! c5 E0 H5 q4.  将下列内容加入到blacklist中以避免被宿主机占用，编辑文件
vim  /etc/modprobe.d/blacklist.conf
blacklist snd_hda_intel
blacklist amd76x_edac
blacklist vga16fb
9 ?- O; Z7 G: _: J* ^0 ]' ~/ lblacklist nouveau
blacklist rivafb
. p: U, N$ b. O2 ~8 X$ Rblacklist nvidiafb
blacklist rivatv
blacklist nvidia
, }- k  S4 {2 i9 F; a
5.  查找显卡的Product ID 以及 Vendor ID：
4 S: i5 I$ t1 Z: B1 Myum install pciutils -y
lspci -nn | grep NVIDIA
如下：
[root@stein-a ~]#
2 I  {. n) N* p9 }3 }03:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP104GL [Quadro P4000] [10de:1bb1] (rev a1)
4 V4 U: k4 |6 G4 h5 O* v( y03:00.1 Audio device [0403]: NVIDIA Corporation GP104 High Definition Audio Controller [10de:10f0] (rev a1)

6.  编辑
vim /etc/modprobe.d/vfio.conf
+ s  O3 [/ e, v1 I& k2 ~( b# create new: for [ids=***], specify [vendor-ID:device-ID]
% I1 M# ?# T/ ]8 j6 U; Doptions vfio-pci ids=10de:1bb1,10de:10f0
1 Y3 ^; v( V* e) t/ K
7.  写入到系统启动项
echo 'vfio-pci' > /etc/modules-load.d/vfio-pci.conf

- v  T: Y2 G# _8.  重新生成initramfs
mv /boot/initramfs-$(uname -r).img /boot/initramfs-$(uname -r).img.bak
8 Y6 l5 B$ _4 j$ u' J% wdracut -v /boot/initramfs-$(uname -r).img $(uname -r)
9 S, o) U6 @" S: L
6 r3 d0 @" j/ ~% O. s3 ?, j9.  重启系统
8 }+ O0 g% C* M0 w* R/ S% g7 r- y+ j" Ureboot
/ w; N. s; d1 t5 ^% `
1 ~! p" `* B* I1 c8 V6 f: J2 h" u10. 验证
' b6 H. T* x, L) `3 A- C3 W1 olspci -nnk -d 10de:1bb1
dmesg | grep -i vfio
[root@stein-a ~]# lspci -nnk -d 10de:1bb1
6 j6 n5 R6 L. ^, u5 X03:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP104GL [Quadro P4000] [10de:1bb1] (rev a1)
        Subsystem: NVIDIA Corporation Device [10de:11a3]
        Kernel driver in use: vfio-pci
. s" f2 k" x0 a0 o( K5 S( q$ O& D        Kernel modules: nouveau
[root@stein-a ~]# dmesg | grep -i vfio
* ?9 e. ~: _: ~3 ~& c. ~6 z2 i' u[    2.503115] VFIO - User Level meta-driver version: 0.3
[    2.515645] vfio_pci: add [10de:1bb1[ffff:ffff]] class 0x000000/00000000
[    2.515752] vfio_pci: add [10de:10f0[ffff:ffff]] class 0x000000/00000000
$ {- D) I/ M) V+ t4 d[root@stein-a ~]#
( M8 u& w3 e4 [8 n7 l3 V复制代码

) |' B: v. F/ }8 @- [2. Ubuntu18.04系列为虚拟机配置GPU直通
复制代码
1. 编辑文件vim  /etc/modules， 添加以下内容：
pci_stub
vfio
vfio_iommu_type1
vfio_pci
kvm
% n! {/ k" Q; f; ~2 D: Tkvm_intel

; L+ Q% e0 a: r' @: W% \2. 在KVM主机上启用IOMMU
#对于Intel芯片：
GRUB_CMDLINE_LINUX_DEFAULT="quiet splash intel_iommu=on"
+ q+ B! [( c/ C8 l' X#对于AMD芯片：
& Y9 o! k. I3 e- I/ aGRUB_CMDLINE_LINUX_DEFAULT="iommu=pt iommu=1"
' m4 ~3 c; n2 t; X
vim /etc/default/grub
% g8 C: w: S! n/ f" F
GRUB_DEFAULT=0
' Z9 O+ `- B' i& |1 ~8 S' U! DGRUB_TIMEOUT_STYLE=hidden
GRUB_TIMEOUT=0
GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian`
GRUB_CMDLINE_LINUX_DEFAULT="quiet splash intel_iommu=on"
GRUB_CMDLINE_LINUX=""

* g5 j( |4 b0 o   3.  重新生成grub
+ ^5 O& l9 O1 V- T6 z   EFI
7 l+ w& U! P( v5 }3 v   grub2-mkconfig -o /boot/efi/EFI/centos/grub.cfg
   非EFI
   grub2-mkconfig -o /boot/grub2/grub.cfg
5 J6 l! E  f; `, `" E  h4.  将下列内容加入到blacklist中以避免被宿主机占用，编辑文件
# V' |  F5 ]' L- x- V3 Evim  /etc/modprobe.d/blacklist.conf
blacklist snd_hda_intel
blacklist amd76x_edac
* D, }, V1 j+ cblacklist vga16fb
. V# z' y1 L9 ~, }4 Ablacklist nouveau
! [- `9 @8 C+ n2 wblacklist rivafb
6 d7 C- r$ `7 v. S1 k( _8 Pblacklist nvidiafb
8 x3 x2 E* P) u; U+ [: Gblacklist rivatv
blacklist nvidia
7 [( t' D5 v8 w4 z7 u* B  h7 b' {
# H2 i6 A' ?9 u8 {" K5.  查找显卡的Product ID 以及 Vendor ID：
% Q- T7 b1 C9 x( B- d; O- R- |apt install pciutils -y
lspci -nn | grep NVIDIA
/ K9 Z9 r7 T" J) r, l  Q如下：
: ]+ P- y  c" \) M[root@stein-a ~]# lspci -nn | grep NVIDIA
' _3 q4 g# c& _% r) p8 Q03:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP104GL [Quadro P4000] [10de:1bb1] (rev a1)
03:00.1 Audio device [0403]: NVIDIA Corporation GP104 High Definition Audio Controller [10de:10f0] (rev a1)

2 i( g* B/ x5 c6.  编辑
vim /etc/modprobe.d/vfio.conf
* [6 a! n3 ~% |" u0 b2 G# create new: for [ids=***], specify [vendor-ID:device-ID]
! Y9 i7 s9 I. ooptions vfio-pci ids=10de:1bb1,10de:10f0

7.  写入到系统启动项
echo 'vfio-pci' > /etc/modules-load.d/vfio-pci.conf
3 J$ k# P8 J% V
8.  重新生成initramfs
dracut -v /boot/initramfs-$(uname -r).img $(uname -r)

9.  重启系统
8 U( j$ d$ Q9 \* c6 ireboot
7 C) P. ~" h, |5 x6 B" u" \
10. 验证
lspci -nnk -d 10de:1bb1
dmesg | grep -i vfio
' _- m" E8 k1 S8 c- _, M% _root@kvm:~# lspci -nnk -d 10de:1bb1
' u$ v6 j, ^' f. `5 tdmesg | grep -i vfio
: _3 q$ T: y4 w& P1 K" W03:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP104GL [Quadro P4000] [10de:1bb1] (rev a1)
        Subsystem: NVIDIA Corporation GP104GL [Quadro P4000] [10de:11a3]
        Kernel driver in use: vfio-pci
" Y8 P" Y  @% W: H% {        Kernel modules: nvidiafb, nouveau
root@kvm:~# dmesg | grep -i vfio
, ?, K- c& ]3 M[    3.838714] VFIO - User Level meta-driver version: 0.3
/ ~* _7 n5 i, o. Q[    3.846238] vfio-pci 0000:03:00.0: vgaarb: changed VGA decodes: olddecodes=io+mem,decodes=io+mem:owns=none
! {  W1 Z8 a. C# K. r& l[    3.866370] vfio_pci: add [10de:1bb1[ffffffff:ffffffff]] class 0x000000/00000000
% ?8 y, y0 Z# G, ]6 F1 E. g+ x% ~[    3.886375] vfio_pci: add [10de:10f0[ffffffff:ffffffff]] class 0x000000/00000000
复制代码

. g( p. q+ b9 _复制代码
#如果你单机部署的，在单机下配置。
#如果你是高可用部署的，在三台控制节点配置
1. 添加pci
* ^' C' p: v8 ^7 s6 S7 k8 X9 {% Fvim /etc/kolla/config/nova/nova-compute.conf
# r) s% M1 x4 E[libvirt]
inject_password=true
cpu_mode=host-passthrough
virt_type = kvm
[pci]
passthrough_whitelist: { "vendor_id": "10de", "product_id": "1bb1" }
: c2 p( l# T1 I( D% Y8 c2 m
2. 修改nova.conf
vim /etc/kolla/config/nova.conf
[DEFAULT]
1 @$ A# l$ {- I4 P% Z5 G  m; g! lservice_down_time = 120
cpu_allocation_ratio = 4.0   
9 k) n3 M' [" _! t: E7 m. j. Cdisk_allocation_ratio=1.0
ram_allocation_ratio = 1.0   
, k: t4 L7 A) M- _reserved_host_disk_mb = 4096
; ?+ k5 j0 V7 @reserved_host_memory_mb = 4096
7 X( \; K& D" a, fallow_resize_to_same_host = True   
remove_unused_base_images = False
& A( a5 D6 q: Q' S5 _* yimage_cache_manager_interval = 0
. E& X2 J3 S+ b1 h- ?; c( u7 Bresume_guests_state_on_host_boot = True
. u3 D( n- n# R6 R& {) I% |
. g0 k4 U) A* e9 O3 c; P' O[PCI]
alias: { "vendor_id":"10de", "product_id":"1bb1", "device_type":"type-PCI", "name":"quadro-p4000" }
$ Y2 ~. z) W* ^; N[filter_scheduler]
enabled_filters = RetryFilter, AvailabilityZoneFilter, ComputeFilter, ComputeCapabilitiesFilter, ImagePropertiesFilter, ServerGroupAntiAffinityFilter, ServerGroupAffinityFilter, PciPassthroughFilter
1 O1 ]/ _1 p# J# |available_filters = nova.scheduler.filters.all_filters

3. GPU 类型实例创建  
openstack flavor create --vcpus 4 --ram 8192 --disk 30 --property "pci_passthrough:alias"="quadro-p400:1" g1.4c.8m.p400
4 m  |2 {! Q8 F3 h% R' c复制代码

& f5 f$ Z, ~# N! a- O: |& `% m" g0 J  u3. CentOS7.x系列 安装显卡驱动
复制代码
1.  查看是否含有英伟达显卡
lspci | grep -i NVIDIA
6 S2 U7 F: q$ {) {- U! R4 V#下面说明有1块英伟达的显卡
[root@train-all ~]#  lspci | grep -i NVIDIA
2 z/ _1 D2 O, p0 K: E; \* X( K04:00.0 VGA compatible controller: NVIDIA Corporation GP104GL [Quadro P4000] (rev a1)
8 K# g3 ^( _" b3 M3 X: f/ ~04:00.1 Audio device: NVIDIA Corporation GP104 High Definition Audio Controller (rev a1)
[root@train-all ~]#

2.  添加ELRepo源
; c* u- Q8 Y& I+ v1 P' q9 N6 `2 brpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
1 P. D0 N- I- c) O& w/ J
$ E! l. t. w" c  j+ i* ~3.  安装ELRepo
rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-2.el7.elrepo.noarch.rpm

5 D( j% G. y- w  a0 Y) n% `; t  u4.  安装nvidia-detect
% A* Z$ a4 S) \& @+ r8 p$ Zyum install nvidia-detect -y
2 }- L3 v6 L* V
5.  运行nvidia-detect
nvidia-detect -v

( L; {' `9 N$ @% ]2 ]* z6.  查找驱动程序
, {  C1 p+ [5 d3 [: H6 v8 O9 l4 Yyum search kmod-nvidia
: ]" I; v% [# \9 E' L2 S6 t
5 B! D+ w% z& \! _+ [$ s7.  安装驱动程序
% w+ d2 |3 _1 e% ], R2 k# M" {yum install kmod-nvidia.x86_64 -y
2 v/ ~! F8 d! J* p. W
8.  查看禁用Nouveau
, ~' z6 S  k% Ilsmod | grep nouveau
. B: s3 m1 C  N' J# t, |#若没有输出 则说明禁用成功，否则执行下面的命令

9.  在/etc/modprobe.d/blacklist-nouveau.conf中创建一个文件，其内容如下：
vi /etc/modprobe.d/blacklist-nouveau.conf
添加
" H% W4 {$ J6 S# jblacklist nouveau
& c0 p2 \9 u9 r3 S. zoptions nouveau modeset=0
$ s) U- [3 ~. k. w% |! j
10. 重新生成内核initramfs
dracut --force

11.  重启系统
! d5 m+ u4 ^' }& o3 b. y, J, C8 l: Wreboot

& o! r6 G1 r( e  k4 J+ e12.  测试
nvidia-smi