Jump to content
Калькуляторы

Высокая загрузка прерываниями терминирующего сервера freebsd 8.2-p4 amd64 + igb (ET Dual Port)

Доброе время суток!

 

Есть сервер:

- Supermicro 6016T-URF

- CPU: 2 x Intel® Xeon® CPU E5620@2.40GHz (2 x 4 cores, hyper threading выключен)

- RAM: 6Gb (3x2Gb DDR3)

- HDD: Flash 8Gb

- OS: NanoBSD 8.2 p4 amd64

- NETWORK: 2 x Intel ET Dual Port Server Adapter

 

igb0 - vlan пользователей

igb1 - vlan национальный трафик

igb2 - vlan пользователей

igb3 - vlan мировой трафик

 

При нагрузке в 530M симметричного трафика 81 kpps сервер целиком загружен прерываниями:

# top -aSP
last pid:  4616;  load averages:  1.91,  1.19,  0.67                                                                                                                                                                                                                                                 up 0+12:49:46  15:44:28
188 processes: 36 running, 118 sleeping, 34 waiting
CPU 0:  0.0% user,  0.0% nice,  2.7% system, 97.3% interrupt,  0.0% idle
CPU 1:  0.0% user,  0.0% nice,  6.0% system, 93.3% interrupt,  0.7% idle
CPU 2:  0.0% user,  0.0% nice,  8.1% system, 91.9% interrupt,  0.0% idle
CPU 3:  0.0% user,  0.0% nice,  2.0% system, 98.0% interrupt,  0.0% idle
CPU 4:  0.0% user,  0.0% nice, 17.4% system, 77.2% interrupt,  5.4% idle
CPU 5:  2.0% user,  0.0% nice, 26.8% system, 63.8% interrupt,  7.4% idle
CPU 6:  0.0% user,  0.0% nice,  8.1% system, 87.2% interrupt,  4.7% idle
CPU 7:  2.7% user,  0.0% nice, 30.9% system, 55.0% interrupt, 11.4% idle
Mem: 38M Active, 13M Inact, 519M Wired, 1128K Cache, 77M Buf, 5338M Free
Swap:
 PID USERNAME  THR PRI NICE   SIZE    RES STATE   C   TIME   WCPU COMMAND
  12 root       58 -60    -     0K   928K WAIT    0  21.3H 686.23% [intr]
   0 root       40 -68    0     0K   624K -       7  26:11 36.52% [kernel]
  11 root        8 171 ki31     0K   128K RUN     0  80.1H 36.04% [idle]
...

 

Конфигурация:

 #cat /boot/loader.conf
loader_logo="none"        
autoboot_delay="2"        
hw.ata.atapi_dma="0"      
hw.ata.ata_dma="0"        
hw.ata.wc="0"            
net.fibs=16               
kern.cam.boot_delay=10000 
hw.igb.rxd=4096
hw.igb.txd=4096
hw.igb.max_interrupt_rate=1000

 

 #cat /etc/sysctl.conf
net.inet.tcp.blackhole=2
net.inet.udp.blackhole=1
net.inet.icmp.drop_redirect=1
net.inet.icmp.log_redirect=1
net.inet.ip.redirect=0
net.inet.tcp.drop_synfin=1
net.inet.ip.forwarding=1
net.inet.flowtable.enable=0

kern.coredump=0
kern.ipc.nmbclusters=512000

dev.igb.0.rx_processing_limit=4096
dev.igb.0.enable_aim=0
dev.igb.0.flow_control=0
dev.igb.1.rx_processing_limit=4096
dev.igb.1.enable_aim=0
dev.igb.1.flow_control=0
dev.igb.2.rx_processing_limit=4096
dev.igb.2.enable_aim=0
dev.igb.2.flow_control=0
dev.igb.3.rx_processing_limit=4096
dev.igb.3.enable_aim=0
dev.igb.3.flow_control=0

 

1# cat /etc/start_if.igb0 
ifconfig igb0 -rxcsum -txcsum -lro -tso up
ifconfig igb1 -rxcsum -txcsum -lro -tso up
ifconfig igb2 -rxcsum -txcsum -lro -tso up
ifconfig igb3 -rxcsum -txcsum -lro -tso up
....

 

Для балансировки очередей по ядрам процессора используется скрипт:

# cat /usr/local/etc/rc.d/cpuset-igb
#!/bin/sh

# PROVIDE: cpuset-igb
# REQUIRE: FILESYSTEMS
# BEFORE:  netif
# KEYWORD: nojail

case "$1" in
*start)
 echo "Binding igb(4) IRQs to CPUs"
 cpus=`sysctl -n kern.smp.cpus`
 vmstat -ai | sed -E '/^irq.*que/!d; s/^irq([0-9]+): igb([0-9]+):que ([0-9]+).*/\1 \2 \3/' |\
 while read irq igb que
 do
   cpuset -l $(( ($igb+$que) % $cpus )) -x $irq
 done
 ;;
esac

 

Сетевые карты настроены идентично:

# sysctl -a | grep igb.0
dev.igb.0.%desc: Intel(R) PRO/1000 Network Connection version - 2.0.7
dev.igb.0.%driver: igb
dev.igb.0.%location: slot=0 function=0
dev.igb.0.%pnpinfo: vendor=0x8086 device=0x10c9 subvendor=0x15d9 subdevice=0x0600 class=0x020000
dev.igb.0.%parent: pci1
dev.igb.0.nvm: -1
dev.igb.0.flow_control: 0
dev.igb.0.enable_aim: 0
dev.igb.0.rx_processing_limit: 4096
dev.igb.0.link_irq: 2
dev.igb.0.dropped: 0
dev.igb.0.tx_dma_fail: 0
dev.igb.0.rx_overruns: 0
dev.igb.0.watchdog_timeouts: 0
dev.igb.0.device_control: 1087373889
dev.igb.0.rx_control: 67141634
dev.igb.0.interrupt_mask: 4
dev.igb.0.extended_int_mask: 2147483941
dev.igb.0.tx_buf_alloc: 0
dev.igb.0.rx_buf_alloc: 0
dev.igb.0.fc_high_water: 58976
dev.igb.0.fc_low_water: 58960
dev.igb.0.queue0.interrupt_rate: 1000
dev.igb.0.queue0.txd_head: 2334
dev.igb.0.queue0.txd_tail: 2334
dev.igb.0.queue0.no_desc_avail: 0
dev.igb.0.queue0.tx_packets: 48850063
dev.igb.0.queue0.rxd_head: 3402
dev.igb.0.queue0.rxd_tail: 3401
dev.igb.0.queue0.rx_packets: 34090314
dev.igb.0.queue0.rx_bytes: 14927930908
dev.igb.0.queue0.lro_queued: 0
dev.igb.0.queue0.lro_flushed: 0
dev.igb.0.queue1.interrupt_rate: 1000
dev.igb.0.queue1.txd_head: 2948
dev.igb.0.queue1.txd_tail: 2948
dev.igb.0.queue1.no_desc_avail: 0
dev.igb.0.queue1.tx_packets: 54633922
dev.igb.0.queue1.rxd_head: 170
dev.igb.0.queue1.rxd_tail: 167
dev.igb.0.queue1.rx_packets: 31957162
dev.igb.0.queue1.rx_bytes: 16608051512
dev.igb.0.queue1.lro_queued: 0
dev.igb.0.queue1.lro_flushed: 0
dev.igb.0.queue2.interrupt_rate: 1000
dev.igb.0.queue2.txd_head: 592
dev.igb.0.queue2.txd_tail: 592
dev.igb.0.queue2.no_desc_avail: 0
dev.igb.0.queue2.tx_packets: 34478376
dev.igb.0.queue2.rxd_head: 2247
dev.igb.0.queue2.rxd_tail: 2246
dev.igb.0.queue2.rx_packets: 29161671
dev.igb.0.queue2.rx_bytes: 14408816351
dev.igb.0.queue2.lro_queued: 0
dev.igb.0.queue2.lro_flushed: 0
dev.igb.0.queue3.interrupt_rate: 1000
dev.igb.0.queue3.txd_head: 1648
dev.igb.0.queue3.txd_tail: 1648
dev.igb.0.queue3.no_desc_avail: 0
dev.igb.0.queue3.tx_packets: 29875000
dev.igb.0.queue3.rxd_head: 2972
dev.igb.0.queue3.rxd_tail: 2971
dev.igb.0.queue3.rx_packets: 67283868
dev.igb.0.queue3.rx_bytes: 39140146916
dev.igb.0.queue3.lro_queued: 0
dev.igb.0.queue3.lro_flushed: 0
dev.igb.0.queue4.interrupt_rate: 1000
dev.igb.0.queue4.txd_head: 700
dev.igb.0.queue4.txd_tail: 700
dev.igb.0.queue4.no_desc_avail: 0
dev.igb.0.queue4.tx_packets: 69646686
dev.igb.0.queue4.rxd_head: 1781
dev.igb.0.queue4.rxd_tail: 1780
dev.igb.0.queue4.rx_packets: 31586037
dev.igb.0.queue4.rx_bytes: 15191217027
dev.igb.0.queue4.lro_queued: 0
dev.igb.0.queue4.lro_flushed: 0
dev.igb.0.queue5.interrupt_rate: 1000
dev.igb.0.queue5.txd_head: 1440
dev.igb.0.queue5.txd_tail: 1440
dev.igb.0.queue5.no_desc_avail: 0
dev.igb.0.queue5.tx_packets: 51096272
dev.igb.0.queue5.rxd_head: 2764
dev.igb.0.queue5.rxd_tail: 2762
dev.igb.0.queue5.rx_packets: 33131211
dev.igb.0.queue5.rx_bytes: 18297721719
dev.igb.0.queue5.lro_queued: 0
dev.igb.0.queue5.lro_flushed: 0
dev.igb.0.queue6.interrupt_rate: 1000
dev.igb.0.queue6.txd_head: 612
dev.igb.0.queue6.txd_tail: 612
dev.igb.0.queue6.no_desc_avail: 0
dev.igb.0.queue6.tx_packets: 29516082
dev.igb.0.queue6.rxd_head: 2998
dev.igb.0.queue6.rxd_tail: 2997
dev.igb.0.queue6.rx_packets: 27859894
dev.igb.0.queue6.rx_bytes: 14019910034
dev.igb.0.queue6.lro_queued: 0
dev.igb.0.queue6.lro_flushed: 0
dev.igb.0.queue7.interrupt_rate: 1000
dev.igb.0.queue7.txd_head: 1032
dev.igb.0.queue7.txd_tail: 1032
dev.igb.0.queue7.no_desc_avail: 0
dev.igb.0.queue7.tx_packets: 43895300
dev.igb.0.queue7.rxd_head: 3235
dev.igb.0.queue7.rxd_tail: 3234
dev.igb.0.queue7.rx_packets: 36359331
dev.igb.0.queue7.rx_bytes: 19704509697
dev.igb.0.queue7.lro_queued: 0
dev.igb.0.queue7.lro_flushed: 0
dev.igb.0.mac_stats.excess_coll: 0
dev.igb.0.mac_stats.single_coll: 0
dev.igb.0.mac_stats.multiple_coll: 0
dev.igb.0.mac_stats.late_coll: 0
dev.igb.0.mac_stats.collision_count: 0
dev.igb.0.mac_stats.symbol_errors: 0
dev.igb.0.mac_stats.sequence_errors: 0
dev.igb.0.mac_stats.defer_count: 0
dev.igb.0.mac_stats.missed_packets: 0
dev.igb.0.mac_stats.recv_no_buff: 0
dev.igb.0.mac_stats.recv_undersize: 0
dev.igb.0.mac_stats.recv_fragmented: 0
dev.igb.0.mac_stats.recv_oversize: 0
dev.igb.0.mac_stats.recv_jabber: 0
dev.igb.0.mac_stats.recv_errs: 0
dev.igb.0.mac_stats.crc_errs: 0
dev.igb.0.mac_stats.alignment_errs: 0
dev.igb.0.mac_stats.coll_ext_errs: 0
dev.igb.0.mac_stats.xon_recvd: 0
dev.igb.0.mac_stats.xon_txd: 0
dev.igb.0.mac_stats.xoff_recvd: 0
dev.igb.0.mac_stats.xoff_txd: 0
dev.igb.0.mac_stats.total_pkts_recvd: 293679048
dev.igb.0.mac_stats.good_pkts_recvd: 291424047
dev.igb.0.mac_stats.bcast_pkts_recvd: 2081336
dev.igb.0.mac_stats.mcast_pkts_recvd: 26520
dev.igb.0.mac_stats.rx_frames_64: 0
dev.igb.0.mac_stats.rx_frames_65_127: 179877996
dev.igb.0.mac_stats.rx_frames_128_255: 9035781
dev.igb.0.mac_stats.rx_frames_256_511: 4633737
dev.igb.0.mac_stats.rx_frames_512_1023: 8620689
dev.igb.0.mac_stats.rx_frames_1024_1522: 89255844
dev.igb.0.mac_stats.good_octets_recvd: 154627546923
dev.igb.0.mac_stats.good_octets_txd: 395386062810
dev.igb.0.mac_stats.total_pkts_txd: 361984953
dev.igb.0.mac_stats.good_pkts_txd: 361984953
dev.igb.0.mac_stats.bcast_pkts_txd: 9648
dev.igb.0.mac_stats.mcast_pkts_txd: 342
dev.igb.0.mac_stats.tx_frames_64: 31408955
dev.igb.0.mac_stats.tx_frames_65_127: 52243337
dev.igb.0.mac_stats.tx_frames_128_255: 9840673
dev.igb.0.mac_stats.tx_frames_256_511: 5953245
dev.igb.0.mac_stats.tx_frames_512_1023: 8397409
dev.igb.0.mac_stats.tx_frames_1024_1522: 254141334
dev.igb.0.mac_stats.tso_txd: 0
dev.igb.0.mac_stats.tso_ctx_fail: 0
dev.igb.0.interrupts.asserts: 193999211
dev.igb.0.interrupts.rx_pkt_timer: 291420061
dev.igb.0.interrupts.rx_abs_timer: 0
dev.igb.0.interrupts.tx_pkt_timer: 0
dev.igb.0.interrupts.tx_abs_timer: 291424047
dev.igb.0.interrupts.tx_queue_empty: 361980717
dev.igb.0.interrupts.tx_queue_min_thresh: 0
dev.igb.0.interrupts.rx_desc_min_thresh: 0
dev.igb.0.interrupts.rx_overrun: 0
dev.igb.0.host.breaker_tx_pkt: 0
dev.igb.0.host.host_tx_pkt_discard: 0
dev.igb.0.host.rx_pkt: 3986
dev.igb.0.host.breaker_rx_pkts: 0
dev.igb.0.host.breaker_rx_pkt_drop: 0
dev.igb.0.host.tx_good_pkt: 3895
dev.igb.0.host.breaker_tx_pkt_drop: 0
dev.igb.0.host.rx_good_bytes: 154627546993
dev.igb.0.host.tx_good_bytes: 395385950536
dev.igb.0.host.length_errors: 0
dev.igb.0.host.serdes_violation_pkt: 0
dev.igb.0.host.header_redir_missed: 0

 

Как можно побороть нагрузку прерываниями сервера?

Share this post


Link to post
Share on other sites

А что он собственно терминирует, и чем занимается?(шейпинг/нат/коллекторы трафика етц)

Share this post


Link to post
Share on other sites

Create an account or sign in to comment

You need to be a member in order to leave a comment

Create an account

Sign up for a new account in our community. It's easy!

Register a new account

Sign in

Already have an account? Sign in here.

Sign In Now
Sign in to follow this