Jump to content
Калькуляторы

Fatal trap 18: integer divide fault while in kernel mode паника и перезагрузка сервера

Столкнулся с новой для себя проблемой на pppoe сервере с FreeBSD 10.1-RELEASE amd 64, время от времени а точнее раз в неделю полторы вне зависимости от нагрузки происходит kernel panic с таким выводом:

Fatal trap 18: integer divide fault while in kernel mode
cpuid = 0; apic id = 00
instruction pointer     = 0x20:0xffffffff80785266
stack pointer           = 0x28:0xfffffe0000283050
frame pointer           = 0x28:0xfffffe00002830e0
code segment            = base 0x0, limit 0xfffff, type 0x1b
                       = DPL 0, pres 1, long 1, def32 0, gran 1
processor eflags        = interrupt enabled, resume, IOPL = 0
current process         = 12 (irq286: igb3:que 0)
trap number             = 18
panic: integer divide fault
cpuid = 0

 

На сервере имеется 4 сетевых карты igb и из них собраны 2 lagg интерфейса, из установленного:

 

autoconf-2.69 Automatically configure source code on many Un*x platforms

autoconf-wrapper-20131203 Wrapper script for GNU autoconf

automake-1.14_1 GNU Standards-compliant Makefile generator

automake-wrapper-20131203 Wrapper script for GNU automake

bash-4.3.30_1 The GNU Project's Bourne Again SHell

bison-2.7.1,1 Parser generator from FSF, (mostly) compatible with Yacc

dialog4ports-0.1.5_2 Console Interface to configure ports

expat-2.1.0_2 XML 1.0 parser written in C

gawk-4.1.1_1 The GNU version of Awk

gettext-runtime-0.19.3 GNU gettext runtime libraries and programs

gettext-tools-0.19.3 GNU gettext development and translation tools

gmake-4.1_1 GNU version of 'make' utility

gmake-lite-4.1_1 Minimalist version of gnu make

help2man-1.43.3_1 Automatically generating simple manual pages from program output

indexinfo-0.2 Utility to regenerate the GNU info page index

libsigsegv-2.10_1 Handling page faults in user mode

libtool-2.4.2.418 Generic shared library support script

m4-1.4.17_1,1 GNU m4

mpd5-5.7_1 Multi-link PPP daemon based on netgraph(4)

net-snmp-5.7.2_17 Extendable SNMP implementation

ng_ipacct-20061223_3 Netgraph IP accounting

p5-Locale-gettext-1.05_4 Message handling functions

perl5-5.18.4_10 Practical Extraction and Report Language

pkg-1.3.8_3 Package manager

quagga-0.99.23.1_2 Free RIPv1, RIPv2, OSPFv2, BGP4, IS-IS route software

 

 

 

kernel config

 

 

 

options CONFIG_AUTOGENERATED

ident GENERIC

machine amd64

cpu HAMMER

makeoptions WITH_CTF=1

makeoptions DEBUG=-g

options RADIX_MPATH

options HZ=1000

options DEVICE_POLLING

options NETGRAPH_NETFLOW

options NETGRAPH_IPFW

options NETGRAPH_VJC

options NETGRAPH_UI

options NETGRAPH_TTY

options NETGRAPH_TEE

options NETGRAPH_RFC1490

options NETGRAPH_PPP

options NETGRAPH_ONE2MANY

options NETGRAPH_MPPC_ENCRYPTION

options NETGRAPH_LMI

options NETGRAPH_L2TP

options NETGRAPH_KSOCKET

options NETGRAPH_IFACE

options NETGRAPH_HOLE

options NETGRAPH_FRAME_RELAY

options NETGRAPH_ECHO

options NETGRAPH_BPF

options NETGRAPH_PPTPGRE

options NETGRAPH_PPPOE

options NETGRAPH_SOCKET

options NETGRAPH_ETHER

options NETGRAPH

options DUMMYNET

options IPDIVERT

options IPFIREWALL_VERBOSE_LIMIT=100

options IPFIREWALL_VERBOSE

options IPFIREWALL

options XENHVM

options USB_DEBUG

options SC_PIXEL_MODE

options VESA

options AHD_REG_PRETTY_PRINT

options AHC_REG_PRETTY_PRINT

options ATA_STATIC_ID

options ACPI_DMAR

options SMP

options KDB_TRACE

options KDB

options INCLUDE_CONFIG_FILE

options DDB_CTF

options KDTRACE_HOOKS

options KDTRACE_FRAME

options MAC

options PROCDESC

options CAPABILITIES

options CAPABILITY_MODE

options AUDIT

options HWPMC_HOOKS

options KBD_INSTALL_CDEV

options PRINTF_BUFR_SIZE=128

options _KPOSIX_PRIORITY_SCHEDULING

options SYSVSEM

options SYSVMSG

options SYSVSHM

options STACK

options KTRACE

options SCSI_DELAY=5000

options COMPAT_FREEBSD7

options COMPAT_FREEBSD6

options COMPAT_FREEBSD5

options COMPAT_FREEBSD4

options COMPAT_FREEBSD32

options GEOM_LABEL

options GEOM_RAID

options GEOM_PART_GPT

options PSEUDOFS

options PROCFS

options CD9660

options MSDOSFS

options NFS_ROOT

options NFSLOCKD

options NFSD

options NFSCL

options MD_ROOT

options QUOTA

options UFS_GJOURNAL

options UFS_DIRHASH

options UFS_ACL

options SOFTUPDATES

options FFS

options SCTP

options TCP_OFFLOAD

options INET

options PREEMPTION

options SCHED_ULE

options NEW_PCIB

options GEOM_PART_MBR

options GEOM_PART_EBR_COMPAT

options GEOM_PART_EBR

options GEOM_PART_BSD

device isa

device mem

device io

device uart_ns8250

device cpufreq

device acpi

device pci

device fdc

device ahci

device ata

device mvs

device siis

device ahc

device ahd

device esp

device hptiop

device isp

device mpt

device mps

device mpr

device sym

device trm

device adv

device adw

device aic

device bt

device isci

device scbus

device ch

device da

device sa

device cd

device pass

device ses

device amr

device arcmsr

device ciss

device dpt

device hptmv

device hptnr

device hptrr

device hpt27xx

device iir

device ips

device mly

device twa

device tws

device aac

device aacp

device aacraid

device ida

device mfi

device mlx

device mrsas

device twe

device atkbdc

device atkbd

device psm

device kbdmux

device vga

device splash

device sc

device vt

device vt_vga

device vt_efifb

device agp

device cbb

device pccard

device cardbus

device uart

device ppc

device ppbus

device lpt

device ppi

device puc

device em

device igb

device ixgbe

device miibus

device loop

device random

device padlock_rng

device rdrand_rng

device ether

device vlan

device tun

device md

device gif

device faith

device firmware

device bpf

device uhci

device ohci

device ehci

device xhci

device usb

device ukbd

device umass

device mmc

device mmcsd

device sdhci

device virtio

device virtio_pci

device vtnet

device virtio_blk

device virtio_scsi

device virtio_balloon

device hyperv

device xenpci

device vmx

device lagg

 

 

 

rc.conf

 

 

ihostname="nas"

 

gateway_enable="YES"

 

ifconfig_igb0="up"

ifconfig_igb1="up"

ifconfig_igb2="up"

ifconfig_igb3="up"

 

ifconfig_lagg0="laggproto lacp laggport igb0 laggport igb1 xx.xx.xx.xx netmask xx.xx.xx.xx"

ifconfig_lagg1="laggproto lacp laggport igb2 laggport igb3 xx.xx.xx.xx netmask xx.xx.xx.xx"

 

defaultrouter="xx.xx.xx.xx"

 

cloned_interfaces="lagg0 lagg1 vlan51 vlan52 vlan53 vlan54 vlan55 vlan56 vlan57 vlan58 vlan59 vlan60 vlan61 vlan62 vlan63 vlan64 vlan65 vlan66 vlan67 vlan68 vlan69 vlan70 vlan71 vlan101 vlan102 vlan103 vlan104 vlan105 vlan106 vlan107 vlan108 vlan109 vlan110 vlan151 vlan152 vlan153 vlan154 vlan155 vlan156 vlan157 vlan158 vlan159 vlan160 vlan161 vlan162 vlan163 vlan164 vlan165 vlan166 vlan167 vlan201 vlan202 vlan203 vlan204 vlan205 vlan206 vlan207 vlan208 vlan209 vlan210 vlan211 vlan212 vlan213 vlan214 vlan215 vlan216 vlan217 vlan218 vlan219 vlan220 vlan221 vlan222 vlan223 vlan224 vlan225 vlan226 vlan227 vlan228 vlan229 vlan230 vlan231 vlan232 vlan233 vlan234 vlan235 vlan251 vlan252 vlan253 vlan254 vlan255 vlan256 vlan257 vlan258 vlan259 vlan260 vlan261 vlan262 vlan263 vlan264 vlan265 vlan266 vlan267 vlan268 vlan269 vlan270 vlan271 vlan272 vlan273 vlan274 vlan275 vlan276 vlan277 vlan278 vlan279 vlan280 vlan281 vlan305 vlan306 vlan307 vlan308 vlan309 vlan310 vlan311 vlan312 vlan313 vlan314 vlan315 vlan316 vlan317 vlan318 vlan319 vlan320 vlan321 vlan322 vlan323 vlan324 vlan325 vlan326 vlan327 vlan328 vlan329 vlan330 vlan331 vlan332 vlan333 vlan334 vlan335 vlan351 vlan352 vlan353 vlan354 vlan355 vlan356 vlan357 vlan358 vlan359 vlan360 vlan361 vlan362 vlan363 vlan364 vlan365 vlan366 vlan367 vlan368 vlan369 vlan370 vlan371 vlan372 vlan373 vlan374 vlan375"

 

ifconfig_vlan51="vlan 51 vlandev lagg1"

ifconfig_vlan52="vlan 52 vlandev lagg1"

ifconfig_vlan53="vlan 53 vlandev lagg1"

ifconfig_vlan54="vlan 54 vlandev lagg1"

ifconfig_vlan55="vlan 55 vlandev lagg1"

ifconfig_vlan56="vlan 56 vlandev lagg1"

ifconfig_vlan57="vlan 57 vlandev lagg1"

ifconfig_vlan58="vlan 58 vlandev lagg1"

ifconfig_vlan59="vlan 59 vlandev lagg1"

ifconfig_vlan60="vlan 60 vlandev lagg1"

ifconfig_vlan61="vlan 61 vlandev lagg1"

ifconfig_vlan62="vlan 62 vlandev lagg1"

ifconfig_vlan63="vlan 63 vlandev lagg1"

ifconfig_vlan64="vlan 64 vlandev lagg1"

ifconfig_vlan65="vlan 65 vlandev lagg1"

ifconfig_vlan66="vlan 66 vlandev lagg1"

ifconfig_vlan67="vlan 67 vlandev lagg1"

ifconfig_vlan68="vlan 68 vlandev lagg1"

ifconfig_vlan69="vlan 69 vlandev lagg1"

ifconfig_vlan70="vlan 70 vlandev lagg1"

ifconfig_vlan71="vlan 71 vlandev lagg1"

 

ifconfig_vlan101="vlan 101 vlandev lagg1"

ifconfig_vlan102="vlan 102 vlandev lagg1"

ifconfig_vlan103="vlan 103 vlandev lagg1"

ifconfig_vlan104="vlan 104 vlandev lagg1"

ifconfig_vlan105="vlan 105 vlandev lagg1"

ifconfig_vlan106="vlan 106 vlandev lagg1"

ifconfig_vlan107="vlan 107 vlandev lagg1"

ifconfig_vlan108="vlan 108 vlandev lagg1"

ifconfig_vlan109="vlan 109 vlandev lagg1"

ifconfig_vlan110="vlan 110 vlandev lagg1"

 

ifconfig_vlan151="vlan 151 vlandev lagg1"

ifconfig_vlan152="vlan 152 vlandev lagg1"

ifconfig_vlan153="vlan 153 vlandev lagg1"

ifconfig_vlan154="vlan 154 vlandev lagg1"

ifconfig_vlan155="vlan 155 vlandev lagg1"

ifconfig_vlan156="vlan 156 vlandev lagg1"

ifconfig_vlan157="vlan 157 vlandev lagg1"

ifconfig_vlan158="vlan 158 vlandev lagg1"

ifconfig_vlan159="vlan 159 vlandev lagg1"

ifconfig_vlan160="vlan 160 vlandev lagg1"

ifconfig_vlan161="vlan 161 vlandev lagg1"

ifconfig_vlan162="vlan 162 vlandev lagg1"

ifconfig_vlan163="vlan 163 vlandev lagg1"

ifconfig_vlan164="vlan 164 vlandev lagg1"

ifconfig_vlan165="vlan 165 vlandev lagg1"

ifconfig_vlan166="vlan 166 vlandev lagg1"

ifconfig_vlan167="vlan 167 vlandev lagg1"

 

ifconfig_vlan201="vlan 201 vlandev lagg1"

ifconfig_vlan202="vlan 202 vlandev lagg1"

ifconfig_vlan203="vlan 203 vlandev lagg1"

ifconfig_vlan204="vlan 204 vlandev lagg1"

ifconfig_vlan205="vlan 205 vlandev lagg1"

ifconfig_vlan206="vlan 206 vlandev lagg1"

ifconfig_vlan207="vlan 207 vlandev lagg1"

ifconfig_vlan208="vlan 208 vlandev lagg1"

ifconfig_vlan209="vlan 209 vlandev lagg1"

ifconfig_vlan210="vlan 210 vlandev lagg1"

ifconfig_vlan211="vlan 211 vlandev lagg1"

ifconfig_vlan212="vlan 212 vlandev lagg1"

ifconfig_vlan213="vlan 213 vlandev lagg1"

ifconfig_vlan214="vlan 214 vlandev lagg1"

ifconfig_vlan215="vlan 215 vlandev lagg1"

ifconfig_vlan216="vlan 216 vlandev lagg1"

ifconfig_vlan217="vlan 217 vlandev lagg1"

ifconfig_vlan218="vlan 218 vlandev lagg1"

ifconfig_vlan219="vlan 219 vlandev lagg1"

ifconfig_vlan220="vlan 220 vlandev lagg1"

ifconfig_vlan221="vlan 221 vlandev lagg1"

ifconfig_vlan222="vlan 222 vlandev lagg1"

ifconfig_vlan223="vlan 223 vlandev lagg1"

ifconfig_vlan224="vlan 224 vlandev lagg1"

ifconfig_vlan225="vlan 225 vlandev lagg1"

ifconfig_vlan226="vlan 226 vlandev lagg1"

ifconfig_vlan227="vlan 227 vlandev lagg1"

ifconfig_vlan228="vlan 228 vlandev lagg1"

ifconfig_vlan229="vlan 229 vlandev lagg1"

ifconfig_vlan230="vlan 230 vlandev lagg1"

ifconfig_vlan231="vlan 231 vlandev lagg1"

ifconfig_vlan232="vlan 232 vlandev lagg1"

ifconfig_vlan233="vlan 233 vlandev lagg1"

ifconfig_vlan234="vlan 234 vlandev lagg1"

ifconfig_vlan235="vlan 235 vlandev lagg1"

 

ifconfig_vlan251="vlan 251 vlandev lagg1"

ifconfig_vlan252="vlan 252 vlandev lagg1"

ifconfig_vlan253="vlan 253 vlandev lagg1"

ifconfig_vlan254="vlan 254 vlandev lagg1"

ifconfig_vlan255="vlan 255 vlandev lagg1"

ifconfig_vlan256="vlan 256 vlandev lagg1"

ifconfig_vlan257="vlan 257 vlandev lagg1"

ifconfig_vlan258="vlan 258 vlandev lagg1"

ifconfig_vlan259="vlan 259 vlandev lagg1"

ifconfig_vlan260="vlan 260 vlandev lagg1"

ifconfig_vlan261="vlan 261 vlandev lagg1"

ifconfig_vlan262="vlan 262 vlandev lagg1"

ifconfig_vlan263="vlan 263 vlandev lagg1"

ifconfig_vlan264="vlan 264 vlandev lagg1"

ifconfig_vlan265="vlan 265 vlandev lagg1"

ifconfig_vlan266="vlan 266 vlandev lagg1"

ifconfig_vlan267="vlan 267 vlandev lagg1"

ifconfig_vlan268="vlan 268 vlandev lagg1"

ifconfig_vlan269="vlan 269 vlandev lagg1"

ifconfig_vlan270="vlan 270 vlandev lagg1"

ifconfig_vlan271="vlan 271 vlandev lagg1"

ifconfig_vlan272="vlan 272 vlandev lagg1"

ifconfig_vlan273="vlan 273 vlandev lagg1"

ifconfig_vlan274="vlan 274 vlandev lagg1"

ifconfig_vlan275="vlan 275 vlandev lagg1"

ifconfig_vlan276="vlan 276 vlandev lagg1"

ifconfig_vlan277="vlan 277 vlandev lagg1"

ifconfig_vlan278="vlan 278 vlandev lagg1"

ifconfig_vlan279="vlan 279 vlandev lagg1"

ifconfig_vlan280="vlan 280 vlandev lagg1"

ifconfig_vlan281="vlan 281 vlandev lagg1"

 

ifconfig_vlan305="vlan 305 vlandev lagg1"

ifconfig_vlan306="vlan 306 vlandev lagg1"

ifconfig_vlan307="vlan 307 vlandev lagg1"

ifconfig_vlan308="vlan 308 vlandev lagg1"

ifconfig_vlan309="vlan 309 vlandev lagg1"

ifconfig_vlan310="vlan 310 vlandev lagg1"

ifconfig_vlan311="vlan 311 vlandev lagg1"

ifconfig_vlan312="vlan 312 vlandev lagg1"

ifconfig_vlan313="vlan 313 vlandev lagg1"

ifconfig_vlan314="vlan 314 vlandev lagg1"

ifconfig_vlan315="vlan 315 vlandev lagg1"

ifconfig_vlan316="vlan 316 vlandev lagg1"

ifconfig_vlan317="vlan 317 vlandev lagg1"

ifconfig_vlan318="vlan 318 vlandev lagg1"

ifconfig_vlan319="vlan 319 vlandev lagg1"

ifconfig_vlan320="vlan 320 vlandev lagg1"

ifconfig_vlan321="vlan 321 vlandev lagg1"

ifconfig_vlan322="vlan 322 vlandev lagg1"

ifconfig_vlan323="vlan 323 vlandev lagg1"

ifconfig_vlan324="vlan 324 vlandev lagg1"

ifconfig_vlan325="vlan 325 vlandev lagg1"

ifconfig_vlan326="vlan 326 vlandev lagg1"

ifconfig_vlan327="vlan 327 vlandev lagg1"

ifconfig_vlan328="vlan 328 vlandev lagg1"

ifconfig_vlan329="vlan 329 vlandev lagg1"

ifconfig_vlan330="vlan 330 vlandev lagg1"

ifconfig_vlan331="vlan 331 vlandev lagg1"

ifconfig_vlan332="vlan 332 vlandev lagg1"

ifconfig_vlan333="vlan 333 vlandev lagg1"

ifconfig_vlan334="vlan 334 vlandev lagg1"

ifconfig_vlan335="vlan 335 vlandev lagg1"

 

ifconfig_vlan351="vlan 351 vlandev lagg1"

ifconfig_vlan352="vlan 352 vlandev lagg1"

ifconfig_vlan353="vlan 353 vlandev lagg1"

ifconfig_vlan354="vlan 354 vlandev lagg1"

ifconfig_vlan355="vlan 355 vlandev lagg1"

ifconfig_vlan356="vlan 356 vlandev lagg1"

ifconfig_vlan357="vlan 357 vlandev lagg1"

ifconfig_vlan358="vlan 358 vlandev lagg1"

ifconfig_vlan359="vlan 359 vlandev lagg1"

ifconfig_vlan360="vlan 360 vlandev lagg1"

ifconfig_vlan361="vlan 361 vlandev lagg1"

ifconfig_vlan362="vlan 362 vlandev lagg1"

ifconfig_vlan363="vlan 363 vlandev lagg1"

ifconfig_vlan364="vlan 364 vlandev lagg1"

ifconfig_vlan365="vlan 365 vlandev lagg1"

ifconfig_vlan366="vlan 366 vlandev lagg1"

ifconfig_vlan367="vlan 367 vlandev lagg1"

ifconfig_vlan368="vlan 368 vlandev lagg1"

ifconfig_vlan369="vlan 369 vlandev lagg1"

ifconfig_vlan370="vlan 370 vlandev lagg1"

ifconfig_vlan371="vlan 371 vlandev lagg1"

ifconfig_vlan372="vlan 372 vlandev lagg1"

ifconfig_vlan373="vlan 373 vlandev lagg1"

ifconfig_vlan374="vlan 374 vlandev lagg1"

ifconfig_vlan375="vlan 375 vlandev lagg1"

 

 

#SNMP

snmpd_enable="YES"

 

#MPD

mpd_enable="YES"

mpd_flags="-b"

 

#IPFW

firewall_enable="YES"

 

quagga_enable=YES

quagga_daemons="zebra bgpd"

 

#keymap="us.iso.kbd"

sshd_enable="YES"

ntpd_enable="YES"

 

 

# Set dumpdev to "AUTO" to enable crash dumps, "NO" to disable

dumpdev="AUTO"

 

 

 

dump

 

 

FreeBSD 10.1-RELEASE FreeBSD 10.1-RELEASE amd64

 

panic: integer divide fault

 

GNU gdb 6.1.1 [FreeBSD]

Copyright 2004 Free Software Foundation, Inc.

GDB is free software, covered by the GNU General Public License, and you are

welcome to change it and/or distribute copies of it under certain conditions.

Type "show copying" to see the conditions.

There is absolutely no warranty for GDB. Type "show warranty" for details.

This GDB was configured as "amd64-marcel-freebsd"...

 

Unread portion of the kernel message buffer:

 

 

Fatal trap 18: integer divide fault while in kernel mode

cpuid = 0; apic id = 00

instruction pointer = 0x20:0xffffffff80785266

stack pointer = 0x28:0xfffffe0000283050

frame pointer = 0x28:0xfffffe00002830e0

code segment = base 0x0, limit 0xfffff, type 0x1b

= DPL 0, pres 1, long 1, def32 0, gran 1

processor eflags = interrupt enabled, resume, IOPL = 0

current process = 12 (irq286: igb3:que 0)

trap number = 18

panic: integer divide fault

cpuid = 0

KDB: stack backtrace:

#0 0xffffffff80705900 at kdb_backtrace+0x60

#1 0xffffffff806caa25 at panic+0x155

#2 0xffffffff80a8585f at trap_fatal+0x38f

#3 0xffffffff80a854bc at trap+0x75c

#4 0xffffffff80a6b0c2 at calltrap+0x8

#5 0xffffffff807b607f at ng_bpf_rcvdata+0x8f

#6 0xffffffff807b359c at ng_apply_item+0x22c

#7 0xffffffff807b31b3 at ng_snd_item+0x1a3

#8 0xffffffff807b359c at ng_apply_item+0x22c

#9 0xffffffff807b31b3 at ng_snd_item+0x1a3

#10 0xffffffff807b359c at ng_apply_item+0x22c

#11 0xffffffff807b31b3 at ng_snd_item+0x1a3

#12 0xffffffff807c3d68 at ng_ppp_comp_recv+0x148

#13 0xffffffff807b359c at ng_apply_item+0x22c

#14 0xffffffff807b31b3 at ng_snd_item+0x1a3

#15 0xffffffff807b359c at ng_apply_item+0x22c

#16 0xffffffff807b31b3 at ng_snd_item+0x1a3

#17 0xffffffff807c6e0c at ng_pppoe_rcvdata_ether+0x27c

Uptime: 5d21h31m53s

Dumping 834 out of 8139 MB:..2%..12%..22%..31%..41%..52%..62%..71%..81%..91%

 

Reading symbols from /boot/kernel/ng_tcpmss.ko.symbols...done.

Loaded symbols for /boot/kernel/ng_tcpmss.ko.symbols

Reading symbols from /boot/kernel/ng_car.ko.symbols...done.

Loaded symbols for /boot/kernel/ng_car.ko.symbols

#0 doadump (textdump=<value optimized out>) at pcpu.h:219

219 pcpu.h: No such file or directory.

in pcpu.h

(kgdb) #0 doadump (textdump=<value optimized out>) at pcpu.h:219

#1 0xffffffff806ca6a2 in kern_reboot (howto=260)

at /usr/src/sys/kern/kern_shutdown.c:452

#2 0xffffffff806caa64 in panic (fmt=<value optimized out>)

at /usr/src/sys/kern/kern_shutdown.c:759

#3 0xffffffff80a8585f in trap_fatal (frame=<value optimized out>,

eva=<value optimized out>) at /usr/src/sys/amd64/amd64/trap.c:865

#4 0xffffffff80a854bc in trap (frame=<value optimized out>)

at /usr/src/sys/amd64/amd64/trap.c:203

#5 0xffffffff80a6b0c2 in calltrap ()

at /usr/src/sys/amd64/amd64/exception.S:232

#6 0xffffffff80785266 in bpf_filter (pc=0xfffff80016a14e04,

p=0xfffff8000ba97816 "E", wirelen=327, buflen=327)

at /usr/src/sys/net/bpf_filter.c:466

#7 0xffffffff807b607f in ng_bpf_rcvdata (hook=<value optimized out>,

item=<value optimized out>) at /usr/src/sys/netgraph/ng_bpf.c:476

#8 0xffffffff807b359c in ng_apply_item (node=0xfffff8014684e600,

item=0xfffff80146021380, rw=0) at /usr/src/sys/netgraph/ng_base.c:2397

#9 0xffffffff807b31b3 in ng_snd_item (item=0xfffff80146021380,

flags=<value optimized out>) at /usr/src/sys/netgraph/ng_base.c:2314

#10 0xffffffff807b359c in ng_apply_item (node=0xfffff80146830500,

item=0xfffff80146021380, rw=0) at /usr/src/sys/netgraph/ng_base.c:2397

#11 0xffffffff807b31b3 in ng_snd_item (item=0xfffff80146021380,

flags=<value optimized out>) at /usr/src/sys/netgraph/ng_base.c:2314

#12 0xffffffff807b359c in ng_apply_item (node=0xfffff80146fb7800,

item=0xfffff80146021380, rw=0) at /usr/src/sys/netgraph/ng_base.c:2397

#13 0xffffffff807b31b3 in ng_snd_item (item=0xfffff80146021380,

flags=<value optimized out>) at /usr/src/sys/netgraph/ng_base.c:2314

#14 0xffffffff807c3d68 in ng_ppp_comp_recv (node=<value optimized out>,

item=0xfffff80146021380, proto=<value optimized out>,

linkNum=<value optimized out>) at /usr/src/sys/netgraph/ng_ppp.c:947

#15 0xffffffff807b359c in ng_apply_item (node=0xfffff801468bc200,

item=0xfffff80146021380, rw=0) at /usr/src/sys/netgraph/ng_base.c:2397

#16 0xffffffff807b31b3 in ng_snd_item (item=0xfffff80146021380,

flags=<value optimized out>) at /usr/src/sys/netgraph/ng_base.c:2314

#17 0xffffffff807b359c in ng_apply_item (node=0xfffff8014679d700,

item=0xfffff80146021380, rw=0) at /usr/src/sys/netgraph/ng_base.c:2397

#18 0xffffffff807b31b3 in ng_snd_item (item=0xfffff80146021380,

flags=<value optimized out>) at /usr/src/sys/netgraph/ng_base.c:2314

#19 0xffffffff807c6e0c in ng_pppoe_rcvdata_ether (hook=<value optimized out>,

item=0xfffff80146021380) at /usr/src/sys/netgraph/ng_pppoe.c:1387

#20 0xffffffff807b359c in ng_apply_item (node=0xfffff80146d06000,

item=0xfffff80146021380, rw=0) at /usr/src/sys/netgraph/ng_base.c:2397

#21 0xffffffff807b31b3 in ng_snd_item (item=0xfffff80146021380,

flags=<value optimized out>) at /usr/src/sys/netgraph/ng_base.c:2314

#22 0xffffffff80791016 in ether_demux (ifp=0xfffff8000b548800,

m=0xfffff80218ae1e00) at /usr/src/sys/net/if_ethersubr.c:868

#23 0xffffffff80791b4e in ether_nh_input (m=<value optimized out>)

at /usr/src/sys/net/if_ethersubr.c:646

#24 0xffffffff8079cba2 in netisr_dispatch_src (proto=<value optimized out>,

source=<value optimized out>, m=0x3730316c)

at /usr/src/sys/net/netisr.c:972

#25 0xffffffff80790ed1 in ether_demux (ifp=0xfffff80009eaa000,

m=0xfffff80218ae1e00) at /usr/src/sys/net/if_ethersubr.c:760

#26 0xffffffff80791b4e in ether_nh_input (m=<value optimized out>)

at /usr/src/sys/net/if_ethersubr.c:646

#27 0xffffffff8079cba2 in netisr_dispatch_src (proto=<value optimized out>,

source=<value optimized out>, m=0x3730316c)

at /usr/src/sys/net/netisr.c:972

#28 0xffffffff803e53e8 in igb_rxeof (que=0xfffff800044dfc00, count=99,

done=0x0) at /usr/src/sys/dev/e1000/if_igb.c:4797

#29 0xffffffff803e5993 in igb_msix_que (arg=0xfffff800044dfc00)

at /usr/src/sys/dev/e1000/if_igb.c:1621

#30 0xffffffff8069e51b in intr_event_execute_handlers (

p=<value optimized out>, ie=0xfffff800044da300)

at /usr/src/sys/kern/kern_intr.c:1263

#31 0xffffffff8069e966 in ithread_loop (arg=0xfffff800044fe0e0)

at /usr/src/sys/kern/kern_intr.c:1276

#32 0xffffffff8069c13a in fork_exit (

callout=0xffffffff8069e8d0 <ithread_loop>, arg=0xfffff800044fe0e0,

frame=0xfffffe0000283ac0) at /usr/src/sys/kern/kern_fork.c:996

#33 0xffffffff80a6b5fe in fork_trampoline ()

at /usr/src/sys/amd64/amd64/exception.S:606

#34 0x0000000000000000 in ?? ()

Current language: auto; currently minimal

(kgdb)

 

------------------------------------------------------------------------

ps -axlww

 

UID PID PPID CPU PRI NI VSZ RSS MWCHAN STAT TT TIME COMMAND

0 0 0 0 -8 0 0 18014389971124704 - DLs - 21:13.74 [kernel]

0 1 0 0 20 0 9472 17979214879562752 wait DLs - 0:00.36 [init]

0 2 0 0 -16 0 0 18014389971124704 - DL - 0:02.44 [cam]

0 3 0 0 -16 0 0 18014389971124704 waiting_ DL - 0:00.01 [sctp_iterator]

0 4 0 0 -16 0 0 18014389971124704 idle DL - 0:00.22 [enc_daemon0]

0 5 0 0 -16 0 0 18014389971124704 psleep DL - 0:05.13 [pagedaemon]

0 6 0 0 -16 0 0 18014389971124704 psleep DL - 0:00.00 [vmdaemon]

0 7 0 0 -16 0 0 18014389971124704 pollid DL - 0:00.40 [idlepoll]

0 8 0 0 155 0 0 18014389971124704 pgzero DL - 0:00.00 [pagezero]

0 9 0 0 20 0 0 18014389971124704 sdflush DL - 0:15.97 [bufdaemon]

0 10 0 0 -16 0 0 18014389971124704 audit_wo DL - 0:00.00 [audit]

0 11 0 0 155 0 0 18014389971124704 - RL - 43748:37.72 [idle]

0 12 0 0 -76 0 0 18014389971124704 - WL - 6091:01.22 [intr]

0 13 0 0 -16 0 0 18014389971124704 sleep DL - 11:36.24 [ng_queue]

0 14 0 0 -8 0 0 18014389971124704 - DL - 0:00.02 [geom]

0 15 0 0 -16 0 0 18014389971124704 - DL - 103:12.93 [rand_harvestq]

0 16 0 0 -68 0 0 18014389971124704 - DL - 0:39.91 [usb]

0 17 0 0 -16 0 0 18014389971124704 tzpoll DL - 0:02.57 [acpi_thermal]

0 18 0 0 -16 0 0 18014389971124704 cooling DL - 0:00.24 [acpi_cooling0]

0 19 0 0 -16 0 0 18014389971124704 vlruwt DL - 0:01.38 [vnlru]

0 20 0 0 16 0 0 18014389971124704 syncer DL - 4:21.01 [syncer]

101 6790 1 0 20 0 31840 17979214901499136 select Ds - 56:55.69 [zebra]

101 6796 1 0 20 0 34800 17979214881672704 select Ds - 2:40.56 [bgpd]

0 7857 1 0 52 0 13164 0 select Ds - 0:06.30 [devd]

0 8315 1 0 20 0 14492 17979214894690560 select Ds - 0:03.84 [syslogd]

0 8490 1 0 23 0 281368 17979214898344192 select Ds - 583:16.16 [mpd5]

0 8667 1 0 24 0 76796 0 select D - 290:31.62 [snmpd]

0 8695 1 0 20 0 25448 17979214897788928 select Ds - 3:18.19 [ntpd]

0 8736 1 0 20 0 61204 17979214881648896 select Ds - 0:05.24 [sshd]

0 8739 1 0 20 0 24104 17979214894067968 select Ds - 0:05.45 [sendmail]

25 8742 1 0 20 0 24104 17979214899163392 pause Ds - 0:00.09 [sendmail]

0 8746 1 0 52 0 16592 17979214897892864 nanslp Ds - 0:01.02 [cron]

0 8792 1 0 52 0 14488 17979214879546368 ttyin Ds+ - 0:00.00 [getty]

0 8793 1 0 52 0 14488 17979214881681664 ttyin Ds+ - 0:00.00 [getty]

0 8794 1 0 52 0 14488 17979214894277376 ttyin Ds+ - 0:00.00 [getty]

0 8795 1 0 52 0 14488 17979214896412672 ttyin Ds+ - 0:00.00 [getty]

0 8796 1 0 52 0 14488 0 ttyin Ds+ - 0:00.00 [getty]

0 8797 1 0 52 0 14488 17979214881639936 ttyin Ds+ - 0:00.00 [getty]

0 8798 1 0 52 0 14488 17979214898337024 ttyin Ds+ - 0:00.00 [getty]

0 8799 1 0 52 0 14488 17979214879569920 ttyin Ds+ - 0:00.00 [getty]

 

------------------------------------------------------------------------

vmstat -s

 

764749945 cpu context switches

439100610 device interrupts

497473979 software interrupts

369010503 traps

2463897317 system calls

20 kernel threads created

285323 fork() calls

74865 vfork() calls

0 rfork() calls

0 swap pager pageins

0 swap pager pages paged in

0 swap pager pageouts

0 swap pager pages paged out

1006 vnode pager pageins

8193 vnode pager pages paged in

0 vnode pager pageouts

0 vnode pager pages paged out

0 page daemon wakeups

0 pages examined by the page daemon

792 pages reactivated

12404768 copy-on-write faults

20508 copy-on-write optimized faults

23168251 zero fill pages zeroed

0 zero fill pages prezeroed

0 intransit blocking page faults

56707364 total VM faults taken

792 page faults requiring I/O

0 pages affected by kernel thread creation

10714372 pages affected by fork()

2616845 pages affected by vfork()

0 pages affected by rfork()

0 pages cached

75208405 pages freed

0 pages freed by daemon

0 pages freed by exiting processes

4401 pages active

280985 pages inactive

0 pages in VM cache

148799 pages wired down

1589428 pages free

4096 bytes per page

19109796 total name lookups

cache hits (76% pos + 6% neg) system 0% per-directory

deletions 0%, falsehits 0%, toolong 0%

 

------------------------------------------------------------------------

vmstat -m

 

Type InUse MemUse HighUse Requests Size(s)

acpiintr 1 1K - 1 64

cdev 5 2K - 5 256

acpica 4733 438K - 372424 16,32,64,128,256,512,1024,2048

filedesc 48 309K - 360552 16,32,64,128,256,2048,4096,8192,16384,32768,65536

sigio 1 1K - 1 64

filecaps 0 0K - 10 16,64

kdtrace 274 61K - 7269254 64,256

kenv 74 11K - 79 16,32,64,128,8192

kqueue 0 0K - 46465 256,2048

proc-args 20 2K - 342316 16,32,64,128,256

hhook 2 1K - 2 256

ithread 203 34K - 203 32,128,256

KTRACE 100 13K - 100 128

CAM path 17 1K - 111 32

linker 154 31K - 168 16,32,64,128,256,512,1024,2048

lockf 26 3K - 8148 64,128

loginclass 2 1K - 3633 64

devbuf 19190 40887K - 19613 16,32,64,128,256,512,1024,2048,4096,8192,32768,65536

temp 2522 83K - 123076135 16,32,64,128,256,1024,2048,4096

acpitask 1 16K - 1 16384

module 387 49K - 387 128

mtx_pool 2 16K - 2 8192

acpisem 20 3K - 20 128

CAM periph 10 3K - 35 16,32,64,128,256

pmchooks 1 1K - 1 128

entropy 1026 65K - 1877 32,64,4096

acpidev 89 6K - 89 64

pgrp 20 3K - 13149 128

session 20 3K - 13126 128

proc 2 32K - 2 16384

subproc 159 220K - 360328 512,4096

cred 68 11K - 4318774 64,256

plimit 12 3K - 12545 256

uidinfo 4 5K - 11517 128,4096

CAM queue 23 38K - 98 16,32,512,16384

sysctl 0 0K - 532791 16,32,64

sysctloid 6510 326K - 6761 16,32,64,128

sysctltmp 0 0K - 6967562 16,32,64,128

CAM dev queue 10 1K - 10 64

tidhash 1 32K - 1 32768

callout 7 2696K - 7

umtx 2106 264K - 2106 128

p1003.1b 1 1K - 1 16

SWAP 2 2189K - 2 64

bus 1084 107K - 57135 16,32,64,128,256,1024

bus-sc 90 371K - 4732 16,32,64,128,256,512,1024,2048,4096,8192,16384,32768,65536

devstat 14 29K - 14 32,4096

eventhandler 100 8K - 100 64,128

DEVFS3 151 38K - 184 256

DEVFS1 125 63K - 151 512

kobj 243 972K - 620 4096

Per-cpu 1 1K - 1 32

DEVFS_RULE 55 26K - 55 64,512

DEVFS 17 1K - 18 16,128

rman 313 36K - 731 16,32,128

sbuf 0 0K - 143190 16,32,64,128,256,512,1024,2048,4096,8192,16384,32768,65536

taskqueue 69 10K - 69 16,32,256

Unitno 212 8K - 13961236 32,64

vmem 3 400K - 8 1024,2048,4096,8192,16384

ioctlops 0 0K - 229246551 16,32,64,128,256,512,1024

select 914 115K - 914 128

iov 0 0K - 6063697 16,64,128,256,512

msg 4 30K - 4 2048,4096,8192,16384

sem 4 106K - 4 2048,4096

shm 1 32K - 1 32768

tty 19 19K - 21 1024

pts 0 0K - 2 256

mbuf_tag 681 22K - 16941246458 32

shmfd 1 8K - 1 8192

soname 6 1K - 271676131 16,32,128

pcb 391 1197K - 7017046 16,32,64,128,1024,2048,8192

vfscache 1 2048K - 1

cl_savebuf 0 0K - 58 64

vfs_hash 1 1024K - 1

vnodes 1 1K - 1 256

mount 61 3K - 164 16,32,64,128,256

vnodemarker 0 0K - 131001 512

BPF 1073 135K - 12998 128

ifnet 1074 2162K - 13006 128,256,512,1024,2048,4096,8192,16384

ifaddr 3077 1025K - 44395 32,512,4096

ether_multi 1818 72K - 36730 16,64

clone 8 1K - 8 128

arpcom 177 3K - 177 16

lltable 1083 539K - 13149 256,512

vlan 173 22K - 174 64,128,256

routetbl 7123 313K - 820024 32,128,256,512

netflow_hash 1 3072K - 1

netflow_general 1 1K - 1 256

netgraph 7469 20369K - 22338355 16,64,128,256,4096

netgraph_msg 0 0K - 204032335 64,128,256,512,1024

netgraph_hook 38044 4756K - 89575184 128

netgraph_node 9602 2784K - 22364352 128,256,512,1024,2048,4096,8192,16384,32768,65536

netgraph_bpf 34344 6516K - 923186 128,512,1024

netgraph_iface 895 112K - 12820 128

netgraph_ksock 1 1K - 1 128

netgraph_mppc 0 0K - 1 1024

netgraph_ppp 895 14320K - 12820 16384

netgraph_pppoe 1644 2955K - 22206987 64,512,16384

netgraph_path 0 0K - 156715418 16,32

netgraph_sock 1535 113K - 22244578 32,128,256,512,1024,2048,4096,8192,16384,32768

igmp 1073 269K - 12998 256

in_multi 898 225K - 18353 256

sctp_a_it 0 0K - 1605 16

sctp_vrf 1 1K - 1 64

sctp_ifa 3 1K - 804 128

sctp_ifn 3 1K - 18353 128

sctp_iter 0 0K - 1605 256

hostcache 1 32K - 1 32768

syncache 1 64K - 1 65536

dummynet 3 3K - 3 512,1024

IpFw/IpAcct 15 36K - 24 16,32,64,128,1024,32768

ipfw_tbl 30 8K - 30 256

rpc 2 16K - 2 8192

audit_evclass 187 6K - 228 32

pagedep 5 1025K - 1173 256

inodedep 13 4101K - 10769 512

bmsafemap 5 33K - 2905 256,8192

newblk 5 8193K - 28371 256

indirdep 0 0K - 368 128,32768

freefrag 0 0K - 386 128

freeblks 1 1K - 1413 256

freefile 0 0K - 1315 64

diradd 8 1K - 7658 128

mkdir 0 0K - 16 128

dirrem 8 1K - 7654 128

newdirblk 0 0K - 8 64

freework 5 1K - 1908 64,128

freedep 0 0K - 8 64

jaddref 0 0K - 7674 128

jremref 0 0K - 7670 128

jnewblk 0 0K - 28367 128

jfreefrag 0 0K - 386 128

jseg 1 1K - 4087 128

jsegdep 17 2K - 44097 64

sbdep 0 0K - 4063 64

savedino 0 0K - 1143 256

jblocks 8 2K - 8 128,256

softdep 4 2K - 4 512

ufs_dirhash 585 138K - 585 16,32,64,128,256,512,1024,8192,16384

ufs_quota 1 1024K - 1

ufs_mount 12 66K - 13 512,4096,8192,16384

vm_pgdata 2 1025K - 2 128

UMAHash 6 11K - 15 512,1024,2048,4096

pfs_nodes 21 6K - 21 256

GEOM 151 40K - 1709 16,32,64,128,256,512,1024,2048,8192,16384

CAM SIM 10 3K - 10 256

memdesc 1 4K - 1 4096

scsi_cd 0 0K - 5 16

UART 6 5K - 6 16,1024

atkbddev 2 1K - 2 64

kbdmux 6 22K - 6 16,512,1024,2048,16384

LED 32 3K - 32 16,128

raid_data 0 0K - 258 32,128,256

scsi_da 0 0K - 111 16,32,64,512

CAM XPT 49 5K - 220 16,32,64,128,256,512,1024,2048,65536

SCSI ENC 25 100K - 76446 16,64,256,2048,32768

pci_link 16 2K - 16 16,128

CAM DEV 13 26K - 37 2048

md_nvidia_data 0 0K - 42 512

mps 13 479K - 29 16,32,64,128,256,512,1024,4096,8192,65536

md_sii_data 0 0K - 42 512

USB 50 94K - 60 16,32,128,256,512,1024,4096,16384,32768

USBdev 35 4K - 35 32,64,128,256,512

apmdev 1 1K - 1 128

madt_table 0 0K - 1 4096

io_apic 2 4K - 2 2048

MCA 18 3K - 18 128

msi 38 5K - 38 128

nexusdev 3 1K - 3 16

isadev 7 1K - 7 128

CAM CCB 0 0K - 394283 2048

 

------------------------------------------------------------------------

vmstat -z

 

ITEM SIZE LIMIT USED FREE REQ FAIL SLEEP

 

UMA Kegs: 384, 0, 109, 1, 109, 0, 0

UMA Zones: 1408, 0, 109, 1, 109, 0, 0

UMA Slabs: 80, 0, 9971, 29, 44122, 0, 0

UMA RCntSlabs: 88, 0, 14340, 15, 14340, 0, 0

UMA Hash: 256, 0, 3, 12, 9, 0, 0

4 Bucket: 32, 0, 194, 2181, 181116, 0, 0

6 Bucket: 48, 0, 0, 2075, 51934, 0, 0

8 Bucket: 64, 0, 79, 2587, 322891, 11, 0

12 Bucket: 96, 0, 183, 1375, 1184143, 0, 0

16 Bucket: 128, 0, 179, 1061, 57103, 0, 0

32 Bucket: 256, 0, 155, 1450, 77854, 49, 0

64 Bucket: 512, 0, 426, 390, 57902, 49, 0

128 Bucket: 1024, 0, 404, 352, 236336, 0, 0

256 Bucket: 2048, 0, 347, 135,405046114, 0, 0

vmem btag: 56, 0, 38844, 1839, 64664, 287, 0

VM OBJECT: 256, 0, 49787, 1093,12736130, 0, 0

RADIX NODE: 144, 0, 23302, 1160,18922650, 0, 0

MAP: 240, 0, 3, 61, 3, 0, 0

KMAP ENTRY: 128, 0, 7, 148, 7, 0, 0

MAP ENTRY: 128, 0, 1389, 1122,16233219, 0, 0

VMSPACE: 448, 0, 20, 277, 360190, 0, 0

fakepg: 104, 0, 0, 0, 0, 0, 0

mt_zone: 4112, 0, 346, 0, 346, 0, 0

16: 16, 0, 4169, 2357,420106948, 0, 0

32: 32, 0, 16127, 3123,17075281501, 0, 0

64: 64, 0, 16601, 21157,256138645, 0, 0

128: 128, 0, 81315, 8833,196830127, 0, 0

256: 256, 0, 13810, 12425,77157379, 0, 0

512: 512, 0, 5728, 656,11414749, 0, 0

1024: 1024, 0, 1873, 259,24896167, 0, 0

2048: 2048, 0, 1166, 134, 786677, 0, 0

4096: 4096, 0, 4821, 619,92632052, 0, 0

8192: 8192, 0, 27, 12, 247, 0, 0

16384: 16384, 0, 1085, 95, 13628, 0, 0

32768: 32768, 0, 58, 11, 27058, 0, 0

65536: 65536, 0, 33, 11, 18536, 0, 0

SLEEPQUEUE: 80, 0, 1054, 465, 1054, 0, 0

64 pcpu: 8, 0, 7670, 522, 7670, 0, 0

Files: 80, 0, 452, 1448,155645130, 0, 0

TURNSTILE: 136, 0, 1054, 166, 1054, 0, 0

rl_entry: 40, 0, 99, 1501, 99, 0, 0

umtx pi: 96, 0, 0, 0, 0, 0, 0

MAC labels: 40, 0, 0, 0, 0, 0, 0

PROC: 1216, 0, 39, 78, 360208, 0, 0

THREAD: 1168, 0, 233, 820, 6909044, 0, 0

cpuset: 72, 0, 130, 420, 154, 0, 0

audit_record: 1248, 0, 0, 0, 0, 0, 0

mbuf_packet: 256, 3237795, 25586, 3009,30158359799, 0, 0

mbuf: 256, 3237795, 1, 2535,46892563122, 0, 0

mbuf_cluster: 2048, 505904, 28589, 11, 28589, 0, 0

mbuf_jumbo_page: 4096, 252951, 0, 40, 52956, 0, 0

mbuf_jumbo_9k: 9216, 224844, 0, 0, 0, 0, 0

mbuf_jumbo_16k: 16384, 168632, 0, 0, 0, 0, 0

mbuf_ext_refcnt: 4, 0, 0, 0, 0, 0, 0

NetGraph items: 72, 4123, 174, 1469,3736959070, 0, 0

NetGraph data items: 72, 527, 2, 525,33963205599,2096006, 600

ttyinq: 160, 0, 120, 255, 375, 0, 0

ttyoutq: 256, 0, 64, 251, 198, 0, 0

g_bio: 248, 0, 0, 928, 655616, 0, 0

DMAR_MAP_ENTRY: 120, 0, 0, 0, 0, 0, 0

ata_request: 336, 0, 0, 0, 0, 0, 0

vtnet_tx_hdr: 24, 0, 0, 0, 0, 0, 0

FPU_save_area: 832, 0, 0, 0, 0, 0, 0

VNODE: 472, 0, 101821, 203, 3049311, 0, 0

VNODEPOLL: 112, 0, 0, 0, 0, 0, 0

BUF TRIE: 144, 0, 509, 52249, 146300, 0, 0

NAMEI: 1024, 0, 0, 120,11438836, 0, 0

S VFS Cache: 108, 0, 85057, 24948, 2772689, 0, 0

STS VFS Cache: 148, 0, 0, 0, 0, 0, 0

L VFS Cache: 328, 0, 24922, 230, 337307, 0, 0

LTS VFS Cache: 368, 0, 0, 0, 0, 0, 0

NCLNODE: 528, 0, 0, 0, 0, 0, 0

DIRHASH: 1024, 0, 1900, 40, 1900, 0, 0

pipe: 744, 0, 5, 115, 214585, 0, 0

procdesc: 128, 0, 0, 0, 0, 0, 0

Mountpoints: 816, 0, 5, 10, 5, 0, 0

ksiginfo: 112, 0, 106, 944, 21192, 0, 0

itimer: 352, 0, 1, 32, 1, 0, 0

KNOTE: 128, 0, 0, 651, 38060, 0, 0

socket: 696, 260470, 396, 854,151597581, 0, 0

unpcb: 240, 260480, 11, 485, 2248301, 0, 0

ipq: 56, 15833, 0, 284, 18586, 0, 0

udp_inpcb: 392, 260470, 16, 914,149325109, 0, 0

udpcb: 16, 260538, 16, 2745,149325109, 0, 0

tcp_inpcb: 392, 260470, 18, 272, 23754, 0, 0

tcpcb: 1024, 260472, 18, 86, 23754, 0, 0

tcptw: 88, 27810, 0, 630, 587, 0, 0

syncache: 160, 15375, 0, 375, 14309, 0, 0

hostcache: 136, 15370, 0, 377, 278, 0, 0

tcpreass: 40, 31700, 0, 1600, 494, 0, 0

sackhole: 32, 0, 0, 1625, 439, 0, 0

sctp_ep: 1400, 260470, 0, 0, 0, 0, 0

sctp_asoc: 2408, 40000, 0, 0, 0, 0, 0

sctp_laddr: 48, 80012, 0, 1577, 1605, 0, 0

sctp_raddr: 720, 80000, 0, 0, 0, 0, 0

sctp_chunk: 136, 400026, 0, 0, 0, 0, 0

sctp_readq: 104, 400026, 0, 0, 0, 0, 0

sctp_stream_msg_out: 104, 400026, 0, 0, 0, 0, 0

sctp_asconf: 40, 400000, 0, 0, 0, 0, 0

sctp_asconf_ack: 48, 400060, 0, 0, 0, 0, 0

udplite_inpcb: 392, 260470, 0, 0, 0, 0, 0

ripcb: 392, 260470, 0, 180, 59, 0, 0

rtentry: 200, 0, 7064, 336, 95683, 0, 0

IPFW dynamic rule: 120, 4125, 0, 0, 0, 0, 0

divcb: 392, 260470, 0, 0, 0, 0, 0

selfd: 56, 0, 1338, 3277,33232376812, 0, 0

SWAPMETA: 288, 1011816, 0, 0, 0, 0, 0

FFS inode: 168, 0, 101792, 466, 3049223, 0, 0

FFS1 dinode: 128, 0, 0, 0, 0, 0, 0

FFS2 dinode: 256, 0, 101792, 328, 3049223, 0, 0

NetFlow IPv4 cache: 88, 1048606, 11696, 12267,1687062925, 0, 0

 

 

------------------------------------------------------------------------

vmstat -i

 

interrupt total rate

irq1: atkbd0 18 0

irq20: ehci1 1783366 22016

irq21: ehci0 764300 9435

cpu0:timer 573905864 7085257

irq264: mps0 215697 2662

irq265: igb0:que 0 5116774278 63170052

irq266: igb0:que 1 965101817 11914837

irq267: igb0:que 2 1004307976 12398863

irq268: igb0:que 3 873135428 10779449

irq269: igb0:que 4 906809984 11195184

irq270: igb0:que 5 929111202 11470508

irq271: igb0:link 2 0

irq272: igb1:que 0 887549483 10957401

irq273: igb1:que 1 923241933 11398048

irq274: igb1:que 2 871187089 10755396

irq275: igb1:que 3 862168757 10644058

irq276: igb1:que 4 871874088 10763877

irq277: igb1:que 5 894344374 11041288

irq278: igb1:link 3 0

irq279: igb2:que 0 3796559067 46871099

irq280: igb2:que 1 1240655816 15316738

irq281: igb2:que 2 1217730053 15033704

irq282: igb2:que 3 1143788663 14120847

irq283: igb2:que 4 1159196347 14311066

irq284: igb2:que 5 1180726459 14576869

irq285: igb2:link 2 0

irq286: igb3:que 0 3916945630 48357353

irq287: igb3:que 1 1217085413 15025745

irq288: igb3:que 2 1232307777 15213676

irq289: igb3:que 3 1150802266 14207435

irq290: igb3:que 4 1199016877 14802677

irq291: igb3:que 5 1235654787 15254997

irq292: igb3:link 3 0

irq293: ahci0 23 0

cpu4:timer 499826582 6170698

cpu3:timer 491639350 6069621

cpu2:timer 510401440 6301252

cpu1:timer 498680834 6156553

cpu5:timer 513536932 6339962

Total 37886829980 467738641

 

------------------------------------------------------------------------

pstat -T

 

452/260468 files

0M/16383M swap space

 

------------------------------------------------------------------------

pstat -s

 

Device 512-blocks Used Avail Capacity

/dev/da0p3 33554176 0 33554176 0%

 

------------------------------------------------------------------------

iostat

 

iostat: kvm_read(_tk_nin): invalid address (0x0)

iostat: disabling TTY statistics

da0 da1 cd0 cpu

KB/t tps MB/s KB/t tps MB/s KB/t tps MB/s us ni sy in id

29.41 2635 75.68 2.34 1 0.00 0.00 0 0.00 1 0 1 12 85

 

------------------------------------------------------------------------

ipcs -a

 

Message Queues:

T ID KEY MODE OWNER GROUP CREATOR CGROUP CBYTES QNUM QBYTES LSPID LRPID

 

STIME RTIME CTIME

 

Shared Memory:

T ID KEY MODE OWNER GROUP CREATOR CGROUP NATTCH SEGSZ CPID LPID ATIME DTIME CTIME

 

Semaphores:

T ID KEY MODE OWNER GROUP CREATOR CGROUP NSEMS OTIME CTIME

 

 

------------------------------------------------------------------------

ipcs -T

 

msginfo:

msgmax: 16384 (max characters in a message)

msgmni: 40 (# of message queues)

msgmnb: 2048 (max characters in a message queue)

msgtql: 40 (max # of messages in system)

msgssz: 8 (size of a message segment)

msgseg: 2048 (# of message segments in system)

 

shminfo:

shmmax: 536870912 (max shared memory segment size)

shmmin: 1 (min shared memory segment size)

shmmni: 192 (max number of shared memory identifiers)

shmseg: 128 (max shared memory segments per process)

shmall: 131072 (max amount of shared memory in pages)

 

seminfo:

semmni: 50 (# of semaphore identifiers)

semmns: 340 (# of semaphores in system)

semmnu: 150 (# of undo structures in system)

semmsl: 340 (max # of semaphores per id)

semopm: 100 (max # of operations per semop call)

semume: 50 (max # of undo entries per process)

semusz: 632 (size in bytes of undo structure)

semvmx: 32767 (semaphore maximum value)

semaem: 16384 (adjust on exit max value)

 

 

------------------------------------------------------------------------

nfsstat

 

Client Info:

Rpc Counts:

Getattr Setattr Lookup Readlink Read Write Create Remove

0 0 0 0 0 0 0 0

Rename Link Symlink Mkdir Rmdir Readdir RdirPlus Access

0 0 0 0 0 0 0 0

Mknod Fsstat Fsinfo PathConf Commit

0 0 0 0 0

Rpc Info:

TimedOut Invalid X Replies Retries Requests

0 0 0 0 0

Cache Info:

Attr Hits Misses Lkup Hits Misses BioR Hits Misses BioW Hits Misses

0 0 0 0 0 0 0 0

BioRLHits Misses BioD Hits Misses DirE Hits Misses Accs Hits Misses

0 0 0 0 0 0 0 0

 

Server Info:

Getattr Setattr Lookup Readlink Read Write Create Remove

0 0 0 0 0 0 0 0

Rename Link Symlink Mkdir Rmdir Readdir RdirPlus Access

0 0 0 0 0 0 0 0

Mknod Fsstat Fsinfo PathConf Commit

0 0 0 0 0

Server Ret-Failed

0

Server Faults

0

Server Cache Stats:

Inprog Idem Non-idem Misses

0 0 0 0

Server Write Gathering:

WriteOps WriteRPC Opsaved

0 0 0

 

------------------------------------------------------------------------

netstat -s

 

tcp:

855358 packets sent

449592 data packets (54704894 bytes)

1177 data packets (458611 bytes) retransmitted

97 data packets unnecessarily retransmitted

0 resends initiated by MTU discovery

391135 ack-only packets (366105 delayed)

0 URG only packets

0 window probe packets

31 window update packets

13429 control packets

892129 packets received

472415 acks (for 54684803 bytes)

9161 duplicate acks

0 acks for unsent data

448632 packets (37143127 bytes) received in-sequence

510 completely duplicate packets (81521 bytes)

3 old duplicate packets

55 packets with some dup. data (26807 bytes duped)

494 out-of-order packets (13968 bytes)

0 packets (0 bytes) of data after window

0 window probes

34 window update packets

4199 packets received after close

2 discarded for bad checksums

0 discarded for bad header offset fields

0 discarded because packet too short

0 discarded due to memory problems

48 connection requests

14234 connection accepts

0 bad connection attempts

0 listen queue overflows

323 ignored RSTs in the windows

14263 connections established (including accepts)

23736 connections closed (including 1022 drops)

8813 connections updated cached RTT on close

8814 connections updated cached RTT variance on close

3052 connections updated cached ssthresh on close

14 embryonic connections dropped

470254 segments updated rtt (of 461352 attempts)

1318 retransmit timeouts

1 connection dropped by rexmit timeout

0 persist timeouts

0 connections dropped by persist timeout

0 Connections (fin_wait_2) dropped because of timeout

0 keepalive timeouts

0 keepalive probes sent

0 connections dropped by keepalive

347534 correct ACK header predictions

379142 correct data packet header predictions

14309 syncache entries added

611 retransmitted

399 dupsyn

0 dropped

14234 completed

0 bucket overflow

0 cache overflow

314 reset

38 stale

0 aborted

0 badack

13 unreach

0 zone failures

14309 cookies sent

290 cookies received

278 hostcache entries added

0 bucket overflow

12 SACK recovery episodes

8 segment rexmits in SACK recovery episodes

5200 byte rexmits in SACK recovery episodes

751 SACK options (SACK blocks) received

3 SACK options (SACK blocks) sent

0 SACK scoreboard overflow

0 packets with ECN CE bit set

0 packets with ECN ECT(0) bit set

0 packets with ECN ECT(1) bit set

0 successful ECN handshakes

0 times ECN reduced the congestion window

0 packets with valid tcp-md5 signature received

0 packets with invalid tcp-md5 signature received

0 packets with tcp-md5 signature mismatch

0 packets with unexpected tcp-md5 signature received

0 packets without expected tcp-md5 signature received

udp:

38509015 datagrams received

0 with incomplete header

0 with bad data length field

0 with bad checksum

12658 with no checksum

3073 dropped due to no socket

31454405 broadcast/multicast datagrams undelivered

2 dropped due to full socket buffers

0 not for hashed pcb

7051535 delivered

63838730 datagrams output

0 times multicast source filter matched

ip:

29970680408 total packets received

220 bad header checksums

0 with size smaller than minimum

1027 with data size < data length

0 with ip length > max ip packet size

0 with header length < data size

0 with data length < header length

0 with bad options

1 with incorrect version number

55758 fragments received

0 fragments dropped (dup or out of space)

0 fragments dropped after timeout

18586 packets reassembled ok

40250640 packets for this host

691511 packets for unknown/unsupported protocol

29916204550 packets forwarded (0 packets fast forwarded)

663841 packets not forwardable

0 packets received for unknown multicast group

0 redirects sent

66221784 packets sent from this host

0 packets sent with fabricated ip header

0 output packets dropped due to no bufs, etc.

0 output packets discarded due to no route

5530041 output datagrams fragmented

11060498 fragments created

153568 datagrams that can't be fragmented

0 tunneling packets that can't find gif

4397 datagrams with bad address in header

icmp:

729253 calls to icmp_error

903 errors not generated in response to an icmp message

Output histogram:

echo reply: 790839

destination unreachable: 616720

time exceeded: 105551

0 messages with bad code fields

0 messages less than the minimum length

1 message with bad checksum

1 message with bad length

0 multicast echo requests ignored

0 multicast timestamp requests ignored

Input histogram:

echo reply: 1

destination unreachable: 517129

routing redirect: 186

echo: 849487

time exceeded: 1368

790839 message responses generated

0 invalid return addresses

0 no return routes

igmp:

172835 messages received

0 messages received with too few bytes

0 messages received with wrong TTL

0 messages received with bad checksum

114 V1/V2 membership queries received

0 V3 membership queries received

114 membership queries received with invalid field(s)

0 general queries received

0 group queries received

0 group-source queries received

0 group-source queries dropped

6336 membership reports received

0 membership reports received with invalid field(s)

0 membership reports received for groups to which we belong

278 V3 reports received without Router Alert

0 membership reports sent

arp:

1288 ARP requests sent

124 ARP replies sent

80065 ARP requests received

1322 ARP replies received

17797441 ARP packets received

8 total packets dropped due to no ARP entry

140 ARP entrys timed out

0 Duplicate IPs seen

 

------------------------------------------------------------------------

netstat -m

 

netstat: invalid address (0x0)

25587/5544/31131 mbufs in use (current/cache/total)

25580/3020/28600/505904 mbuf clusters in use (current/cache/total/max)

25586/3009 mbuf+clusters out of packet secondary zone in use (current/cache)

0/40/40/252951 4k (page size) jumbo clusters in use (current/cache/total/max)

0/0/0/224844 9k jumbo clusters in use (current/cache/total/max)

0/0/0/168632 16k jumbo clusters in use (current/cache/total/max)

57578K/7586K/65164K bytes allocated to network (current/cache/total)

0/0/0 requests for mbufs denied (mbufs/clusters/mbuf+clusters)

0/0/0 requests for mbufs delayed (mbufs/clusters/mbuf+clusters)

0/0/0 requests for jumbo clusters delayed (4k/9k/16k)

0/0/0 requests for jumbo clusters denied (4k/9k/16k)

 

------------------------------------------------------------------------

 

fstat

 

fstat: can't read file 1 at 0x200007fffffffff

fstat: can't read file 2 at 0x4000000001fffff

fstat: can't read file 1 at 0x200007fffffffff

fstat: can't read file 2 at 0x4000000001fffff

fstat: can't read file 1 at 0x200007fffffffff

fstat: can't read file 2 at 0x4000000001fffff

fstat: can't read file 1 at 0x200007fffffffff

fstat: can't read file 2 at 0x4000000001fffff

fstat: can't read file 1 at 0x200007fffffffff

fstat: can't read file 2 at 0x4000000001fffff

fstat: can't read file 1 at 0x200007fffffffff

fstat: can't read file 2 at 0x4000000001fffff

fstat: can't read file 1 at 0x200007fffffffff

fstat: can't read file 2 at 0x4000000001fffff

fstat: can't read file 1 at 0x200007fffffffff

fstat: can't read file 2 at 0x4000000001fffff

fstat: can't read file 1 at 0x200007fffffffff

fstat: can't read file 2 at 0x4000000001fffff

fstat: can't read file 1 at 0x200007fffffffff

fstat: can't read file 2 at 0x4000000001fffff

fstat: can't read file 4 at 0x780000ffff

fstat: can't read file 1 at 0x200007fffffffff

fstat: can't read file 2 at 0x4000000001fffff

fstat: can't read file 4 at 0x780000ffff

fstat: can't read file 5 at 0x600000000

fstat: can't read file 1 at 0x200007fffffffff

fstat: can't read file 2 at 0x4000000001fffff

fstat: can't read file 1 at 0x200007fffffffff

fstat: can't read file 2 at 0x4000000001fffff

fstat: can't read file 4 at 0x780000ffff

fstat: can't read file 5 at 0x400000000

fstat: can't read file 7 at 0x200007fffffffff

fstat: can't read file 8 at 0x4000000001fffff

fstat: can't read file 10 at 0x780000ffff

fstat: can't read file 11 at 0x400000000

fstat: can't read file 13 at 0x200007fffffffff

fstat: can't read file 14 at 0x4000000001fffff

fstat: can't read file 16 at 0x780000ffff

fstat: can't read file 17 at 0x400000000

fstat: can't read file 19 at 0x200007fffffffff

fstat: can't read file 20 at 0x4000000001fffff

fstat: can't read file 22 at 0x780000ffff

fstat: can't read file 23 at 0x2200000001

fstat: can't read file 1 at 0x200007fffffffff

fstat: can't read file 2 at 0x4000000001fffff

fstat: can't read file 4 at 0x780000ffff

fstat: can't read file 5 at 0x600000000

fstat: can't read file 7 at 0x200007fffffffff

fstat: can't read file 8 at 0x4000000001fffff

fstat: can't read file 10 at 0x780000ffff

fstat: can't read file 11 at 0x600000000

fstat: can't read file 13 at 0x200007fffffffff

fstat: can't read file 1 at 0x200007fffffffff

fstat: can't read file 2 at 0x4000000001fffff

fstat: can't read file 4 at 0x780000ffff

fstat: can't read file 5 at 0x400000000

fstat: can't read file 7 at 0x200007fffffffff

fstat: can't read file 8 at 0x4000000001fffff

fstat: can't read file 10 at 0x780000ffff

fstat: can't read file 11 at 0x400000000

fstat: can't read file 13 at 0x200007fffffffff

fstat: can't read file 14 at 0x4000000001fffff

fstat: can't read file 16 at 0x780000ffff

fstat: can't read file 17 at 0x400000000

fstat: can't read file 19 at 0x200007fffffffff

fstat: can't read file 20 at 0x4000000001fffff

fstat: can't read file 22 at 0x780000ffff

fstat: can't read file 23 at 0xe00000001

fstat: can't read file 25 at 0x200007fffffffff

fstat: can't read file 26 at 0x4000000001fffff

fstat: can't read file 28 at 0x780000ffff

fstat: can't read file 29 at 0x200000001

fstat: can't read file 31 at 0x200007fffffffff

fstat: can't read file 32 at 0x4000000001fffff

fstat: can't read file 34 at 0x780000ffff

fstat: can't read file 35 at 0xa00000001

fstat: can't read file 37 at 0x200007fffffffff

fstat: can't read file 38 at 0x4000000001fffff

fstat: can't read file 40 at 0x780000ffff

fstat: can't read file 41 at 0x600000001

fstat: can't read file 43 at 0x200007fffffffff

fstat: can't read file 44 at 0x4000000001fffff

fstat: can't read file 46 at 0x780000ffff

fstat: can't read file 47 at 0x200000000

fstat: can't read file 49 at 0x200007fffffffff

fstat: can't read file 50 at 0x4000000001fffff

fstat: can't read file 52 at 0x780000ffff

fstat: can't read file 53 at 0x200000000

fstat: can't read file 55 at 0x200007fffffffff

fstat: can't read file 56 at 0x4000000001fffff

fstat: can't read file 58 at 0x780000ffff

fstat: can't read file 59 at 0x200000001

fstat: can't read file 61 at 0x200007fffffffff

fstat: can't read file 62 at 0x4000000001fffff

fstat: can't read file 64 at 0x780000ffff

fstat: can't read file 65 at 0x200000001

fstat: can't read file 67 at 0x200007fffffffff

fstat: can't read file 68 at 0x4000000001fffff

fstat: can't read file 70 at 0x780000ffff

fstat: can't read file 71 at 0x200000001

fstat: can't read file 73 at 0x200007fffffffff

fstat: can't read file 74 at 0x4000000001fffff

fstat: can't read file 76 at 0x780000ffff

fstat: can't read file 77 at 0x200000001

fstat: can't read file 79 at 0x200007fffffffff

fstat: can't read file 80 at 0x4000000001fffff

fstat: can't read file 82 at 0x780000ffff

fstat: can't read file 83 at 0x200000000

fstat: can't read file 85 at 0x200007fffffffff

fstat: can't read file 86 at 0x4000000001fffff

fstat: can't read file 88 at 0x780000ffff

fstat: can't read file 89 at 0x200000000

fstat: can't read file 91 at 0x200007fffffffff

fstat: can't read file 92 at 0x4000000001fffff

fstat: can't read file 94 at 0x780000ffff

fstat: can't read file 95 at 0x9200000001

fstat: can't read file 97 at 0x200007fffffffff

fstat: can't read file 98 at 0x4000000001fffff

fstat: can't read file 100 at 0x780000ffff

fstat: can't read file 101 at 0x200000001

fstat: can't read file 103 at 0x200007fffffffff

fstat: can't read file 104 at 0x4000000001fffff

fstat: can't read file 106 at 0x780000ffff

fstat: can't read file 107 at 0x200000000

fstat: can't read file 109 at 0x200007fffffffff

fstat: can't read file 110 at 0x4000000001fffff

fstat: can't read file 112 at 0x780000ffff

fstat: can't read file 113 at 0x200000000

fstat: can't read file 115 at 0x200007fffffffff

fstat: can't read file 116 at 0x4000000001fffff

fstat: can't read file 118 at 0x780000ffff

fstat: can't read file 119 at 0x200000001

fstat: can't read file 121 at 0x200007fffffffff

fstat: can't read file 122 at 0x4000000001fffff

fstat: can't read file 124 at 0x780000ffff

fstat: can't read file 125 at 0x200000000

fstat: can't read file 131 at 0xcd163400000000

fstat: can't read file 133 at 0x200007fffffffff

fstat: can't read file 134 at 0x4000000001fffff

fstat: can't read file 136 at 0x780000ffff

fstat: can't read file 137 at 0x200000001

fstat: can't read file 139 at 0x200007fffffffff

fstat: can't read file 140 at 0x4000000001fffff

fstat: can't read file 142 at 0x780000ffff

fstat: can't read file 143 at 0x200000001

fstat: can't read file 145 at 0x200007fffffffff

fstat: can't read file 146 at 0x4000000001fffff

fstat: can't read file 148 at 0x780000ffff

fstat: can't read file 149 at 0xa00000001

fstat: can't read file 151 at 0x200007fffffffff

fstat: can't read file 152 at 0x4000000001fffff

fstat: can't read file 154 at 0x780000ffff

fstat: can't read file 155 at 0x600000001

fstat: can't read file 157 at 0x200007fffffffff

fstat: can't read file 158 at 0x4000000001fffff

fstat: can't read file 160 at 0x780000ffff

fstat: can't read file 161 at 0x200000001

fstat: can't read file 163 at 0x200007fffffffff

fstat: can't read file 164 at 0x4000000001fffff

fstat: can't read file 166 at 0x780000ffff

fstat: can't read file 167 at 0x200000001

fstat: can't read file 169 at 0x200007fffffffff

fstat: can't read file 170 at 0x4000000001fffff

fstat: can't read file 172 at 0x780000ffff

fstat: can't read file 173 at 0x200000001

fstat: can't read file 175 at 0x200007fffffffff

fstat: can't read file 176 at 0x4000000001fffff

fstat: can't read file 178 at 0x780000ffff

fstat: can't read file 179 at 0x200000001

fstat: can't read file 181 at 0x200007fffffffff

fstat: can't read file 182 at 0x4000000001fffff

fstat: can't read file 184 at 0x780000ffff

fstat: can't read file 185 at 0x200000001

fstat: can't read file 187 at 0x200007fffffffff

fstat: can't read file 188 at 0x4000000001fffff

fstat: can't read file 190 at 0x780000ffff

fstat: can't read file 191 at 0x200000001

fstat: can't read file 193 at 0x200007fffffffff

fstat: can't read file 194 at 0x4000000001fffff

fstat: can't read file 196 at 0x780000ffff

fstat: can't read file 197 at 0x200000001

fstat: can't read file 199 at 0x200007fffffffff

fstat: can't read file 200 at 0x4000000001fffff

fstat: can't read file 202 at 0x780000ffff

fstat: can't read file 203 at 0x200000001

fstat: can't read file 205 at 0x200007fffffffff

fstat: can't read file 206 at 0x4000000001fffff

fstat: can't read file 208 at 0x780000ffff

fstat: can't read file 209 at 0x200000001

fstat: can't read file 211 at 0x200007fffffffff

fstat: can't read file 212 at 0x4000000001fffff

fstat: can't read file 214 at 0x780000ffff

fstat: can't read file 215 at 0x200000001

fstat: can't read file 217 at 0x200007fffffffff

fstat: can't read file 218 at 0x4000000001fffff

fstat: can't read file 220 at 0x780000ffff

fstat: can't read file 221 at 0x200000001

fstat: can't read file 223 at 0x200007fffffffff

fstat: can't read file 224 at 0x4000000001fffff

fstat: can't read file 226 at 0x780000ffff

fstat: can't read file 227 at 0x200000001

fstat: can't read file 229 at 0x200007fffffffff

fstat: can't read file 230 at 0x4000000001fffff

fstat: can't read file 232 at 0x780000ffff

fstat: can't read file 233 at 0x200000001

fstat: can't read file 235 at 0x200007fffffffff

fstat: can't read file 236 at 0x4000000001fffff

fstat: can't read file 238 at 0x780000ffff

fstat: can't read file 239 at 0x200000001

fstat: can't read file 241 at 0x200007fffffffff

fstat: can't read file 242 at 0x4000000001fffff

fstat: can't read file 244 at 0x780000ffff

fstat: can't read file 245 at 0x200000001

fstat: can't read file 247 at 0x200007fffffffff

fstat: can't read file 248 at 0x4000000001fffff

fstat: can't read file 250 at 0x780000ffff

fstat: can't read file 251 at 0x200000001

fstat: can't read file 253 at 0x200007fffffffff

fstat: can't read file 254 at 0x4000000001fffff

fstat: can't read file 256 at 0x780000ffff

fstat: can't read file 257 at 0x200000001

fstat: can't read file 259 at 0x200007fffffffff

fstat: can't read file 260 at 0x4000000001fffff

fstat: can't read file 262 at 0x780000ffff

fstat: can't read file 263 at 0x200000001

fstat: can't read file 265 at 0x200007fffffffff

fstat: can't read file 266 at 0x4000000001fffff

fstat: can't read file 268 at 0x780000ffff

fstat: can't read file 269 at 0x200000001

fstat: can't read file 271 at 0x200007fffffffff

fstat: can't read file 272 at 0x4000000001fffff

fstat: can't read file 274 at 0x780000ffff

fstat: can't read file 275 at 0x200000001

fstat: can't read file 277 at 0x200007fffffffff

fstat: can't read file 278 at 0x4000000001fffff

fstat: can't read file 280 at 0x780000ffff

fstat: can't read file 281 at 0x200000001

fstat: can't read file 283 at 0x200007fffffffff

fstat: can't read file 284 at 0x4000000001fffff

fstat: can't read file 286 at 0x780000ffff

fstat: can't read file 287 at 0x200000001

fstat: can't read file 289 at 0x200007fffffffff

fstat: can't read file 290 at 0x4000000001fffff

fstat: can't read file 292 at 0x780000ffff

fstat: can't read file 293 at 0x200000001

fstat: can't read file 295 at 0x200007fffffffff

fstat: can't read file 296 at 0x4000000001fffff

fstat: can't read file 298 at 0x780000ffff

fstat: can't read file 299 at 0x200000001

fstat: can't read file 301 at 0x200007fffffffff

fstat: can't read file 302 at 0x4000000001fffff

fstat: can't read file 304 at 0x780000ffff

fstat: can't read file 305 at 0x200000001

fstat: can't read file 307 at 0x200007fffffffff

fstat: can't read file 308 at 0x4000000001fffff

fstat: can't read file 310 at 0x780000ffff

fstat: can't read file 311 at 0x200000001

fstat: can't read file 313 at 0x200007fffffffff

fstat: can't read file 314 at 0x4000000001fffff

fstat: can't read file 316 at 0x780000ffff

fstat: can't read file 317 at 0x200000001

fstat: can't read file 319 at 0x200007fffffffff

fstat: can't read file 320 at 0x4000000001fffff

fstat: can't read file 322 at 0x780000ffff

fstat: can't read file 323 at 0x200000001

fstat: can't read file 325 at 0x200007fffffffff

fstat: can't read file 326 at 0x4000000001fffff

fstat: can't read file 328 at 0x780000ffff

fstat: can't read file 329 at 0x200000001

fstat: can't read file 331 at 0x200007fffffffff

fstat: can't read file 332 at 0x4000000001fffff

fstat: can't read file 334 at 0x780000ffff

fstat: can't read file 335 at 0x200000001

fstat: can't read file 337 at 0x200007fffffffff

fstat: can't read file 338 at 0x4000000001fffff

fstat: can't read file 340 at 0x780000ffff

fstat: can't read file 341 at 0x200000001

fstat: can't read file 343 at 0x200007fffffffff

fstat: can't read file 344 at 0x4000000001fffff

fstat: can't read file 346 at 0x780000ffff

fstat: can't read file 347 at 0x200000001

fstat: can't read file 349 at 0x200007fffffffff

fstat: can't read file 350 at 0x4000000001fffff

fstat: can't read file 352 at 0x780000ffff

fstat: can't read file 353 at 0x200000001

fstat: can't read file 355 at 0x200007fffffffff

fstat: can't read file 356 at 0x4000000001fffff

fstat: can't read file 358 at 0x780000ffff

fstat: can't read file 359 at 0x200000001

fstat: can't read file 361 at 0x200007fffffffff

fstat: can't read file 362 at 0x4000000001fffff

fstat: can't read file 364 at 0x780000ffff

fstat: can't read file 365 at 0x200000001

fstat: can't read file 367 at 0x200007fffffffff

fstat: can't read file 368 at 0x4000000001fffff

fstat: can't read file 370 at 0x780000ffff

fstat: can't read file 371 at 0x200000001

fstat: can't read file 373 at 0x200007fffffffff

fstat: can't read file 374 at 0x4000000001fffff

fstat: can't read file 376 at 0x780000ffff

fstat: can't read file 377 at 0x200000001

fstat: can't read file 379 at 0x200007fffffffff

fstat: can't read file 380 at 0x4000000001fffff

fstat: can't read file 382 at 0x780000ffff

fstat: can't read file 383 at 0x200000001

fstat: can't read file 385 at 0x200007fffffffff

fstat: can't read file 386 at 0x4000000001fffff

fstat: can't read file 388 at 0x780000ffff

fstat: can't read file 389 at 0x200000001

fstat: can't read file 391 at 0x200007fffffffff

fstat: can't read file 392 at 0x4000000001fffff

fstat: can't read file 394 at 0x780000ffff

fstat: can't read file 395 at 0x200000001

fstat: can't read file 397 at 0x200007fffffffff

fstat: can't read file 398 at 0x4000000001fffff

fstat: can't read file 400 at 0x780000ffff

fstat: can't read file 401 at 0x200000001

fstat: can't read file 403 at 0x200007fffffffff

fstat: can't read file 404 at 0x4000000001fffff

fstat: can't read file 406 at 0x780000ffff

fstat: can't read file 407 at 0x200000001

fstat: can't read file 409 at 0x200007fffffffff

fstat: can't read file 410 at 0x4000000001fffff

fstat: can't read file 412 at 0x780000ffff

fstat: can't read file 413 at 0x200000001

fstat: can't read file 415 at 0x200007fffffffff

fstat: can't read file 416 at 0x4000000001fffff

fstat: can't read file 418 at 0x780000ffff

fstat: can't read file 419 at 0x200000001

fstat: can't read file 421 at 0x200007fffffffff

fstat: can't read file 422 at 0x4000000001fffff

fstat: can't read file 424 at 0x780000ffff

fstat: can't read file 425 at 0x200000001

fstat: can't read file 427 at 0x200007fffffffff

fstat: can't read file 428 at 0x4000000001fffff

fstat: can't read file 430 at 0x780000ffff

fstat: can't read file 431 at 0x200000001

fstat: can't read file 433 at 0x200007fffffffff

fstat: can't read file 434 at 0x4000000001fffff

fstat: can't read file 436 at 0x780000ffff

fstat: can't read file 437 at 0x200000001

fstat: can't read file 439 at 0x200007fffffffff

fstat: can't read file 440 at 0x4000000001fffff

fstat: can't read file 442 at 0x780000ffff

fstat: can't read file 443 at 0x200000001

fstat: can't read file 445 at 0x200007fffffffff

fstat: can't read file 446 at 0x4000000001fffff

fstat: can't read file 1 at 0x200007fffffffff

fstat: can't read file 2 at 0x4000000001fffff

fstat: can't read file 4 at 0x780000ffff

fstat: can't read file 5 at 0x400000000

fstat: can't read file 7 at 0x200007fffffffff

fstat: can't read file 8 at 0x4000000001fffff

fstat: can't read file 10 at 0x780000ffff

fstat: can't read file 11 at 0x400000000

fstat: can't read file 13 at 0x200007fffffffff

fstat: can't read file 14 at 0x4000000001fffff

fstat: can't read file 16 at 0x780000ffff

fstat: can't read file 17 at 0x400000000

fstat: can't read file 1 at 0x200007fffffffff

fstat: can't read file 2 at 0x4000000001fffff

fstat: can't read file 4 at 0x780000ffff

fstat: can't read file 5 at 0x800000000

fstat: can't read file 7 at 0x200007fffffffff

fstat: can't read file 1 at 0x200007fffffffff

fstat: can't read file 2 at 0x4000000001fffff

fstat: can't read file 4 at 0x780000ffff

fstat: can't read file 5 at 0x400000000

fstat: can't read file 7 at 0x200007fffffffff

fstat: can't read file 8 at 0x4000000001fffff

fstat: can't read file 10 at 0x780000ffff

fstat: can't read file 11 at 0x400000000

fstat: can't read file 13 at 0x200007fffffffff

fstat: can't read file 14 at 0x4000000001fffff

fstat: can't read file 1 at 0x200007fffffffff

fstat: can't read file 2 at 0x4000000001fffff

fstat: can't read file 4 at 0x780000ffff

fstat: can't read file 5 at 0x400000000

fstat: can't read file 7 at 0x200007fffffffff

fstat: can't read file 8 at 0x4000000001fffff

USER CMD PID FD MOUNT INUM MODE SZ|DV R/W

root getty 8799 root / 2 drwxr-xr-x 1024 r

root getty 8799 wd / 2 drwxr-xr-x 1024 r

root getty 8799 text /usr 25922735 -r-xr-xr-x 28360 r

root getty 8799 ctty /dev 65 crw------- ttyv7 rw

root getty 8799 0 /dev 65 crw------- ttyv7 rw

root getty 8798 root / 2 drwxr-xr-x 1024 r

root getty 8798 wd / 2 drwxr-xr-x 1024 r

root getty 8798 text /usr 25922735 -r-xr-xr-x 28360 r

root getty 8798 ctty /dev 64 crw------- ttyv6 rw

root getty 8798 0 /dev 64 crw------- ttyv6 rw

root getty 8797 root / 2 drwxr-xr-x 1024 r

root getty 8797 wd / 2 drwxr-xr-x 1024 r

root getty 8797 text /usr 25922735 -r-xr-xr-x 28360 r

root getty 8797 ctty /dev 63 crw------- ttyv5 rw

root getty 8797 0 /dev 63 crw------- ttyv5 rw

root getty 8796 root / 2 drwxr-xr-x 1024 r

root getty 8796 wd / 2 drwxr-xr-x 1024 r

root getty 8796 text /usr 25922735 -r-xr-xr-x 28360 r

root getty 8796 ctty /dev 62 crw------- ttyv4 rw

root getty 8796 0 /dev 62 crw------- ttyv4 rw

root getty 8795 root / 2 drwxr-xr-x 1024 r

root getty 8795 wd / 2 drwxr-xr-x 1024 r

root getty 8795 text /usr 25922735 -r-xr-xr-x 28360 r

root getty 8795 ctty /dev 61 crw------- ttyv3 rw

root getty 8795 0 /dev 61 crw------- ttyv3 rw

root getty 8794 root / 2 drwxr-xr-x 1024 r

root getty 8794 wd / 2 drwxr-xr-x 1024 r

root getty 8794 text /usr 25922735 -r-xr-xr-x 28360 r

root getty 8794 ctty /dev 60 crw------- ttyv2 rw

root getty 8794 0 /dev 60 crw------- ttyv2 rw

root getty 8793 root / 2 drwxr-xr-x 1024 r

root getty 8793 wd / 2 drwxr-xr-x 1024 r

root getty 8793 text /usr 25922735 -r-xr-xr-x 28360 r

root getty 8793 ctty /dev 59 crw------- ttyv1 rw

root getty 8793 0 /dev 59 crw------- ttyv1 rw

root getty 8792 root / 2 drwxr-xr-x 1024 r

root getty 8792 wd / 2 drwxr-xr-x 1024 r

root getty 8792 text /usr 25922735 -r-xr-xr-x 28360 r

root getty 8792 ctty /dev 58 crw------- ttyv0 rw

root getty 8792 0 /dev 58 crw------- ttyv0 rw

root cron 8746 root / 2 drwxr-xr-x 1024 r

root cron 8746 wd /var 28410624 drwxr-x--- 512 r

root cron 8746 text /usr 25200569 -r-xr-xr-x 41352 r

root cron 8746 0 /dev 9 crw-rw-rw- null rw

smmsp sendmail 8742 root / 2 drwxr-xr-x 1024 r

smmsp sendmail 8742 wd /var 13242246 drwxrwx--- 512 r

smmsp sendmail 8742 text /usr 25922903 -r-xr-sr-x 692520 r

smmsp sendmail 8742 0 /dev 9 crw-rw-rw- null r

root sendmail 8739 root / 2 drwxr-xr-x 1024 r

root sendmail 8739 wd /var 13242244 drwxr-xr-x 512 r

root sendmail 8739 text /usr 25922903 -r-xr-sr-x 692520 r

root sendmail 8739 0 /dev 9 crw-rw-rw- null r

root sshd 8736 root / 2 drwxr-xr-x 1024 r

root sshd 8736 wd / 2 drwxr-xr-x 1024 r

root sshd 8736 text /usr 25200639 -r-xr-xr-x 297792 r

root sshd 8736 0 /dev 9 crw-rw-rw- null rw

root ntpd 8695 root / 2 drwxr-xr-x 1024 r

root ntpd 8695 wd / 2 drwxr-xr-x 1024 r

root ntpd 8695 text /usr 25200561 -r-xr-xr-x 432544 r

root ntpd 8695 0 /dev 9 crw-rw-rw- null rw

root ntpd 8695 6 /dev 9 crw-rw-rw- null rw

root ntpd 8695 12 /dev 9 crw-rw-rw- null rw

root ntpd 8695 18* local dgram fffff80218f24690 <-> fffff8000b4aba50

root snmpd 8667 root / 2 drwxr-xr-x 1024 r

root snmpd 8667 wd / 2 drwxr-xr-x 1024 r

root snmpd 8667 text /usr 21850511 -rwxr-xr-x 29128 r

root snmpd 8667 0 /dev 9 crw-rw-rw- null rw

root snmpd 8667 6 /dev 9 crw-rw-rw- null rw

root snmpd 8667 12 /dev 9 crw-rw-rw- null rw

root mpd5 8490 root / 2 drwxr-xr-x 1024 r

root mpd5 8490 wd /usr 21830369 drwxr-xr-x 512 r

root mpd5 8490 text /usr 21830377 -rwxr-xr-x 2556555 r

root mpd5 8490 0 /dev 9 crw-rw-rw- null rw

root mpd5 8490 6 /dev 9 crw-rw-rw- null rw

root mpd5 8490 12 /dev 9 crw-rw-rw- null rw

root mpd5 8490 18* local dgram fffff8000b4ab960 <-> fffff8000b4aba50

root mpd5 8490 24 /var 2889219 -rw-r--r-- 5 rw

root mpd5 8490 30* netgraph dgram 2 fffff8000b4ce000

root mpd5 8490 36* netgraph dgram 1 fffff8000b4ce2b8

root mpd5 8490 42* pipe fffff8000b4175d0 <-> fffff8000b417730 0 rw

root mpd5 8490 48* pipe fffff8000b417730 <-> fffff8000b4175d0 0 rw

root mpd5 8490 54* netgraph dgram 2 fffff8000b4cdae0

root mpd5 8490 60* netgraph dgram 1 fffff8000b4cd828

root mpd5 8490 66* netgraph dgram 2 fffff8000b4cd570

root mpd5 8490 72* netgraph dgram 1 fffff8000b4cd2b8

root mpd5 8490 78* pipe fffff8000b47b000 <-> fffff8000b47b160 0 rw

root mpd5 8490 84* pipe fffff8000b47b160 <-> fffff8000b47b000 0 rw

root mpd5 8490 90* netgraph dgram 2 fffff8007e2112b8

root mpd5 8490 96* internet stream tcp fffff80218fad400

root mpd5 8490 102* pipe fffff80218f35000 <-> fffff80218f35160 0 rw

root mpd5 8490 108* pipe fffff80218f35160 <-> fffff80218f35000 0 rw

root mpd5 8490 114* internet stream tcp fffff80218fad000

root mpd5 8490 120* internet dgram udp fffff8000b546620

root mpd5 8490 132* pipe fffff8000b4165d0 <-> fffff8000b416730 0 rw

root mpd5 8490 138* pipe fffff8000b416730 <-> fffff8000b4165d0 0 rw

root mpd5 8490 144* netgraph dgram 2 fffff8000b5422b8

root mpd5 8490 150* netgraph dgram 1 fffff8000b542000

root mpd5 8490 156* netgraph dgram 2 fffff8000b536ae0

root mpd5 8490 162* netgraph dgram 1 fffff8000b536828

root mpd5 8490 168* netgraph dgram 2 fffff8000b536570

root mpd5 8490 174* netgraph dgram 1 fffff8000b44c000

root mpd5 8490 180* netgraph dgram 2 fffff8000b4d9ae0

root mpd5 8490 186* netgraph dgram 1 fffff8000b4d9828

root mpd5 8490 192* netgraph dgram 2 fffff8000b4d9570

root mpd5 8490 198* netgraph dgram 1 fffff8000b4d92b8

root mpd5 8490 204* netgraph dgram 2 fffff8000b4d9000

root mpd5 8490 210* netgraph dgram 1 fffff80146d36ae0

root mpd5 8490 216* netgraph dgram 2 fffff80146d36828

root mpd5 8490 222* netgraph dgram 1 fffff80146d36570

root mpd5 8490 228* netgraph dgram 2 fffff80146d362b8

root mpd5 8490 234* netgraph dgram 1 fffff80146d36000

root mpd5 8490 240* netgraph dgram 2 fffff8000b571ae0

root mpd5 8490 246* netgraph dgram 1 fffff8000b571828

root mpd5 8490 252* netgraph dgram 2 fffff80146d28828

root mpd5 8490 258* netgraph dgram 1 fffff80146d28570

root mpd5 8490 264* netgraph dgram 2 fffff80146d282b8

root mpd5 8490 270* netgraph dgram 1 fffff80146d28000

root mpd5 8490 276* netgraph dgram 2 fffff80146dd9ae0

root mpd5 8490 282* netgraph dgram 1 fffff80146dd9828

root mpd5 8490 288* netgraph dgram 2 fffff80146dd9570

root mpd5 8490 294* netgraph dgram 1 fffff80146dd92b8

root mpd5 8490 300* netgraph dgram 2 fffff80146dd9000

root mpd5 8490 306* netgraph dgram 1 fffff8000b44cae0

root mpd5 8490 312* netgraph dgram 2 fffff8000b44c828

root mpd5 8490 318* netgraph dgram 1 fffff8000b44c570

root mpd5 8490 324* netgraph dgram 2 fffff8000b44c2b8

root mpd5 8490 330* netgraph dgram 1 fffff80146e662b8

root mpd5 8490 336* netgraph dgram 2 fffff80146e66000

root mpd5 8490 342* netgraph dgram 1 fffff80146e64ae0

root mpd5 8490 348* netgraph dgram 2 fffff80146e64828

root mpd5 8490 354* netgraph dgram 1 fffff80146e64570

root mpd5 8490 360* netgraph dgram 2 fffff80146e642b8

root mpd5 8490 366* netgraph dgram 1 fffff80146e64000

root mpd5 8490 372* netgraph dgram 2 fffff80146e3fae0

root mpd5 8490 378* netgraph dgram 1 fffff80146e3f828

root mpd5 8490 384* netgraph dgram 2 fffff80146e3f570

root mpd5 8490 390* netgraph dgram 1 fffff80146e3f2b8

root mpd5 8490 396* netgraph dgram 2 fffff80146e3f000

root mpd5 8490 402* netgraph dgram 1 fffff80146d28ae0

root mpd5 8490 408* netgraph dgram 2 fffff80146e42ae0

root mpd5 8490 414* netgraph dgram 1 fffff80146e42828

root mpd5 8490 420* netgraph dgram 2 fffff80146e42570

root mpd5 8490 426* netgraph dgram 1 fffff80146e422b8

root mpd5 8490 432* netgraph dgram 2 fffff80146e42000

root mpd5 8490 438* netgraph dgram 1 fffff80146e63ae0

root mpd5 8490 444* netgraph dgram 2 fffff80146e63828

root syslogd 8315 root / 2 drwxr-xr-x 1024 r

root syslogd 8315 wd / 2 drwxr-xr-x 1024 r

root syslogd 8315 text /usr 25200460 -r-xr-xr-x 40352 r

root syslogd 8315 0 /dev 9 crw-rw-rw- null rw

root syslogd 8315 6 /dev 9 crw-rw-rw- null rw

root syslogd 8315 12 /dev 9 crw-rw-rw- null rw

root syslogd 8315 18 /var 2889248 -rw------- 4 w

root devd 7857 root / 2 drwxr-xr-x 1024 r

root devd 7857 wd / 2 drwxr-xr-x 1024 r

root devd 7857 text / 642135 -r-xr-xr-x 634824 r

root devd 7857 0 /dev 9 crw-rw-rw- null rw

root devd 7857 6 /dev 9 crw-rw-rw- null rw

quagga bgpd 6796 root / 2 drwxr-xr-x 1024 r

quagga bgpd 6796 wd / 2 drwxr-xr-x 1024 r

quagga bgpd 6796 text /usr 21848108 -rwxr-xr-x 935248 r

quagga bgpd 6796 0 /dev 9 crw-rw-rw- null rw

quagga bgpd 6796 6 /dev 9 crw-rw-rw- null rw

quagga bgpd 6796 12 /dev 9 crw-rw-rw- null rw

quagga zebra 6790 root / 2 drwxr-xr-x 1024 r

quagga zebra 6790 wd / 2 drwxr-xr-x 1024 r

quagga zebra 6790 text /usr 21848897 -rwxr-xr-x 194240 r

quagga zebra 6790 0 /dev 9 crw-rw-rw- null rw

quagga zebra 6790 6 /dev 9 crw-rw-rw- null rw

root init 1 root / 2 drwxr-xr-x 1024 r

root init 1 wd / 2 drwxr-xr-x 1024 r

root init 1 text / 642054 -r-xr-xr-x 981064 r

root kernel 0 root / 2 drwxr-xr-x 1024 r

root kernel 0 wd / 2 drwxr-xr-x 1024 r

 

------------------------------------------------------------------------

dmesg

 

Fatal trap 18: integer divide fault while in kernel mode

cpuid = 0; apic id = 00

instruction pointer = 0x20:0xffffffff80785266

stack pointer = 0x28:0xfffffe0000283050

frame pointer = 0x28:0xfffffe00002830e0

code segment = base 0x0, limit 0xfffff, type 0x1b

= DPL 0, pres 1, long 1, def32 0, gran 1

processor eflags = interrupt enabled, resume, IOPL = 0

current process = 12 (irq286: igb3:que 0)

trap number = 18

panic: integer divide fault

cpuid = 0

KDB: stack backtrace:

#0 0xffffffff80705900 at kdb_backtrace+0x60

#1 0xffffffff806caa25 at panic+0x155

#2 0xffffffff80a8585f at trap_fatal+0x38f

#3 0xffffffff80a854bc at trap+0x75c

#4 0xffffffff80a6b0c2 at calltrap+0x8

#5 0xffffffff807b607f at ng_bpf_rcvdata+0x8f

#6 0xffffffff807b359c at ng_apply_item+0x22c

#7 0xffffffff807b31b3 at ng_snd_item+0x1a3

#8 0xffffffff807b359c at ng_apply_item+0x22c

#9 0xffffffff807b31b3 at ng_snd_item+0x1a3

#10 0xffffffff807b359c at ng_apply_item+0x22c

#11 0xffffffff807b31b3 at ng_snd_item+0x1a3

#12 0xffffffff807c3d68 at ng_ppp_comp_recv+0x148

#13 0xffffffff807b359c at ng_apply_item+0x22c

#14 0xffffffff807b31b3 at ng_snd_item+0x1a3

#15 0xffffffff807b359c at ng_apply_item+0x22c

#16 0xffffffff807b31b3 at ng_snd_item+0x1a3

#17 0xffffffff807c6e0c at ng_pppoe_rcvdata_ether+0x27c

Uptime: 5d21h31m53s

Dumping 834 out of 8139 MB:..2%..12%..22%..31%..41%..52%..62%..71%..81%..91%

 

------------------------------------------------------------------------

kernel config

 

options CONFIG_AUTOGENERATED

ident GENERIC

machine amd64

cpu HAMMER

makeoptions WITH_CTF=1

makeoptions DEBUG=-g

options RADIX_MPATH

options HZ=1000

options DEVICE_POLLING

options NETGRAPH_NETFLOW

options NETGRAPH_IPFW

options NETGRAPH_VJC

options NETGRAPH_UI

options NETGRAPH_TTY

options NETGRAPH_TEE

options NETGRAPH_RFC1490

options NETGRAPH_PPP

options NETGRAPH_ONE2MANY

options NETGRAPH_MPPC_ENCRYPTION

options NETGRAPH_LMI

options NETGRAPH_L2TP

options NETGRAPH_KSOCKET

options NETGRAPH_IFACE

options NETGRAPH_HOLE

options NETGRAPH_FRAME_RELAY

options NETGRAPH_ECHO

options NETGRAPH_BPF

options NETGRAPH_PPTPGRE

options NETGRAPH_PPPOE

options NETGRAPH_SOCKET

options NETGRAPH_ETHER

options NETGRAPH

options DUMMYNET

options IPDIVERT

options IPFIREWALL_VERBOSE_LIMIT=100

options IPFIREWALL_VERBOSE

options IPFIREWALL

options XENHVM

options USB_DEBUG

options SC_PIXEL_MODE

options VESA

options AHD_REG_PRETTY_PRINT

options AHC_REG_PRETTY_PRINT

options ATA_STATIC_ID

options ACPI_DMAR

options SMP

options KDB_TRACE

options KDB

options INCLUDE_CONFIG_FILE

options DDB_CTF

options KDTRACE_HOOKS

options KDTRACE_FRAME

options MAC

options PROCDESC

options CAPABILITIES

options CAPABILITY_MODE

options AUDIT

options HWPMC_HOOKS

options KBD_INSTALL_CDEV

options PRINTF_BUFR_SIZE=128

options _KPOSIX_PRIORITY_SCHEDULING

options SYSVSEM

options SYSVMSG

options SYSVSHM

options STACK

options KTRACE

options SCSI_DELAY=5000

options COMPAT_FREEBSD7

options COMPAT_FREEBSD6

options COMPAT_FREEBSD5

options COMPAT_FREEBSD4

options COMPAT_FREEBSD32

options GEOM_LABEL

options GEOM_RAID

options GEOM_PART_GPT

options PSEUDOFS

options PROCFS

options CD9660

options MSDOSFS

options NFS_ROOT

options NFSLOCKD

options NFSD

options NFSCL

options MD_ROOT

options QUOTA

options UFS_GJOURNAL

options UFS_DIRHASH

options UFS_ACL

options SOFTUPDATES

options FFS

options SCTP

options TCP_OFFLOAD

options INET

options PREEMPTION

options SCHED_ULE

options NEW_PCIB

options GEOM_PART_MBR

options GEOM_PART_EBR_COMPAT

options GEOM_PART_EBR

options GEOM_PART_BSD

device isa

device mem

device io

device uart_ns8250

device cpufreq

device acpi

device pci

device fdc

device ahci

device ata

device mvs

device siis

device ahc

device ahd

device esp

device hptiop

device isp

device mpt

device mps

device mpr

device sym

device trm

device adv

device adw

device aic

device bt

device isci

device scbus

device ch

device da

device sa

device cd

device pass

device ses

device amr

device arcmsr

device ciss

device dpt

device hptmv

device hptnr

device hptrr

device hpt27xx

device iir

device ips

device mly

device twa

device tws

device aac

device aacp

device aacraid

device ida

device mfi

device mlx

device mrsas

device twe

device atkbdc

device atkbd

device psm

device kbdmux

device vga

device splash

device sc

device vt

device vt_vga

device vt_efifb

device agp

device cbb

device pccard

device cardbus

device uart

device ppc

device ppbus

device lpt

device ppi

device puc

device em

device igb

device ixgbe

device miibus

device loop

device random

device padlock_rng

device rdrand_rng

device ether

device vlan

device tun

device md

device gif

device faith

device firmware

device bpf

device uhci

device ohci

device ehci

device xhci

device usb

device ukbd

device umass

device mmc

device mmcsd

device sdhci

device virtio

device virtio_pci

device vtnet

device virtio_blk

device virtio_scsi

device virtio_balloon

device hyperv

device xenpci

device vmx

device lagg

 

------------------------------------------------------------------------

ddb capture buffer

 

ddb: ddb_capture: kvm_nlist

 

 

Поиски по форуму внятного ответа не дают, может кто то сталкивался и решил проблему?

Если потребуется дополнительная информация из дампа - пишите, спасибо за любую помощь.

Edited by nearia

Share this post


Link to post
Share on other sites

Показывайте, CPU, кол-во CPU, объем памяти, /etc/sysctl.conf и /boot/loader.conf

Share this post


Link to post
Share on other sites

cpu

hw.machine: amd64
hw.model: Intel(R) Xeon(R) CPU E5-2420 0 @ 1.90GHz
hw.ncpu: 6
hw.machine_arch: amd64

 

память

hw.physmem: 8535035904

 

loader.conf

net.link.lagg.0.use_flowid=0

 

sysctl.conf изменений не вносил

Share this post


Link to post
Share on other sites

Ищите рекомендации dadv и настраивайте тюниниг сетевых карт.

Share this post


Link to post
Share on other sites

Топикстартер, а вы конфиг еще с 7.0 за собой таскаете или просто любите устаревшие/нонпродакшен опции? Соберите настоящий GENERIC для начала, без этих чудесных никому не нужных опций, а потом уже будем искать причины.

Share this post


Link to post
Share on other sites

Ищите рекомендации dadv и настраивайте тюниниг сетевых карт.

Спасибо, посмотрю.

Share this post


Link to post
Share on other sites

Топикстартер, а вы конфиг еще с 7.0 за собой таскаете или просто любите устаревшие/нонпродакшен опции? Соберите настоящий GENERIC для начала, без этих чудесных никому не нужных опций, а потом уже будем искать причины.

Может сможете привести пример?

Share this post


Link to post
Share on other sites

Смотрите GENERIC от 10.1 и потом добавляете свои драйвера.

Share this post


Link to post
Share on other sites

Может сможете привести пример?

 

 

поллинг - деприкейтед, не нужен вообще

radix_mpath - вообще не нужен трижды

вписывать руками остальную кучу опций давно нет необходимости, модулями есть все.

 

А если хотите знать более глубже, пожалуйста: сейчас идет активно переделка кода лукапа, и нерабочий до этого mpath более рабочим никто не сделал. Поллинг с момента выхода поддержки MSI-X никто не патчит и никто за ним не следит. Он давно уже не рабочий, блокировки в нем никто не правит, он даже в текущую модель-то не вписывается.

 

В общем собирайте нормальный генерик, а то не ясно вызваны ли трапы одной из устаревших функций или же это железное.

Share this post


Link to post
Share on other sites

Ядро пересобрано, результат пока такой :

kernel panic

 

Fatal trap 18: integer divide fault while in kernel mode

cpuid = 0; apic id = 00

instruction pointer = 0x20:0xffffffff81a4dc76

stack pointer = 0x28:0xfffffe00f63710e0

frame pointer = 0x28:0xfffffe00f6371170

code segment = base 0x0, limit 0xfffff, type 0x1b

= DPL 0, pres 1, long 1, def32 0, gran 1

processor eflags = interrupt enabled, resume, IOPL = 0

current process = 12 (irq264: igb0:que 0)

trap number = 18

panic: integer divide fault

cpuid = 0

KDB: stack backtrace:

#0 0xffffffff8095e880 at kdb_backtrace+0x60

#1 0xffffffff809239a5 at panic+0x155

#2 0xffffffff80cfdbbf at trap_fatal+0x38f

#3 0xffffffff80cfd81c at trap+0x75c

#4 0xffffffff80ce3422 at calltrap+0x8

#5 0xffffffff81a4d45f at ng_bpf_rcvdata+0x8f

#6 0xffffffff81a1b0ca at ng_apply_item+0x21a

#7 0xffffffff81a1ad1a at ng_snd_item+0x38a

#8 0xffffffff81a1b0ca at ng_apply_item+0x21a

#9 0xffffffff81a1ad1a at ng_snd_item+0x38a

#10 0xffffffff81a1b0ca at ng_apply_item+0x21a

#11 0xffffffff81a1ad1a at ng_snd_item+0x38a

#12 0xffffffff81a3c1c8 at ng_ppp_comp_recv+0x148

#13 0xffffffff81a1b0ca at ng_apply_item+0x21a

#14 0xffffffff81a1ad1a at ng_snd_item+0x38a

#15 0xffffffff81a1b0ca at ng_apply_item+0x21a

#16 0xffffffff81a1ad1a at ng_snd_item+0x38a

#17 0xffffffff81a2fa6c at ng_pppoe_rcvdata_ether+0x27c

Uptime: 6d1h20m4s

 

 

 

kernel conf:

 

 

 

options CONFIG_AUTOGENERATED

ident GENERIC

machine amd64

cpu HAMMER

makeoptions WITH_CTF=1

makeoptions DEBUG=-g

options DUMMYNET

options IPDIVERT

options IPFIREWALL_VERBOSE_LIMIT=100

options IPFIREWALL_VERBOSE

options IPFIREWALL

options XENHVM

options USB_DEBUG

options ATH_ENABLE_11N

options AH_AR5416_INTERRUPT_MITIGATION

options AH_SUPPORT_AR5416

options IEEE80211_SUPPORT_MESH

options IEEE80211_AMPDU_AGE

options IEEE80211_DEBUG

options SC_PIXEL_MODE

options VESA

options AHD_REG_PRETTY_PRINT

options AHC_REG_PRETTY_PRINT

options ATA_STATIC_ID

options ACPI_DMAR

options SMP

options KDB_TRACE

options KDB

options INCLUDE_CONFIG_FILE

options DDB_CTF

options KDTRACE_HOOKS

options KDTRACE_FRAME

options MAC

options PROCDESC

options CAPABILITIES

options CAPABILITY_MODE

options AUDIT

options HWPMC_HOOKS

options KBD_INSTALL_CDEV

options PRINTF_BUFR_SIZE=128

options _KPOSIX_PRIORITY_SCHEDULING

options SYSVSEM

options SYSVMSG

options SYSVSHM

options STACK

options KTRACE

options SCSI_DELAY=5000

options COMPAT_FREEBSD7

options COMPAT_FREEBSD6

options COMPAT_FREEBSD5

options COMPAT_FREEBSD4

options COMPAT_FREEBSD32

options GEOM_LABEL

options GEOM_RAID

options GEOM_PART_GPT

options PSEUDOFS

options PROCFS

options CD9660

options MSDOSFS

options NFS_ROOT

options NFSLOCKD

options NFSD

options NFSCL

options MD_ROOT

options QUOTA

options UFS_GJOURNAL

options UFS_DIRHASH

options UFS_ACL

options SOFTUPDATES

options FFS

options SCTP

options TCP_OFFLOAD

options INET

options PREEMPTION

options SCHED_ULE

options NEW_PCIB

options GEOM_PART_MBR

options GEOM_PART_EBR_COMPAT

options GEOM_PART_EBR

options GEOM_PART_BSD

device isa

device mem

device io

device uart_ns8250

device cpufreq

device acpi

device pci

device fdc

device ahci

device ata

device mvs

device siis

device ahc

device ahd

device esp

device hptiop

device isp

device mpt

device mps

device mpr

device sym

device trm

device adv

device adw

device aic

device bt

device isci

device scbus

device ch

device da

device sa

device cd

device pass

device ses

device amr

device arcmsr

device ciss

device dpt

device hptmv

device hptnr

device hptrr

device hpt27xx

device iir

device ips

device mly

device twa

device tws

device aac

device aacp

device aacraid

device ida

device mfi

device mlx

device mrsas

device twe

device atkbdc

device atkbd

device psm

device kbdmux

device vga

device splash

device sc

device vt

device vt_vga

device vt_efifb

device agp

device cbb

device pccard

device cardbus

device uart

device ppc

device ppbus

device lpt

device ppi

device puc

device bxe

device de

device em

device igb

device ixgbe

device ixl

device ixlv

device le

device ti

device txp

device vx

device miibus

device ae

device age

device alc

device ale

device bce

device bfe

device bge

device cas

device dc

device et

device fxp

device gem

device hme

device jme

device lge

device msk

device nfe

device nge

device pcn

device re

device rl

device sf

device sge

device sis

device sk

device ste

device stge

device tl

device tx

device vge

device vr

device wb

device xl

device cs

device ed

device ex

device ep

device fe

device sn

device xe

device wlan

device wlan_wep

device wlan_ccmp

device wlan_tkip

device wlan_amrr

device an

device ath

device ath_pci

device ath_hal

device ath_rate_sample

device ipw

device iwi

device iwn

device malo

device mwl

device ral

device wi

device wpi

device loop

device random

device padlock_rng

device rdrand_rng

device ether

device vlan

device tun

device md

device gif

device faith

device firmware

device bpf

device uhci

device ohci

device ehci

device xhci

device usb

device ukbd

device umass

device sound

device snd_cmi

device snd_csa

device snd_emu10kx

device snd_es137x

device snd_hda

device snd_ich

device snd_via8233

device mmc

device mmcsd

device sdhci

device virtio

device virtio_pci

device vtnet

device virtio_blk

device virtio_scsi

device virtio_balloon

device hyperv

device xenpci

device vmx

device lagg

 

 

Share this post


Link to post
Share on other sites

Явным способом используется ng_bpf? Вероятно надо все же поглядеть что вы делаете в netgraph.

Share this post


Link to post
Share on other sites

ng_bpf - поди где то деление на ноль или ещё что то недозволительное в арифметике.

Share this post


Link to post
Share on other sites

Явным способом используется ng_bpf?

 

Пакетов не сортируем.

Вероятно надо все же поглядеть что вы делаете в netgraph.

# kldstat

Id Refs Address Size Name

1 32 0xffffffff80200000 1728a70 kernel

2 1 0xffffffff81a11000 357f ums.ko

3 1 0xffffffff81a15000 3821 ng_socket.ko

4 12 0xffffffff81a19000 b998 netgraph.ko

5 1 0xffffffff81a25000 2b99 ng_mppc.ko

6 1 0xffffffff81a28000 80d rc4.ko

7 1 0xffffffff81a29000 40a7 ng_ether.ko

8 1 0xffffffff81a2e000 4686 ng_pppoe.ko

9 1 0xffffffff81a33000 17d7 ng_tee.ko

10 1 0xffffffff81a35000 4052 ng_iface.ko

11 1 0xffffffff81a3a000 5817 ng_ppp.ko

12 1 0xffffffff81a40000 5f03 ng_netflow.ko

13 1 0xffffffff81a46000 4aaf ng_ksocket.ko

14 1 0xffffffff81a4b000 18b0 ng_tcpmss.ko

15 1 0xffffffff81a4d000 2952 ng_bpf.ko

16 1 0xffffffff81a50000 22b9 ng_car.ko

 

ng_bpf - поди где то деление на ноль или ещё что то недозволительное в арифметике.

Не подскажете как с этим бороться?

Edited by nearia

Share this post


Link to post
Share on other sites

# kldstat

 

 

ngctl dot | dot -Tpng > dot.netgraph.png

Сделайте картинку, не надо kldstat, он к netgraph отношения не имеет никакого.

Share this post


Link to post
Share on other sites

ngctl dot | dot -Tpng > dot.netgraph.png

После выполнения команды сервер за 3 часа нарисовал файл с прямой линией, в ngctl.dot 240000 строк, есть ещё вывод ngctl list он не подойдёт?

На сервере более 1500 сессий PPPoe, graphviz вообще в состоянии такое нарисовать , или что я мог не правильно сделать?

Share this post


Link to post
Share on other sites

Ну график то красивше :) можно и list, сойдет. Только лучше на пасту куда-нибудь киньте, не рвите форум.

Share this post


Link to post
Share on other sites

С одной стороны я не очень понимаю зачем mpd линкует везде bpf, да еще с кучей хуков. Фильтрация?

С другой же 15 тысяч строк как-то не нормально, вы не находите? Где-то что-то протекло в нетграфе, вероятно.

Share this post


Link to post
Share on other sites

С одной стороны я не очень понимаю зачем mpd линкует везде bpf, да еще с кучей хуков. Фильтрация?

Да, адреса поделены на внешние и локальные, другой какой то выборки нет.

 

С другой же 15 тысяч строк как-то не нормально, вы не находите?

Затрудняюсь ответить на этот вопрос , так как впервые с этим сталкиваюсь и не имею представления сколько будет "нормально" для такого количества соединений.

Share this post


Link to post
Share on other sites

Create an account or sign in to comment

You need to be a member in order to leave a comment

Create an account

Sign up for a new account in our community. It's easy!

Register a new account

Sign in

Already have an account? Sign in here.

Sign In Now
Sign in to follow this