git checkout 374b105797c3d4f29c685f3be535c35f5689b30e Previous HEAD position was b1daf37... Merge 'security/for-linus' into devel-xian-x86_64-201312131052 HEAD is now at 374b105... Linux 3.13-rc3 ls -a /kernel-tests/run-queue/kvm/x86_64-nfsroot/linux-devel:devel-xian-x86_64-201312131052:374b105797c3d4f29c685f3be535c35f5689b30e:bisect-mm SUCCESS bisect: good commit 374b105797c3d4f29c685f3be535c35f5689b30e git bisect start b1daf37ff52cf2ff39b2db2399e052ad7965c6aa 374b105797c3d4f29c685f3be535c35f5689b30e -- Previous HEAD position was 374b105... Linux 3.13-rc3 HEAD is now at c2f84f5... Merge 'net/master' into devel-xian-x86_64-201312131052 ls -a /kernel-tests/run-queue/kvm/x86_64-nfsroot/linux-devel:devel-xian-x86_64-201312131052:c2f84f5177293270c0203fabfd600e11e211fd73:bisect-mm 2013-12-13-11:32:33 c2f84f5177293270c0203fabfd600e11e211fd73 compiling 945 real 1653 user 143 sys 190.09% cpu x86_64-nfsroot 2013-12-13-11:56:25 detecting boot state 3.13.0-rc3-01014-gc2f84f5..................... TEST FAILURE [ 18.400635] [ INFO: inconsistent lock state ] [ 18.400635] 3.13.0-rc3-01014-gc2f84f5 #560 Not tainted [ 18.400635] --------------------------------- [ 18.400635] inconsistent {SOFTIRQ-ON-W} -> {IN-SOFTIRQ-W} usage. [ 18.400635] swapper/1/0 [HC0[0]:SC1[1]:HE1:SE0] takes: [ 18.400635] (&(&sk->sk_dst_lock)->rlock){+.?...}, at: [] __udp4_lib_rcv+0x569/0x79d [ 18.400635] {SOFTIRQ-ON-W} state was registered at: [ 18.400635] [] __lock_acquire+0x975/0x1868 [ 18.400635] [] lock_acquire+0xab/0x12e [ 18.400635] [] _raw_spin_lock+0x3b/0x6d [ 18.400635] [] inet_bind+0x18d/0x1dc [ 18.400635] [] kernel_bind+0x10/0x12 [ 18.400635] [] xs_bind+0xa0/0x11e [ 18.400635] [] xs_create_sock.isra.14+0x20b/0x230 [ 18.400635] [] xs_tcp_setup_socket+0x5f/0x3e5 [ 18.400635] [] process_one_work+0x247/0x443 [ 18.400635] [] worker_thread+0x1d2/0x2bc [ 18.400635] [] kthread+0xf9/0x101 [ 18.400635] [] ret_from_fork+0x7c/0xb0 [ 18.400635] irq event stamp: 384478 [ 18.400635] hardirqs last enabled at (384478): [] local_bh_enable+0xab/0xb5 [ 18.400635] hardirqs last disabled at (384477): [] local_bh_enable+0x56/0xb5 [ 18.400635] softirqs last enabled at (384456): [] _local_bh_enable+0x43/0x45 [ 18.400635] softirqs last disabled at (384457): [] irq_exit+0x59/0xa0 [ 18.400635] [ 18.400635] other info that might help us debug this: [ 18.400635] Possible unsafe locking scenario: [ 18.400635] [ 18.400635] CPU0 [ 18.400635] ---- [ 18.400635] lock(&(&sk->sk_dst_lock)->rlock); [ 18.400635] [ 18.400635] lock(&(&sk->sk_dst_lock)->rlock); [ 18.400635] [ 18.400635] *** DEADLOCK *** [ 18.400635] [ 18.400635] 2 locks held by swapper/1/0: [ 18.400635] #0: (rcu_read_lock){.+.+..}, at: [] __netif_receive_skb_core+0x168/0x58e [ 18.400635] #1: (rcu_read_lock){.+.+..}, at: [] ip_local_deliver_finish+0x2f/0x2a0 [ 18.400635] [ 18.400635] stack backtrace: [ 18.400635] CPU: 1 PID: 0 Comm: swapper/1 Not tainted 3.13.0-rc3-01014-gc2f84f5 #560 [ 18.400635] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 [ 18.400635] 0000000000000002 ffff88001ec03990 ffffffff81ae1b26 ffff88001e7c8000 [ 18.400635] ffff88001ec039e0 ffffffff81ad9edd 0000000000000001 ffff880000000001 [ 18.400635] ffff880000000000 ffff88001e7c8778 0000000000000004 0000000000000006 [ 18.400635] Call Trace: [ 18.400635] [] dump_stack+0x4d/0x66 [ 18.400635] [] print_usage_bug+0x2a8/0x2b9 [ 18.400635] [] ? print_shortest_lock_dependencies+0x193/0x193 [ 18.400635] [] mark_lock+0x33c/0x57c [ 18.400635] [] __lock_acquire+0x903/0x1868 [ 18.400635] [] ? run_posix_cpu_timers+0x3e/0x488 [ 18.400635] [] ? trace_hardirqs_off_caller+0x1f/0xb2 [ 18.400635] [] lock_acquire+0xab/0x12e [ 18.400635] [] ? __udp4_lib_rcv+0x569/0x79d [ 18.400635] [] ? trace_hardirqs_off_caller+0x1f/0xb2 [ 18.400635] [] _raw_spin_lock+0x3b/0x6d [ 18.400635] [] ? __udp4_lib_rcv+0x569/0x79d [ 18.400635] [] __udp4_lib_rcv+0x569/0x79d [ 18.400635] [] udp_rcv+0x1a/0x1c [ 18.400635] [] ip_local_deliver_finish+0x151/0x2a0 [ 18.400635] [] ? ip_local_deliver_finish+0x2f/0x2a0 [ 18.400635] [] ip_local_deliver+0x7a/0x7f [ 18.400635] [] ip_rcv_finish+0x39a/0x461 [ 18.400635] [] ip_rcv+0x298/0x2f7 [ 18.400635] [] __netif_receive_skb_core+0x4cd/0x58e [ 18.400635] [] ? __netif_receive_skb_core+0x168/0x58e [ 18.400635] [] __netif_receive_skb+0x1d/0x5f [ 18.400635] [] netif_receive_skb+0x68/0xf4 [ 18.400635] [] napi_gro_receive+0x3e/0x80 [ 18.400635] [] e1000_clean_rx_irq+0x393/0x415 [ 18.400635] [] e1000_clean+0x59d/0x71f [ 18.400635] [] ? kvm_clock_read+0x27/0x31 [ 18.400635] [] ? sched_clock+0x9/0xd [ 18.400635] [] net_rx_action+0xf6/0x210 [ 18.400635] [] __do_softirq+0xf3/0x27a [ 18.400635] [] irq_exit+0x59/0xa0 [ 18.400635] [] do_IRQ+0x9d/0xb4 [ 18.400635] [] common_interrupt+0x72/0x72 [ 18.400635] [] ? __atomic_notifier_call_chain+0x5/0x9c [ 18.400635] [] ? __atomic_notifier_call_chain+0x5/0x9c [ 18.400635] [] ? native_safe_halt+0x6/0x8 [ 18.400635] [] ? trace_hardirqs_on+0xd/0xf [ 18.400635] [] default_idle+0x38/0xc1 [ 18.400635] [] arch_cpu_idle+0x18/0x28 [ 18.400635] [] cpu_startup_entry+0x15c/0x24d [ 18.400635] [] start_secondary+0x247/0x250 [ 42.155389] random: nonblocking pool is initialized [ 58.301213] NFSD: Using /var/lib/nfs/v4recovery as the NFSv4 state recovery directory [ 58.305201] NFSD: starting 90-second grace period (net ffffffff821869c0) /kernel/x86_64-nfsroot/c2f84f5177293270c0203fabfd600e11e211fd73/dmesg-nfsroot-inn-14:20131213120702:x86_64-nfsroot:3.13.0-rc3-01014-gc2f84f5:560 0:1:1 all_good:bad:all_bad boots [1;35mREPEAT COUNT: 20 # /cc/wfg/mm-bisect/.repeat[0m Previous HEAD position was c2f84f5... Merge 'net/master' into devel-xian-x86_64-201312131052 HEAD is now at 2481b0b... Merge 'arm-soc/to-build' into devel-xian-x86_64-201312131052 ls -a /kernel-tests/run-queue/kvm/x86_64-nfsroot/linux-devel:devel-xian-x86_64-201312131052:2481b0b9a5f7af0f748a18a9710371a9cb22c986:bisect-mm 2013-12-13-12:07:17 2481b0b9a5f7af0f748a18a9710371a9cb22c986 compiling 380 real 833 user 69 sys 237.52% cpu x86_64-nfsroot 2013-12-13-12:21:28 detecting boot state 3.13.0-rc3-00870-g2481b0b........... 1 2 3 4. 6 8..... 9. 10... 11... 12... 17 20 SUCCESS Previous HEAD position was 2481b0b... Merge 'arm-soc/to-build' into devel-xian-x86_64-201312131052 HEAD is now at 3c0eee3... Linux 2.6.37 Bisecting: 71 revisions left to test after this (roughly 6 steps) [c8781cf4a309ae4d1393f5878d4e51987665898c] bnx2x: avoid null pointer dereference when enabling SR-IOV running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/mm/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-nfsroot/linux-devel:devel-xian-x86_64-201312131052:c8781cf4a309ae4d1393f5878d4e51987665898c:bisect-mm 2013-12-13-12:42:13 c8781cf4a309ae4d1393f5878d4e51987665898c compiling 744 real 1678 user 142 sys 244.55% cpu x86_64-nfsroot 2013-12-13-13:02:33 detecting boot state 3.13.0-rc2-00500-gc8781cf....... 6 11 13 18 20 SUCCESS Bisecting: 35 revisions left to test after this (roughly 5 steps) [fffc15a5012e9052d3b236efc56840841a125416] vxlan: release rt when found circular route running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/mm/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-nfsroot/linux-devel:devel-xian-x86_64-201312131052:fffc15a5012e9052d3b236efc56840841a125416:bisect-mm 2013-12-13-13:08:37 fffc15a5012e9052d3b236efc56840841a125416 compiling 520 real 575 user 50 sys 120.33% cpu x86_64-nfsroot 2013-12-13-13:27:23 detecting boot state 3.13.0-rc2-00536-gfffc15a...... 2 4 11. 18. 19.... 20 SUCCESS Bisecting: 17 revisions left to test after this (roughly 4 steps) [4bd7b5127bd02c12c1cc837a7a0b6ce295eb2505] micrel: add support for KSZ8041RNLI running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/mm/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-nfsroot/linux-devel:devel-xian-x86_64-201312131052:4bd7b5127bd02c12c1cc837a7a0b6ce295eb2505:bisect-mm 2013-12-13-13:36:30 4bd7b5127bd02c12c1cc837a7a0b6ce295eb2505 compiling 110 real 52 user 9 sys 55.19% cpu x86_64-nfsroot 2013-12-13-13:45:43 detecting boot state 3.13.0-rc2-00554-g4bd7b51...... 5 12 18 20 SUCCESS Bisecting: 8 revisions left to test after this (roughly 3 steps) [9508fdde4d53f0d9e583e841ed08796d2f310be1] Revert "8390 : Replace ei_debug with msg_enable/NETIF_MSG_* feature" running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/mm/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-nfsroot/linux-devel:devel-xian-x86_64-201312131052:9508fdde4d53f0d9e583e841ed08796d2f310be1:bisect-mm 2013-12-13-13:50:50 9508fdde4d53f0d9e583e841ed08796d2f310be1 reuse /kernel/x86_64-nfsroot/9508fdde4d53f0d9e583e841ed08796d2f310be1/vmlinuz-3.13.0-rc2-00254-g9508fdd 2013-12-13-13:50:50 detecting boot state ........ 1 2. 3... 5 6... 7 8. 10.. 11..... 13.. 14... 16... 17 18. 20 SUCCESS Bisecting: 4 revisions left to test after this (roughly 2 steps) [10574059ce0451c6572c85329c772aa15085f8eb] xen-netback: napi: fix abuse of budget running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/mm/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-nfsroot/linux-devel:devel-xian-x86_64-201312131052:10574059ce0451c6572c85329c772aa15085f8eb:bisect-mm 2013-12-13-14:14:34 10574059ce0451c6572c85329c772aa15085f8eb compiling 37 real 47 user 8 sys 147.04% cpu x86_64-nfsroot 2013-12-13-14:19:28 detecting boot state 3.13.0-rc2-00567-g1057405................. TEST FAILURE [ 21.292309] [ INFO: inconsistent lock state ] [ 21.292309] 3.13.0-rc2-00567-g1057405 #565 Not tainted [ 21.292309] --------------------------------- [ 21.292309] inconsistent {SOFTIRQ-ON-W} -> {IN-SOFTIRQ-W} usage. [ 21.292309] swapper/0/0 [HC0[0]:SC1[1]:HE1:SE0] takes: [ 21.292309] (&(&sk->sk_dst_lock)->rlock){+.?...}, at: [] __udp4_lib_rcv+0x569/0x79d [ 21.292309] {SOFTIRQ-ON-W} state was registered at: [ 21.292309] [] __lock_acquire+0x975/0x1868 [ 21.292309] [] lock_acquire+0xab/0x12e [ 21.292309] [] _raw_spin_lock+0x3b/0x6d [ 21.292309] [] inet_bind+0x18d/0x1dc [ 21.292309] [] kernel_bind+0x10/0x12 [ 21.292309] [] xs_bind+0xa0/0x11e [ 21.292309] [] xs_create_sock.isra.14+0x20b/0x230 [ 21.292309] [] xs_tcp_setup_socket+0x5f/0x3e5 [ 21.292309] [] process_one_work+0x247/0x443 [ 21.292309] [] worker_thread+0x1d2/0x2bc [ 21.292309] [] kthread+0xf9/0x101 [ 21.292309] [] ret_from_fork+0x7c/0xb0 [ 21.292309] irq event stamp: 516516 [ 21.292309] hardirqs last enabled at (516516): [] local_bh_enable+0xab/0xb5 [ 21.292309] hardirqs last disabled at (516515): [] local_bh_enable+0x56/0xb5 [ 21.292309] softirqs last enabled at (516494): [] _local_bh_enable+0x43/0x45 [ 21.292309] softirqs last disabled at (516495): [] irq_exit+0x59/0xa0 [ 21.292309] [ 21.292309] other info that might help us debug this: [ 21.292309] Possible unsafe locking scenario: [ 21.292309] [ 21.292309] CPU0 [ 21.292309] ---- [ 21.292309] lock(&(&sk->sk_dst_lock)->rlock); [ 21.292309] [ 21.292309] lock(&(&sk->sk_dst_lock)->rlock); [ 21.292309] [ 21.292309] *** DEADLOCK *** [ 21.292309] [ 21.292309] 2 locks held by swapper/0/0: [ 21.292309] #0: (rcu_read_lock){.+.+..}, at: [] __netif_receive_skb_core+0x168/0x58e [ 21.292309] #1: (rcu_read_lock){.+.+..}, at: [] ip_local_deliver_finish+0x2f/0x2a0 [ 21.292309] [ 21.292309] stack backtrace: [ 21.292309] CPU: 0 PID: 0 Comm: swapper/0 Not tainted 3.13.0-rc2-00567-g1057405 #565 [ 21.292309] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 [ 21.292309] 0000000000000002 ffff88001ea03990 ffffffff81ae1c36 ffffffff82089490 [ 21.292309] ffff88001ea039e0 ffffffff81ad9fed 0000000000000001 ffff880000000001 [ 21.292309] ffff880000000000 ffffffff82089c08 0000000000000004 0000000000000006 [ 21.292309] Call Trace: [ 21.292309] [] dump_stack+0x4d/0x66 [ 21.292309] [] print_usage_bug+0x2a8/0x2b9 [ 21.292309] [] ? print_shortest_lock_dependencies+0x193/0x193 [ 21.292309] [] mark_lock+0x33c/0x57c [ 21.292309] [] __lock_acquire+0x903/0x1868 [ 21.292309] [] ? run_posix_cpu_timers+0x3e/0x488 [ 21.292309] [] ? trace_hardirqs_off_caller+0x1f/0xb2 [ 21.292309] [] lock_acquire+0xab/0x12e [ 21.292309] [] ? __udp4_lib_rcv+0x569/0x79d [ 21.292309] [] ? trace_hardirqs_off_caller+0x1f/0xb2 [ 21.292309] [] _raw_spin_lock+0x3b/0x6d [ 21.292309] [] ? __udp4_lib_rcv+0x569/0x79d [ 21.292309] [] __udp4_lib_rcv+0x569/0x79d [ 21.292309] [] udp_rcv+0x1a/0x1c [ 21.292309] [] ip_local_deliver_finish+0x151/0x2a0 [ 21.292309] [] ? ip_local_deliver_finish+0x2f/0x2a0 [ 21.292309] [] ip_local_deliver+0x7a/0x7f [ 21.292309] [] ip_rcv_finish+0x39a/0x461 [ 21.292309] [] ip_rcv+0x298/0x2f7 [ 21.292309] [] __netif_receive_skb_core+0x4cd/0x58e [ 21.292309] [] ? __netif_receive_skb_core+0x168/0x58e [ 21.292309] [] __netif_receive_skb+0x1d/0x5f [ 21.292309] [] netif_receive_skb+0x68/0xf4 [ 21.292309] [] napi_gro_receive+0x3e/0x80 [ 21.292309] [] e1000_clean_rx_irq+0x393/0x415 [ 21.292309] [] e1000_clean+0x59d/0x71f [ 21.292309] [] ? kvm_clock_read+0x27/0x31 [ 21.292309] [] ? sched_clock+0x9/0xd [ 21.292309] [] net_rx_action+0xf6/0x210 [ 21.292309] [] __do_softirq+0xf3/0x27a [ 21.292309] [] irq_exit+0x59/0xa0 [ 21.292309] [] do_IRQ+0x9d/0xb4 [ 21.292309] [] common_interrupt+0x72/0x72 [ 21.292309] [] ? __atomic_notifier_call_chain+0x5/0x9c [ 21.292309] [] ? native_safe_halt+0x6/0x8 [ 21.292309] [] default_idle+0x38/0xc1 [ 21.292309] [] arch_cpu_idle+0x18/0x28 [ 21.292309] [] cpu_startup_entry+0x15c/0x24d [ 21.292309] [] rest_init+0xc9/0xcd [ 21.292309] [] ? rest_init+0x5/0xcd [ 21.292309] [] start_kernel+0x3d4/0x3e0 [ 21.292309] [] ? repair_env_string+0x58/0x58 [ 21.292309] [] ? early_idt_handlers+0x120/0x120 [ 21.292309] [] x86_64_start_reservations+0x2a/0x2c [ 21.292309] [] x86_64_start_kernel+0x140/0x14d [ 59.580182] rc (1949) used greatest stack depth: 3672 bytes left [ 63.246539] NFSD: Using /var/lib/nfs/v4recovery as the NFSv4 state recovery directory [ 63.250734] NFSD: starting 90-second grace period (net ffffffff821869c0) /kernel/x86_64-nfsroot/10574059ce0451c6572c85329c772aa15085f8eb/dmesg-nfsroot-cairo-18:20131213142751:x86_64-nfsroot:3.13.0-rc2-00567-g1057405:565 0:1:1 all_good:bad:all_bad boots [1;35mREPEAT COUNT: 20 # /cc/wfg/mm-bisect/.repeat[0m Bisecting: 1 revision left to test after this (roughly 1 step) [975022310233fb0f0193873d79a7b8438070fa82] udp: ipv4: must add synchronization in udp_sk_rx_dst_set() running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/mm/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-nfsroot/linux-devel:devel-xian-x86_64-201312131052:975022310233fb0f0193873d79a7b8438070fa82:bisect-mm TEST FAILURE [ 19.888938] [ INFO: inconsistent lock state ] [ 19.888938] 3.13.0-rc2-00256-g9750223 #794 Not tainted [ 19.888938] --------------------------------- [ 19.888938] inconsistent {SOFTIRQ-ON-W} -> {IN-SOFTIRQ-W} usage. [ 19.888938] swapper/0/0 [HC0[0]:SC1[1]:HE1:SE0] takes: [ 19.888938] (&(&sk->sk_dst_lock)->rlock){+.?...}, at: [] __udp4_lib_rcv+0x569/0x79d [ 19.888938] {SOFTIRQ-ON-W} state was registered at: [ 19.888938] [] __lock_acquire+0x96c/0x185f [ 19.888938] [] lock_acquire+0xab/0x12e [ 19.888938] [] _raw_spin_lock+0x3b/0x6d [ 19.888938] [] inet_bind+0x18d/0x1dc [ 19.888938] [] kernel_bind+0x10/0x12 [ 19.888938] [] xs_bind+0xa0/0x11e [ 19.888938] [] xs_create_sock.isra.14+0x20b/0x230 [ 19.888938] [] xs_tcp_setup_socket+0x5f/0x3e5 [ 19.888938] [] process_one_work+0x247/0x443 [ 19.888938] [] worker_thread+0x1d2/0x2bc [ 19.888938] [] kthread+0xf9/0x101 [ 19.888938] [] ret_from_fork+0x7c/0xb0 [ 19.888938] irq event stamp: 546368 [ 19.888938] hardirqs last enabled at (546368): [] local_bh_enable+0xab/0xb5 [ 19.888938] hardirqs last disabled at (546367): [] local_bh_enable+0x56/0xb5 [ 19.888938] softirqs last enabled at (546346): [] _local_bh_enable+0x43/0x45 [ 19.888938] softirqs last disabled at (546347): [] irq_exit+0x59/0xa0 [ 19.888938] [ 19.888938] other info that might help us debug this: [ 19.888938] Possible unsafe locking scenario: [ 19.888938] [ 19.888938] CPU0 [ 19.888938] ---- [ 19.888938] lock(&(&sk->sk_dst_lock)->rlock); [ 19.888938] [ 19.888938] lock(&(&sk->sk_dst_lock)->rlock); [ 19.888938] [ 19.888938] *** DEADLOCK *** [ 19.888938] [ 19.888938] 2 locks held by swapper/0/0: [ 19.888938] #0: (rcu_read_lock){.+.+..}, at: [] __netif_receive_skb_core+0x168/0x58e [ 19.888938] #1: (rcu_read_lock){.+.+..}, at: [] ip_local_deliver_finish+0x2f/0x2a0 [ 19.888938] [ 19.888938] stack backtrace: [ 19.888938] CPU: 0 PID: 0 Comm: swapper/0 Not tainted 3.13.0-rc2-00256-g9750223 #794 [ 19.888938] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 [ 19.888938] 0000000000000002 ffff88001ea039a0 ffffffff81ae4f2d ffffffff82097490 [ 19.888938] ffff88001ea039f0 ffffffff81add317 0000000000000001 ffff880000000001 [ 19.888938] ffff880000000000 ffffffff82097c08 0000000000000004 0000000000000006 [ 19.888938] Call Trace: [ 19.888938] [] dump_stack+0x4d/0x66 [ 19.888938] [] print_usage_bug+0x2a8/0x2b9 [ 19.888938] [] ? print_shortest_lock_dependencies+0x193/0x193 [ 19.888938] [] mark_lock+0x33c/0x57c [ 19.888938] [] __lock_acquire+0x8fa/0x185f [ 19.888938] [] ? sched_clock_local+0x12/0x72 [ 19.888938] [] ? trace_hardirqs_off_caller+0x1f/0xb2 [ 19.888938] [] lock_acquire+0xab/0x12e [ 19.888938] [] ? __udp4_lib_rcv+0x569/0x79d [ 19.888938] [] ? trace_hardirqs_off_caller+0x1f/0xb2 [ 19.888938] [] _raw_spin_lock+0x3b/0x6d [ 19.888938] [] ? __udp4_lib_rcv+0x569/0x79d [ 19.888938] [] __udp4_lib_rcv+0x569/0x79d [ 19.888938] [] udp_rcv+0x1a/0x1c [ 19.888938] [] ip_local_deliver_finish+0x151/0x2a0 [ 19.888938] [] ? ip_local_deliver_finish+0x2f/0x2a0 [ 19.888938] [] ip_local_deliver+0x7a/0x7f [ 19.888938] [] ip_rcv_finish+0x39a/0x461 [ 19.888938] [] ip_rcv+0x283/0x2e0 [ 19.888938] [] __netif_receive_skb_core+0x4cd/0x58e [ 19.888938] [] ? __netif_receive_skb_core+0x168/0x58e [ 19.888938] [] __netif_receive_skb+0x1d/0x5f [ 19.888938] [] netif_receive_skb+0x68/0xf4 [ 19.888938] [] napi_gro_receive+0x3e/0x80 [ 19.888938] [] e1000_clean_rx_irq+0x393/0x415 [ 19.888938] [] e1000_clean+0x59d/0x71f [ 19.888938] [] ? kvm_clock_read+0x27/0x31 [ 19.888938] [] ? sched_clock+0x9/0xd [ 19.888938] [] net_rx_action+0xf6/0x210 [ 19.888938] [] __do_softirq+0xf3/0x27a [ 19.888938] [] irq_exit+0x59/0xa0 [ 19.888938] [] do_IRQ+0x9d/0xb4 [ 19.888938] [] common_interrupt+0x72/0x72 [ 19.888938] [] ? __atomic_notifier_call_chain+0x5/0x9c [ 19.888938] [] ? native_safe_halt+0x6/0x8 [ 19.888938] [] default_idle+0x38/0xc1 [ 19.888938] [] arch_cpu_idle+0x18/0x28 [ 19.888938] [] cpu_startup_entry+0x15c/0x24d [ 19.888938] [] rest_init+0xc9/0xcd [ 19.888938] [] ? rest_init+0x5/0xcd [ 19.888938] [] start_kernel+0x3d4/0x3e0 [ 19.888938] [] ? repair_env_string+0x58/0x58 [ 19.888938] [] ? early_idt_handlers+0x120/0x120 [ 19.888938] [] x86_64_start_reservations+0x2a/0x2c [ 19.888938] [] x86_64_start_kernel+0x140/0x14d [ 76.415449] NFSD: Using /var/lib/nfs/v4recovery as the NFSv4 state recovery directory [ 76.418975] NFSD: starting 90-second grace period (net ffffffff82196a00) Kernel tests: Boot OK! /kernel/x86_64-nfsroot/975022310233fb0f0193873d79a7b8438070fa82/dmesg-nfsroot-cairo-11:20131212200746:x86_64-nfsroot:3.13.0-rc2-00256-g9750223:794 /kernel/x86_64-nfsroot/975022310233fb0f0193873d79a7b8438070fa82/dmesg-nfsroot-cairo-13:20131212200916:x86_64-nfsroot:3.13.0-rc2-00256-g9750223:794 /kernel/x86_64-nfsroot/975022310233fb0f0193873d79a7b8438070fa82/dmesg-nfsroot-inn-7:20131212200917:x86_64-nfsroot:3.13.0-rc2-00256-g9750223:794 /kernel/x86_64-nfsroot/975022310233fb0f0193873d79a7b8438070fa82/dmesg-nfsroot-xian-11:20131212200746:x86_64-nfsroot:3.13.0-rc2-00256-g9750223:794 /kernel/x86_64-nfsroot/975022310233fb0f0193873d79a7b8438070fa82/dmesg-nfsroot-inn-16:20131212201103:x86_64-nfsroot:3.13.0-rc2-00256-g9750223:794 /kernel/x86_64-nfsroot/975022310233fb0f0193873d79a7b8438070fa82/dmesg-nfsroot-cairo-3:20131212200902:x86_64-nfsroot:3.13.0-rc2-00256-g9750223:794 1:6:8 all_good:bad:all_bad boots [1;35mREPEAT COUNT: 25 # /cc/wfg/mm-bisect/.repeat[0m Bisecting: 0 revisions left to test after this (roughly 0 steps) [a1bf1750871a6f242b0fdb174cc55d2c57e7ed66] net:fec: remove duplicate lines in comment about errata ERR006358 running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/mm/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-nfsroot/linux-devel:devel-xian-x86_64-201312131052:a1bf1750871a6f242b0fdb174cc55d2c57e7ed66:bisect-mm 2013-12-13-14:28:25 a1bf1750871a6f242b0fdb174cc55d2c57e7ed66 reuse /kernel/x86_64-nfsroot/a1bf1750871a6f242b0fdb174cc55d2c57e7ed66/vmlinuz-3.13.0-rc2-00255-ga1bf175 2013-12-13-14:28:25 detecting boot state ......... 11 20 25 SUCCESS 975022310233fb0f0193873d79a7b8438070fa82 is the first bad commit commit 975022310233fb0f0193873d79a7b8438070fa82 Author: Eric Dumazet Date: Wed Dec 11 14:46:51 2013 -0800 udp: ipv4: must add synchronization in udp_sk_rx_dst_set() Unlike TCP, UDP input path does not hold the socket lock. Before messing with sk->sk_rx_dst, we must use a spinlock, otherwise multiple cpus could leak a refcount. This patch also takes care of renewing a stale dst entry. (When the sk->sk_rx_dst would not be used by IP early demux) Fixes: 421b3885bf6d ("udp: ipv4: Add udp early demux") Signed-off-by: Eric Dumazet Cc: Shawn Bohrer Signed-off-by: David S. Miller :040000 040000 ecd3b20121c6e4a844611a788e22c5b14dba250d 6ebc1278621b739a01779e23021bc5bd5353d6a3 M net bisect run success ls -a /kernel-tests/run-queue/kvm/x86_64-nfsroot/linux-devel:devel-xian-x86_64-201312131052:a1bf1750871a6f242b0fdb174cc55d2c57e7ed66:bisect-mm 2013-12-13-14:34:31 a1bf1750871a6f242b0fdb174cc55d2c57e7ed66 reuse /kernel/x86_64-nfsroot/a1bf1750871a6f242b0fdb174cc55d2c57e7ed66/vmlinuz-3.13.0-rc2-00255-ga1bf175 2013-12-13-14:34:31 detecting boot state ....... 28 32 36 39 44 49 55 61 65 67. 68. 71 73 75 SUCCESS ls -a /kernel-tests/run-queue/kvm/x86_64-nfsroot/linux-devel:devel-xian-x86_64-201312131052:b1daf37ff52cf2ff39b2db2399e052ad7965c6aa:bisect-mm TEST FAILURE [ 24.126244] [ INFO: inconsistent lock state ] [ 24.126244] 3.13.0-rc3-00689-gb1daf37 #892 Not tainted [ 24.126244] --------------------------------- [ 24.126244] inconsistent {SOFTIRQ-ON-W} -> {IN-SOFTIRQ-W} usage. [ 24.126244] swapper/1/0 [HC0[0]:SC1[1]:HE1:SE0] takes: [ 24.126244] (&(&sk->sk_dst_lock)->rlock){+.?...}, at: [] __udp4_lib_rcv+0x569/0x79d [ 24.126244] {SOFTIRQ-ON-W} state was registered at: [ 24.126244] [] __lock_acquire+0x96c/0x185f [ 24.126244] [] lock_acquire+0xab/0x12e [ 24.126244] [] _raw_spin_lock+0x3b/0x6d [ 24.126244] [] inet_bind+0x18d/0x1dc [ 24.126244] [] kernel_bind+0x10/0x12 [ 24.126244] [] xs_bind+0xa0/0x11e [ 24.126244] [] xs_create_sock.isra.14+0x20b/0x230 [ 24.126244] [] xs_tcp_setup_socket+0x5f/0x3e5 [ 24.126244] [] process_one_work+0x247/0x443 [ 24.126244] [] worker_thread+0x1d2/0x2bc [ 24.126244] [] kthread+0xf9/0x101 [ 24.126244] [] ret_from_fork+0x7c/0xb0 [ 24.126244] irq event stamp: 364856 [ 24.126244] hardirqs last enabled at (364856): [] local_bh_enable+0xab/0xb5 [ 24.126244] hardirqs last disabled at (364855): [] local_bh_enable+0x56/0xb5 [ 24.126244] softirqs last enabled at (364834): [] _local_bh_enable+0x43/0x45 [ 24.126244] softirqs last disabled at (364835): [] irq_exit+0x59/0xa0 [ 24.126244] [ 24.126244] other info that might help us debug this: [ 24.126244] Possible unsafe locking scenario: [ 24.126244] [ 24.126244] CPU0 [ 24.126244] ---- [ 24.126244] lock(&(&sk->sk_dst_lock)->rlock); [ 24.126244] [ 24.126244] lock(&(&sk->sk_dst_lock)->rlock); [ 24.126244] [ 24.126244] *** DEADLOCK *** [ 24.126244] [ 24.126244] 2 locks held by swapper/1/0: [ 24.126244] #0: (rcu_read_lock){.+.+..}, at: [] __netif_receive_skb_core+0x168/0x58e [ 24.126244] #1: (rcu_read_lock){.+.+..}, at: [] ip_local_deliver_finish+0x2f/0x2a0 [ 24.126244] [ 24.126244] stack backtrace: [ 24.126244] CPU: 1 PID: 0 Comm: swapper/1 Not tainted 3.13.0-rc3-00689-gb1daf37 #892 [ 24.126244] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 [ 24.126244] 0000000000000002 ffff88001ec039a0 ffffffff81ae56fd ffff88001e7c8000 [ 24.126244] ffff88001ec039f0 ffffffff81addae7 0000000000000001 ffff880000000001 [ 24.126244] ffff880000000000 ffff88001e7c8778 0000000000000004 0000000000000006 [ 24.126244] Call Trace: [ 24.126244] [] dump_stack+0x4d/0x66 [ 24.126244] [] print_usage_bug+0x2a8/0x2b9 [ 24.126244] [] ? print_shortest_lock_dependencies+0x193/0x193 [ 24.126244] [] mark_lock+0x33c/0x57c [ 24.126244] [] __lock_acquire+0x8fa/0x185f [ 24.126244] [] ? sched_clock_local+0x12/0x72 [ 24.126244] [] ? trace_hardirqs_off_caller+0x1f/0xb2 [ 24.126244] [] lock_acquire+0xab/0x12e [ 24.126244] [] ? __udp4_lib_rcv+0x569/0x79d [ 24.126244] [] ? trace_hardirqs_off_caller+0x1f/0xb2 [ 24.126244] [] _raw_spin_lock+0x3b/0x6d [ 24.126244] [] ? __udp4_lib_rcv+0x569/0x79d [ 24.126244] [] __udp4_lib_rcv+0x569/0x79d [ 24.126244] [] udp_rcv+0x1a/0x1c [ 24.126244] [] ip_local_deliver_finish+0x151/0x2a0 [ 24.126244] [] ? ip_local_deliver_finish+0x2f/0x2a0 [ 24.126244] [] ip_local_deliver+0x7a/0x7f [ 24.126244] [] ip_rcv_finish+0x39a/0x461 [ 24.126244] [] ip_rcv+0x283/0x2e0 [ 24.126244] [] __netif_receive_skb_core+0x4cd/0x58e [ 24.126244] [] ? __netif_receive_skb_core+0x168/0x58e [ 24.126244] [] __netif_receive_skb+0x1d/0x5f [ 24.126244] [] netif_receive_skb+0x68/0xf4 [ 24.126244] [] napi_gro_receive+0x3e/0x80 [ 24.126244] [] e1000_clean_rx_irq+0x393/0x415 [ 24.126244] [] e1000_clean+0x59d/0x71f [ 24.126244] [] ? kvm_clock_read+0x27/0x31 [ 24.126244] [] ? sched_clock+0x9/0xd [ 24.126244] [] net_rx_action+0xf6/0x210 [ 24.126244] [] __do_softirq+0xf3/0x27a [ 24.126244] [] irq_exit+0x59/0xa0 [ 24.126244] [] do_IRQ+0x9d/0xb4 [ 24.126244] [] common_interrupt+0x72/0x72 [ 24.126244] [] ? __atomic_notifier_call_chain+0x5/0x9c [ 24.126244] [] ? __atomic_notifier_call_chain+0x5/0x9c [ 24.126244] [] ? native_safe_halt+0x6/0x8 [ 24.126244] [] ? trace_hardirqs_on+0xd/0xf [ 24.126244] [] default_idle+0x38/0xc1 [ 24.126244] [] arch_cpu_idle+0x18/0x28 [ 24.126244] [] cpu_startup_entry+0x15c/0x24d [ 24.126244] [] start_secondary+0x247/0x250 [ 58.800756] random: nonblocking pool is initialized [ 65.376172] NFSD: Using /var/lib/nfs/v4recovery as the NFSv4 state recovery directory [ 65.380189] NFSD: starting 90-second grace period (net ffffffff82196a80) /kernel/x86_64-nfsroot/b1daf37ff52cf2ff39b2db2399e052ad7965c6aa/dmesg-nfsroot-athens-16:20131213113211:x86_64-nfsroot:3.13.0-rc3-00689-gb1daf37:892 /kernel/x86_64-nfsroot/b1daf37ff52cf2ff39b2db2399e052ad7965c6aa/dmesg-nfsroot-jaketown-27:20131213113203:x86_64-nfsroot:3.13.0-rc3-00689-gb1daf37:892 /kernel/x86_64-nfsroot/b1daf37ff52cf2ff39b2db2399e052ad7965c6aa/dmesg-nfsroot-xian-29:20131213113113:x86_64-nfsroot:3.13.0-rc3-00689-gb1daf37:892 /kernel/x86_64-nfsroot/b1daf37ff52cf2ff39b2db2399e052ad7965c6aa/dmesg-nfsroot-cairo-7:20131213110650:x86_64-nfsroot:3.13.0-rc3-00689-gb1daf37:892 /kernel/x86_64-nfsroot/b1daf37ff52cf2ff39b2db2399e052ad7965c6aa/dmesg-nfsroot-inn-12:20131213110239:x86_64-nfsroot:3.13.0-rc3-00689-gb1daf37:892 /kernel/x86_64-nfsroot/b1daf37ff52cf2ff39b2db2399e052ad7965c6aa/dmesg-nfsroot-athens-31:20131213113131:x86_64-nfsroot:3.13.0-rc3-00689-gb1daf37:892 /kernel/x86_64-nfsroot/b1daf37ff52cf2ff39b2db2399e052ad7965c6aa/dmesg-nfsroot-snb-30:20131213111411:x86_64-nfsroot:3.13.0-rc3-00689-gb1daf37:892 /kernel/x86_64-nfsroot/b1daf37ff52cf2ff39b2db2399e052ad7965c6aa/dmesg-nfsroot-xian-28:20131213113117:x86_64-nfsroot:3.13.0-rc3-00689-gb1daf37:892 /kernel/x86_64-nfsroot/b1daf37ff52cf2ff39b2db2399e052ad7965c6aa/dmesg-nfsroot-athens-5:20131213113120:x86_64-nfsroot:3.13.0-rc3-00689-gb1daf37:892 /kernel/x86_64-nfsroot/b1daf37ff52cf2ff39b2db2399e052ad7965c6aa/dmesg-nfsroot-snb-16:20131213110747:x86_64-nfsroot:3.13.0-rc3-00689-gb1daf37:892 /kernel/x86_64-nfsroot/b1daf37ff52cf2ff39b2db2399e052ad7965c6aa/dmesg-nfsroot-athens-30:20131213113122:x86_64-nfsroot:3.13.0-rc3-00689-gb1daf37:892 /kernel/x86_64-nfsroot/b1daf37ff52cf2ff39b2db2399e052ad7965c6aa/dmesg-nfsroot-xian-24:20131213111134:x86_64-nfsroot:3.13.0-rc3-00689-gb1daf37:892 /kernel/x86_64-nfsroot/b1daf37ff52cf2ff39b2db2399e052ad7965c6aa/dmesg-nfsroot-xian-4:20131213111125:x86_64-nfsroot:3.13.0-rc3-00689-gb1daf37:892 /kernel/x86_64-nfsroot/b1daf37ff52cf2ff39b2db2399e052ad7965c6aa/dmesg-nfsroot-athens-3:20131213113114:x86_64-nfsroot:3.13.0-rc3-00689-gb1daf37:892 /kernel/x86_64-nfsroot/b1daf37ff52cf2ff39b2db2399e052ad7965c6aa/dmesg-nfsroot-jaketown-17:20131213111049:x86_64-nfsroot:3.13.0-rc3-00689-gb1daf37:892 /kernel/x86_64-nfsroot/b1daf37ff52cf2ff39b2db2399e052ad7965c6aa/dmesg-nfsroot-cairo-7:20131213113105:x86_64-nfsroot:3.13.0-rc3-00689-gb1daf37:892 /kernel/x86_64-nfsroot/b1daf37ff52cf2ff39b2db2399e052ad7965c6aa/dmesg-nfsroot-jaketown-11:20131213111351:x86_64-nfsroot:3.13.0-rc3-00689-gb1daf37:892 /kernel/x86_64-nfsroot/b1daf37ff52cf2ff39b2db2399e052ad7965c6aa/dmesg-nfsroot-athens-17:20131213113207:x86_64-nfsroot:3.13.0-rc3-00689-gb1daf37:892 /kernel/x86_64-nfsroot/b1daf37ff52cf2ff39b2db2399e052ad7965c6aa/dmesg-nfsroot-inn-28:20131213110310:x86_64-nfsroot:3.13.0-rc3-00689-gb1daf37:892 0:19:19 all_good:bad:all_bad boots [detached HEAD 2cd6a37] Revert "udp: ipv4: must add synchronization in udp_sk_rx_dst_set()" 1 file changed, 6 insertions(+), 16 deletions(-) ls -a /kernel-tests/run-queue/kvm/x86_64-nfsroot/linux-devel:devel-xian-x86_64-201312131052:2cd6a37a65f578f7e23e67986107422ad3c012ba:bisect-mm 2013-12-13-14:46:26 2cd6a37a65f578f7e23e67986107422ad3c012ba compiling 2013-12-13-15:00:31 detecting boot state 3.13.0-rc3-01326-g2cd6a37........... 2.... 3 4.. 6 7. 10 15 20 22 24. 27 28 29 33 35 37.. 38 40.... 72 75 SUCCESS ========= upstream ========= Fetching linus Fetching linus ls -a /kernel-tests/run-queue/kvm/x86_64-nfsroot/linux-devel:devel-xian-x86_64-201312131052:8ae516aa8b8161254d3e402b3348b2a9b8d1efd0:bisect-mm 2013-12-13-15:23:52 8ae516aa8b8161254d3e402b3348b2a9b8d1efd0 compiling 2013-12-13-15:37:49 detecting boot state 3.13.0-rc1-00141-g8ae516a...... 1 4. 6 11 12 14 16 18. 20.. 25 29 30. 32 34. 39 47 54 59 68 71 74 75 SUCCESS ========= linux-next ========= Fetching next ls -a /kernel-tests/run-queue/kvm/x86_64-nfsroot/linux-devel:devel-xian-x86_64-201312131052:1a63d5777d237952bb06f1a2481c3acc366aeb8d:bisect-mm TEST FAILURE [ 2.447293] tsc: Refined TSC clocksource calibration: 2892.982 MHz [ 2.450321] virtio-pci 0000:00:04.0: irq 40 for MSI/MSI-X [ 2.451649] virtio-pci 0000:00:04.0: irq 41 for MSI/MSI-X [ 2.453048] BUG: key ffff8800192e6ad0 not in .data! [ 2.454223] ------------[ cut here ]------------ /kernel/x86_64-nfsroot/1a63d5777d237952bb06f1a2481c3acc366aeb8d/dmesg-nfsroot-lkp-tt02-16:20131213142945:x86_64-nfsroot:3.13.0-rc3-next-20131213-04499-g1a63d57:3 0:1:9 all_good:bad:all_bad boots [1;35mREPEAT COUNT: 100 # /cc/wfg/mm-bisect/.repeat[0m