git checkout 8712a00514e50aafa7c9bf5cd3955fa60758e53b ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-c6-0312/tip:master:8712a00514e50aafa7c9bf5cd3955fa60758e53b:bisect-net 2014-03-12-14:44:17 8712a00514e50aafa7c9bf5cd3955fa60758e53b compiling Queued build task to /kernel-tests/build-queue/x86_64-randconfig-c6-0312-8712a00514e50aafa7c9bf5cd3955fa60758e53b Check for kernel in /kernel/x86_64-randconfig-c6-0312/8712a00514e50aafa7c9bf5cd3955fa60758e53b waiting for completion of /kernel-tests/build-queue/x86_64-randconfig-c6-0312-8712a00514e50aafa7c9bf5cd3955fa60758e53b waiting for completion of /kernel-tests/build-queue/.x86_64-randconfig-c6-0312-8712a00514e50aafa7c9bf5cd3955fa60758e53b kernel: /kernel/x86_64-randconfig-c6-0312/8712a00514e50aafa7c9bf5cd3955fa60758e53b/vmlinuz-3.14.0-rc6-00017-g8712a00 2014-03-12-15:25:18 detecting boot state . 1 12 20 SUCCESS bisect: good commit 8712a00514e50aafa7c9bf5cd3955fa60758e53b git bisect start 221f19a662f536aa8640694e744019776d2df584 8712a00514e50aafa7c9bf5cd3955fa60758e53b -- /c/kernel-tests/linear-bisect: ["-b", "221f19a662f536aa8640694e744019776d2df584", "-g", "8712a00514e50aafa7c9bf5cd3955fa60758e53b", "/c/kernel-tests/bisect-test-boot-failure.sh", "/home/wfg/net/obj-bisect"] Bisecting: 1832 revisions left to test after this (roughly 11 steps) [8d8ec3963d9d341ca73b817a931f274807ecc5ae] Merge branch 'x86/asmlinkage' running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/net/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-c6-0312/tip:master:8d8ec3963d9d341ca73b817a931f274807ecc5ae:bisect-net 2014-03-12-15:27:23 8d8ec3963d9d341ca73b817a931f274807ecc5ae compiling Queued build task to /kernel-tests/build-queue/x86_64-randconfig-c6-0312-8d8ec3963d9d341ca73b817a931f274807ecc5ae Check for kernel in /kernel/x86_64-randconfig-c6-0312/8d8ec3963d9d341ca73b817a931f274807ecc5ae waiting for completion of /kernel-tests/build-queue/x86_64-randconfig-c6-0312-8d8ec3963d9d341ca73b817a931f274807ecc5ae waiting for completion of /kernel-tests/build-queue/.x86_64-randconfig-c6-0312-8d8ec3963d9d341ca73b817a931f274807ecc5ae kernel: /kernel/x86_64-randconfig-c6-0312/8d8ec3963d9d341ca73b817a931f274807ecc5ae/vmlinuz-3.14.0-rc6-00341-g8d8ec39 2014-03-12-15:54:23 detecting boot state .. TEST FAILURE [ 7.260972] init: Failed to create pty - disabling logging for job [ 7.263060] init: Temporary process spawn error: No such file or directory [ 7.296442] ------------[ cut here ]------------ [ 7.297016] WARNING: CPU: 1 PID: 104 at kernel/locking/mutex-debug.c:82 debug_mutex_unlock+0x211/0x220() [ 7.297016] DEBUG_LOCKS_WARN_ON(lock->owner != current) [ 7.297016] Modules linked in: [ 7.297016] CPU: 1 PID: 104 Comm: sh Not tainted 3.14.0-rc6-00341-g8d8ec39 #7 [ 7.297016] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 [ 7.297016] 0000000000000009 ffff8800071ebb30 ffffffff816e5145 ffff8800071ebb78 [ 7.297016] ffff8800071ebb68 ffffffff8106a8ad ffff88000c83bdd0 0000000000000001 [ 7.297016] 0000000000000246 000000000000054e ffff8800070ff200 ffff8800071ebbc8 [ 7.297016] Call Trace: [ 7.297016] [] dump_stack+0x4d/0x66 [ 7.297016] [] warn_slowpath_common+0x7d/0xa0 [ 7.297016] [] warn_slowpath_fmt+0x4c/0x50 [ 7.297016] [] debug_mutex_unlock+0x211/0x220 [ 7.297016] [] __mutex_unlock_slowpath+0xdb/0x210 [ 7.297016] [] mutex_unlock+0xe/0x10 [ 7.297016] [] unlink_file_vma+0x4a/0x60 [ 7.297016] [] free_pgtables+0x92/0xe0 [ 7.297016] [] exit_mmap+0xa6/0x190 [ 7.297016] [] mmput+0x61/0xe0 [ 7.297016] [] flush_old_exec+0x6f1/0x9a0 [ 7.297016] [] load_elf_binary+0x318/0xe50 [ 7.297016] [] ? do_raw_read_unlock+0x2c/0x50 [ 7.297016] [] search_binary_handler+0x89/0x1a0 [ 7.297016] [] do_execve+0x620/0x7e0 [ 7.297016] [] ? do_execve+0x58f/0x7e0 [ 7.297016] [] SyS_execve+0x25/0x30 [ 7.297016] [] stub_execve+0x69/0xa0 [ 7.297016] ---[ end trace 7817f91b2bed369b ]--- [ 7.453780] init: Failed to create pty - disabling logging for job /kernel/x86_64-randconfig-c6-0312/8d8ec3963d9d341ca73b817a931f274807ecc5ae/dmesg-quantal-lkp-nex04-64:20140312155514:x86_64-randconfig-c6-0312:: /kernel/x86_64-randconfig-c6-0312/8d8ec3963d9d341ca73b817a931f274807ecc5ae/dmesg-quantal-lkp-nex04-16:20140312155518:x86_64-randconfig-c6-0312:: 0:2:2 all_good:bad:all_bad boots 2014-03-12 15:55:25 REPEAT COUNT: 20 # /cc/wfg/net-bisect/.repeat Bisecting: 324 revisions left to test after this (roughly 9 steps) [b4a4183f983cfed78029b7ce9a7b809da742b794] Merge branch 'perf/urgent' running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/net/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-c6-0312/tip:master:b4a4183f983cfed78029b7ce9a7b809da742b794:bisect-net 2014-03-12-15:55:26 b4a4183f983cfed78029b7ce9a7b809da742b794 compiling Queued build task to /kernel-tests/build-queue/x86_64-randconfig-c6-0312-b4a4183f983cfed78029b7ce9a7b809da742b794 Check for kernel in /kernel/x86_64-randconfig-c6-0312/b4a4183f983cfed78029b7ce9a7b809da742b794 waiting for completion of /kernel-tests/build-queue/x86_64-randconfig-c6-0312-b4a4183f983cfed78029b7ce9a7b809da742b794 waiting for completion of /kernel-tests/build-queue/.x86_64-randconfig-c6-0312-b4a4183f983cfed78029b7ce9a7b809da742b794 kernel: /kernel/x86_64-randconfig-c6-0312/b4a4183f983cfed78029b7ce9a7b809da742b794/vmlinuz-3.14.0-rc6-00204-gb4a4183 2014-03-12-16:31:26 detecting boot state .. TEST FAILURE [ 3.510235] debug: unmapping init [mem 0xffffffff81c24000-0xffffffff81d58fff] [ 3.528891] random: init urandom read with 5 bits of entropy available [ 3.545373] ------------[ cut here ]------------ [ 3.546019] WARNING: CPU: 0 PID: 1 at kernel/locking/mutex-debug.c:82 debug_mutex_unlock+0x211/0x220() [ 3.546019] DEBUG_LOCKS_WARN_ON(lock->owner != current) [ 3.546019] Modules linked in: [ 3.546019] CPU: 0 PID: 1 Comm: init Not tainted 3.14.0-rc6-00204-gb4a4183 #10 [ 3.546019] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 [ 3.546019] 0000000000000009 ffff8800120cfcc8 ffffffff816e4925 ffff8800120cfd10 [ 3.546019] ffff8800120cfd00 ffffffff8106a8ad ffff88001083add0 0000000000000001 [ 3.546019] 0000000000000246 000000000000003c ffff88000b0ba000 ffff8800120cfd60 [ 3.546019] Call Trace: [ 3.546019] [] dump_stack+0x4d/0x66 [ 3.546019] [] warn_slowpath_common+0x7d/0xa0 [ 3.546019] [] warn_slowpath_fmt+0x4c/0x50 [ 3.546019] [] debug_mutex_unlock+0x211/0x220 [ 3.546019] [] __mutex_unlock_slowpath+0xdb/0x210 [ 3.546019] [] mutex_unlock+0xe/0x10 [ 3.546019] [] copy_process.part.47+0xcef/0x1cc0 [ 3.546019] [] do_fork+0x100/0x4a0 [ 3.546019] [] ? might_fault+0x5f/0xb0 [ 3.546019] [] ? sysret_check+0x22/0x5d [ 3.546019] [] SyS_clone+0x16/0x20 [ 3.546019] [] stub_clone+0x69/0x90 [ 3.546019] [] ? system_call_fastpath+0x16/0x1b [ 3.546019] ---[ end trace b1d382112acbf185 ]--- [ 3.730518] init: Failed to create pty - disabling logging for job /kernel/x86_64-randconfig-c6-0312/b4a4183f983cfed78029b7ce9a7b809da742b794/dmesg-quantal-ivytown2-12:20140312163214:x86_64-randconfig-c6-0312:3.14.0-rc6-00204-gb4a4183:10 /kernel/x86_64-randconfig-c6-0312/b4a4183f983cfed78029b7ce9a7b809da742b794/dmesg-quantal-ivytown2-18:20140312163208:x86_64-randconfig-c6-0312:3.14.0-rc6-00204-gb4a4183:10 /kernel/x86_64-randconfig-c6-0312/b4a4183f983cfed78029b7ce9a7b809da742b794/dmesg-yocto-ivytown2-10:20140312163212:x86_64-randconfig-c6-0312:3.14.0-rc6-00204-gb4a4183:10 /kernel/x86_64-randconfig-c6-0312/b4a4183f983cfed78029b7ce9a7b809da742b794/dmesg-quantal-ivytown2-29:20140312163216:x86_64-randconfig-c6-0312:3.14.0-rc6-00204-gb4a4183:10 /kernel/x86_64-randconfig-c6-0312/b4a4183f983cfed78029b7ce9a7b809da742b794/dmesg-yocto-ivytown2-19:20140312163217:x86_64-randconfig-c6-0312:3.14.0-rc6-00204-gb4a4183:10 0:5:5 all_good:bad:all_bad boots 2014-03-12 16:32:28 REPEAT COUNT: 20 # /cc/wfg/net-bisect/.repeat Bisecting: 187 revisions left to test after this (roughly 8 steps) [319309fdafea7717522b94e406ea3429fd9498b4] Merge branch 'core/types' running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/net/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-c6-0312/tip:master:319309fdafea7717522b94e406ea3429fd9498b4:bisect-net 2014-03-12-16:32:29 319309fdafea7717522b94e406ea3429fd9498b4 compiling Queued build task to /kernel-tests/build-queue/x86_64-randconfig-c6-0312-319309fdafea7717522b94e406ea3429fd9498b4 Check for kernel in /kernel/x86_64-randconfig-c6-0312/319309fdafea7717522b94e406ea3429fd9498b4 waiting for completion of /kernel-tests/build-queue/x86_64-randconfig-c6-0312-319309fdafea7717522b94e406ea3429fd9498b4 waiting for completion of /kernel-tests/build-queue/.x86_64-randconfig-c6-0312-319309fdafea7717522b94e406ea3429fd9498b4 kernel: /kernel/x86_64-randconfig-c6-0312/319309fdafea7717522b94e406ea3429fd9498b4/vmlinuz-3.14.0-rc6-00123-g319309f 2014-03-12-16:43:29 detecting boot state ... TEST FAILURE [ 2.228796] init: Failed to create pty - disabling logging for job [ 2.229961] init: Temporary process spawn error: No such file or directory [ 2.244211] ------------[ cut here ]------------ [ 2.245015] WARNING: CPU: 0 PID: 103 at kernel/locking/mutex-debug.c:82 debug_mutex_unlock+0x211/0x220() [ 2.245015] DEBUG_LOCKS_WARN_ON(lock->owner != current) [ 2.245015] Modules linked in: [ 2.245015] CPU: 0 PID: 103 Comm: sh Not tainted 3.14.0-rc6-00123-g319309f #11 [ 2.245015] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 [ 2.245015] 0000000000000009 ffff88000b1c5b30 ffffffff816e3757 ffff88000b1c5b78 [ 2.245015] ffff88000b1c5b68 ffffffff810698ad ffff88001083cdd0 0000000000000001 [ 2.245015] 0000000000000246 0000000000000556 ffff88000b16d100 ffff88000b1c5bc8 [ 2.245015] Call Trace: [ 2.245015] [] dump_stack+0x4d/0x66 [ 2.245015] [] warn_slowpath_common+0x7d/0xa0 [ 2.245015] [] warn_slowpath_fmt+0x4c/0x50 [ 2.245015] [] debug_mutex_unlock+0x211/0x220 [ 2.245015] [] __mutex_unlock_slowpath+0xdb/0x210 [ 2.245015] [] mutex_unlock+0xe/0x10 [ 2.245015] [] unlink_file_vma+0x4a/0x60 [ 2.245015] [] free_pgtables+0x92/0xe0 [ 2.245015] [] exit_mmap+0xa6/0x190 [ 2.245015] [] mmput+0x61/0xe0 [ 2.245015] [] flush_old_exec+0x6f1/0x9a0 [ 2.245015] [] load_elf_binary+0x318/0xe50 [ 2.245015] [] ? do_raw_read_unlock+0x2c/0x50 [ 2.245015] [] search_binary_handler+0x89/0x1a0 [ 2.245015] [] do_execve+0x620/0x7e0 [ 2.245015] [] ? do_execve+0x58f/0x7e0 [ 2.245015] [] SyS_execve+0x25/0x30 [ 2.245015] [] stub_execve+0x69/0xa0 [ 2.245015] ---[ end trace 8da6bacc3782e050 ]--- [ 2.282610] init: Failed to create pty - disabling logging for job /kernel/x86_64-randconfig-c6-0312/319309fdafea7717522b94e406ea3429fd9498b4/dmesg-quantal-ivytown2-31:20140312164428:x86_64-randconfig-c6-0312:3.14.0-rc6-00123-g319309f:11 /kernel/x86_64-randconfig-c6-0312/319309fdafea7717522b94e406ea3429fd9498b4/dmesg-quantal-xgwo-4:20140312164447:x86_64-randconfig-c6-0312:3.14.0-rc6-00123-g319309f:11 0:2:2 all_good:bad:all_bad boots 2014-03-12 16:45:01 REPEAT COUNT: 20 # /cc/wfg/net-bisect/.repeat Bisecting: 106 revisions left to test after this (roughly 7 steps) [cbdc4aaf1932bbf23934443126f610b64d53e7ce] manual merge of core/locking running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/net/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-c6-0312/tip:master:cbdc4aaf1932bbf23934443126f610b64d53e7ce:bisect-net 2014-03-12-16:45:02 cbdc4aaf1932bbf23934443126f610b64d53e7ce compiling Queued build task to /kernel-tests/build-queue/x86_64-randconfig-c6-0312-cbdc4aaf1932bbf23934443126f610b64d53e7ce Check for kernel in /kernel/x86_64-randconfig-c6-0312/cbdc4aaf1932bbf23934443126f610b64d53e7ce waiting for completion of /kernel-tests/build-queue/x86_64-randconfig-c6-0312-cbdc4aaf1932bbf23934443126f610b64d53e7ce waiting for completion of /kernel-tests/build-queue/.x86_64-randconfig-c6-0312-cbdc4aaf1932bbf23934443126f610b64d53e7ce kernel: /kernel/x86_64-randconfig-c6-0312/cbdc4aaf1932bbf23934443126f610b64d53e7ce/vmlinuz-3.14.0-rc6-00041-gcbdc4aa 2014-03-12-17:38:02 detecting boot state .. TEST FAILURE [ 2.734615] init: Failed to create pty - disabling logging for job [ 2.736404] init: Temporary process spawn error: No such file or directory [ 2.781281] ------------[ cut here ]------------ [ 2.782025] WARNING: CPU: 1 PID: 121 at kernel/locking/mutex-debug.c:82 debug_mutex_unlock+0x211/0x220() [ 2.782025] DEBUG_LOCKS_WARN_ON(lock->owner != current) [ 2.782025] Modules linked in: [ 2.782025] CPU: 1 PID: 121 Comm: stop Not tainted 3.14.0-rc6-00041-gcbdc4aa #13 [ 2.782025] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 [ 2.782025] 0000000000000009 ffff88000b201c78 ffffffff816e3757 ffff88000b201cc0 [ 2.782025] ffff88000b201cb0 ffffffff810698ad ffff880010839dd0 0000000000000001 [ 2.782025] 0000000000000246 000000000000084c ffff88000b182d00 ffff88000b201d10 [ 2.782025] Call Trace: [ 2.782025] [] dump_stack+0x4d/0x66 [ 2.782025] [] warn_slowpath_common+0x7d/0xa0 [ 2.782025] [] warn_slowpath_fmt+0x4c/0x50 [ 2.782025] [] debug_mutex_unlock+0x211/0x220 [ 2.782025] [] __mutex_unlock_slowpath+0xdb/0x210 [ 2.782025] [] mutex_unlock+0xe/0x10 [ 2.782025] [] unlink_file_vma+0x4a/0x60 [ 2.782025] [] free_pgtables+0x92/0xe0 [ 2.782025] [] exit_mmap+0xa6/0x190 [ 2.782025] [] mmput+0x61/0xe0 [ 2.782025] [] do_exit+0x3df/0xba0 [ 2.782025] [] ? sysret_check+0x22/0x5d [ 2.782025] [] do_group_exit+0x64/0xe0 [ 2.782025] [] SyS_exit_group+0x14/0x20 [ 2.782025] [] system_call_fastpath+0x16/0x1b [ 2.782025] ---[ end trace d8ef062ed8d351d2 ]--- [ 2.843891] init: Failed to create pty - disabling logging for job /kernel/x86_64-randconfig-c6-0312/cbdc4aaf1932bbf23934443126f610b64d53e7ce/dmesg-quantal-ivytown2-9:20140312173833:x86_64-randconfig-c6-0312:3.14.0-rc6-00041-gcbdc4aa:13 0:1:1 all_good:bad:all_bad boots 2014-03-12 17:39:04 REPEAT COUNT: 20 # /cc/wfg/net-bisect/.repeat linear-bisect: handle over to git bisect linear-bisect: git bisect start cbdc4aaf1932bbf23934443126f610b64d53e7ce 8712a00514e50aafa7c9bf5cd3955fa60758e53b -- Previous HEAD position was cbdc4aa... manual merge of core/locking Switched to branch 'master' Your branch is behind 'origin/master' by 41213 commits, and can be fast-forwarded. (use "git pull" to update your local branch) Bisecting: 11 revisions left to test after this (roughly 4 steps) [47be1c1a0e188232b5e5962917b21750053cd3f8] lockdep: Change lockdep_set_novalidate_class() to use _and_name linear-bisect: git bisect run /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/net/obj-bisect running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/net/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-c6-0312/tip:master:47be1c1a0e188232b5e5962917b21750053cd3f8:bisect-net 2014-03-12-17:39:43 47be1c1a0e188232b5e5962917b21750053cd3f8 compiling Queued build task to /kernel-tests/build-queue/x86_64-randconfig-c6-0312-47be1c1a0e188232b5e5962917b21750053cd3f8 Check for kernel in /kernel/x86_64-randconfig-c6-0312/47be1c1a0e188232b5e5962917b21750053cd3f8 waiting for completion of /kernel-tests/build-queue/x86_64-randconfig-c6-0312-47be1c1a0e188232b5e5962917b21750053cd3f8 waiting for completion of /kernel-tests/build-queue/.x86_64-randconfig-c6-0312-47be1c1a0e188232b5e5962917b21750053cd3f8 kernel: /kernel/x86_64-randconfig-c6-0312/47be1c1a0e188232b5e5962917b21750053cd3f8/vmlinuz-3.13.0-02713-g47be1c1 2014-03-12-18:06:43 detecting boot state .. 6 18 20 SUCCESS Bisecting: 5 revisions left to test after this (roughly 3 steps) [c9122da1e2d29bd6a1475a0d1ce2aa6ac6ea25fa] locking: Move mcs_spinlock.h into kernel/locking/ running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/net/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-c6-0312/tip:master:c9122da1e2d29bd6a1475a0d1ce2aa6ac6ea25fa:bisect-net 2014-03-12-18:09:15 c9122da1e2d29bd6a1475a0d1ce2aa6ac6ea25fa compiling Queued build task to /kernel-tests/build-queue/x86_64-randconfig-c6-0312-c9122da1e2d29bd6a1475a0d1ce2aa6ac6ea25fa Check for kernel in /kernel/x86_64-randconfig-c6-0312/c9122da1e2d29bd6a1475a0d1ce2aa6ac6ea25fa waiting for completion of /kernel-tests/build-queue/x86_64-randconfig-c6-0312-c9122da1e2d29bd6a1475a0d1ce2aa6ac6ea25fa waiting for completion of /kernel-tests/build-queue/.x86_64-randconfig-c6-0312-c9122da1e2d29bd6a1475a0d1ce2aa6ac6ea25fa kernel: /kernel/x86_64-randconfig-c6-0312/c9122da1e2d29bd6a1475a0d1ce2aa6ac6ea25fa/vmlinuz-3.13.0-02719-gc9122da 2014-03-12-18:42:15 detecting boot state .. 1 2 7 10 13 20 SUCCESS Bisecting: 2 revisions left to test after this (roughly 2 steps) [1d8fe7dc8078b23e060ec62ccb4cdc1ac3c41bf8] locking/mutexes: Unlock the mutex without the wait_lock running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/net/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-c6-0312/tip:master:1d8fe7dc8078b23e060ec62ccb4cdc1ac3c41bf8:bisect-net 2014-03-12-18:46:16 1d8fe7dc8078b23e060ec62ccb4cdc1ac3c41bf8 compiling Queued build task to /kernel-tests/build-queue/x86_64-randconfig-c6-0312-1d8fe7dc8078b23e060ec62ccb4cdc1ac3c41bf8 Check for kernel in /kernel/x86_64-randconfig-c6-0312/1d8fe7dc8078b23e060ec62ccb4cdc1ac3c41bf8 waiting for completion of /kernel-tests/build-queue/x86_64-randconfig-c6-0312-1d8fe7dc8078b23e060ec62ccb4cdc1ac3c41bf8 waiting for completion of /kernel-tests/build-queue/.x86_64-randconfig-c6-0312-1d8fe7dc8078b23e060ec62ccb4cdc1ac3c41bf8 kernel: /kernel/x86_64-randconfig-c6-0312/1d8fe7dc8078b23e060ec62ccb4cdc1ac3c41bf8/vmlinuz-3.13.0-02722-g1d8fe7d 2014-03-12-19:21:17 detecting boot state .. TEST FAILURE [ 2.677569] debug: unmapping init [mem 0xffffffff81c21000-0xffffffff81d55fff] [ 2.689702] random: init urandom read with 5 bits of entropy available [ 2.716575] ------------[ cut here ]------------ [ 2.717019] WARNING: CPU: 0 PID: 75 at kernel/locking/mutex-debug.c:82 debug_mutex_unlock+0x211/0x220() [ 2.717019] DEBUG_LOCKS_WARN_ON(lock->owner != current) [ 2.717019] Modules linked in: [ 2.717019] CPU: 0 PID: 75 Comm: 99-trinity Not tainted 3.13.0-02722-g1d8fe7d #19 [ 2.717019] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 [ 2.717019] 0000000000000009 ffff88000b119cc8 ffffffff816e3697 ffff88000b119d10 [ 2.717019] ffff88000b119d00 ffffffff81069a6d ffff88001083bdd0 0000000000000001 [ 2.717019] 0000000000000246 000000000000016a ffff88000b0c8000 ffff88000b119d60 [ 2.717019] Call Trace: [ 2.717019] [] dump_stack+0x4d/0x66 [ 2.717019] [] warn_slowpath_common+0x7d/0xa0 [ 2.717019] [] warn_slowpath_fmt+0x4c/0x50 [ 2.717019] [] debug_mutex_unlock+0x211/0x220 [ 2.717019] [] __mutex_unlock_slowpath+0xdb/0x210 [ 2.717019] [] mutex_unlock+0xe/0x10 [ 2.717019] [] copy_process.part.47+0xcef/0x1cc0 [ 2.717019] [] do_fork+0x100/0x4a0 [ 2.717019] [] ? do_raw_spin_unlock+0xc7/0x110 [ 2.717019] [] ? _raw_spin_unlock+0x27/0x40 [ 2.717019] [] ? sysret_check+0x22/0x5d [ 2.717019] [] SyS_clone+0x16/0x20 [ 2.717019] [] stub_clone+0x69/0x90 [ 2.717019] [] ? system_call_fastpath+0x16/0x1b [ 2.717019] ---[ end trace c8f271bdcd115ac5 ]--- [ 2.811382] init: Failed to create pty - disabling logging for job /kernel/x86_64-randconfig-c6-0312/1d8fe7dc8078b23e060ec62ccb4cdc1ac3c41bf8/dmesg-quantal-ivytown2-11:20140312192206:x86_64-randconfig-c6-0312:3.13.0-02722-g1d8fe7d:19 /kernel/x86_64-randconfig-c6-0312/1d8fe7dc8078b23e060ec62ccb4cdc1ac3c41bf8/dmesg-quantal-ivytown2-16:20140312192208:x86_64-randconfig-c6-0312:3.13.0-02722-g1d8fe7d:19 /kernel/x86_64-randconfig-c6-0312/1d8fe7dc8078b23e060ec62ccb4cdc1ac3c41bf8/dmesg-quantal-ivytown2-19:20140312192207:x86_64-randconfig-c6-0312:3.13.0-02722-g1d8fe7d:19 /kernel/x86_64-randconfig-c6-0312/1d8fe7dc8078b23e060ec62ccb4cdc1ac3c41bf8/dmesg-quantal-ivytown2-29:20140312192207:x86_64-randconfig-c6-0312:3.13.0-02722-g1d8fe7d:19 /kernel/x86_64-randconfig-c6-0312/1d8fe7dc8078b23e060ec62ccb4cdc1ac3c41bf8/dmesg-quantal-ivytown2-30:20140312192204:x86_64-randconfig-c6-0312:3.13.0-02722-g1d8fe7d:19 /kernel/x86_64-randconfig-c6-0312/1d8fe7dc8078b23e060ec62ccb4cdc1ac3c41bf8/dmesg-yocto-ivytown2-23:20140312192210:x86_64-randconfig-c6-0312:3.13.0-02722-g1d8fe7d:19 /kernel/x86_64-randconfig-c6-0312/1d8fe7dc8078b23e060ec62ccb4cdc1ac3c41bf8/dmesg-yocto-ivytown2-24:20140312192209:x86_64-randconfig-c6-0312:3.13.0-02722-g1d8fe7d:19 /kernel/x86_64-randconfig-c6-0312/1d8fe7dc8078b23e060ec62ccb4cdc1ac3c41bf8/dmesg-yocto-ivytown2-2:20140312192209:x86_64-randconfig-c6-0312:3.13.0-02722-g1d8fe7d:19 /kernel/x86_64-randconfig-c6-0312/1d8fe7dc8078b23e060ec62ccb4cdc1ac3c41bf8/dmesg-yocto-ivytown2-6:20140312192209:x86_64-randconfig-c6-0312:3.13.0-02722-g1d8fe7d:19 0:9:9 all_good:bad:all_bad boots 2014-03-12 19:22:19 REPEAT COUNT: 20 # /cc/wfg/net-bisect/.repeat Bisecting: 0 revisions left to test after this (roughly 1 step) [47667fa1502e4d759df87e9cc7fbc0f202483361] locking/mutexes: Modify the way optimistic spinners are queued running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/net/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-c6-0312/tip:master:47667fa1502e4d759df87e9cc7fbc0f202483361:bisect-net 2014-03-12-19:22:20 47667fa1502e4d759df87e9cc7fbc0f202483361 compiling Queued build task to /kernel-tests/build-queue/x86_64-randconfig-c6-0312-47667fa1502e4d759df87e9cc7fbc0f202483361 Check for kernel in /kernel/x86_64-randconfig-c6-0312/47667fa1502e4d759df87e9cc7fbc0f202483361 waiting for completion of /kernel-tests/build-queue/x86_64-randconfig-c6-0312-47667fa1502e4d759df87e9cc7fbc0f202483361 waiting for completion of /kernel-tests/build-queue/.x86_64-randconfig-c6-0312-47667fa1502e4d759df87e9cc7fbc0f202483361 kernel: /kernel/x86_64-randconfig-c6-0312/47667fa1502e4d759df87e9cc7fbc0f202483361/vmlinuz-3.13.0-02721-g47667fa 2014-03-12-20:12:20 detecting boot state 2 12 16 18 20 SUCCESS 1d8fe7dc8078b23e060ec62ccb4cdc1ac3c41bf8 is the first bad commit commit 1d8fe7dc8078b23e060ec62ccb4cdc1ac3c41bf8 Author: Jason Low Date: Tue Jan 28 11:13:14 2014 -0800 locking/mutexes: Unlock the mutex without the wait_lock When running workloads that have high contention in mutexes on an 8 socket machine, mutex spinners would often spin for a long time with no lock owner. The main reason why this is occuring is in __mutex_unlock_common_slowpath(), if __mutex_slowpath_needs_to_unlock(), then the owner needs to acquire the mutex->wait_lock before releasing the mutex (setting lock->count to 1). When the wait_lock is contended, this delays the mutex from being released. We should be able to release the mutex without holding the wait_lock. Signed-off-by: Jason Low Cc: chegu_vinod@hp.com Cc: paulmck@linux.vnet.ibm.com Cc: Waiman.Long@hp.com Cc: torvalds@linux-foundation.org Cc: tglx@linutronix.de Cc: riel@redhat.com Cc: akpm@linux-foundation.org Cc: davidlohr@hp.com Cc: hpa@zytor.com Cc: andi@firstfloor.org Cc: aswin@hp.com Cc: scott.norton@hp.com Signed-off-by: Peter Zijlstra Link: http://lkml.kernel.org/r/1390936396-3962-4-git-send-email-jason.low2@hp.com Signed-off-by: Ingo Molnar :040000 040000 6bad053bcd9e8addbbc365b274a9fd758ccdaac6 35657361771160f8624a7ab125b69e64103b129e M kernel bisect run success ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-c6-0312/tip:master:47667fa1502e4d759df87e9cc7fbc0f202483361:bisect-net 2014-03-12-20:14:52 47667fa1502e4d759df87e9cc7fbc0f202483361 reuse /kernel/x86_64-randconfig-c6-0312/47667fa1502e4d759df87e9cc7fbc0f202483361/vmlinuz-3.13.0-02721-g47667fa 2014-03-12-20:14:52 detecting boot state ..... 21.. 22. 24. 25 27 30 34 37 40 46 51... 52 56 57 60 SUCCESS ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-c6-0312/tip:master:221f19a662f536aa8640694e744019776d2df584:bisect-net TEST FAILURE [ 2.756070] dgap: dgap-1.3-16, Digi International Part Number 40002347_C [ 2.756078] dgap: For the tools package or updated drivers please visit http://www.digi.com [ 2.795708] ------------[ cut here ]------------ [ 2.796082] WARNING: CPU: 1 PID: 22 at kernel/locking/mutex-debug.c:82 debug_mutex_unlock+0x211/0x220() [ 2.796082] DEBUG_LOCKS_WARN_ON(lock->owner != current) [ 2.796082] Modules linked in: [ 2.796082] CPU: 1 PID: 22 Comm: kworker/1:1 Not tainted 3.14.0-rc6-01849-g221f19a #2 [ 2.796082] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 [ 2.796082] Workqueue: events_long serio_handle_event [ 2.796082] 0000000000000009 ffff88000d045b70 ffffffff816e5665 ffff88000d045bb8 [ 2.796082] ffff88000d045ba8 ffffffff8106ad6d ffffffff81b2f120 0000000000000001 [ 2.796082] 0000000000000246 0000000000002fa9 0000000000000000 ffff88000d045c08 [ 2.796082] Call Trace: [ 2.796082] [] dump_stack+0x4d/0x66 [ 2.796082] [] warn_slowpath_common+0x7d/0xa0 [ 2.796082] [] warn_slowpath_fmt+0x4c/0x50 [ 2.796082] [] debug_mutex_unlock+0x211/0x220 [ 2.796082] [] __mutex_unlock_slowpath+0xdb/0x210 [ 2.796082] [] mutex_unlock+0xe/0x10 [ 2.796082] [] kernfs_find_and_get_ns+0x4a/0x60 [ 2.796082] [] sysfs_merge_group+0x21/0xd0 [ 2.796082] [] dpm_sysfs_add+0x8f/0xb0 [ 2.796082] [] device_add+0x466/0x5d0 [ 2.796082] [] ? __list_add+0xbf/0x170 [ 2.796082] [] serio_handle_event+0x100/0x200 [ 2.796082] [] process_one_work+0x3eb/0x690 [ 2.796082] [] ? process_one_work+0x329/0x690 [ 2.796082] [] worker_thread+0x36e/0x690 [ 2.796082] [] ? manage_workers.isra.24+0x350/0x350 [ 2.796082] [] kthread+0xed/0x110 [ 2.796082] [] ? _raw_spin_unlock_irq+0x2c/0x40 [ 2.796082] [] ? kthread_create_on_node+0x230/0x230 [ 2.796082] [] ret_from_fork+0x7c/0xb0 [ 2.796082] [] ? kthread_create_on_node+0x230/0x230 [ 2.796082] ---[ end trace f4e010a8ac52dfd5 ]--- [ 2.861602] netem: version 1.3 /kernel/x86_64-randconfig-c6-0312/221f19a662f536aa8640694e744019776d2df584/dmesg-quantal-xgwo-5:20140312144513:x86_64-randconfig-c6-0312:3.14.0-rc6-01849-g221f19a:2 /kernel/x86_64-randconfig-c6-0312/221f19a662f536aa8640694e744019776d2df584/dmesg-quantal-jaketown-35:20140312142835:x86_64-randconfig-c6-0312:: /kernel/x86_64-randconfig-c6-0312/221f19a662f536aa8640694e744019776d2df584/dmesg-yocto-xgwo-3:20140312142827:x86_64-randconfig-c6-0312:3.14.0-rc6-01849-g221f19a:2 /kernel/x86_64-randconfig-c6-0312/221f19a662f536aa8640694e744019776d2df584/dmesg-quantal-xgwo-2:20140312144439:x86_64-randconfig-c6-0312:3.14.0-rc6-01849-g221f19a:2 /kernel/x86_64-randconfig-c6-0312/221f19a662f536aa8640694e744019776d2df584/dmesg-quantal-xgwo-3:20140312144628:x86_64-randconfig-c6-0312:3.14.0-rc6-01849-g221f19a:2 /kernel/x86_64-randconfig-c6-0312/221f19a662f536aa8640694e744019776d2df584/dmesg-yocto-bens-7:20140312144639:x86_64-randconfig-c6-0312:3.14.0-rc6-01849-g221f19a:2 /kernel/x86_64-randconfig-c6-0312/221f19a662f536aa8640694e744019776d2df584/dmesg-quantal-xian-18:20140312144346:x86_64-randconfig-c6-0312:3.14.0-rc6-01849-g221f19a:2 /kernel/x86_64-randconfig-c6-0312/221f19a662f536aa8640694e744019776d2df584/dmesg-quantal-stoakley-4:20140312142713:x86_64-randconfig-c6-0312:3.14.0-rc6-01849-g221f19a:2 /kernel/x86_64-randconfig-c6-0312/221f19a662f536aa8640694e744019776d2df584/dmesg-yocto-xps-3:20140312143056:x86_64-randconfig-c6-0312:3.14.0-rc6-01849-g221f19a:2 /kernel/x86_64-randconfig-c6-0312/221f19a662f536aa8640694e744019776d2df584/dmesg-quantal-xgwo-4:20140312142827:x86_64-randconfig-c6-0312:3.14.0-rc6-01849-g221f19a:2 /kernel/x86_64-randconfig-c6-0312/221f19a662f536aa8640694e744019776d2df584/dmesg-quantal-xian-22:20140312142722:x86_64-randconfig-c6-0312:3.14.0-rc6-01849-g221f19a:2 /kernel/x86_64-randconfig-c6-0312/221f19a662f536aa8640694e744019776d2df584/dmesg-quantal-snb-14:20140312144314:x86_64-randconfig-c6-0312:: /kernel/x86_64-randconfig-c6-0312/221f19a662f536aa8640694e744019776d2df584/dmesg-quantal-bens-5:20140312143118:x86_64-randconfig-c6-0312:3.14.0-rc6-01849-g221f19a:2 /kernel/x86_64-randconfig-c6-0312/221f19a662f536aa8640694e744019776d2df584/dmesg-yocto-jaketown-33:20140312142726:x86_64-randconfig-c6-0312:: /kernel/x86_64-randconfig-c6-0312/221f19a662f536aa8640694e744019776d2df584/dmesg-quantal-bens-5:20140312144556:x86_64-randconfig-c6-0312:3.14.0-rc6-01849-g221f19a:2 /kernel/x86_64-randconfig-c6-0312/221f19a662f536aa8640694e744019776d2df584/dmesg-quantal-athens-29:20140312144300:x86_64-randconfig-c6-0312:3.14.0-rc6-01849-g221f19a:2 /kernel/x86_64-randconfig-c6-0312/221f19a662f536aa8640694e744019776d2df584/dmesg-quantal-snb-27:20140312144250:x86_64-randconfig-c6-0312:: /kernel/x86_64-randconfig-c6-0312/221f19a662f536aa8640694e744019776d2df584/dmesg-yocto-bens-6:20140312142917:x86_64-randconfig-c6-0312:3.14.0-rc6-01849-g221f19a:2 1:18:18 all_good:bad:all_bad boots [detached HEAD 3ef4a3c] Revert "locking/mutexes: Unlock the mutex without the wait_lock" 1 file changed, 4 insertions(+), 4 deletions(-) ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-c6-0312/tip:master:3ef4a3c1eb3e9e6f82b0541a9d8120afe1c800e8:bisect-net 2014-03-12-20:28:31 3ef4a3c1eb3e9e6f82b0541a9d8120afe1c800e8 compiling Queued build task to /kernel-tests/build-queue/x86_64-randconfig-c6-0312-3ef4a3c1eb3e9e6f82b0541a9d8120afe1c800e8 Check for kernel in /kernel/x86_64-randconfig-c6-0312/3ef4a3c1eb3e9e6f82b0541a9d8120afe1c800e8 waiting for completion of /kernel-tests/build-queue/x86_64-randconfig-c6-0312-3ef4a3c1eb3e9e6f82b0541a9d8120afe1c800e8 kernel: /kernel/x86_64-randconfig-c9-0312/89e5b5f8aaaa9ab6ef8aab85589b72c4e6e0bcbf/vmlinuz-3.14.0-rc1-00113-g89e5b5f status: FAIL: no commit 3ef4a3c1eb3e9e6f82b0541a9d8120afe1c800e8 for build task x86_64-randconfig-c6-0312-3ef4a3c1eb3e9e6f82b0541a9d8120afe1c800e8 2014-03-12-21:10:31 3ef4a3c1eb3e9e6f82b0541a9d8120afe1c800e8 SKIP BROKEN BUILD No kernel in /kernel/x86_64-randconfig-c6-0312/3ef4a3c1eb3e9e6f82b0541a9d8120afe1c800e8 ========= upstream ========= Warning: you are leaving 1 commit behind, not connected to any of your branches: 3ef4a3c Revert "locking/mutexes: Unlock the mutex without the wait_lock" If you want to keep them by creating a new branch, this may be a good time to do so with: git branch new_branch_name 3ef4a3c HEAD is now at 33807f4... Merge branch 'for-next' of git://git.samba.org/sfrench/cifs-2.6 ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-c6-0312/tip:master:33807f4f0daec3b00565c2932d95f614f5833adf:bisect-net 2014-03-12-21:10:34 33807f4f0daec3b00565c2932d95f614f5833adf compiling Queued build task to /kernel-tests/build-queue/x86_64-randconfig-c6-0312-33807f4f0daec3b00565c2932d95f614f5833adf Check for kernel in /kernel/x86_64-randconfig-c6-0312/33807f4f0daec3b00565c2932d95f614f5833adf waiting for completion of /kernel-tests/build-queue/x86_64-randconfig-c6-0312-33807f4f0daec3b00565c2932d95f614f5833adf waiting for completion of /kernel-tests/build-queue/.x86_64-randconfig-c6-0312-33807f4f0daec3b00565c2932d95f614f5833adf kernel: /kernel/x86_64-randconfig-c6-0312/33807f4f0daec3b00565c2932d95f614f5833adf/vmlinuz-3.14.0-rc6-00026-g33807f4 2014-03-12-21:35:34 detecting boot state ... 12 20.. 22 27. 35 43 57 59 60 SUCCESS ========= linux-next ========= Fetching next From git://gitmirror/next + 56032fc5...58eac4d akpm-base -> next/akpm-base (forced update) + 1b0a7e3...69d5cdd master -> next/master (forced update) c3bebc7..33807f4 stable -> next/stable * [new tag] next-20140311 -> next-20140311 Previous HEAD position was 33807f4... Merge branch 'for-next' of git://git.samba.org/sfrench/cifs-2.6 HEAD is now at 69d5cdd... Add linux-next specific files for 20140311 ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-c6-0312/tip:master:69d5cdda16b3c9767b82d80b07edb9bb056448e3:bisect-net 2014-03-12-21:43:52 69d5cdda16b3c9767b82d80b07edb9bb056448e3 compiling Queued build task to /kernel-tests/build-queue/x86_64-randconfig-c6-0312-69d5cdda16b3c9767b82d80b07edb9bb056448e3 Check for kernel in /kernel/x86_64-randconfig-c6-0312/69d5cdda16b3c9767b82d80b07edb9bb056448e3 waiting for completion of /kernel-tests/build-queue/x86_64-randconfig-c6-0312-69d5cdda16b3c9767b82d80b07edb9bb056448e3 waiting for completion of /kernel-tests/build-queue/.x86_64-randconfig-c6-0312-69d5cdda16b3c9767b82d80b07edb9bb056448e3 kernel: /kernel/x86_64-randconfig-c6-0312/69d5cdda16b3c9767b82d80b07edb9bb056448e3/vmlinuz-3.14.0-rc6-next-20140311 2014-03-12-22:29:52 detecting boot state . 1. 2 3 5. 7.. 12 17 22 26 34 35 37 43 50 53 58 60 SUCCESS ========= tip/master ========= Fetching tip From git://gitmirror/tip 7ff4247..1a75b8e irq/core -> tip/irq/core 221f19a..6670a8a master -> tip/master 6bedfab..4191c29 perf/core -> tip/perf/core 0708268..6037dd1 sched/core -> tip/sched/core f9a8a0a..09e1517 timers/core -> tip/timers/core Previous HEAD position was 69d5cdd... Add linux-next specific files for 20140311 HEAD is now at 6670a8a... Merge branch 'timers/core' ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-c6-0312/tip:master:6670a8a6f7e711b45114e6c66871d8bbc22a59ed:bisect-net TEST FAILURE [ 4.173359] 1-Wire driver for the DS2760 battery monitor chip - (c) 2004-2005, Szabolcs Gyurko [ 4.177161] wafer5823wdt: WDT driver for Wafer 5823 single board computer initialising [ 4.180405] ------------[ cut here ]------------ [ 4.181070] WARNING: CPU: 0 PID: 1 at kernel/locking/mutex-debug.c:82 debug_mutex_unlock+0x211/0x220() [ 4.181070] DEBUG_LOCKS_WARN_ON(lock->owner != current) [ 4.181070] Modules linked in: [ 4.181070] CPU: 0 PID: 1 Comm: swapper/0 Not tainted 3.14.0-rc6-01875-g6670a8a #24 [ 4.181070] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 [ 4.181070] 0000000000000009 ffff8800120cfbd8 ffffffff816e5726 ffff8800120cfc20 [ 4.181070] ffff8800120cfc10 ffffffff8106ad6d ffffffff81b2f120 0000000000000001 [ 4.181070] 0000000000000246 0000000000002dd2 ffff88001229c500 ffff8800120cfc70 [ 4.181070] Call Trace: [ 4.181070] [] dump_stack+0x4d/0x66 [ 4.181070] [] warn_slowpath_common+0x7d/0xa0 [ 4.181070] [] warn_slowpath_fmt+0x4c/0x50 [ 4.181070] [] debug_mutex_unlock+0x211/0x220 [ 4.181070] [] __mutex_unlock_slowpath+0xdb/0x210 [ 4.181070] [] mutex_unlock+0xe/0x10 [ 4.181070] [] kernfs_find_and_get_ns+0x4a/0x60 [ 4.181070] [] sysfs_merge_group+0x21/0xd0 [ 4.181070] [] dpm_sysfs_add+0x8f/0xb0 [ 4.181070] [] device_add+0x466/0x5d0 [ 4.181070] [] device_create_groups_vargs+0x120/0x140 [ 4.181070] [] device_create+0x41/0x50 [ 4.181070] [] misc_register+0xd2/0x140 [ 4.181070] [] ? fitpc2_wdt_init+0x14e/0x14e [ 4.181070] [] wafwdt_init+0xef/0x166 [ 4.181070] [] do_one_initcall+0x122/0x180 [ 4.181070] [] ? parse_args+0x1f8/0x330 [ 4.181070] [] kernel_init_freeable+0x1ed/0x272 [ 4.181070] [] ? do_early_param+0x88/0x88 [ 4.181070] [] ? rest_init+0xd0/0xd0 [ 4.181070] [] kernel_init+0xe/0xf0 [ 4.181070] [] ret_from_fork+0x7c/0xb0 [ 4.181070] [] ? rest_init+0xd0/0xd0 [ 4.181070] ---[ end trace 2e1725245623f36e ]--- [ 4.255988] wafer5823wdt: initialized. timeout=60 sec (nowayout=0) /kernel/x86_64-randconfig-c6-0312/6670a8a6f7e711b45114e6c66871d8bbc22a59ed/dmesg-quantal-xgwo-7:20140312213519:x86_64-randconfig-c6-0312:3.14.0-rc6-01875-g6670a8a:24 /kernel/x86_64-randconfig-c6-0312/6670a8a6f7e711b45114e6c66871d8bbc22a59ed/dmesg-yocto-bens-1:20140312213459:x86_64-randconfig-c6-0312:3.14.0-rc6-01875-g6670a8a:24 /kernel/x86_64-randconfig-c6-0312/6670a8a6f7e711b45114e6c66871d8bbc22a59ed/dmesg-quantal-xps-1:20140312213526:x86_64-randconfig-c6-0312:3.14.0-rc6-01875-g6670a8a:24 /kernel/x86_64-randconfig-c6-0312/6670a8a6f7e711b45114e6c66871d8bbc22a59ed/dmesg-quantal-xps-5:20140312213525:x86_64-randconfig-c6-0312:3.14.0-rc6-01875-g6670a8a:24 /kernel/x86_64-randconfig-c6-0312/6670a8a6f7e711b45114e6c66871d8bbc22a59ed/dmesg-quantal-ivytown2-11:20140312213511:x86_64-randconfig-c6-0312:3.14.0-rc6-01875-g6670a8a:24 /kernel/x86_64-randconfig-c6-0312/6670a8a6f7e711b45114e6c66871d8bbc22a59ed/dmesg-quantal-bens-5:20140312213533:x86_64-randconfig-c6-0312:3.14.0-rc6-01875-g6670a8a:24 /kernel/x86_64-randconfig-c6-0312/6670a8a6f7e711b45114e6c66871d8bbc22a59ed/dmesg-yocto-ivytown2-3:20140312213510:x86_64-randconfig-c6-0312:3.14.0-rc6-01875-g6670a8a:24 /kernel/x86_64-randconfig-c6-0312/6670a8a6f7e711b45114e6c66871d8bbc22a59ed/dmesg-quantal-xgwo-1:20140312213509:x86_64-randconfig-c6-0312:3.14.0-rc6-01875-g6670a8a:24 /kernel/x86_64-randconfig-c6-0312/6670a8a6f7e711b45114e6c66871d8bbc22a59ed/dmesg-yocto-ivytown2-13:20140312213459:x86_64-randconfig-c6-0312:3.14.0-rc6-01875-g6670a8a:24 0:9:9 all_good:bad:all_bad boots