sendfile to nonblocking socket
From: voron
Date: Mon Apr 23 2007 - 17:39:33 EST
Hello
I'm testing a web server nginx for films sharing in my LAN. And I've got
some interesting results. When I tried to download film or another big
file via gigabit link, I've got sendfile block with nonblocking
socket. Strace log in attach. Some commens
#enabling nonblock on fd 3
20:51:20 ioctl(3, FIONBIO, [1]) = 0
#normal nonblocking sendfile, asked 2147480274 bytes, sent 236164
bytes, so nonblocking works
20:51:20 sendfile(3, 8, [847150], 2147480274) = 236164
#sendfile 390 M, 6 seconds
20:51:22 sendfile(3, 8, [102578266], 2147481510) = 390115144
#sendfile 1000 M, 15 seconds
20:51:40 sendfile(3, 8, [1303409692], 2147482596) = 1008100764
#sendfile ~2G, 30 seconds
20:51:55 sendfile(3, 8, [2312288408], 1982678888) = 1982678888
As I see, nonblocking mode is enabled - sendfile sends less than asked.
But 2G via single 30 seconds sendfile call - this is blocking call. How
can I avoid that? I prefer sendfile as fastest way to send file
content to network socket. The problem with sendfile block on
nonblocking socket has place only when I'm using network connection,
that is faster than my hard disk, for example gigabit NIC or localhost.
When I use 100Mbit NIC, which is slower, than my hard disk, I got small
and fast sendfile calls without blocking.
My kernel is Linux 2.6.20.3-grsec x86_64. I verified that also on
2.6.18 - same results. Please advise.
ps: I've did same tests with lighttpd's sendfile and got same results -
block on nonblocking socket, when network is faster than disk.
Thank you,
Alex
strace -tp 10305
Process 10305 attached - interrupt to quit
20:51:17 write(13, "2007/04/23 20:51:17 [info] 10305"..., 85) = 85
20:51:17 epoll_wait(7, {{EPOLLIN, {u32=1054142480, u64=50471214837776}}}, 512, 4294967295) = 1
20:51:20 accept(10, {sa_family=AF_INET, sin_port=htons(55446), sin_addr=inet_addr("192.168.78.1")}, [5056729052170158096]) = 3
20:51:20 ioctl(3, FIONBIO, [1]) = 0
20:51:20 epoll_ctl(7, EPOLL_CTL_ADD, 3, {EPOLLIN|EPOLLET, {u32=1054142784, u64=50471214838080}}) = 0
20:51:20 epoll_wait(7, {{EPOLLIN, {u32=1054142784, u64=50471214838080}}}, 512, 600000) = 1
20:51:20 recvfrom(3, "GET /3.tmp HTTP/1.0\r\nUser-Agent:"..., 1024, 0, NULL, NULL) = 147
20:51:20 open("/var/www/cacti/htdocs/3.tmp", O_RDONLY) = 8
20:51:20 fstat(8, {st_mode=S_IFREG|0644, st_size=4294967296, ...}) = 0
20:51:20 setsockopt(3, SOL_TCP, TCP_CORK, [1], 4) = 0
20:51:20 writev(3, [{"HTTP/1.1 200 OK\r\nServer: nginx/0"..., 262}], 1) = 262
20:51:20 sendfile(3, 8, [0], 2147479552) = 20576
20:51:20 epoll_ctl(7, EPOLL_CTL_MOD, 3, {EPOLLIN|EPOLLOUT|EPOLLET, {u32=1054142784, u64=50471214838080}}) = 0
20:51:20 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 600000) = 1
20:51:20 sendfile(3, 8, [20576], 2147483552) = 55568
20:51:20 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599981) = 1
20:51:20 sendfile(3, 8, [76144], 2147481232) = 27784
20:51:20 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599979) = 1
20:51:20 sendfile(3, 8, [103928], 2147482120) = 90298
20:51:20 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599979) = 1
20:51:20 sendfile(3, 8, [194226], 2147481934) = 652924
20:51:20 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599978) = 1
20:51:20 sendfile(3, 8, [847150], 2147480274) = 236164
20:51:20 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599970) = 1
20:51:21 sendfile(3, 8, [1083314], 2147481678) = 4549630
20:51:21 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599960) = 1
20:51:21 sendfile(3, 8, [5632944], 2147482704) = 38529462
20:51:21 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599894) = 1
20:51:21 sendfile(3, 8, [44162406], 2147480218) = 33451936
20:51:22 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 600000) = 1
20:51:22 sendfile(3, 8, [77614342], 2147480314) = 20393456
20:51:22 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 600000) = 1
20:51:22 sendfile(3, 8, [98007798], 2147480842) = 583464
20:51:22 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 600000) = 1
20:51:22 sendfile(3, 8, [98591262], 2147483106) = 687654
20:51:22 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599959) = 1
20:51:22 sendfile(3, 8, [99278916], 2147483580) = 798790
20:51:22 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599945) = 1
20:51:22 sendfile(3, 8, [100077706], 2147483510) = 2500560
20:51:22 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599931) = 1
20:51:22 sendfile(3, 8, [102578266], 2147481510) = 390115144
20:51:28 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599910) = 1
20:51:28 sendfile(3, 8, [492693410], 2147481694) = 1007170
20:51:28 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 600000) = 1
20:51:28 sendfile(3, 8, [493700580], 2147482140) = 1028008
20:51:28 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599984) = 1
20:51:28 sendfile(3, 8, [494728588], 2147482228) = 804430152
20:51:40 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599971) = 1
20:51:40 sendfile(3, 8, [1299158740], 2147481900) = 1069684
20:51:40 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 600000) = 1
20:51:40 sendfile(3, 8, [1300228424], 2147481272) = 1062738
20:51:40 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599976) = 1
20:51:40 sendfile(3, 8, [1301291162], 2147483494) = 1007170
20:51:40 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599954) = 1
20:51:40 sendfile(3, 8, [1302298332], 2147479844) = 1111360
20:51:40 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599931) = 1
20:51:40 sendfile(3, 8, [1303409692], 2147482596) = 1008100764
20:51:55 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599905) = 1
20:51:55 sendfile(3, 8, [2311510456], 1983456840) = 777952
20:51:55 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 600000) = 1
20:51:55 sendfile(3, 8, [2312288408], 1982678888) = 1982678888
20:52:25 write(16, "192.168.78.1 - voron [23/Apr/200"..., 114) = 114
20:52:25 close(8) = 0
20:52:25 setsockopt(3, SOL_TCP, TCP_CORK, [0], 4) = 0
20:52:25 recvfrom(3, 0x57d880, 1024, 0, 0, 0) = -1 EAGAIN (Resource temporarily unavailable)
20:52:25 epoll_wait(7, {{EPOLLIN|EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 75000) = 1
20:52:25 recvfrom(3, "", 1024, 0, NULL, NULL) = 0
20:52:25 close(3)