NFS: low read/stat performance on small files

From: Vivek Trivedi
Date: Fri Mar 23 2012 - 07:17:18 EST


Hi,
we are facing below 2 performance issue on NFS:

1. Read speed is low for small files
==========================

[ Log on NFS Client]

$ echo 3 > /proc/sys/vm/drop_caches
$ dd if=200KBfile.txt of=/dev/null
400+0 records in
400+0 records out
204800 bytes (200.0KB) copied, 0.027074 seconds, 7.2MB/s


Read speed for 200KB file is 7.2 MB


[ Log on NFS Client]

$ echo 3 > /proc/sys/vm/drop_caches
$ dd if=100MBfile.txt of=/dev/null
204800+0 records in
204800+0 records out
104857600 bytes (100.0MB) copied, 9.351221 seconds, 10.7MB/s

Read speed for 100MB file is 10.7 MB

As you see read speed for 200KB file is only 7.2MB/sec while it is
10.7 MB/sec when we read 100MB file.
Why there is so much difference in read performance ?
Is there any way to achieve high read speed for small files ?


2. Read/stat for a directory tree is slow on NFS than local
==========================================

we have lot of *.jpg files in a directory. If we try to "stat" and
"read" from this directory,
performannce is very slow on NFS Client compared to Local(NFS server)
"stat" and "read"


[ Log on Local (NFS Server) ]

$ echo 3 > /proc/sys/vm/drop_caches
$ ./stat_read_files_test ./lot_of_jpg_files/
Time Taken : 9288 msec


[ Log on NFS Client]

$ echo 3 > /proc/sys/vm/drop_caches
$ ./stat_read_files_test ./lot_of_jpg_files/
Time Taken : 19966 msec

As you see, on NFS client time taken is almost *double* than that of
local(NFS server)
We are using UDP with rsize,wsize=32k on 100MB ethernet link.
I am attaching read/stat testcase.

Is there any way to improve this performance ?


Thanks,
Vivek
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <fcntl.h>
#include <string.h>
#include <sys/time.h>

#define BUFFERSIZE 4096
char READBUFFER[BUFFERSIZE];

#include <dirent.h>
int TraversePath(char *path)
{
struct dirent *d = NULL;
DIR *dir = NULL; /* pointer to directory head*/
char buf[255]={0}; /* buffer to store the complete file/dir name*/
struct stat statbuf; /* to obtain the statistics of file/dir */
int retval =0; /* to hold the return value*/
int fd = 0;

memset(&statbuf,0,sizeof(struct stat));

retval = stat(path,&statbuf);

/* if the stat returned success and path provided is of valid directory*/
if(S_ISDIR(statbuf.st_mode) && (retval==0))
{
dir = opendir(path); /* open the directory*/
/* reads entry one by one*/
while((d = readdir(dir)) != NULL)
{
if((strcmp(d->d_name,".")!=0) && (strcmp(d->d_name,"..")!=0))
{
sprintf(buf,"%s/%s",path,d->d_name);
retval = stat(buf,&statbuf);
if(retval == 0)
{
if(!S_ISDIR(statbuf.st_mode))
{
/* This is file - read from this, Since read ahead
* will itself bring 128KB, so we can just read 4KB
* to start with */
fd = open(buf,O_RDONLY,(mode_t)777);
if(fd) {
read(fd, READBUFFER, BUFFERSIZE);
close(fd);
}
}
else
{
/*
This is a directory, recursive search in it
once all files are read
*/
TraversePath(buf);
}
}
else
{
perror("stat failed\n");
}
}
}
}
else
{
perror("Failed");
}
return retval;
}

int main(int argc, char **argv)
{
struct timeval rv;
struct timeval rv1;

int stat_time = 0;

if(argc < 2) {
printf("./TraversePath <path> \n");
return 0;
}

//Traverse the complete path inside timing unit
gettimeofday(&rv, 0);
TraversePath(argv[1]);
gettimeofday(&rv1, 0);

stat_time = (rv1.tv_sec * 1000 + rv1.tv_usec / 1000) - (rv.tv_sec * 1000 + rv.tv_usec / 1000);

printf(" Traversed Path : %s \n", argv[1]);
printf(" Time Taken : %d msec \n",stat_time);

return 0;
}