#include "globals.h" #if (LINUX_20 | LINUX_22 | LINUX_24) extern ULONG insert_io(ULONG disk, ASYNCH_IO *io); extern void RunAsynchIOQueue(ULONG disk); extern void profile_complete(void); #if (LINUX_22 | LINUX_24) kmem_cache_t *bh_p = 0; #endif // // low level disk sector interface routines // #define AIO_SIGNATURE (ULONG)"AIOD" typedef struct _AIO_DESCRIPTOR { struct buffer_head bh; ASYNCH_IO *io; ULONG signature; } AIO_DESCRIPTOR; struct buffer_head *bh_head = 0; struct buffer_head *bh_tail = 0; ULONG bh_count = 0; #if (LINUX_SLEEP) NWFSInitMutex(bh_semaphore); #endif void lock_bh_free(void) { #if (LINUX_SLEEP) if (WaitOnSemaphore(&bh_semaphore) == -EINTR) NWFSPrint("lock bh was interrupted\n"); #endif } void unlock_bh_free(void) { #if (LINUX_SLEEP) SignalSemaphore(&bh_semaphore); #endif } void put_bh(struct buffer_head *bh) { lock_bh_free(); #if (CONSERVE_MEMORY) if (bh_count > MAX_BUFFER_HEADS) { unlock_bh_free(); #if (LINUX_20) kfree(bh); #elif (LINUX_22 | LINUX_24) if (bh_p) kmem_cache_free(bh_p, bh); #else NWFSFree(bh); #endif return; } #endif if (!bh_head) { bh_head = bh_tail = bh; bh->b_next_free = bh->b_prev_free = 0; } else { bh_tail->b_next_free = bh; bh->b_next_free = 0; bh->b_prev_free = bh_tail; bh_tail = bh; } bh_count++; unlock_bh_free(); return; } struct buffer_head *get_bh(void) { struct buffer_head *bh; #if (LINUX_22 | LINUX_24) if (!bh_p) { bh_p = kmem_cache_create("nwfs_buffer_head", sizeof(AIO_DESCRIPTOR), 0, SLAB_HWCACHE_ALIGN, NULL, NULL); if (!bh_p) { NWFSPrint("Cannot create buffer head SLAB cache\n"); return 0; } } #endif lock_bh_free(); if (bh_head) { bh = bh_head; bh_head = bh->b_next_free; if (bh_head) bh_head->b_prev_free = NULL; else bh_tail = NULL; if (bh_count) bh_count--; bh->b_next_free = bh->b_prev_free = 0; unlock_bh_free(); NWFSSet(bh, 0, sizeof(AIO_DESCRIPTOR)); return bh; } else { unlock_bh_free(); #if (LINUX_20) bh = (struct buffer_head *)kmalloc(sizeof(AIO_DESCRIPTOR), GFP_BUFFER); #elif (LINUX_22 | LINUX_24) bh = (struct buffer_head *)kmem_cache_alloc(bh_p, SLAB_BUFFER); #else bh = (struct buffer_head *)NWFSIOAlloc(sizeof(AIO_DESCRIPTOR), BH_TAG); #endif if (!bh) return 0; NWFSSet(bh, 0, sizeof(AIO_DESCRIPTOR)); return bh; } unlock_bh_free(); return 0; } void free_bh_list(void) { struct buffer_head *bh; lock_bh_free(); while (bh_head) { bh = bh_head; bh_head = bh->b_next_free; if (bh_count) bh_count--; #if (LINUX_20) kfree(bh); #elif (LINUX_22 | LINUX_24) if (bh_p) kmem_cache_free(bh_p, bh); #else NWFSFree(bh); #endif } bh_head = bh_tail = 0; unlock_bh_free(); return; } #if (LINUX_BUFFER_CACHE) // // The 2.2.X buffer cache is not SMP thread safe, // so lock the kernel when we enter it to prevent memory corruption // from occurring on SMP systems. // ULONG pReadDiskSectors(ULONG disk, ULONG StartingLBA, BYTE *Sector, ULONG sectors, ULONG readAhead) { register ULONG i, bytesRead = 0; register ULONG bps; register NWDISK *NWDisk; register ULONG read_error = 0; struct buffer_head *bh[sectors]; NWDisk = SystemDisk[disk]; bps = NWDisk->BytesPerSector; #if (VERBOSE) NWFSPrint("read disk-%d lba-%d for %d sectors\n", (int)disk, (int)StartingLBA, (int)sectors); #endif #if (LINUX_22) lock_kernel(); #endif for (i=0; i < sectors; i++) { if (!(bh[i] = getblk((ULONG)NWDisk->PhysicalDiskHandle, (StartingLBA + i), bps))) { for (i=0; i < sectors; i++) if (bh[i]) brelse(bh[i]); #if (LINUX_22) unlock_kernel(); #endif return 0; } } ll_rw_block(READ, sectors, bh); for(i=0; i < sectors; i++) { wait_on_buffer(bh[i]); if (buffer_uptodate(bh[i])) { NWFSCopy(&Sector[i * bps], bh[i]->b_data, bps); bytesRead += bps; } else read_error = 1; brelse(bh[i]); } #if (LINUX_22) unlock_kernel(); #endif #if (PROFILE_AIO) profile_complete(); #endif return (read_error ? 0 : bytesRead); } ULONG pWriteDiskSectors(ULONG disk, ULONG StartingLBA, BYTE *Sector, ULONG sectors, ULONG readAhead) { register ULONG i, bytesWritten = 0; register ULONG bps; struct buffer_head *bh[sectors]; register ULONG write_error = 0; register NWDISK *NWDisk; NWDisk = SystemDisk[disk]; bps = NWDisk->BytesPerSector; #if (VERBOSE) NWFSPrint("write disk-%d lba-%d for %d sectors\n", (int)disk, (int)StartingLBA, (int)sectors); #endif #if (LINUX_22) lock_kernel(); #endif for (i=0; i < sectors; i++) { if (!(bh[i] = getblk((ULONG)NWDisk->PhysicalDiskHandle, (StartingLBA + i), bps))) { for (i=0; i < sectors; i++) if (bh[i]) brelse(bh[i]); #if (LINUX_22) unlock_kernel(); #endif return 0; } NWFSCopy(bh[i]->b_data, &Sector[i * bps], bps); mark_buffer_uptodate(bh[i], 1); mark_buffer_dirty(bh[i], 0); bytesWritten += bps; } ll_rw_block(WRITE, sectors, bh); for (i=0; i < sectors; i++) { wait_on_buffer(bh[i]); if (!buffer_uptodate(bh[i])) write_error = 1; brelse(bh[i]); } #if (LINUX_22) unlock_kernel(); #endif #if (PROFILE_AIO) profile_complete(); #endif return (write_error ? 0 : bytesWritten); } ULONG pZeroFillDiskSectors(ULONG disk, ULONG StartingLBA, ULONG sectors, ULONG readAhead) { register ULONG i, bytesWritten = 0; register ULONG bps; struct buffer_head *bh[sectors]; register ULONG write_error = 0; register NWDISK *NWDisk; NWDisk = SystemDisk[disk]; bps = NWDisk->BytesPerSector; #if (VERBOSE) NWFSPrint("zero disk-%d lba-%d for %d sectors\n", (int)disk, (int)StartingLBA, (int)sectors); #endif #if (LINUX_22) lock_kernel(); #endif for (i=0; i < sectors; i++) { if (!(bh[i] = getblk((ULONG)NWDisk->PhysicalDiskHandle, (StartingLBA + i), bps))) { for (i=0; i < sectors; i++) if (bh[i]) brelse(bh[i]); #if (LINUX_22) unlock_kernel(); #endif return 0; } NWFSSet(bh[i]->b_data, 0, bps); mark_buffer_uptodate(bh[i], 1); mark_buffer_dirty(bh[i], 0); bytesWritten += bps; } ll_rw_block(WRITE, sectors, bh); for (i=0; i < sectors; i++) { wait_on_buffer(bh[i]); if (!buffer_uptodate(bh[i])) write_error = 1; brelse(bh[i]); } #if (LINUX_22) unlock_kernel(); #endif #if (PROFILE_AIO) profile_complete(); #endif return (write_error ? 0 : bytesWritten); } #else // // this case assumes we will use the NWFS buffer cache and not allow // cache sharing with the linux buffer cache. // // // Linux 2.0 // #if (LINUX_20) ULONG pReadDiskSectors(ULONG disk, ULONG StartingLBA, BYTE *Sector, ULONG sectors, ULONG readAhead) { register ULONG i, j, bytesRead = 0; register ULONG bps; register NWDISK *NWDisk; register ULONG read_error = 0; struct buffer_head *bh[sectors]; NWDisk = SystemDisk[disk]; bps = NWDisk->BytesPerSector; for (i=0; i < sectors; i++) { bh[i] = get_bh(); if (!bh[i]) { for (j=0; j < i; j++) if (bh[j]) put_bh(bh[j]); return 0; } } for (i=0; i < sectors; i++) { bh[i]->b_this_page = bh[(i + 1) % sectors]; // create circular list bh[i]->b_state = 0; bh[i]->b_next_free = (struct buffer_head *)NULL; bh[i]->b_size = 512; bh[i]->b_data = (char *)&Sector[i * bps]; bh[i]->b_list = BUF_CLEAN; bh[i]->b_dev = (int)NWDisk->PhysicalDiskHandle; bh[i]->b_blocknr = (StartingLBA + i); bh[i]->b_count = 1; bh[i]->b_flushtime = 0; clear_bit(BH_Uptodate, &bh[i]->b_state); } ll_rw_block(READ, sectors, bh); for(i=0; i < sectors; i++) { wait_on_buffer(bh[i]); if (buffer_uptodate(bh[i])) bytesRead += bps; else read_error = 1; put_bh(bh[i]); } #if (PROFILE_AIO) profile_complete(); #endif return (read_error ? 0 : bytesRead); } ULONG pWriteDiskSectors(ULONG disk, ULONG StartingLBA, BYTE *Sector, ULONG sectors, ULONG readAhead) { register ULONG i, j, bytesWritten = 0; register ULONG bps; register ULONG write_error = 0; register NWDISK *NWDisk; struct buffer_head *bh[sectors]; NWDisk = SystemDisk[disk]; bps = NWDisk->BytesPerSector; for (i=0; i < sectors; i++) { bh[i] = get_bh(); if (!bh[i]) { for (j=0; j < i; j++) if (bh[j]) put_bh(bh[j]); return 0; } } for (i=0; i < sectors; i++) { bh[i]->b_this_page = bh[(i + 1) % sectors]; // create circular list bh[i]->b_state = 0; bh[i]->b_next_free = (struct buffer_head *) NULL; bh[i]->b_size = 512; bh[i]->b_data = (char *)&Sector[i * bps]; bh[i]->b_list = BUF_CLEAN; bh[i]->b_dev = (int)NWDisk->PhysicalDiskHandle; bh[i]->b_blocknr = (StartingLBA + i); bh[i]->b_count = 1; bh[i]->b_flushtime = 0; set_bit(BH_Uptodate, &bh[i]->b_state); set_bit(BH_Dirty, &bh[i]->b_state); } ll_rw_block(WRITE, sectors, bh); for (i=0; i < sectors; i++) { wait_on_buffer(bh[i]); if (buffer_uptodate(bh[i])) bytesWritten += bps; else write_error = 1; put_bh(bh[i]); } #if (PROFILE_AIO) profile_complete(); #endif return (write_error ? 0 : bytesWritten); } ULONG pZeroFillDiskSectors(ULONG disk, ULONG StartingLBA, ULONG sectors, ULONG readAhead) { register ULONG i, j, bytesWritten = 0; register ULONG bps; register ULONG write_error = 0; register NWDISK *NWDisk; struct buffer_head *bh[sectors]; NWDisk = SystemDisk[disk]; bps = NWDisk->BytesPerSector; for (i=0; i < sectors; i++) { bh[i] = get_bh(); if (!bh[i]) { for (j=0; j < i; j++) if (bh[j]) put_bh(bh[j]); return 0; } } for (i=0; i < sectors; i++) { bh[i]->b_this_page = bh[(i + 1) % sectors]; // create circular list bh[i]->b_state = 0; bh[i]->b_next_free = (struct buffer_head *) NULL; bh[i]->b_size = 512; bh[i]->b_data = (char *) ZeroBuffer; bh[i]->b_list = BUF_CLEAN; bh[i]->b_dev = (int)NWDisk->PhysicalDiskHandle; bh[i]->b_blocknr = (StartingLBA + i); bh[i]->b_count = 1; bh[i]->b_flushtime = 0; set_bit(BH_Uptodate, &bh[i]->b_state); set_bit(BH_Dirty, &bh[i]->b_state); } ll_rw_block(WRITE, sectors, bh); for (i=0; i < sectors; i++) { wait_on_buffer(bh[i]); if (buffer_uptodate(bh[i])) bytesWritten += bps; else write_error = 1; put_bh(bh[i]); } #if (PROFILE_AIO) profile_complete(); #endif return (write_error ? 0 : bytesWritten); } #endif // // Linux 2.2 // #if (LINUX_22) void end_io(struct buffer_head *bh, int uptodate) { mark_buffer_uptodate(bh, uptodate); unlock_buffer(bh); return; } ULONG pReadDiskSectors(ULONG disk, ULONG StartingLBA, BYTE *Sector, ULONG sectors, ULONG readAhead) { register ULONG i, j, bytesRead = 0; register ULONG bps; register NWDISK *NWDisk; register ULONG read_error = 0; struct buffer_head *bh[sectors]; NWDisk = SystemDisk[disk]; bps = NWDisk->BytesPerSector; for (i=0; i < sectors; i++) { bh[i] = get_bh(); if (!bh[i]) { for (j=0; j < i; j++) if (bh[j]) put_bh(bh[j]); return 0; } } for (i=0; i < sectors; i++) { bh[i]->b_this_page = bh[(i + 1) % sectors]; // create circular list bh[i]->b_state = 0; bh[i]->b_next_free = (struct buffer_head *)NULL; bh[i]->b_size = 512; bh[i]->b_data = (char *)&Sector[i * bps]; bh[i]->b_list = BUF_CLEAN; bh[i]->b_dev = (int)NWDisk->PhysicalDiskHandle; bh[i]->b_blocknr = (StartingLBA + i); bh[i]->b_count = 1; bh[i]->b_flushtime = 0; bh[i]->b_end_io = end_io; clear_bit(BH_Uptodate, &bh[i]->b_state); } lock_kernel(); ll_rw_block(READ, sectors, bh); for(i=0; i < sectors; i++) { wait_on_buffer(bh[i]); if (buffer_uptodate(bh[i])) bytesRead += bps; else read_error = 1; put_bh(bh[i]); } unlock_kernel(); #if (PROFILE_AIO) profile_complete(); #endif return (read_error ? 0 : bytesRead); } ULONG pWriteDiskSectors(ULONG disk, ULONG StartingLBA, BYTE *Sector, ULONG sectors, ULONG readAhead) { register ULONG i, j, bytesWritten = 0; register ULONG bps; register ULONG write_error = 0; register NWDISK *NWDisk; struct buffer_head *bh[sectors]; NWDisk = SystemDisk[disk]; bps = NWDisk->BytesPerSector; for (i=0; i < sectors; i++) { bh[i] = get_bh(); if (!bh[i]) { for (j=0; j < i; j++) if (bh[j]) put_bh(bh[j]); return 0; } } for (i=0; i < sectors; i++) { bh[i]->b_this_page = bh[(i + 1) % sectors]; // create circular list bh[i]->b_state = 0; bh[i]->b_next_free = (struct buffer_head *) NULL; bh[i]->b_size = 512; bh[i]->b_data = (char *)&Sector[i * bps]; bh[i]->b_list = BUF_CLEAN; bh[i]->b_dev = (int)NWDisk->PhysicalDiskHandle; bh[i]->b_blocknr = (StartingLBA + i); bh[i]->b_count = 1; bh[i]->b_flushtime = 0; bh[i]->b_end_io = end_io; set_bit(BH_Uptodate, &bh[i]->b_state); set_bit(BH_Dirty, &bh[i]->b_state); } lock_kernel(); ll_rw_block(WRITE, sectors, bh); for (i=0; i < sectors; i++) { wait_on_buffer(bh[i]); if (buffer_uptodate(bh[i])) bytesWritten += bps; else write_error = 1; put_bh(bh[i]); } unlock_kernel(); #if (PROFILE_AIO) profile_complete(); #endif return (write_error ? 0 : bytesWritten); } ULONG pZeroFillDiskSectors(ULONG disk, ULONG StartingLBA, ULONG sectors, ULONG readAhead) { register ULONG i, j, bytesWritten = 0; register ULONG bps; register ULONG write_error = 0; register NWDISK *NWDisk; struct buffer_head *bh[sectors]; NWDisk = SystemDisk[disk]; bps = NWDisk->BytesPerSector; for (i=0; i < sectors; i++) { bh[i] = get_bh(); if (!bh[i]) { for (j=0; j < i; j++) if (bh[j]) put_bh(bh[j]); return 0; } } for (i=0; i < sectors; i++) { bh[i]->b_this_page = bh[(i + 1) % sectors]; // create circular list bh[i]->b_state = 0; bh[i]->b_next_free = (struct buffer_head *) NULL; bh[i]->b_size = 512; bh[i]->b_data = (char *) ZeroBuffer; bh[i]->b_list = BUF_CLEAN; bh[i]->b_dev = (int)NWDisk->PhysicalDiskHandle; bh[i]->b_blocknr = (StartingLBA + i); bh[i]->b_count = 1; bh[i]->b_flushtime = 0; bh[i]->b_end_io = end_io; set_bit(BH_Uptodate, &bh[i]->b_state); set_bit(BH_Dirty, &bh[i]->b_state); } lock_kernel(); ll_rw_block(WRITE, sectors, bh); for (i=0; i < sectors; i++) { wait_on_buffer(bh[i]); if (buffer_uptodate(bh[i])) bytesWritten += bps; else write_error = 1; put_bh(bh[i]); } unlock_kernel(); #if (PROFILE_AIO) profile_complete(); #endif return (write_error ? 0 : bytesWritten); } // // // void end_asynch_io(struct buffer_head *bh, int uptodate) { ULONG flags; register int i; AIO_DESCRIPTOR *aio = (AIO_DESCRIPTOR *)bh; ASYNCH_IO *io; mark_buffer_uptodate(bh, uptodate); unlock_buffer(bh); if (!aio->io) { NWFSPrint("nwfs: aio callback has NULL handle\n"); return; } io = aio->io; if (!test_bit(BH_Uptodate, &bh->b_state)) io->ccode = ASIO_IO_ERROR; io->complete++; if (io->complete >= io->count) { save_flags(flags); cli(); for (i=0; i < io->count; i++) put_bh(io->bh[i]); restore_flags(flags); if (io->call_back_routine) (io->call_back_routine)(io); io->signature = 0; #if (PROFILE_AIO) profile_complete(); #endif } return; } ULONG aReadDiskSectors(ULONG disk, ULONG StartingLBA, BYTE *Sector, ULONG sectors, ULONG readAhead, ASYNCH_IO *io) { register ULONG i, j, bytesRead = 0; register ULONG bps; register NWDISK *NWDisk; AIO_DESCRIPTOR *aio; if (sectors > 8) return 0; NWDisk = SystemDisk[disk]; bps = NWDisk->BytesPerSector; for (i=0; i < sectors; i++) { io->bh[i] = get_bh(); if (!io->bh[i]) { for (j=0; j < i; j++) if (io->bh[j]) put_bh(io->bh[j]); return 0; } } io->ccode = 0; io->count = sectors; io->complete = 0; for (i=0; i < sectors; i++) { aio = (AIO_DESCRIPTOR *)io->bh[i]; aio->io = io; aio->signature = AIO_SIGNATURE; io->bh[i]->b_this_page = io->bh[(i + 1) % sectors]; // create circular list io->bh[i]->b_state = 0; io->bh[i]->b_next_free = (struct buffer_head *)NULL; io->bh[i]->b_size = 512; io->bh[i]->b_data = (char *)&Sector[i * bps]; io->bh[i]->b_list = BUF_CLEAN; io->bh[i]->b_dev = (int)NWDisk->PhysicalDiskHandle; io->bh[i]->b_blocknr = (StartingLBA + i); io->bh[i]->b_count = 1; io->bh[i]->b_flushtime = 0; io->bh[i]->b_end_io = end_asynch_io; clear_bit(BH_Uptodate, &io->bh[i]->b_state); } lock_kernel(); ll_rw_block(READ, sectors, &io->bh[0]); run_task_queue(&tq_disk); unlock_kernel(); #if (DEBUG_AIO) NWFSPrint("io asynch read disk-%d lba-%d\n", (int)disk, (int)StartingLBA); #endif bytesRead = (sectors * bps); return (bytesRead); } ULONG aWriteDiskSectors(ULONG disk, ULONG StartingLBA, BYTE *Sector, ULONG sectors, ULONG readAhead, ASYNCH_IO *io) { register ULONG i, j, bytesWritten = 0; register ULONG bps; register NWDISK *NWDisk; AIO_DESCRIPTOR *aio; if (sectors > 8) return 0; NWDisk = SystemDisk[disk]; bps = NWDisk->BytesPerSector; for (i=0; i < sectors; i++) { io->bh[i] = get_bh(); if (!io->bh[i]) { for (j=0; j < i; j++) if (io->bh[j]) put_bh(io->bh[j]); return 0; } } io->ccode = 0; io->count = sectors; io->complete = 0; for (i=0; i < sectors; i++) { aio = (AIO_DESCRIPTOR *)io->bh[i]; aio->io = io; aio->signature = AIO_SIGNATURE; io->bh[i]->b_this_page = io->bh[(i + 1) % sectors]; // create circular list io->bh[i]->b_state = 0; io->bh[i]->b_next_free = (struct buffer_head *) NULL; io->bh[i]->b_size = 512; io->bh[i]->b_data = (char *)&Sector[i * bps]; io->bh[i]->b_list = BUF_CLEAN; io->bh[i]->b_dev = (int)NWDisk->PhysicalDiskHandle; io->bh[i]->b_blocknr = (StartingLBA + i); io->bh[i]->b_count = 1; io->bh[i]->b_flushtime = 0; io->bh[i]->b_end_io = end_asynch_io; set_bit(BH_Uptodate, &io->bh[i]->b_state); set_bit(BH_Dirty, &io->bh[i]->b_state); } lock_kernel(); ll_rw_block(WRITE, sectors, &io->bh[0]); run_task_queue(&tq_disk); unlock_kernel(); #if (DEBUG_AIO) NWFSPrint("io asynch write disk-%d lba-%d\n", (int)disk, (int)StartingLBA); #endif bytesWritten = (sectors * bps); return (bytesWritten); } ULONG aZeroFillDiskSectors(ULONG disk, ULONG StartingLBA, ULONG sectors, ULONG readAhead, ASYNCH_IO *io) { register ULONG i, j, bytesWritten = 0; register ULONG bps; register NWDISK *NWDisk; AIO_DESCRIPTOR *aio; if (sectors > 8) return 0; NWDisk = SystemDisk[disk]; bps = NWDisk->BytesPerSector; for (i=0; i < sectors; i++) { io->bh[i] = get_bh(); if (!io->bh[i]) { for (j=0; j < i; j++) if (io->bh[j]) put_bh(io->bh[j]); return 0; } } io->ccode = 0; io->count = sectors; io->complete = 0; for (i=0; i < sectors; i++) { aio = (AIO_DESCRIPTOR *)io->bh[i]; aio->io = io; aio->signature = AIO_SIGNATURE; io->bh[i]->b_this_page = io->bh[(i + 1) % sectors]; // create circular list io->bh[i]->b_state = 0; io->bh[i]->b_next_free = (struct buffer_head *) NULL; io->bh[i]->b_size = 512; io->bh[i]->b_data = (char *) ZeroBuffer; io->bh[i]->b_list = BUF_CLEAN; io->bh[i]->b_dev = (int)NWDisk->PhysicalDiskHandle; io->bh[i]->b_blocknr = (StartingLBA + i); io->bh[i]->b_count = 1; io->bh[i]->b_flushtime = 0; io->bh[i]->b_end_io = end_asynch_io; set_bit(BH_Uptodate, &io->bh[i]->b_state); set_bit(BH_Dirty, &io->bh[i]->b_state); } lock_kernel(); ll_rw_block(WRITE, sectors, &io->bh[0]); run_task_queue(&tq_disk); unlock_kernel(); #if (DEBUG_AIO) NWFSPrint("io asynch fill disk-%d lba-%d\n", (int)disk, (int)StartingLBA); #endif bytesWritten = (sectors * bps); return (bytesWritten); } #endif