| Message ID | 20250304011036.1373650-1-jaegeuk@kernel.org (mailing list archive) |
|---|---|
| State | New |
| Headers | show |
| Series | [f2fs-dev] f2fs_io: add fragread command to evaluate fragmented buffer for reads | expand |
On 3/4/25 09:10, Jaegeuk Kim via Linux-f2fs-devel wrote: > From: Jaegeuk Kim <jaegeuk@google.com> > > This adds a fragread command in f2fs_io, which is able to measure the > read performance on fragmented data buffer. > > Signed-off-by: Jaegeuk Kim <jaegeuk@google.com> > --- > tools/f2fs_io/f2fs_io.c | 110 ++++++++++++++++++++++++++++++++++++++++ > 1 file changed, 110 insertions(+) > > diff --git a/tools/f2fs_io/f2fs_io.c b/tools/f2fs_io/f2fs_io.c > index b72c26648f56..8431262575e0 100644 > --- a/tools/f2fs_io/f2fs_io.c > +++ b/tools/f2fs_io/f2fs_io.c > @@ -1018,6 +1018,115 @@ static void do_read(int argc, char **argv, const struct cmd_desc *cmd) > exit(0); > } > > +#define fragread_desc "read data with a fragmented buffer from file" > +#define fragread_help \ > +"f2fs_io fragread [chunk_size in 4kb] [offset in chunk_size] [count] [advice] [file_path]\n\n" \ > +"Read data in file_path and print nbytes\n" \ > +"advice can be\n" \ > +" 1 : set sequential|willneed\n" \ > +" 0 : none\n" \ > + > +#ifndef PAGE_SIZE > +#define PAGE_SIZE sysconf(_SC_PAGESIZE) > +#endif > +#define ALLOC_SIZE (2 * 1024 * 1024 - 4 * 1024) // 2MB - 4KB > + > +static void do_fragread(int argc, char **argv, const struct cmd_desc *cmd) > +{ > + u64 buf_size = 0, ret = 0, read_cnt = 0; > + u64 offset; > + char *buf = NULL; > + uintptr_t idx, ptr; > + unsigned bs, count, i; > + u64 total_time = 0; > + int flags = 0, alloc_count = 0; > + void *mem_hole, **mem_holes; > + int fd, advice; > + > + if (argc != 6) { > + fputs("Excess arguments\n\n", stderr); > + fputs(cmd->cmd_help, stderr); > + exit(1); > + } > + > + bs = atoi(argv[1]); > + if (bs > 256 * 1024) > + die("Too big chunk size - limit: 1GB"); > + buf_size = bs * F2FS_DEFAULT_BLKSIZE; > + > + offset = atoi(argv[2]) * buf_size; > + count = atoi(argv[3]); > + advice = atoi(argv[4]); > + mem_holes = xmalloc(sizeof(void *) * (buf_size / PAGE_SIZE)); > + > + /* 1. Allocate the buffer using mmap. */ > + buf = mmap(NULL, buf_size, PROT_READ | PROT_WRITE, > + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); > + > + /* 2. Loop and touch each page. */ > + for (idx = (uintptr_t)buf; idx < (uintptr_t)buf + buf_size; > + idx += PAGE_SIZE) > + { > + /* Touch the current page. */ > + volatile char *page = (volatile char *)idx; > + *page; > + > + /* 3. Allocate (2M - 4K) memory using mmap and touch all of it. */ > + mem_hole = mmap(NULL, ALLOC_SIZE, PROT_READ | PROT_WRITE, > + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); > + if (mem_hole == MAP_FAILED) > + die_errno("map failed"); > + > + /* Store the allocated memory pointer. */ > + mem_holes[alloc_count++] = mem_hole; Do we need to call mlock to hold allocated cache in the memory, if administrator enables swap, memory fragment will be gone once anonymous pages are swapped to device? Thanks, > + > + /* Touch all allocated memory. */ > + for (ptr = (uintptr_t)mem_hole; > + ptr < (uintptr_t)mem_hole + ALLOC_SIZE; > + ptr += PAGE_SIZE) { > + volatile char *alloc_page = (volatile char *)ptr; > + *alloc_page; > + } > + } > + printf("Touched allocated memory: count = %u\n", alloc_count); > + printf(" - allocated memory: = "); > + for (idx = 0; idx < 5; idx++) > + printf(" %p", mem_holes[idx]); > + printf("\n"); > + > + fd = xopen(argv[5], O_RDONLY | flags, 0); > + > + if (advice) { > + if (posix_fadvise(fd, 0, F2FS_DEFAULT_BLKSIZE, > + POSIX_FADV_SEQUENTIAL) != 0) > + die_errno("fadvise failed"); > + if (posix_fadvise(fd, 0, F2FS_DEFAULT_BLKSIZE, > + POSIX_FADV_WILLNEED) != 0) > + die_errno("fadvise failed"); > + printf("fadvise SEQUENTIAL|WILLNEED to a file: %s\n", argv[5]); > + } > + > + total_time = get_current_us(); > + > + for (i = 0; i < count; i++) { > + ret = pread(fd, buf, buf_size, offset + buf_size * i); > + if (ret != buf_size) { > + printf("pread expected: %"PRIu64", readed: %"PRIu64"\n", > + buf_size, ret); > + if (ret > 0) > + read_cnt += ret; > + break; > + } > + > + read_cnt += ret; > + } > + printf("Fragmented_Read %"PRIu64" bytes total_time = %"PRIu64" us, BW = %.Lf MB/s\n", > + read_cnt, get_current_us() - total_time, > + ((long double)read_cnt / (get_current_us() - total_time))); > + printf("\n"); > + exit(0); > +} > + > #define randread_desc "random read data from file" > #define randread_help \ > "f2fs_io randread [chunk_size in 4kb] [count] [IO] [advise] [file_path]\n\n" \ > @@ -2002,6 +2111,7 @@ const struct cmd_desc cmd_list[] = { > CMD(write_advice), > CMD(read), > CMD(randread), > + CMD(fragread), > CMD(fiemap), > CMD(gc_urgent), > CMD(defrag_file),
On 03/11, Chao Yu wrote: > On 3/4/25 09:10, Jaegeuk Kim via Linux-f2fs-devel wrote: > > From: Jaegeuk Kim <jaegeuk@google.com> > > > > This adds a fragread command in f2fs_io, which is able to measure the > > read performance on fragmented data buffer. > > > > Signed-off-by: Jaegeuk Kim <jaegeuk@google.com> > > --- > > tools/f2fs_io/f2fs_io.c | 110 ++++++++++++++++++++++++++++++++++++++++ > > 1 file changed, 110 insertions(+) > > > > diff --git a/tools/f2fs_io/f2fs_io.c b/tools/f2fs_io/f2fs_io.c > > index b72c26648f56..8431262575e0 100644 > > --- a/tools/f2fs_io/f2fs_io.c > > +++ b/tools/f2fs_io/f2fs_io.c > > @@ -1018,6 +1018,115 @@ static void do_read(int argc, char **argv, const struct cmd_desc *cmd) > > exit(0); > > } > > > > +#define fragread_desc "read data with a fragmented buffer from file" > > +#define fragread_help \ > > +"f2fs_io fragread [chunk_size in 4kb] [offset in chunk_size] [count] [advice] [file_path]\n\n" \ > > +"Read data in file_path and print nbytes\n" \ > > +"advice can be\n" \ > > +" 1 : set sequential|willneed\n" \ > > +" 0 : none\n" \ > > + > > +#ifndef PAGE_SIZE > > +#define PAGE_SIZE sysconf(_SC_PAGESIZE) > > +#endif > > +#define ALLOC_SIZE (2 * 1024 * 1024 - 4 * 1024) // 2MB - 4KB > > + > > +static void do_fragread(int argc, char **argv, const struct cmd_desc *cmd) > > +{ > > + u64 buf_size = 0, ret = 0, read_cnt = 0; > > + u64 offset; > > + char *buf = NULL; > > + uintptr_t idx, ptr; > > + unsigned bs, count, i; > > + u64 total_time = 0; > > + int flags = 0, alloc_count = 0; > > + void *mem_hole, **mem_holes; > > + int fd, advice; > > + > > + if (argc != 6) { > > + fputs("Excess arguments\n\n", stderr); > > + fputs(cmd->cmd_help, stderr); > > + exit(1); > > + } > > + > > + bs = atoi(argv[1]); > > + if (bs > 256 * 1024) > > + die("Too big chunk size - limit: 1GB"); > > + buf_size = bs * F2FS_DEFAULT_BLKSIZE; > > + > > + offset = atoi(argv[2]) * buf_size; > > + count = atoi(argv[3]); > > + advice = atoi(argv[4]); > > + mem_holes = xmalloc(sizeof(void *) * (buf_size / PAGE_SIZE)); > > + > > + /* 1. Allocate the buffer using mmap. */ > > + buf = mmap(NULL, buf_size, PROT_READ | PROT_WRITE, > > + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); > > + > > + /* 2. Loop and touch each page. */ > > + for (idx = (uintptr_t)buf; idx < (uintptr_t)buf + buf_size; > > + idx += PAGE_SIZE) > > + { > > + /* Touch the current page. */ > > + volatile char *page = (volatile char *)idx; > > + *page; > > + > > + /* 3. Allocate (2M - 4K) memory using mmap and touch all of it. */ > > + mem_hole = mmap(NULL, ALLOC_SIZE, PROT_READ | PROT_WRITE, > > + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); > > + if (mem_hole == MAP_FAILED) > > + die_errno("map failed"); > > + > > + /* Store the allocated memory pointer. */ > > + mem_holes[alloc_count++] = mem_hole; > > Do we need to call mlock to hold allocated cache in the memory, if > administrator enables swap, memory fragment will be gone once anonymous > pages are swapped to device? Hmm, the test is going to use this very intensively, so is it possible? > > Thanks, > > > + > > + /* Touch all allocated memory. */ > > + for (ptr = (uintptr_t)mem_hole; > > + ptr < (uintptr_t)mem_hole + ALLOC_SIZE; > > + ptr += PAGE_SIZE) { > > + volatile char *alloc_page = (volatile char *)ptr; > > + *alloc_page; > > + } > > + } > > + printf("Touched allocated memory: count = %u\n", alloc_count); > > + printf(" - allocated memory: = "); > > + for (idx = 0; idx < 5; idx++) > > + printf(" %p", mem_holes[idx]); > > + printf("\n"); > > + > > + fd = xopen(argv[5], O_RDONLY | flags, 0); > > + > > + if (advice) { > > + if (posix_fadvise(fd, 0, F2FS_DEFAULT_BLKSIZE, > > + POSIX_FADV_SEQUENTIAL) != 0) > > + die_errno("fadvise failed"); > > + if (posix_fadvise(fd, 0, F2FS_DEFAULT_BLKSIZE, > > + POSIX_FADV_WILLNEED) != 0) > > + die_errno("fadvise failed"); > > + printf("fadvise SEQUENTIAL|WILLNEED to a file: %s\n", argv[5]); > > + } > > + > > + total_time = get_current_us(); > > + > > + for (i = 0; i < count; i++) { > > + ret = pread(fd, buf, buf_size, offset + buf_size * i); > > + if (ret != buf_size) { > > + printf("pread expected: %"PRIu64", readed: %"PRIu64"\n", > > + buf_size, ret); > > + if (ret > 0) > > + read_cnt += ret; > > + break; > > + } > > + > > + read_cnt += ret; > > + } > > + printf("Fragmented_Read %"PRIu64" bytes total_time = %"PRIu64" us, BW = %.Lf MB/s\n", > > + read_cnt, get_current_us() - total_time, > > + ((long double)read_cnt / (get_current_us() - total_time))); > > + printf("\n"); > > + exit(0); > > +} > > + > > #define randread_desc "random read data from file" > > #define randread_help \ > > "f2fs_io randread [chunk_size in 4kb] [count] [IO] [advise] [file_path]\n\n" \ > > @@ -2002,6 +2111,7 @@ const struct cmd_desc cmd_list[] = { > > CMD(write_advice), > > CMD(read), > > CMD(randread), > > + CMD(fragread), > > CMD(fiemap), > > CMD(gc_urgent), > > CMD(defrag_file),
On 3/12/25 03:36, Jaegeuk Kim wrote: > On 03/11, Chao Yu wrote: >> On 3/4/25 09:10, Jaegeuk Kim via Linux-f2fs-devel wrote: >>> From: Jaegeuk Kim <jaegeuk@google.com> >>> >>> This adds a fragread command in f2fs_io, which is able to measure the >>> read performance on fragmented data buffer. >>> >>> Signed-off-by: Jaegeuk Kim <jaegeuk@google.com> >>> --- >>> tools/f2fs_io/f2fs_io.c | 110 ++++++++++++++++++++++++++++++++++++++++ >>> 1 file changed, 110 insertions(+) >>> >>> diff --git a/tools/f2fs_io/f2fs_io.c b/tools/f2fs_io/f2fs_io.c >>> index b72c26648f56..8431262575e0 100644 >>> --- a/tools/f2fs_io/f2fs_io.c >>> +++ b/tools/f2fs_io/f2fs_io.c >>> @@ -1018,6 +1018,115 @@ static void do_read(int argc, char **argv, const struct cmd_desc *cmd) >>> exit(0); >>> } >>> >>> +#define fragread_desc "read data with a fragmented buffer from file" >>> +#define fragread_help \ >>> +"f2fs_io fragread [chunk_size in 4kb] [offset in chunk_size] [count] [advice] [file_path]\n\n" \ >>> +"Read data in file_path and print nbytes\n" \ >>> +"advice can be\n" \ >>> +" 1 : set sequential|willneed\n" \ >>> +" 0 : none\n" \ >>> + >>> +#ifndef PAGE_SIZE >>> +#define PAGE_SIZE sysconf(_SC_PAGESIZE) >>> +#endif >>> +#define ALLOC_SIZE (2 * 1024 * 1024 - 4 * 1024) // 2MB - 4KB >>> + >>> +static void do_fragread(int argc, char **argv, const struct cmd_desc *cmd) >>> +{ >>> + u64 buf_size = 0, ret = 0, read_cnt = 0; >>> + u64 offset; >>> + char *buf = NULL; >>> + uintptr_t idx, ptr; >>> + unsigned bs, count, i; >>> + u64 total_time = 0; >>> + int flags = 0, alloc_count = 0; >>> + void *mem_hole, **mem_holes; >>> + int fd, advice; >>> + >>> + if (argc != 6) { >>> + fputs("Excess arguments\n\n", stderr); >>> + fputs(cmd->cmd_help, stderr); >>> + exit(1); >>> + } >>> + >>> + bs = atoi(argv[1]); >>> + if (bs > 256 * 1024) >>> + die("Too big chunk size - limit: 1GB"); >>> + buf_size = bs * F2FS_DEFAULT_BLKSIZE; >>> + >>> + offset = atoi(argv[2]) * buf_size; >>> + count = atoi(argv[3]); >>> + advice = atoi(argv[4]); >>> + mem_holes = xmalloc(sizeof(void *) * (buf_size / PAGE_SIZE)); >>> + >>> + /* 1. Allocate the buffer using mmap. */ >>> + buf = mmap(NULL, buf_size, PROT_READ | PROT_WRITE, >>> + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); >>> + >>> + /* 2. Loop and touch each page. */ >>> + for (idx = (uintptr_t)buf; idx < (uintptr_t)buf + buf_size; >>> + idx += PAGE_SIZE) >>> + { >>> + /* Touch the current page. */ >>> + volatile char *page = (volatile char *)idx; >>> + *page; >>> + >>> + /* 3. Allocate (2M - 4K) memory using mmap and touch all of it. */ >>> + mem_hole = mmap(NULL, ALLOC_SIZE, PROT_READ | PROT_WRITE, >>> + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); >>> + if (mem_hole == MAP_FAILED) >>> + die_errno("map failed"); >>> + >>> + /* Store the allocated memory pointer. */ >>> + mem_holes[alloc_count++] = mem_hole; >> >> Do we need to call mlock to hold allocated cache in the memory, if >> administrator enables swap, memory fragment will be gone once anonymous >> pages are swapped to device? > > Hmm, the test is going to use this very intensively, so is it possible? Not sure, but I doubt it is possible to happen in low-end device? due to it has very less memory and system may has already token the most of memory. Thanks, > >> >> Thanks, >> >>> + >>> + /* Touch all allocated memory. */ >>> + for (ptr = (uintptr_t)mem_hole; >>> + ptr < (uintptr_t)mem_hole + ALLOC_SIZE; >>> + ptr += PAGE_SIZE) { >>> + volatile char *alloc_page = (volatile char *)ptr; >>> + *alloc_page; >>> + } >>> + } >>> + printf("Touched allocated memory: count = %u\n", alloc_count); >>> + printf(" - allocated memory: = "); >>> + for (idx = 0; idx < 5; idx++) >>> + printf(" %p", mem_holes[idx]); >>> + printf("\n"); >>> + >>> + fd = xopen(argv[5], O_RDONLY | flags, 0); >>> + >>> + if (advice) { >>> + if (posix_fadvise(fd, 0, F2FS_DEFAULT_BLKSIZE, >>> + POSIX_FADV_SEQUENTIAL) != 0) >>> + die_errno("fadvise failed"); >>> + if (posix_fadvise(fd, 0, F2FS_DEFAULT_BLKSIZE, >>> + POSIX_FADV_WILLNEED) != 0) >>> + die_errno("fadvise failed"); >>> + printf("fadvise SEQUENTIAL|WILLNEED to a file: %s\n", argv[5]); >>> + } >>> + >>> + total_time = get_current_us(); >>> + >>> + for (i = 0; i < count; i++) { >>> + ret = pread(fd, buf, buf_size, offset + buf_size * i); >>> + if (ret != buf_size) { >>> + printf("pread expected: %"PRIu64", readed: %"PRIu64"\n", >>> + buf_size, ret); >>> + if (ret > 0) >>> + read_cnt += ret; >>> + break; >>> + } >>> + >>> + read_cnt += ret; >>> + } >>> + printf("Fragmented_Read %"PRIu64" bytes total_time = %"PRIu64" us, BW = %.Lf MB/s\n", >>> + read_cnt, get_current_us() - total_time, >>> + ((long double)read_cnt / (get_current_us() - total_time))); >>> + printf("\n"); >>> + exit(0); >>> +} >>> + >>> #define randread_desc "random read data from file" >>> #define randread_help \ >>> "f2fs_io randread [chunk_size in 4kb] [count] [IO] [advise] [file_path]\n\n" \ >>> @@ -2002,6 +2111,7 @@ const struct cmd_desc cmd_list[] = { >>> CMD(write_advice), >>> CMD(read), >>> CMD(randread), >>> + CMD(fragread), >>> CMD(fiemap), >>> CMD(gc_urgent), >>> CMD(defrag_file),
diff --git a/tools/f2fs_io/f2fs_io.c b/tools/f2fs_io/f2fs_io.c index b72c26648f56..8431262575e0 100644 --- a/tools/f2fs_io/f2fs_io.c +++ b/tools/f2fs_io/f2fs_io.c @@ -1018,6 +1018,115 @@ static void do_read(int argc, char **argv, const struct cmd_desc *cmd) exit(0); } +#define fragread_desc "read data with a fragmented buffer from file" +#define fragread_help \ +"f2fs_io fragread [chunk_size in 4kb] [offset in chunk_size] [count] [advice] [file_path]\n\n" \ +"Read data in file_path and print nbytes\n" \ +"advice can be\n" \ +" 1 : set sequential|willneed\n" \ +" 0 : none\n" \ + +#ifndef PAGE_SIZE +#define PAGE_SIZE sysconf(_SC_PAGESIZE) +#endif +#define ALLOC_SIZE (2 * 1024 * 1024 - 4 * 1024) // 2MB - 4KB + +static void do_fragread(int argc, char **argv, const struct cmd_desc *cmd) +{ + u64 buf_size = 0, ret = 0, read_cnt = 0; + u64 offset; + char *buf = NULL; + uintptr_t idx, ptr; + unsigned bs, count, i; + u64 total_time = 0; + int flags = 0, alloc_count = 0; + void *mem_hole, **mem_holes; + int fd, advice; + + if (argc != 6) { + fputs("Excess arguments\n\n", stderr); + fputs(cmd->cmd_help, stderr); + exit(1); + } + + bs = atoi(argv[1]); + if (bs > 256 * 1024) + die("Too big chunk size - limit: 1GB"); + buf_size = bs * F2FS_DEFAULT_BLKSIZE; + + offset = atoi(argv[2]) * buf_size; + count = atoi(argv[3]); + advice = atoi(argv[4]); + mem_holes = xmalloc(sizeof(void *) * (buf_size / PAGE_SIZE)); + + /* 1. Allocate the buffer using mmap. */ + buf = mmap(NULL, buf_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + + /* 2. Loop and touch each page. */ + for (idx = (uintptr_t)buf; idx < (uintptr_t)buf + buf_size; + idx += PAGE_SIZE) + { + /* Touch the current page. */ + volatile char *page = (volatile char *)idx; + *page; + + /* 3. Allocate (2M - 4K) memory using mmap and touch all of it. */ + mem_hole = mmap(NULL, ALLOC_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mem_hole == MAP_FAILED) + die_errno("map failed"); + + /* Store the allocated memory pointer. */ + mem_holes[alloc_count++] = mem_hole; + + /* Touch all allocated memory. */ + for (ptr = (uintptr_t)mem_hole; + ptr < (uintptr_t)mem_hole + ALLOC_SIZE; + ptr += PAGE_SIZE) { + volatile char *alloc_page = (volatile char *)ptr; + *alloc_page; + } + } + printf("Touched allocated memory: count = %u\n", alloc_count); + printf(" - allocated memory: = "); + for (idx = 0; idx < 5; idx++) + printf(" %p", mem_holes[idx]); + printf("\n"); + + fd = xopen(argv[5], O_RDONLY | flags, 0); + + if (advice) { + if (posix_fadvise(fd, 0, F2FS_DEFAULT_BLKSIZE, + POSIX_FADV_SEQUENTIAL) != 0) + die_errno("fadvise failed"); + if (posix_fadvise(fd, 0, F2FS_DEFAULT_BLKSIZE, + POSIX_FADV_WILLNEED) != 0) + die_errno("fadvise failed"); + printf("fadvise SEQUENTIAL|WILLNEED to a file: %s\n", argv[5]); + } + + total_time = get_current_us(); + + for (i = 0; i < count; i++) { + ret = pread(fd, buf, buf_size, offset + buf_size * i); + if (ret != buf_size) { + printf("pread expected: %"PRIu64", readed: %"PRIu64"\n", + buf_size, ret); + if (ret > 0) + read_cnt += ret; + break; + } + + read_cnt += ret; + } + printf("Fragmented_Read %"PRIu64" bytes total_time = %"PRIu64" us, BW = %.Lf MB/s\n", + read_cnt, get_current_us() - total_time, + ((long double)read_cnt / (get_current_us() - total_time))); + printf("\n"); + exit(0); +} + #define randread_desc "random read data from file" #define randread_help \ "f2fs_io randread [chunk_size in 4kb] [count] [IO] [advise] [file_path]\n\n" \ @@ -2002,6 +2111,7 @@ const struct cmd_desc cmd_list[] = { CMD(write_advice), CMD(read), CMD(randread), + CMD(fragread), CMD(fiemap), CMD(gc_urgent), CMD(defrag_file),