diff mbox series

[v7,3/3] shmem: stable directory offsets

Message ID 168814734331.530310.3911190551060453102.stgit@manet.1015granger.net (mailing list archive)
State New, archived
Headers show
Series shmemfs stable directory offsets | expand

Commit Message

Chuck Lever June 30, 2023, 5:49 p.m. UTC
From: Chuck Lever <chuck.lever@oracle.com>

The current cursor-based directory offset mechanism doesn't work
when a tmpfs filesystem is exported via NFS. This is because NFS
clients do not open directories. Each server-side READDIR operation
has to open the directory, read it, then close it. The cursor state
for that directory, being associated strictly with the opened
struct file, is thus discarded after each NFS READDIR operation.

Directory offsets are cached not only by NFS clients, but also by
user space libraries on those clients. Essentially there is no way
to invalidate those caches when directory offsets have changed on
an NFS server after the offset-to-dentry mapping changes. Thus the
whole application stack depends on unchanging directory offsets.

The solution we've come up with is to make the directory offset for
each file in a tmpfs filesystem stable for the life of the directory
entry it represents.

shmem_readdir() and shmem_dir_llseek() now use an xarray to map each
directory offset (an loff_t integer) to the memory address of a
struct dentry.

Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
---
 include/linux/shmem_fs.h |    1 +
 mm/shmem.c               |   47 +++++++++++++++++++++++++++++++++++++++-------
 2 files changed, 41 insertions(+), 7 deletions(-)

Comments

kernel test robot July 13, 2023, 1:30 p.m. UTC | #1
Hello,

kernel test robot noticed a 3.0% improvement of stress-ng.handle.ops_per_sec on:


commit: a1a690e009744e4526526b2f838beec5ef9233cc ("[PATCH v7 3/3] shmem: stable directory offsets")
url: https://github.com/intel-lab-lkp/linux/commits/Chuck-Lever/libfs-Add-directory-operations-for-stable-offsets/20230701-014925
base: https://git.kernel.org/cgit/linux/kernel/git/akpm/mm.git mm-everything
patch link: https://lore.kernel.org/all/168814734331.530310.3911190551060453102.stgit@manet.1015granger.net/
patch subject: [PATCH v7 3/3] shmem: stable directory offsets

testcase: stress-ng
test machine: 36 threads 1 sockets Intel(R) Core(TM) i9-9980XE CPU @ 3.00GHz (Skylake) with 32G memory
parameters:

	nr_threads: 10%
	disk: 1SSD
	testtime: 60s
	fs: ext4
	class: filesystem
	test: handle
	cpufreq_governor: performance






Details are as below:
-------------------------------------------------------------------------------------------------->


To reproduce:

        git clone https://github.com/intel/lkp-tests.git
        cd lkp-tests
        sudo bin/lkp install job.yaml           # job file is attached in this email
        bin/lkp split-job --compatible job.yaml # generate the yaml file for lkp run
        sudo bin/lkp run generated-yaml-file

        # if come across any failure that blocks the test,
        # please remove ~/.lkp and /lkp dir to run from a clean state.

=========================================================================================
class/compiler/cpufreq_governor/disk/fs/kconfig/nr_threads/rootfs/tbox_group/test/testcase/testtime:
  filesystem/gcc-12/performance/1SSD/ext4/x86_64-rhel-8.3/10%/debian-11.1-x86_64-20220510.cgz/lkp-skl-d08/handle/stress-ng/60s

commit: 
  c2c3172c26 ("shmem: Refactor shmem_symlink()")
  a1a690e009 ("shmem: stable directory offsets")

c2c3172c269f808d a1a690e009744e4526526b2f838 
---------------- --------------------------- 
         %stddev     %change         %stddev
             \          |                \  
      5.92            -1.2%       5.85        iostat.cpu.system
     88.56 ± 21%     -29.4%      62.51 ± 39%  sched_debug.cfs_rq:/.util_est_enqueued.avg
      1294 ± 10%     -21.2%       1020 ± 13%  sched_debug.cpu.curr->pid.avg
  11210546            +3.0%   11542342        stress-ng.handle.ops
    186841            +3.0%     192371        stress-ng.handle.ops_per_sec
    100.40 ± 53%     -51.2%      49.00 ± 74%  proc-vmstat.nr_dirtied
    488851 ±  5%      -8.4%     447861        proc-vmstat.numa_hit
    480285 ±  5%      -6.8%     447764        proc-vmstat.numa_local
    750318 ±  6%      -8.4%     687348        proc-vmstat.pgalloc_normal
    710717 ±  7%      -9.0%     646894        proc-vmstat.pgfree
  1.43e+09            -1.7%  1.406e+09        perf-stat.i.branch-instructions
      1.25            -0.0        1.20        perf-stat.i.branch-miss-rate%
  19188631            -5.1%   18207135        perf-stat.i.branch-misses
      0.15            +0.0        0.15        perf-stat.i.dTLB-load-miss-rate%
   2781970            +2.6%    2854604        perf-stat.i.dTLB-load-misses
 1.904e+09            -1.5%  1.875e+09        perf-stat.i.dTLB-loads
 1.139e+09            -1.7%   1.12e+09        perf-stat.i.dTLB-stores
     67.67 ± 10%      +7.1       74.72        perf-stat.i.iTLB-load-miss-rate%
   1568674 ± 46%     -32.7%    1055235 ± 14%  perf-stat.i.iTLB-loads
 7.352e+09            -1.4%  7.247e+09        perf-stat.i.instructions
    124.64            -1.9%     122.30        perf-stat.i.metric.M/sec
     41311 ±  5%     -11.0%      36758 ±  2%  perf-stat.i.node-stores
      1.34            -0.0        1.29        perf-stat.overall.branch-miss-rate%
      0.15            +0.0        0.15        perf-stat.overall.dTLB-load-miss-rate%
     68.34 ± 10%      +7.3       75.68 ±  2%  perf-stat.overall.iTLB-load-miss-rate%
 1.407e+09            -1.7%  1.383e+09        perf-stat.ps.branch-instructions
  18876614            -5.1%   17906237        perf-stat.ps.branch-misses
   2737798            +2.6%    2809345        perf-stat.ps.dTLB-load-misses
 1.873e+09            -1.5%  1.845e+09        perf-stat.ps.dTLB-loads
 1.121e+09            -1.7%  1.102e+09        perf-stat.ps.dTLB-stores
   1543675 ± 46%     -32.8%    1037667 ± 14%  perf-stat.ps.iTLB-loads
 7.235e+09            -1.4%   7.13e+09        perf-stat.ps.instructions
     40647 ±  5%     -11.1%      36139 ±  3%  perf-stat.ps.node-stores
 4.557e+11            -1.2%  4.502e+11        perf-stat.total.instructions
      0.01           -20.0%       0.00        perf-sched.sch_delay.avg.ms.smpboot_thread_fn.kthread.ret_from_fork
      0.08 ± 10%     -30.6%       0.06 ±  9%  perf-sched.wait_and_delay.avg.ms.exit_to_user_mode_loop.exit_to_user_mode_prepare.irqentry_exit_to_user_mode.asm_sysvec_apic_timer_interrupt
      0.19 ±  5%     -26.8%       0.14 ±  3%  perf-sched.wait_and_delay.max.ms.exit_to_user_mode_loop.exit_to_user_mode_prepare.irqentry_exit_to_user_mode.asm_sysvec_apic_timer_interrupt
      0.08 ± 20%     -30.2%       0.06 ± 15%  perf-sched.wait_time.avg.ms.__cond_resched.__fput.task_work_run.exit_to_user_mode_loop.exit_to_user_mode_prepare
      0.08 ± 10%     -31.9%       0.06 ±  9%  perf-sched.wait_time.avg.ms.__cond_resched.__kmem_cache_alloc_node.__kmalloc.do_sys_name_to_handle.__x64_sys_name_to_handle_at
      0.08 ± 12%     -31.4%       0.06 ±  8%  perf-sched.wait_time.avg.ms.__cond_resched.__kmem_cache_alloc_node.__kmalloc.handle_to_path.do_handle_open
      0.08 ±  9%     -29.4%       0.06 ± 14%  perf-sched.wait_time.avg.ms.__cond_resched.apparmor_file_alloc_security.security_file_alloc.init_file.alloc_empty_file
      0.08 ±  9%    -100.0%       0.00        perf-sched.wait_time.avg.ms.__cond_resched.dentry_kill.dput.dcache_dir_close.__fput
      0.08 ± 10%     -30.5%       0.06 ±  7%  perf-sched.wait_time.avg.ms.__cond_resched.dput.__fput.task_work_run.exit_to_user_mode_loop
      0.08 ± 11%    -100.0%       0.00        perf-sched.wait_time.avg.ms.__cond_resched.dput.dcache_dir_close.__fput.task_work_run
      0.08 ± 10%     -30.3%       0.06 ±  8%  perf-sched.wait_time.avg.ms.__cond_resched.dput.path_put.__x64_sys_name_to_handle_at.do_syscall_64
      0.08 ±  9%     -34.9%       0.05 ±  8%  perf-sched.wait_time.avg.ms.__cond_resched.dput.path_put.do_handle_open.do_syscall_64
      0.08 ± 10%     -29.6%       0.05 ± 10%  perf-sched.wait_time.avg.ms.__cond_resched.dput.terminate_walk.path_openat.do_file_open_root
      0.08 ±  9%     -30.6%       0.06 ± 10%  perf-sched.wait_time.avg.ms.__cond_resched.dput.terminate_walk.path_openat.do_filp_open
      0.08 ± 10%     -29.1%       0.06 ±  9%  perf-sched.wait_time.avg.ms.__cond_resched.ilookup5.shmem_fh_to_dentry.exportfs_decode_fh_raw.exportfs_decode_fh
      0.08 ± 18%     -30.9%       0.05 ± 10%  perf-sched.wait_time.avg.ms.__cond_resched.kmem_cache_alloc.alloc_empty_file.path_openat.do_file_open_root
      0.07 ±  7%     -23.8%       0.06 ± 10%  perf-sched.wait_time.avg.ms.__cond_resched.kmem_cache_alloc.alloc_empty_file.path_openat.do_filp_open
      0.08 ± 10%     -32.2%       0.05 ±  9%  perf-sched.wait_time.avg.ms.__cond_resched.kmem_cache_alloc.getname_flags.part.0
      0.08 ± 11%     -33.4%       0.05 ± 12%  perf-sched.wait_time.avg.ms.__cond_resched.kmem_cache_alloc.getname_kernel.do_file_open_root.file_open_root
      0.08 ± 11%     -30.8%       0.06 ±  8%  perf-sched.wait_time.avg.ms.__cond_resched.kmem_cache_alloc.security_file_alloc.init_file.alloc_empty_file
      0.08 ± 10%    -100.0%       0.00        perf-sched.wait_time.avg.ms.__cond_resched.slab_pre_alloc_hook.constprop.0.kmem_cache_alloc_lru
      0.08 ± 11%     -34.6%       0.05 ± 11%  perf-sched.wait_time.avg.ms.__cond_resched.task_work_run.exit_to_user_mode_loop.exit_to_user_mode_prepare.syscall_exit_to_user_mode
      0.08 ± 10%     -30.6%       0.06 ±  9%  perf-sched.wait_time.avg.ms.exit_to_user_mode_loop.exit_to_user_mode_prepare.irqentry_exit_to_user_mode.asm_sysvec_apic_timer_interrupt
      0.08 ± 11%     -32.3%       0.05 ±  9%  perf-sched.wait_time.avg.ms.exit_to_user_mode_loop.exit_to_user_mode_prepare.syscall_exit_to_user_mode.do_syscall_64
      0.13 ± 29%     -43.2%       0.07 ± 22%  perf-sched.wait_time.max.ms.__cond_resched.__fput.task_work_run.exit_to_user_mode_loop.exit_to_user_mode_prepare
      0.18 ± 11%     -22.3%       0.14 ± 12%  perf-sched.wait_time.max.ms.__cond_resched.__kmem_cache_alloc_node.__kmalloc.do_sys_name_to_handle.__x64_sys_name_to_handle_at
      0.17 ±  9%     -28.2%       0.12 ±  6%  perf-sched.wait_time.max.ms.__cond_resched.__kmem_cache_alloc_node.__kmalloc.handle_to_path.do_handle_open
      0.13 ± 14%    -100.0%       0.00        perf-sched.wait_time.max.ms.__cond_resched.dentry_kill.dput.dcache_dir_close.__fput
      0.15 ±  9%     -26.8%       0.11 ± 16%  perf-sched.wait_time.max.ms.__cond_resched.dput.__fput.task_work_run.exit_to_user_mode_loop
      0.11 ± 32%    -100.0%       0.00        perf-sched.wait_time.max.ms.__cond_resched.dput.dcache_dir_close.__fput.task_work_run
      0.17 ±  2%     -26.6%       0.12 ±  7%  perf-sched.wait_time.max.ms.__cond_resched.dput.path_put.__x64_sys_name_to_handle_at.do_syscall_64
      0.16 ± 10%     -42.7%       0.09 ± 18%  perf-sched.wait_time.max.ms.__cond_resched.dput.path_put.do_handle_open.do_syscall_64
      0.16 ± 10%     -27.0%       0.12 ±  6%  perf-sched.wait_time.max.ms.__cond_resched.dput.terminate_walk.path_openat.do_file_open_root
      0.16 ±  6%     -24.1%       0.12 ±  9%  perf-sched.wait_time.max.ms.__cond_resched.ilookup5.shmem_fh_to_dentry.exportfs_decode_fh_raw.exportfs_decode_fh
      0.13 ± 27%     -36.6%       0.08 ± 24%  perf-sched.wait_time.max.ms.__cond_resched.kmem_cache_alloc.alloc_empty_file.path_openat.do_file_open_root
      0.13 ± 16%     -27.5%       0.10 ± 20%  perf-sched.wait_time.max.ms.__cond_resched.kmem_cache_alloc.alloc_empty_file.path_openat.do_filp_open
      0.18 ±  9%     -23.2%       0.14 ± 13%  perf-sched.wait_time.max.ms.__cond_resched.kmem_cache_alloc.getname_flags.part.0
      0.15 ± 11%     -35.8%       0.10 ± 15%  perf-sched.wait_time.max.ms.__cond_resched.kmem_cache_alloc.getname_kernel.do_file_open_root.file_open_root
      0.16 ±  7%     -35.5%       0.10 ± 24%  perf-sched.wait_time.max.ms.__cond_resched.kmem_cache_alloc.security_file_alloc.init_file.alloc_empty_file
      0.17 ± 10%    -100.0%       0.00        perf-sched.wait_time.max.ms.__cond_resched.slab_pre_alloc_hook.constprop.0.kmem_cache_alloc_lru
      0.16 ±  8%     -36.2%       0.10 ± 21%  perf-sched.wait_time.max.ms.__cond_resched.task_work_run.exit_to_user_mode_loop.exit_to_user_mode_prepare.syscall_exit_to_user_mode
      0.19 ±  5%     -26.8%       0.14 ±  3%  perf-sched.wait_time.max.ms.exit_to_user_mode_loop.exit_to_user_mode_prepare.irqentry_exit_to_user_mode.asm_sysvec_apic_timer_interrupt
      0.18 ±  4%     -31.4%       0.12 ±  8%  perf-sched.wait_time.max.ms.exit_to_user_mode_loop.exit_to_user_mode_prepare.syscall_exit_to_user_mode.do_syscall_64
      1.42 ±  6%      -0.8        0.59 ±  3%  perf-profile.calltrace.cycles-pp.do_dentry_open.do_open.path_openat.do_filp_open.do_sys_openat2
      1.89 ±  4%      -0.8        1.07 ±  5%  perf-profile.calltrace.cycles-pp.do_open.path_openat.do_filp_open.do_sys_openat2.__x64_sys_openat
      4.24 ±  3%      -0.8        3.43 ±  2%  perf-profile.calltrace.cycles-pp.__x64_sys_openat.do_syscall_64.entry_SYSCALL_64_after_hwframe.open64
      4.19 ±  3%      -0.8        3.38 ±  2%  perf-profile.calltrace.cycles-pp.do_sys_openat2.__x64_sys_openat.do_syscall_64.entry_SYSCALL_64_after_hwframe.open64
      3.46 ±  3%      -0.8        2.68 ±  3%  perf-profile.calltrace.cycles-pp.do_filp_open.do_sys_openat2.__x64_sys_openat.do_syscall_64.entry_SYSCALL_64_after_hwframe
      3.36 ±  3%      -0.8        2.59 ±  3%  perf-profile.calltrace.cycles-pp.path_openat.do_filp_open.do_sys_openat2.__x64_sys_openat.do_syscall_64
      6.64 ±  3%      -0.8        5.88 ±  2%  perf-profile.calltrace.cycles-pp.open64
      5.39 ±  3%      -0.8        4.63 ±  2%  perf-profile.calltrace.cycles-pp.entry_SYSCALL_64_after_hwframe.open64
      4.95 ±  3%      -0.8        4.19 ±  2%  perf-profile.calltrace.cycles-pp.do_syscall_64.entry_SYSCALL_64_after_hwframe.open64
      5.48 ±  4%      -0.7        4.80 ±  3%  perf-profile.calltrace.cycles-pp.do_syscall_64.entry_SYSCALL_64_after_hwframe.__close
      6.32 ±  3%      -0.6        5.68 ±  2%  perf-profile.calltrace.cycles-pp.entry_SYSCALL_64_after_hwframe.__close
      8.83 ±  3%      -0.5        8.29        perf-profile.calltrace.cycles-pp.__close
      4.33 ±  4%      -0.5        3.82 ±  5%  perf-profile.calltrace.cycles-pp.syscall_exit_to_user_mode.do_syscall_64.entry_SYSCALL_64_after_hwframe.__close
      2.36 ±  5%      -0.5        1.86 ±  8%  perf-profile.calltrace.cycles-pp.__fput.task_work_run.exit_to_user_mode_loop.exit_to_user_mode_prepare.syscall_exit_to_user_mode
      2.93 ±  5%      -0.5        2.45 ±  6%  perf-profile.calltrace.cycles-pp.exit_to_user_mode_loop.exit_to_user_mode_prepare.syscall_exit_to_user_mode.do_syscall_64.entry_SYSCALL_64_after_hwframe
      2.76 ±  5%      -0.5        2.28 ±  7%  perf-profile.calltrace.cycles-pp.task_work_run.exit_to_user_mode_loop.exit_to_user_mode_prepare.syscall_exit_to_user_mode.do_syscall_64
      2.98 ±  5%      -0.5        2.50 ±  6%  perf-profile.calltrace.cycles-pp.exit_to_user_mode_prepare.syscall_exit_to_user_mode.do_syscall_64.entry_SYSCALL_64_after_hwframe.__close
      1.17 ± 13%      -0.3        0.85 ± 17%  perf-profile.calltrace.cycles-pp.rcu_core.__do_softirq.run_ksoftirqd.smpboot_thread_fn.kthread
      1.00 ±  3%      -0.1        0.85 ±  9%  perf-profile.calltrace.cycles-pp.__x64_sys_close.do_syscall_64.entry_SYSCALL_64_after_hwframe.__close
      0.77 ±  4%      -0.1        0.66 ± 13%  perf-profile.calltrace.cycles-pp.filp_close.__x64_sys_close.do_syscall_64.entry_SYSCALL_64_after_hwframe.__close
      1.17 ±  6%      +0.1        1.29 ±  5%  perf-profile.calltrace.cycles-pp.do_dentry_open.do_open.path_openat.do_file_open_root.file_open_root
      3.33 ±  5%      +0.2        3.56 ±  2%  perf-profile.calltrace.cycles-pp.do_file_open_root.file_open_root.do_handle_open.do_syscall_64.entry_SYSCALL_64_after_hwframe
      3.38 ±  5%      +0.2        3.61 ±  2%  perf-profile.calltrace.cycles-pp.file_open_root.do_handle_open.do_syscall_64.entry_SYSCALL_64_after_hwframe.open_by_handle_at
      4.26 ±  3%      -0.8        3.44 ±  2%  perf-profile.children.cycles-pp.__x64_sys_openat
      4.21 ±  3%      -0.8        3.40 ±  2%  perf-profile.children.cycles-pp.do_sys_openat2
      3.47 ±  3%      -0.8        2.69 ±  3%  perf-profile.children.cycles-pp.do_filp_open
      6.69 ±  3%      -0.8        5.93 ±  2%  perf-profile.children.cycles-pp.open64
      2.59 ±  5%      -0.7        1.89 ±  3%  perf-profile.children.cycles-pp.do_dentry_open
      3.35 ±  4%      -0.7        2.66 ±  3%  perf-profile.children.cycles-pp.do_open
      6.40 ±  4%      -0.6        5.80 ±  2%  perf-profile.children.cycles-pp.path_openat
      8.94 ±  3%      -0.5        8.41 ±  2%  perf-profile.children.cycles-pp.__close
      2.38 ±  5%      -0.5        1.88 ±  8%  perf-profile.children.cycles-pp.__fput
      2.95 ±  5%      -0.5        2.46 ±  6%  perf-profile.children.cycles-pp.exit_to_user_mode_loop
      2.77 ±  5%      -0.5        2.30 ±  7%  perf-profile.children.cycles-pp.task_work_run
      3.20 ±  5%      -0.4        2.75 ±  5%  perf-profile.children.cycles-pp.exit_to_user_mode_prepare
      2.89 ±  7%      -0.4        2.44 ±  4%  perf-profile.children.cycles-pp.dput
      0.36 ±  8%      -0.2        0.14 ± 16%  perf-profile.children.cycles-pp.lockref_put_return
      1.00 ±  3%      -0.1        0.86 ± 10%  perf-profile.children.cycles-pp.__x64_sys_close
      0.77 ±  4%      -0.1        0.66 ± 13%  perf-profile.children.cycles-pp.filp_close
      0.35 ± 15%      -0.1        0.24 ± 16%  perf-profile.children.cycles-pp.__slab_free
      0.44 ± 13%      -0.1        0.33 ±  8%  perf-profile.children.cycles-pp.shmem_encode_fh
      0.49 ±  6%      -0.1        0.39 ±  3%  perf-profile.children.cycles-pp.memcg_slab_post_alloc_hook
      0.70 ±  7%      -0.1        0.60 ±  9%  perf-profile.children.cycles-pp.lockref_get
      0.39 ±  8%      -0.1        0.30 ± 24%  perf-profile.children.cycles-pp.locks_remove_posix
      0.34 ±  7%      -0.1        0.26 ±  5%  perf-profile.children.cycles-pp.__call_rcu_common
      0.13 ±  8%      -0.1        0.07 ±  9%  perf-profile.children.cycles-pp.___slab_alloc
      0.25 ± 12%      -0.0        0.20 ± 11%  perf-profile.children.cycles-pp.get_obj_cgroup_from_current
      0.37 ± 10%      -0.0        0.33 ±  9%  perf-profile.children.cycles-pp.rep_movs_alternative
      0.08 ± 16%      -0.0        0.05 ± 54%  perf-profile.children.cycles-pp.obj_cgroup_charge
      0.16 ± 12%      -0.0        0.13 ±  4%  perf-profile.children.cycles-pp.close_fd
      0.13 ±  4%      -0.0        0.10 ± 14%  perf-profile.children.cycles-pp.is_vmalloc_addr
      3.34 ±  5%      +0.2        3.57 ±  2%  perf-profile.children.cycles-pp.do_file_open_root
      0.36 ±  8%      -0.2        0.13 ± 19%  perf-profile.self.cycles-pp.lockref_put_return
      0.34 ± 15%      -0.1        0.23 ± 16%  perf-profile.self.cycles-pp.__slab_free
      0.44 ± 13%      -0.1        0.33 ± 10%  perf-profile.self.cycles-pp.shmem_encode_fh
      0.69 ±  7%      -0.1        0.60 ±  9%  perf-profile.self.cycles-pp.lockref_get
      0.39 ±  7%      -0.1        0.30 ± 25%  perf-profile.self.cycles-pp.locks_remove_posix
      0.37 ±  7%      -0.1        0.29 ±  5%  perf-profile.self.cycles-pp.memcg_slab_post_alloc_hook
      0.20 ± 10%      -0.1        0.15 ±  9%  perf-profile.self.cycles-pp.__call_rcu_common
      0.13 ± 21%      -0.0        0.09 ± 28%  perf-profile.self.cycles-pp.__legitimize_path
      0.13 ±  6%      -0.0        0.10 ± 14%  perf-profile.self.cycles-pp.is_vmalloc_addr
      0.10 ±  7%      -0.0        0.08 ±  7%  perf-profile.self.cycles-pp.do_handle_open
      0.15 ± 11%      +0.0        0.18 ±  5%  perf-profile.self.cycles-pp.init_file




Disclaimer:
Results have been estimated based on internal Intel analysis and are provided
for informational purposes only. Any difference in system hardware or software
design or configuration may affect actual performance.
kernel test robot July 17, 2023, 6:46 a.m. UTC | #2
hi, Chuck Lever,

we reported a 3.0% improvement of stress-ng.handle.ops_per_sec for this commit
on
https://lore.kernel.org/oe-lkp/202307132153.a52cdb2d-oliver.sang@intel.com/

but now we noticed a regression, detail as below, FYI

Hello,

kernel test robot noticed a -15.5% regression of will-it-scale.per_thread_ops on:


commit: a1a690e009744e4526526b2f838beec5ef9233cc ("[PATCH v7 3/3] shmem: stable directory offsets")
url: https://github.com/intel-lab-lkp/linux/commits/Chuck-Lever/libfs-Add-directory-operations-for-stable-offsets/20230701-014925
base: https://git.kernel.org/cgit/linux/kernel/git/akpm/mm.git mm-everything
patch link: https://lore.kernel.org/all/168814734331.530310.3911190551060453102.stgit@manet.1015granger.net/
patch subject: [PATCH v7 3/3] shmem: stable directory offsets

testcase: will-it-scale
test machine: 104 threads 2 sockets (Skylake) with 192G memory
parameters:

	nr_task: 16
	mode: thread
	test: unlink2
	cpufreq_governor: performance


In addition to that, the commit also has significant impact on the following tests:

+------------------+-------------------------------------------------------------------------------------------------+
| testcase: change | will-it-scale: will-it-scale.per_thread_ops -40.0% regression                                   |
| test machine     | 36 threads 1 sockets Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz (Cascade Lake) with 128G memory |
| test parameters  | cpufreq_governor=performance                                                                    |
|                  | mode=thread                                                                                     |
|                  | nr_task=16                                                                                      |
|                  | test=unlink2                                                                                    |
+------------------+-------------------------------------------------------------------------------------------------+
| testcase: change | stress-ng: stress-ng.handle.ops_per_sec 3.0% improvement                                        |
| test machine     | 36 threads 1 sockets Intel(R) Core(TM) i9-9980XE CPU @ 3.00GHz (Skylake) with 32G memory        |
| test parameters  | class=filesystem                                                                                |
|                  | cpufreq_governor=performance                                                                    |
|                  | disk=1SSD                                                                                       |
|                  | fs=ext4                                                                                         |
|                  | nr_threads=10%                                                                                  |
|                  | test=handle                                                                                     |
|                  | testtime=60s                                                                                    |
+------------------+-------------------------------------------------------------------------------------------------+


If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <oliver.sang@intel.com>
| Closes: https://lore.kernel.org/oe-lkp/202307171436.29248fcf-oliver.sang@intel.com


Details are as below:
-------------------------------------------------------------------------------------------------->


To reproduce:

        git clone https://github.com/intel/lkp-tests.git
        cd lkp-tests
        sudo bin/lkp install job.yaml           # job file is attached in this email
        bin/lkp split-job --compatible job.yaml # generate the yaml file for lkp run
        sudo bin/lkp run generated-yaml-file

        # if come across any failure that blocks the test,
        # please remove ~/.lkp and /lkp dir to run from a clean state.

=========================================================================================
compiler/cpufreq_governor/kconfig/mode/nr_task/rootfs/tbox_group/test/testcase:
  gcc-12/performance/x86_64-rhel-8.3/thread/16/debian-11.1-x86_64-20220510.cgz/lkp-skl-fpga01/unlink2/will-it-scale

commit: 
  c2c3172c26 ("shmem: Refactor shmem_symlink()")
  a1a690e009 ("shmem: stable directory offsets")

c2c3172c269f808d a1a690e009744e4526526b2f838 
---------------- --------------------------- 
         %stddev     %change         %stddev
             \          |                \  
      1991           +50.9%       3005        vmstat.system.cs
      0.81            +1.4        2.19        mpstat.cpu.all.soft%
      1.67 ±  2%      -0.2        1.45        mpstat.cpu.all.usr%
  24095642 ±  2%    +113.9%   51539271        numa-numastat.node0.local_node
  24164359 ±  2%    +113.5%   51582554        numa-numastat.node0.numa_hit
    555.33 ±  6%    +108.7%       1158 ±  6%  perf-c2c.DRAM.local
     25.17 ± 16%     +46.4%      36.83 ± 14%  perf-c2c.DRAM.remote
     13.83 ± 20%     +78.3%      24.67 ± 19%  perf-c2c.HITM.remote
    255956 ± 24%     -92.7%      18574 ± 69%  sched_debug.cfs_rq:/.spread0.max
      1659 ±  6%     +59.1%       2640 ±  8%  sched_debug.cpu.clock_task.stddev
      4686 ±  4%     +27.0%       5952 ±  6%  sched_debug.cpu.nr_switches.avg
   1144102 ±  2%     -15.5%     967020        will-it-scale.16.threads
     71505 ±  2%     -15.5%      60438        will-it-scale.per_thread_ops
   1144102 ±  2%     -15.5%     967020        will-it-scale.workload
    131875           +60.2%     211239        meminfo.KReclaimable
    131875           +60.2%     211239        meminfo.SReclaimable
    258837           -14.7%     220833        meminfo.SUnreclaim
    390713           +10.6%     432074        meminfo.Slab
     95496 ±  5%     +86.8%     178379 ±  3%  numa-meminfo.node0.KReclaimable
     95496 ±  5%     +86.8%     178379 ±  3%  numa-meminfo.node0.SReclaimable
    158003 ±  5%     -27.0%     115367 ±  3%  numa-meminfo.node0.SUnreclaim
    253500 ±  4%     +15.9%     293747 ±  2%  numa-meminfo.node0.Slab
     23915 ±  5%     +86.8%      44685 ±  3%  numa-vmstat.node0.nr_slab_reclaimable
     39711 ±  5%     -27.3%      28867 ±  3%  numa-vmstat.node0.nr_slab_unreclaimable
  24164365 ±  2%    +113.5%   51582656        numa-vmstat.node0.numa_hit
  24095648 ±  2%    +113.9%   51539372        numa-vmstat.node0.numa_local
     83317 ±  4%     +83.8%     153145 ±  3%  turbostat.C1E
      0.10 ±  7%      +0.1        0.19 ±  6%  turbostat.C1E%
    236.45            +2.2%     241.56        turbostat.PkgWatt
     22.00           +28.8%      28.34        turbostat.RAMWatt
     32975           +61.6%      53290        proc-vmstat.nr_slab_reclaimable
     64774           -14.6%      55295        proc-vmstat.nr_slab_unreclaimable
  24757758 ±  2%    +110.7%   52155982        proc-vmstat.numa_hit
  24647533 ±  2%    +111.2%   52048781        proc-vmstat.numa_local
     19233            +7.0%      20587        proc-vmstat.pgactivate
  93642540 ±  2%    +258.2%  3.354e+08        proc-vmstat.pgalloc_normal
  93561023 ±  2%    +258.3%  3.352e+08        proc-vmstat.pgfree
     26.98 ±  6%      -8.8%      24.60 ±  2%  perf-stat.i.MPKI
 3.913e+09 ±  2%      +9.7%  4.292e+09        perf-stat.i.branch-instructions
      1.39            +0.1        1.52        perf-stat.i.branch-miss-rate%
  54513120           +20.1%   65495253        perf-stat.i.branch-misses
      6.32 ±  6%     +14.8       21.14 ±  2%  perf-stat.i.cache-miss-rate%
  33974168 ±  3%    +245.6%  1.174e+08        perf-stat.i.cache-misses
      1923           +53.1%       2943        perf-stat.i.context-switches
      2.37 ±  2%      -8.8%       2.16        perf-stat.i.cpi
 4.736e+10            +3.0%  4.879e+10        perf-stat.i.cpu-cycles
    124.48          +149.0%     309.91        perf-stat.i.cpu-migrations
      1399 ±  3%     -70.2%     416.95        perf-stat.i.cycles-between-cache-misses
      0.09 ±  2%      -0.0        0.07        perf-stat.i.dTLB-load-miss-rate%
   4979142 ±  3%      -9.9%    4484285        perf-stat.i.dTLB-load-misses
 5.496e+09 ±  2%     +11.3%   6.12e+09        perf-stat.i.dTLB-loads
 3.487e+09 ±  2%     +13.7%  3.963e+09        perf-stat.i.dTLB-stores
     47.74 ±  3%     -11.2       36.50        perf-stat.i.iTLB-load-miss-rate%
   3839118 ±  2%      -8.6%    3508856        perf-stat.i.iTLB-load-misses
   4222137 ±  4%     +44.8%    6112207        perf-stat.i.iTLB-loads
 2.006e+10 ±  2%     +12.8%  2.262e+10        perf-stat.i.instructions
      5231           +23.4%       6457        perf-stat.i.instructions-per-iTLB-miss
      0.42 ±  2%      +9.5%       0.46        perf-stat.i.ipc
      0.46            +3.0%       0.47        perf-stat.i.metric.GHz
    152.98          +243.5%     525.55        perf-stat.i.metric.K/sec
    129.18 ±  2%     +11.1%     143.54        perf-stat.i.metric.M/sec
      4.45 ±  6%      -1.3        3.15 ±  9%  perf-stat.i.node-load-miss-rate%
    165920 ±  5%     +62.8%     270176 ± 10%  perf-stat.i.node-load-misses
   3567118 ±  3%    +133.8%    8340048        perf-stat.i.node-loads
      1.29 ± 11%      -0.8        0.53 ± 13%  perf-stat.i.node-store-miss-rate%
    100988 ± 10%    +105.0%     207062 ± 13%  perf-stat.i.node-store-misses
   7747689 ±  2%    +410.5%   39548289        perf-stat.i.node-stores
     26.93 ±  6%      -8.8%      24.56 ±  2%  perf-stat.overall.MPKI
      1.39            +0.1        1.53        perf-stat.overall.branch-miss-rate%
      6.32 ±  6%     +14.8       21.14 ±  2%  perf-stat.overall.cache-miss-rate%
      2.36 ±  2%      -8.7%       2.16        perf-stat.overall.cpi
      1395 ±  3%     -70.2%     415.55        perf-stat.overall.cycles-between-cache-misses
      0.09 ±  2%      -0.0        0.07        perf-stat.overall.dTLB-load-miss-rate%
     47.64 ±  2%     -11.2       36.47        perf-stat.overall.iTLB-load-miss-rate%
      5223           +23.4%       6446        perf-stat.overall.instructions-per-iTLB-miss
      0.42 ±  2%      +9.5%       0.46        perf-stat.overall.ipc
      4.45 ±  6%      -1.3        3.14 ±  9%  perf-stat.overall.node-load-miss-rate%
      1.29 ± 11%      -0.8        0.52 ± 13%  perf-stat.overall.node-store-miss-rate%
   5273317           +33.4%    7034229        perf-stat.overall.path-length
   3.9e+09 ±  2%      +9.7%  4.277e+09        perf-stat.ps.branch-instructions
  54319018           +20.1%   65250768        perf-stat.ps.branch-misses
  33859327 ±  3%    +245.6%   1.17e+08        perf-stat.ps.cache-misses
      1916           +53.1%       2933        perf-stat.ps.context-switches
  4.72e+10            +3.0%  4.862e+10        perf-stat.ps.cpu-cycles
    124.04          +149.0%     308.91        perf-stat.ps.cpu-migrations
   4962883 ±  3%     -10.0%    4468932        perf-stat.ps.dTLB-load-misses
 5.477e+09 ±  2%     +11.3%  6.098e+09        perf-stat.ps.dTLB-loads
 3.475e+09 ±  2%     +13.6%   3.95e+09        perf-stat.ps.dTLB-stores
   3826238 ±  2%      -8.6%    3496835        perf-stat.ps.iTLB-load-misses
   4207943 ±  4%     +44.8%    6091870        perf-stat.ps.iTLB-loads
 1.999e+10 ±  2%     +12.8%  2.254e+10        perf-stat.ps.instructions
    165300 ±  5%     +62.8%     269153 ± 10%  perf-stat.ps.node-load-misses
   3555089 ±  3%    +133.8%    8312313        perf-stat.ps.node-loads
    100616 ± 10%    +105.1%     206337 ± 13%  perf-stat.ps.node-store-misses
   7721701 ±  2%    +410.5%   39418013        perf-stat.ps.node-stores
 6.033e+12 ±  2%     +12.7%  6.802e+12        perf-stat.total.instructions
      0.10 ± 33%    +476.0%       0.57 ±  8%  perf-sched.sch_delay.avg.ms.smpboot_thread_fn.kthread.ret_from_fork
      3.94 ± 15%    +142.4%       9.55 ± 30%  perf-sched.sch_delay.max.ms.smpboot_thread_fn.kthread.ret_from_fork
    147.35 ±  2%     -30.4%     102.60 ±  3%  perf-sched.total_wait_and_delay.average.ms
      6589           +55.3%      10232        perf-sched.total_wait_and_delay.count.ms
    147.30 ±  2%     -30.5%     102.37 ±  3%  perf-sched.total_wait_time.average.ms
      0.45 ± 21%    +166.5%       1.20 ± 10%  perf-sched.wait_and_delay.avg.ms.__cond_resched.dput.do_unlinkat.__x64_sys_unlink.do_syscall_64
      0.34 ± 16%    +262.0%       1.24 ±  3%  perf-sched.wait_and_delay.avg.ms.__cond_resched.dput.open_last_lookups.path_openat.do_filp_open
      0.03 ±223%   +3401.6%       1.11 ± 10%  perf-sched.wait_and_delay.avg.ms.__cond_resched.kmem_cache_alloc.alloc_empty_file.path_openat.do_filp_open
      0.06 ±223%   +1645.0%       1.13 ± 13%  perf-sched.wait_and_delay.avg.ms.__cond_resched.mnt_want_write.do_unlinkat.__x64_sys_unlink.do_syscall_64
      0.06 ±223%   +1863.7%       1.11 ±  3%  perf-sched.wait_and_delay.avg.ms.__cond_resched.mnt_want_write.open_last_lookups.path_openat.do_filp_open
      0.32 ± 18%    +255.2%       1.13 ±  5%  perf-sched.wait_and_delay.avg.ms.__cond_resched.shmem_undo_range.shmem_evict_inode.evict.do_unlinkat
      0.35 ± 50%    +247.3%       1.21 ±  9%  perf-sched.wait_and_delay.avg.ms.__cond_resched.slab_pre_alloc_hook.constprop.0.kmem_cache_alloc_lru
      0.31 ± 39%    +275.2%       1.16 ± 10%  perf-sched.wait_and_delay.avg.ms.exit_to_user_mode_loop.exit_to_user_mode_prepare.irqentry_exit_to_user_mode.asm_sysvec_apic_timer_interrupt
      0.31 ± 25%    +268.5%       1.15 ±  7%  perf-sched.wait_and_delay.avg.ms.exit_to_user_mode_loop.exit_to_user_mode_prepare.syscall_exit_to_user_mode.do_syscall_64
      3.73           -15.7%       3.15 ±  5%  perf-sched.wait_and_delay.avg.ms.schedule_timeout.rcu_gp_fqs_loop.rcu_gp_kthread.kthread
    521.10 ±  5%     -56.7%     225.46 ±  4%  perf-sched.wait_and_delay.avg.ms.smpboot_thread_fn.kthread.ret_from_fork
     44.83 ±  9%    +258.7%     160.83 ±  8%  perf-sched.wait_and_delay.count.__cond_resched.dput.do_unlinkat.__x64_sys_unlink.do_syscall_64
    107.17 ± 17%     +45.3%     155.67 ±  4%  perf-sched.wait_and_delay.count.__cond_resched.dput.open_last_lookups.path_openat.do_filp_open
      5.33 ±223%   +2481.2%     137.67 ± 13%  perf-sched.wait_and_delay.count.__cond_resched.kmem_cache_alloc.alloc_empty_file.path_openat.do_filp_open
      5.50 ±223%   +1233.3%      73.33 ±  6%  perf-sched.wait_and_delay.count.__cond_resched.mnt_want_write.do_unlinkat.__x64_sys_unlink.do_syscall_64
      5.83 ±223%   +1091.4%      69.50 ± 10%  perf-sched.wait_and_delay.count.__cond_resched.mnt_want_write.open_last_lookups.path_openat.do_filp_open
     93.67 ± 24%    +305.3%     379.67 ±  4%  perf-sched.wait_and_delay.count.__cond_resched.shmem_undo_range.shmem_evict_inode.evict.do_unlinkat
     33.00 ± 48%    +296.0%     130.67 ±  7%  perf-sched.wait_and_delay.count.__cond_resched.slab_pre_alloc_hook.constprop.0.kmem_cache_alloc_lru
     42.17 ± 12%    +308.3%     172.17 ±  6%  perf-sched.wait_and_delay.count.exit_to_user_mode_loop.exit_to_user_mode_prepare.irqentry_exit_to_user_mode.asm_sysvec_apic_timer_interrupt
     44.17 ± 10%    +395.5%     218.83 ± 10%  perf-sched.wait_and_delay.count.exit_to_user_mode_loop.exit_to_user_mode_prepare.syscall_exit_to_user_mode.do_syscall_64
      1336           +18.8%       1587 ±  6%  perf-sched.wait_and_delay.count.schedule_timeout.rcu_gp_fqs_loop.rcu_gp_kthread.kthread
      1153 ±  3%    +150.7%       2890 ±  2%  perf-sched.wait_and_delay.count.smpboot_thread_fn.kthread.ret_from_fork
      2.77 ± 10%    +152.0%       6.98 ±101%  perf-sched.wait_and_delay.max.ms.__cond_resched.dput.do_unlinkat.__x64_sys_unlink.do_syscall_64
      2.58 ± 14%    +281.7%       9.86 ± 89%  perf-sched.wait_and_delay.max.ms.__cond_resched.dput.open_last_lookups.path_openat.do_filp_open
      0.22 ±223%   +1611.5%       3.77 ±  3%  perf-sched.wait_and_delay.max.ms.__cond_resched.kmem_cache_alloc.alloc_empty_file.path_openat.do_filp_open
      0.44 ±223%    +765.1%       3.79 ±  2%  perf-sched.wait_and_delay.max.ms.__cond_resched.mnt_want_write.do_unlinkat.__x64_sys_unlink.do_syscall_64
      0.42 ±223%    +791.4%       3.75 ±  7%  perf-sched.wait_and_delay.max.ms.__cond_resched.mnt_want_write.open_last_lookups.path_openat.do_filp_open
      2.61 ± 13%    +124.3%       5.85 ± 68%  perf-sched.wait_and_delay.max.ms.__cond_resched.shmem_undo_range.shmem_evict_inode.evict.do_unlinkat
      1.82 ± 46%    +301.8%       7.30 ±106%  perf-sched.wait_and_delay.max.ms.__cond_resched.slab_pre_alloc_hook.constprop.0.kmem_cache_alloc_lru
      2.11 ± 37%     +84.4%       3.89 ±  2%  perf-sched.wait_and_delay.max.ms.exit_to_user_mode_loop.exit_to_user_mode_prepare.irqentry_exit_to_user_mode.asm_sysvec_apic_timer_interrupt
      2.68 ± 52%     -40.4%       1.59 ±  3%  perf-sched.wait_and_delay.max.ms.schedule_timeout.__wait_for_common.wait_for_completion_state.kernel_clone
      0.05 ±158%   +2025.6%       0.97 ± 44%  perf-sched.wait_time.avg.ms.__cond_resched.__fput.task_work_run.exit_to_user_mode_loop.exit_to_user_mode_prepare
      0.28 ±119%    +202.7%       0.85 ± 38%  perf-sched.wait_time.avg.ms.__cond_resched.apparmor_file_alloc_security.security_file_alloc.init_file.alloc_empty_file
      0.39 ± 85%    +249.6%       1.35 ± 41%  perf-sched.wait_time.avg.ms.__cond_resched.dentry_kill.dput.do_unlinkat.__x64_sys_unlink
      0.41 ± 69%    +203.9%       1.25 ± 30%  perf-sched.wait_time.avg.ms.__cond_resched.down_write.do_unlinkat.__x64_sys_unlink.do_syscall_64
      0.26 ± 72%    +331.6%       1.14 ± 15%  perf-sched.wait_time.avg.ms.__cond_resched.down_write.open_last_lookups.path_openat.do_filp_open
      0.37 ± 60%    +265.4%       1.34 ± 18%  perf-sched.wait_time.avg.ms.__cond_resched.down_write.vfs_unlink.do_unlinkat.__x64_sys_unlink
      0.32 ± 54%    +286.7%       1.23 ± 44%  perf-sched.wait_time.avg.ms.__cond_resched.dput.__fput.task_work_run.exit_to_user_mode_loop
      0.45 ± 21%    +166.5%       1.20 ± 10%  perf-sched.wait_time.avg.ms.__cond_resched.dput.do_unlinkat.__x64_sys_unlink.do_syscall_64
      0.34 ± 16%    +262.0%       1.24 ±  3%  perf-sched.wait_time.avg.ms.__cond_resched.dput.open_last_lookups.path_openat.do_filp_open
      0.30 ± 46%    +308.9%       1.21 ±  8%  perf-sched.wait_time.avg.ms.__cond_resched.dput.shmem_unlink.vfs_unlink.do_unlinkat
      0.48 ± 33%    +152.9%       1.22 ±  6%  perf-sched.wait_time.avg.ms.__cond_resched.dput.terminate_walk.path_openat.do_filp_open
      0.39 ± 47%    +186.6%       1.11 ± 10%  perf-sched.wait_time.avg.ms.__cond_resched.kmem_cache_alloc.alloc_empty_file.path_openat.do_filp_open
      0.35 ± 32%    +247.3%       1.21 ±  7%  perf-sched.wait_time.avg.ms.__cond_resched.kmem_cache_alloc.getname_flags.part.0
      0.36 ± 46%    +205.4%       1.11 ± 21%  perf-sched.wait_time.avg.ms.__cond_resched.kmem_cache_alloc.security_file_alloc.init_file.alloc_empty_file
      0.37 ± 22%    +202.2%       1.13 ± 13%  perf-sched.wait_time.avg.ms.__cond_resched.mnt_want_write.do_unlinkat.__x64_sys_unlink.do_syscall_64
      0.29 ± 31%    +284.1%       1.11 ±  3%  perf-sched.wait_time.avg.ms.__cond_resched.mnt_want_write.open_last_lookups.path_openat.do_filp_open
      0.32 ± 18%    +255.2%       1.13 ±  5%  perf-sched.wait_time.avg.ms.__cond_resched.shmem_undo_range.shmem_evict_inode.evict.do_unlinkat
      0.42 ± 20%    +187.5%       1.21 ±  9%  perf-sched.wait_time.avg.ms.__cond_resched.slab_pre_alloc_hook.constprop.0.kmem_cache_alloc_lru
      0.33 ± 85%    +228.8%       1.10 ± 30%  perf-sched.wait_time.avg.ms.__cond_resched.task_work_run.exit_to_user_mode_loop.exit_to_user_mode_prepare.syscall_exit_to_user_mode
      0.31 ± 39%    +275.1%       1.16 ± 10%  perf-sched.wait_time.avg.ms.exit_to_user_mode_loop.exit_to_user_mode_prepare.irqentry_exit_to_user_mode.asm_sysvec_apic_timer_interrupt
      0.31 ± 26%    +269.6%       1.15 ±  7%  perf-sched.wait_time.avg.ms.exit_to_user_mode_loop.exit_to_user_mode_prepare.syscall_exit_to_user_mode.do_syscall_64
      3.72           -15.9%       3.13 ±  5%  perf-sched.wait_time.avg.ms.schedule_timeout.rcu_gp_fqs_loop.rcu_gp_kthread.kthread
    521.00 ±  5%     -56.8%     224.89 ±  4%  perf-sched.wait_time.avg.ms.smpboot_thread_fn.kthread.ret_from_fork
      0.11 ±192%   +2041.9%       2.34 ± 50%  perf-sched.wait_time.max.ms.__cond_resched.__fput.task_work_run.exit_to_user_mode_loop.exit_to_user_mode_prepare
      0.39 ±131%    +373.0%       1.86 ± 35%  perf-sched.wait_time.max.ms.__cond_resched.apparmor_file_alloc_security.security_file_alloc.init_file.alloc_empty_file
      0.85 ± 86%    +414.9%       4.38 ± 65%  perf-sched.wait_time.max.ms.__cond_resched.dentry_kill.dput.do_unlinkat.__x64_sys_unlink
      1.10 ± 56%    +221.1%       3.54 ±  8%  perf-sched.wait_time.max.ms.__cond_resched.down_write.do_unlinkat.__x64_sys_unlink.do_syscall_64
      0.79 ± 96%    +270.2%       2.94 ± 29%  perf-sched.wait_time.max.ms.__cond_resched.down_write.open_last_lookups.path_openat.do_filp_open
      1.36 ± 41%    +181.2%       3.83 ±  2%  perf-sched.wait_time.max.ms.__cond_resched.down_write.vfs_unlink.do_unlinkat.__x64_sys_unlink
      0.97 ± 61%    +516.1%       5.98 ±104%  perf-sched.wait_time.max.ms.__cond_resched.dput.__fput.task_work_run.exit_to_user_mode_loop
      2.77 ± 10%    +152.0%       6.98 ±101%  perf-sched.wait_time.max.ms.__cond_resched.dput.do_unlinkat.__x64_sys_unlink.do_syscall_64
      2.58 ± 14%    +281.7%       9.86 ± 89%  perf-sched.wait_time.max.ms.__cond_resched.dput.open_last_lookups.path_openat.do_filp_open
      1.25 ± 44%    +457.7%       6.95 ±102%  perf-sched.wait_time.max.ms.__cond_resched.dput.shmem_unlink.vfs_unlink.do_unlinkat
      2.38 ± 30%    +125.8%       5.38 ± 71%  perf-sched.wait_time.max.ms.__cond_resched.dput.terminate_walk.path_openat.do_filp_open
      2.08 ± 25%     +81.3%       3.77 ±  3%  perf-sched.wait_time.max.ms.__cond_resched.kmem_cache_alloc.alloc_empty_file.path_openat.do_filp_open
      1.80 ± 22%    +102.3%       3.65 ±  3%  perf-sched.wait_time.max.ms.__cond_resched.kmem_cache_alloc.getname_flags.part.0
      1.62 ± 63%    +116.0%       3.50 ± 10%  perf-sched.wait_time.max.ms.__cond_resched.kmem_cache_alloc.security_file_alloc.init_file.alloc_empty_file
      1.97 ± 28%     +92.0%       3.79 ±  2%  perf-sched.wait_time.max.ms.__cond_resched.mnt_want_write.do_unlinkat.__x64_sys_unlink.do_syscall_64
      2.15 ± 37%     +74.3%       3.75 ±  7%  perf-sched.wait_time.max.ms.__cond_resched.mnt_want_write.open_last_lookups.path_openat.do_filp_open
      2.61 ± 13%    +124.3%       5.85 ± 68%  perf-sched.wait_time.max.ms.__cond_resched.shmem_undo_range.shmem_evict_inode.evict.do_unlinkat
      2.14 ± 11%    +241.1%       7.30 ±106%  perf-sched.wait_time.max.ms.__cond_resched.slab_pre_alloc_hook.constprop.0.kmem_cache_alloc_lru
      1.05 ± 92%    +213.1%       3.29 ± 13%  perf-sched.wait_time.max.ms.__cond_resched.task_work_run.exit_to_user_mode_loop.exit_to_user_mode_prepare.syscall_exit_to_user_mode
      2.11 ± 37%     +84.4%       3.89 ±  2%  perf-sched.wait_time.max.ms.exit_to_user_mode_loop.exit_to_user_mode_prepare.irqentry_exit_to_user_mode.asm_sysvec_apic_timer_interrupt
      2.61 ± 54%     -39.4%       1.58 ±  3%  perf-sched.wait_time.max.ms.schedule_timeout.__wait_for_common.wait_for_completion_state.kernel_clone
     26.45 ±  2%      -2.1       24.32        perf-profile.calltrace.cycles-pp.unlink
     12.59 ±  8%      -2.1       10.49 ±  2%  perf-profile.calltrace.cycles-pp.shmem_get_inode.shmem_mknod.lookup_open.open_last_lookups.path_openat
     24.50 ±  2%      -1.9       22.62        perf-profile.calltrace.cycles-pp.entry_SYSCALL_64_after_hwframe.unlink
     23.89 ±  2%      -1.8       22.10        perf-profile.calltrace.cycles-pp.do_syscall_64.entry_SYSCALL_64_after_hwframe.unlink
      9.52 ± 11%      -1.7        7.85 ±  3%  perf-profile.calltrace.cycles-pp.new_inode.shmem_get_inode.shmem_mknod.lookup_open.open_last_lookups
     22.73 ±  2%      -1.6       21.09        perf-profile.calltrace.cycles-pp.__x64_sys_unlink.do_syscall_64.entry_SYSCALL_64_after_hwframe.unlink
     21.93 ±  2%      -1.5       20.40        perf-profile.calltrace.cycles-pp.do_unlinkat.__x64_sys_unlink.do_syscall_64.entry_SYSCALL_64_after_hwframe.unlink
     27.94 ±  2%      -1.3       26.65        perf-profile.calltrace.cycles-pp.intel_idle_ibrs.cpuidle_enter_state.cpuidle_enter.cpuidle_idle_call.do_idle
      8.02 ±  2%      -1.0        7.06 ±  4%  perf-profile.calltrace.cycles-pp.__close
      3.26 ± 11%      -0.8        2.50 ±  2%  perf-profile.calltrace.cycles-pp.do_open.path_openat.do_filp_open.do_sys_openat2.__x64_sys_openat
      6.07 ±  4%      -0.7        5.38 ±  5%  perf-profile.calltrace.cycles-pp.entry_SYSCALL_64_after_hwframe.__close
      2.66 ± 11%      -0.6        2.04 ±  2%  perf-profile.calltrace.cycles-pp.do_dentry_open.do_open.path_openat.do_filp_open.do_sys_openat2
      2.90            -0.5        2.37        perf-profile.calltrace.cycles-pp.syscall_exit_to_user_mode.do_syscall_64.entry_SYSCALL_64_after_hwframe.__close
      2.25 ±  3%      -0.5        1.75 ±  2%  perf-profile.calltrace.cycles-pp.path_parentat.__filename_parentat.do_unlinkat.__x64_sys_unlink.do_syscall_64
      2.40 ±  3%      -0.4        1.95 ±  2%  perf-profile.calltrace.cycles-pp.__filename_parentat.do_unlinkat.__x64_sys_unlink.do_syscall_64.entry_SYSCALL_64_after_hwframe
      1.76 ±  6%      -0.4        1.34 ±  2%  perf-profile.calltrace.cycles-pp.link_path_walk.path_openat.do_filp_open.do_sys_openat2.__x64_sys_openat
      2.60 ±  7%      -0.4        2.20 ±  2%  perf-profile.calltrace.cycles-pp.alloc_inode.new_inode.shmem_get_inode.shmem_mknod.lookup_open
      1.95 ±  3%      -0.4        1.57        perf-profile.calltrace.cycles-pp.exit_to_user_mode_prepare.syscall_exit_to_user_mode.do_syscall_64.entry_SYSCALL_64_after_hwframe.__close
      1.76 ±  7%      -0.4        1.38 ±  3%  perf-profile.calltrace.cycles-pp.link_path_walk.path_parentat.__filename_parentat.do_unlinkat.__x64_sys_unlink
      1.90 ±  3%      -0.4        1.54        perf-profile.calltrace.cycles-pp.exit_to_user_mode_loop.exit_to_user_mode_prepare.syscall_exit_to_user_mode.do_syscall_64.entry_SYSCALL_64_after_hwframe
      1.38 ± 15%      -0.3        1.04 ±  3%  perf-profile.calltrace.cycles-pp.destroy_inode.do_unlinkat.__x64_sys_unlink.do_syscall_64.entry_SYSCALL_64_after_hwframe
      1.77 ±  4%      -0.3        1.42        perf-profile.calltrace.cycles-pp.task_work_run.exit_to_user_mode_loop.exit_to_user_mode_prepare.syscall_exit_to_user_mode.do_syscall_64
      0.68 ± 11%      -0.3        0.35 ± 70%  perf-profile.calltrace.cycles-pp.apparmor_path_unlink.security_path_unlink.do_unlinkat.__x64_sys_unlink.do_syscall_64
      0.67 ± 17%      -0.3        0.34 ± 70%  perf-profile.calltrace.cycles-pp.may_delete.vfs_unlink.do_unlinkat.__x64_sys_unlink.do_syscall_64
      1.18 ±  9%      -0.3        0.86 ±  4%  perf-profile.calltrace.cycles-pp.__destroy_inode.destroy_inode.do_unlinkat.__x64_sys_unlink.do_syscall_64
      1.06 ±  9%      -0.3        0.75 ±  3%  perf-profile.calltrace.cycles-pp.inode_permission.link_path_walk.path_openat.do_filp_open.do_sys_openat2
      1.00 ±  9%      -0.3        0.69 ±  3%  perf-profile.calltrace.cycles-pp.generic_permission.inode_permission.link_path_walk.path_openat.do_filp_open
      1.45 ±  5%      -0.3        1.15 ±  2%  perf-profile.calltrace.cycles-pp.__fput.task_work_run.exit_to_user_mode_loop.exit_to_user_mode_prepare.syscall_exit_to_user_mode
      1.12 ± 11%      -0.3        0.86 ±  2%  perf-profile.calltrace.cycles-pp.inode_permission.link_path_walk.path_parentat.__filename_parentat.do_unlinkat
      1.06 ± 12%      -0.2        0.81 ±  3%  perf-profile.calltrace.cycles-pp.generic_permission.inode_permission.link_path_walk.path_parentat.__filename_parentat
      1.24 ±  2%      -0.2        1.00        perf-profile.calltrace.cycles-pp.syscall_exit_to_user_mode.do_syscall_64.entry_SYSCALL_64_after_hwframe.__open64
      0.93 ± 11%      -0.2        0.70        perf-profile.calltrace.cycles-pp.inode_wait_for_writeback.evict.do_unlinkat.__x64_sys_unlink.do_syscall_64
      0.90 ± 11%      -0.2        0.67 ±  2%  perf-profile.calltrace.cycles-pp._raw_spin_lock.inode_wait_for_writeback.evict.do_unlinkat.__x64_sys_unlink
      1.66 ±  2%      -0.2        1.44 ±  2%  perf-profile.calltrace.cycles-pp.shmem_alloc_inode.alloc_inode.new_inode.shmem_get_inode.shmem_mknod
      1.64 ±  3%      -0.2        1.42 ±  2%  perf-profile.calltrace.cycles-pp.kmem_cache_alloc_lru.shmem_alloc_inode.alloc_inode.new_inode.shmem_get_inode
      0.63 ±  4%      -0.2        0.43 ± 44%  perf-profile.calltrace.cycles-pp.simple_lookup.lookup_open.open_last_lookups.path_openat.do_filp_open
      2.02 ±  4%      -0.2        1.82 ±  2%  perf-profile.calltrace.cycles-pp.alloc_empty_file.path_openat.do_filp_open.do_sys_openat2.__x64_sys_openat
      1.12 ±  4%      -0.2        0.92        perf-profile.calltrace.cycles-pp.syscall_return_via_sysret.__close
      1.14 ±  4%      -0.2        0.96        perf-profile.calltrace.cycles-pp.syscall_return_via_sysret.__open64
      1.12 ±  4%      -0.2        0.94 ±  4%  perf-profile.calltrace.cycles-pp.syscall_return_via_sysret.unlink
      0.69 ± 12%      -0.2        0.52 ±  3%  perf-profile.calltrace.cycles-pp.security_path_unlink.do_unlinkat.__x64_sys_unlink.do_syscall_64.entry_SYSCALL_64_after_hwframe
      1.02 ±  3%      -0.2        0.86 ±  3%  perf-profile.calltrace.cycles-pp.syscall_exit_to_user_mode.do_syscall_64.entry_SYSCALL_64_after_hwframe.unlink
      1.23 ±  6%      -0.1        1.10 ±  4%  perf-profile.calltrace.cycles-pp.init_file.alloc_empty_file.path_openat.do_filp_open.do_sys_openat2
      1.13 ±  5%      -0.1        1.00        perf-profile.calltrace.cycles-pp.__d_alloc.d_alloc.d_alloc_parallel.lookup_open.open_last_lookups
      1.48 ±  3%      -0.1        1.36        perf-profile.calltrace.cycles-pp.d_alloc_parallel.lookup_open.open_last_lookups.path_openat.do_filp_open
      0.87 ±  6%      -0.1        0.76 ±  5%  perf-profile.calltrace.cycles-pp.try_to_unlazy.open_last_lookups.path_openat.do_filp_open.do_sys_openat2
      0.82 ±  6%      -0.1        0.72 ±  5%  perf-profile.calltrace.cycles-pp.__legitimize_path.try_to_unlazy.open_last_lookups.path_openat.do_filp_open
      0.71 ±  4%      -0.1        0.61 ±  5%  perf-profile.calltrace.cycles-pp.shmem_undo_range.shmem_evict_inode.evict.do_unlinkat.__x64_sys_unlink
      0.66 ±  9%      -0.1        0.56 ±  6%  perf-profile.calltrace.cycles-pp.security_file_alloc.init_file.alloc_empty_file.path_openat.do_filp_open
      0.71 ±  3%      -0.1        0.62 ±  4%  perf-profile.calltrace.cycles-pp.getname_flags.__x64_sys_unlink.do_syscall_64.entry_SYSCALL_64_after_hwframe.unlink
      0.71 ±  4%      -0.1        0.61 ±  3%  perf-profile.calltrace.cycles-pp.allocate_slab.___slab_alloc.kmem_cache_alloc_lru.shmem_alloc_inode.alloc_inode
      0.66 ±  4%      -0.1        0.57 ±  4%  perf-profile.calltrace.cycles-pp.slab_pre_alloc_hook.kmem_cache_alloc_lru.shmem_alloc_inode.alloc_inode.new_inode
      0.72 ±  4%      -0.1        0.63 ±  3%  perf-profile.calltrace.cycles-pp.___slab_alloc.kmem_cache_alloc_lru.shmem_alloc_inode.alloc_inode.new_inode
      0.67 ±  4%      -0.1        0.57 ±  3%  perf-profile.calltrace.cycles-pp.shuffle_freelist.allocate_slab.___slab_alloc.kmem_cache_alloc_lru.shmem_alloc_inode
      0.81 ±  7%      -0.1        0.72 ±  2%  perf-profile.calltrace.cycles-pp.terminate_walk.path_openat.do_filp_open.do_sys_openat2.__x64_sys_openat
      0.69 ±  5%      -0.1        0.60 ±  2%  perf-profile.calltrace.cycles-pp.getname_flags.do_sys_openat2.__x64_sys_openat.do_syscall_64.entry_SYSCALL_64_after_hwframe
      0.62 ±  5%      -0.1        0.53 ±  3%  perf-profile.calltrace.cycles-pp.inode_init_once.shuffle_freelist.allocate_slab.___slab_alloc.kmem_cache_alloc_lru
      0.68 ±  3%      -0.1        0.60 ±  2%  perf-profile.calltrace.cycles-pp.dput.do_unlinkat.__x64_sys_unlink.do_syscall_64.entry_SYSCALL_64_after_hwframe
      0.79 ±  2%      -0.1        0.73 ±  2%  perf-profile.calltrace.cycles-pp.kmem_cache_alloc_lru.__d_alloc.d_alloc.d_alloc_parallel.lookup_open
      0.92 ±  5%      +0.2        1.13 ±  6%  perf-profile.calltrace.cycles-pp._raw_spin_lock.shmem_evict_inode.evict.do_unlinkat.__x64_sys_unlink
      0.77 ±  6%      +0.2        1.00 ±  6%  perf-profile.calltrace.cycles-pp.native_queued_spin_lock_slowpath._raw_spin_lock.shmem_evict_inode.evict.do_unlinkat
      0.27 ±100%      +0.6        0.84 ±  4%  perf-profile.calltrace.cycles-pp.asm_sysvec_apic_timer_interrupt.native_queued_spin_lock_slowpath._raw_spin_lock.inode_sb_list_add.new_inode
      0.27 ±100%      +0.6        0.84 ±  4%  perf-profile.calltrace.cycles-pp.sysvec_apic_timer_interrupt.asm_sysvec_apic_timer_interrupt.native_queued_spin_lock_slowpath._raw_spin_lock.inode_sb_list_add
      1.05 ±  7%      +0.6        1.63 ±  9%  perf-profile.calltrace.cycles-pp.sysvec_apic_timer_interrupt.asm_sysvec_apic_timer_interrupt.cpuidle_enter_state.cpuidle_enter.cpuidle_idle_call
      1.13 ±  7%      +0.6        1.72 ±  9%  perf-profile.calltrace.cycles-pp.asm_sysvec_apic_timer_interrupt.cpuidle_enter_state.cpuidle_enter.cpuidle_idle_call.do_idle
      0.00            +0.6        0.62 ± 13%  perf-profile.calltrace.cycles-pp.__slab_free.rcu_do_batch.rcu_core.__do_softirq.__irq_exit_rcu
      0.00            +0.7        0.65 ±  4%  perf-profile.calltrace.cycles-pp.kmem_cache_free.rcu_do_batch.rcu_core.__do_softirq.run_ksoftirqd
      0.00            +0.7        0.67 ±  6%  perf-profile.calltrace.cycles-pp.__do_softirq.__irq_exit_rcu.sysvec_apic_timer_interrupt.asm_sysvec_apic_timer_interrupt.cpuidle_enter_state
      4.07 ± 12%      +0.7        4.76        perf-profile.calltrace.cycles-pp.vfs_unlink.do_unlinkat.__x64_sys_unlink.do_syscall_64.entry_SYSCALL_64_after_hwframe
      0.00            +0.7        0.70 ±  7%  perf-profile.calltrace.cycles-pp.__irq_exit_rcu.sysvec_apic_timer_interrupt.asm_sysvec_apic_timer_interrupt.cpuidle_enter_state.cpuidle_enter
      0.20 ±141%      +1.0        1.16 ± 10%  perf-profile.calltrace.cycles-pp.asm_sysvec_apic_timer_interrupt.native_queued_spin_lock_slowpath._raw_spin_lock.evict.do_unlinkat
      0.20 ±141%      +1.0        1.16 ± 10%  perf-profile.calltrace.cycles-pp.sysvec_apic_timer_interrupt.asm_sysvec_apic_timer_interrupt.native_queued_spin_lock_slowpath._raw_spin_lock.evict
      0.00            +1.0        0.99 ±  2%  perf-profile.calltrace.cycles-pp.__call_rcu_common.xas_store.__xa_erase.xa_erase.simple_offset_remove
      0.00            +1.3        1.32        perf-profile.calltrace.cycles-pp.allocate_slab.___slab_alloc.kmem_cache_alloc_lru.xas_alloc.xas_create
      1.41 ± 10%      +1.3        2.74        perf-profile.calltrace.cycles-pp.shmem_unlink.vfs_unlink.do_unlinkat.__x64_sys_unlink.do_syscall_64
      0.00            +1.4        1.36        perf-profile.calltrace.cycles-pp.___slab_alloc.kmem_cache_alloc_lru.xas_alloc.xas_create.xas_store
      0.00            +1.5        1.53 ±  3%  perf-profile.calltrace.cycles-pp.xas_store.__xa_erase.xa_erase.simple_offset_remove.shmem_unlink
      0.00            +1.5        1.55 ±  2%  perf-profile.calltrace.cycles-pp.__xa_erase.xa_erase.simple_offset_remove.shmem_unlink.vfs_unlink
      0.37 ±112%      +1.6        1.95 ±  7%  perf-profile.calltrace.cycles-pp.__irq_exit_rcu.sysvec_apic_timer_interrupt.asm_sysvec_apic_timer_interrupt.native_queued_spin_lock_slowpath._raw_spin_lock
      0.37 ±112%      +1.6        1.95 ±  7%  perf-profile.calltrace.cycles-pp.__do_softirq.__irq_exit_rcu.sysvec_apic_timer_interrupt.asm_sysvec_apic_timer_interrupt.native_queued_spin_lock_slowpath
      0.00            +1.6        1.61 ±  2%  perf-profile.calltrace.cycles-pp.xa_erase.simple_offset_remove.shmem_unlink.vfs_unlink.do_unlinkat
      0.00            +1.6        1.63 ±  2%  perf-profile.calltrace.cycles-pp.simple_offset_remove.shmem_unlink.vfs_unlink.do_unlinkat.__x64_sys_unlink
      0.00            +1.7        1.68 ±  4%  perf-profile.calltrace.cycles-pp.__slab_free.rcu_do_batch.rcu_core.__do_softirq.run_ksoftirqd
      0.00            +1.7        1.71        perf-profile.calltrace.cycles-pp.kmem_cache_alloc_lru.xas_alloc.xas_create.xas_store.__xa_alloc
      0.00            +1.8        1.82        perf-profile.calltrace.cycles-pp.xas_alloc.xas_create.xas_store.__xa_alloc.__xa_alloc_cyclic
      0.37 ±112%      +1.9        2.22 ± 13%  perf-profile.calltrace.cycles-pp.rcu_do_batch.rcu_core.__do_softirq.__irq_exit_rcu.sysvec_apic_timer_interrupt
      0.37 ±112%      +1.9        2.23 ± 13%  perf-profile.calltrace.cycles-pp.rcu_core.__do_softirq.__irq_exit_rcu.sysvec_apic_timer_interrupt.asm_sysvec_apic_timer_interrupt
      0.00            +2.1        2.10        perf-profile.calltrace.cycles-pp.allocate_slab.___slab_alloc.kmem_cache_alloc_lru.xas_alloc.xas_expand
      0.00            +2.2        2.16        perf-profile.calltrace.cycles-pp.___slab_alloc.kmem_cache_alloc_lru.xas_alloc.xas_expand.xas_create
     19.64 ±  3%      +2.4       22.08        perf-profile.calltrace.cycles-pp.open_last_lookups.path_openat.do_filp_open.do_sys_openat2.__x64_sys_openat
      0.00            +2.7        2.68        perf-profile.calltrace.cycles-pp.kmem_cache_alloc_lru.xas_alloc.xas_expand.xas_create.xas_store
      0.00            +2.8        2.84        perf-profile.calltrace.cycles-pp.xas_alloc.xas_expand.xas_create.xas_store.__xa_alloc
      0.00            +3.0        2.98        perf-profile.calltrace.cycles-pp.radix_tree_node_ctor.shuffle_freelist.allocate_slab.___slab_alloc.kmem_cache_alloc_lru
     17.20 ±  4%      +3.0       20.21        perf-profile.calltrace.cycles-pp.lookup_open.open_last_lookups.path_openat.do_filp_open.do_sys_openat2
      0.00            +3.2        3.16        perf-profile.calltrace.cycles-pp.xas_expand.xas_create.xas_store.__xa_alloc.__xa_alloc_cyclic
      0.00            +3.3        3.30        perf-profile.calltrace.cycles-pp.shuffle_freelist.allocate_slab.___slab_alloc.kmem_cache_alloc_lru.xas_alloc
      0.00            +3.6        3.59 ±  4%  perf-profile.calltrace.cycles-pp.rcu_do_batch.rcu_core.__do_softirq.run_ksoftirqd.smpboot_thread_fn
      0.00            +3.6        3.60 ±  4%  perf-profile.calltrace.cycles-pp.rcu_core.__do_softirq.run_ksoftirqd.smpboot_thread_fn.kthread
      0.00            +3.6        3.60 ±  4%  perf-profile.calltrace.cycles-pp.__do_softirq.run_ksoftirqd.smpboot_thread_fn.kthread.ret_from_fork
      0.00            +3.6        3.60 ±  4%  perf-profile.calltrace.cycles-pp.run_ksoftirqd.smpboot_thread_fn.kthread.ret_from_fork
      0.00            +3.6        3.62 ±  4%  perf-profile.calltrace.cycles-pp.smpboot_thread_fn.kthread.ret_from_fork
      0.00            +3.7        3.66 ±  4%  perf-profile.calltrace.cycles-pp.ret_from_fork
      0.00            +3.7        3.66 ±  4%  perf-profile.calltrace.cycles-pp.kthread.ret_from_fork
     13.57 ±  6%      +3.7       17.24        perf-profile.calltrace.cycles-pp.shmem_mknod.lookup_open.open_last_lookups.path_openat.do_filp_open
      0.00            +5.3        5.34        perf-profile.calltrace.cycles-pp.xas_create.xas_store.__xa_alloc.__xa_alloc_cyclic.simple_offset_add
      0.00            +5.4        5.43        perf-profile.calltrace.cycles-pp.xas_store.__xa_alloc.__xa_alloc_cyclic.simple_offset_add.shmem_mknod
      0.00            +5.5        5.53        perf-profile.calltrace.cycles-pp.__xa_alloc.__xa_alloc_cyclic.simple_offset_add.shmem_mknod.lookup_open
      0.00            +5.6        5.57        perf-profile.calltrace.cycles-pp.__xa_alloc_cyclic.simple_offset_add.shmem_mknod.lookup_open.open_last_lookups
      0.00            +5.7        5.66        perf-profile.calltrace.cycles-pp.simple_offset_add.shmem_mknod.lookup_open.open_last_lookups.path_openat
     64.05            -2.4       61.65        perf-profile.children.cycles-pp.entry_SYSCALL_64_after_hwframe
     26.54 ±  2%      -2.1       24.39        perf-profile.children.cycles-pp.unlink
     12.60 ±  7%      -2.1       10.50 ±  2%  perf-profile.children.cycles-pp.shmem_get_inode
     62.12            -2.1       60.04        perf-profile.children.cycles-pp.do_syscall_64
      9.52 ± 11%      -1.7        7.85 ±  3%  perf-profile.children.cycles-pp.new_inode
     22.73 ±  2%      -1.6       21.10        perf-profile.children.cycles-pp.__x64_sys_unlink
     21.95 ±  2%      -1.5       20.42        perf-profile.children.cycles-pp.do_unlinkat
     28.01 ±  2%      -1.3       26.71        perf-profile.children.cycles-pp.intel_idle_ibrs
      8.10 ±  2%      -1.0        7.13 ±  4%  perf-profile.children.cycles-pp.__close
      5.19            -0.9        4.25        perf-profile.children.cycles-pp.syscall_exit_to_user_mode
      3.40 ± 14%      -0.9        2.54 ±  2%  perf-profile.children.cycles-pp.inode_permission
      3.54 ±  6%      -0.8        2.74 ±  2%  perf-profile.children.cycles-pp.link_path_walk
      3.26 ± 10%      -0.8        2.50 ±  2%  perf-profile.children.cycles-pp.do_open
      2.71 ± 10%      -0.7        1.98 ±  2%  perf-profile.children.cycles-pp.generic_permission
      2.66 ± 11%      -0.6        2.04 ±  2%  perf-profile.children.cycles-pp.do_dentry_open
      3.43 ±  4%      -0.6        2.86        perf-profile.children.cycles-pp.syscall_return_via_sysret
      2.26 ±  3%      -0.5        1.76 ±  2%  perf-profile.children.cycles-pp.path_parentat
      2.40 ±  3%      -0.4        1.96        perf-profile.children.cycles-pp.__filename_parentat
      2.60 ±  8%      -0.4        2.20 ±  2%  perf-profile.children.cycles-pp.alloc_inode
      1.98 ±  3%      -0.4        1.61        perf-profile.children.cycles-pp.exit_to_user_mode_prepare
      1.92 ±  3%      -0.4        1.55        perf-profile.children.cycles-pp.exit_to_user_mode_loop
      1.36 ± 23%      -0.4        1.00 ±  4%  perf-profile.children.cycles-pp.__mnt_want_write
      1.38 ± 15%      -0.3        1.04 ±  3%  perf-profile.children.cycles-pp.destroy_inode
      1.78 ±  4%      -0.3        1.43        perf-profile.children.cycles-pp.task_work_run
      1.38 ± 20%      -0.3        1.05 ±  2%  perf-profile.children.cycles-pp.mnt_want_write
      1.18 ±  9%      -0.3        0.87 ±  4%  perf-profile.children.cycles-pp.__destroy_inode
      1.46 ±  5%      -0.3        1.16        perf-profile.children.cycles-pp.__fput
      1.65 ±  3%      -0.3        1.38        perf-profile.children.cycles-pp.__entry_text_start
      0.75 ±  2%      -0.2        0.51 ±  2%  perf-profile.children.cycles-pp.file_free_rcu
      0.93 ± 11%      -0.2        0.70 ±  2%  perf-profile.children.cycles-pp.inode_wait_for_writeback
      1.66 ±  2%      -0.2        1.44 ±  2%  perf-profile.children.cycles-pp.shmem_alloc_inode
      0.58 ± 10%      -0.2        0.38 ±  5%  perf-profile.children.cycles-pp.inode_init_owner
      2.02 ±  4%      -0.2        1.82        perf-profile.children.cycles-pp.alloc_empty_file
      1.69 ±  4%      -0.2        1.50 ±  2%  perf-profile.children.cycles-pp.dput
      1.41 ±  4%      -0.2        1.23 ±  2%  perf-profile.children.cycles-pp.getname_flags
      0.52 ± 12%      -0.2        0.34 ±  5%  perf-profile.children.cycles-pp.drop_nlink
      0.69 ± 12%      -0.2        0.52 ±  3%  perf-profile.children.cycles-pp.security_path_unlink
      0.68 ± 11%      -0.2        0.51 ±  3%  perf-profile.children.cycles-pp.apparmor_path_unlink
      0.68 ± 18%      -0.2        0.51 ±  4%  perf-profile.children.cycles-pp.may_delete
      0.56 ±  8%      -0.1        0.41 ± 10%  perf-profile.children.cycles-pp.filp_close
      1.24 ±  6%      -0.1        1.10 ±  4%  perf-profile.children.cycles-pp.init_file
      0.89 ±  3%      -0.1        0.76 ±  2%  perf-profile.children.cycles-pp.inode_maybe_inc_iversion
      0.56 ±  9%      -0.1        0.42 ±  3%  perf-profile.children.cycles-pp.may_open
      0.94 ±  4%      -0.1        0.81 ±  3%  perf-profile.children.cycles-pp.strncpy_from_user
      1.05 ±  5%      -0.1        0.92 ±  4%  perf-profile.children.cycles-pp.try_to_unlazy
      0.98 ±  5%      -0.1        0.85 ±  4%  perf-profile.children.cycles-pp.__legitimize_path
      0.69 ± 10%      -0.1        0.57 ±  5%  perf-profile.children.cycles-pp.iput
      0.64 ±  4%      -0.1        0.52 ±  2%  perf-profile.children.cycles-pp.simple_lookup
      1.48 ±  3%      -0.1        1.36        perf-profile.children.cycles-pp.d_alloc_parallel
      0.38 ± 12%      -0.1        0.26 ± 16%  perf-profile.children.cycles-pp.locks_remove_posix
      0.36 ± 17%      -0.1        0.25 ±  8%  perf-profile.children.cycles-pp.fsnotify_perm
      0.36 ±  8%      -0.1        0.26 ±  6%  perf-profile.children.cycles-pp.security_path_mknod
      0.72 ±  4%      -0.1        0.62 ±  4%  perf-profile.children.cycles-pp.shmem_undo_range
      0.67 ±  9%      -0.1        0.56 ±  6%  perf-profile.children.cycles-pp.security_file_alloc
      0.34 ±  9%      -0.1        0.24 ±  9%  perf-profile.children.cycles-pp.apparmor_path_mknod
      1.34 ±  3%      -0.1        1.25        perf-profile.children.cycles-pp.d_alloc
      0.62 ±  4%      -0.1        0.53 ±  4%  perf-profile.children.cycles-pp.__check_object_size
      0.70 ±  5%      -0.1        0.60 ±  3%  perf-profile.children.cycles-pp.memcg_list_lru_alloc
      0.29 ±  5%      -0.1        0.20 ±  6%  perf-profile.children.cycles-pp._atomic_dec_and_lock
      0.52 ± 10%      -0.1        0.43 ±  5%  perf-profile.children.cycles-pp.apparmor_file_open
      0.87 ±  7%      -0.1        0.78 ±  3%  perf-profile.children.cycles-pp.terminate_walk
      0.54 ± 10%      -0.1        0.45 ±  5%  perf-profile.children.cycles-pp.security_file_open
      0.55 ± 11%      -0.1        0.47 ±  6%  perf-profile.children.cycles-pp.apparmor_file_alloc_security
      0.62 ±  4%      -0.1        0.54 ±  3%  perf-profile.children.cycles-pp.inode_init_once
      0.24 ± 22%      -0.1        0.16 ±  4%  perf-profile.children.cycles-pp.__legitimize_mnt
      0.48 ±  4%      -0.1        0.40 ±  7%  perf-profile.children.cycles-pp._raw_spin_lock_irq
      0.50 ±  6%      -0.1        0.42 ±  4%  perf-profile.children.cycles-pp.__list_add_valid
      0.40 ±  3%      -0.1        0.33 ±  2%  perf-profile.children.cycles-pp.__d_add
      0.41 ±  4%      -0.1        0.34 ±  8%  perf-profile.children.cycles-pp.clear_inode
      0.34 ±  3%      -0.1        0.29 ±  2%  perf-profile.children.cycles-pp.lockref_put_or_lock
      0.44 ±  5%      -0.1        0.38 ±  4%  perf-profile.children.cycles-pp.dentry_kill
      0.45 ±  4%      -0.1        0.40 ±  2%  perf-profile.children.cycles-pp.mod_objcg_state
      0.38 ±  4%      -0.1        0.33 ±  5%  perf-profile.children.cycles-pp.check_heap_object
      0.33 ±  8%      -0.0        0.28 ±  5%  perf-profile.children.cycles-pp.get_obj_cgroup_from_current
      0.30 ±  3%      -0.0        0.25 ±  4%  perf-profile.children.cycles-pp.entry_SYSCALL_64_safe_stack
      0.33 ± 10%      -0.0        0.28 ±  7%  perf-profile.children.cycles-pp.xa_load
      0.28 ±  4%      -0.0        0.23 ±  4%  perf-profile.children.cycles-pp.walk_component
      0.28 ±  5%      -0.0        0.24 ±  6%  perf-profile.children.cycles-pp.__virt_addr_valid
      0.10 ± 17%      -0.0        0.06 ± 15%  perf-profile.children.cycles-pp._find_next_zero_bit
      0.14 ±  5%      -0.0        0.11 ±  7%  perf-profile.children.cycles-pp.d_lookup
      0.17 ± 14%      -0.0        0.14 ±  5%  perf-profile.children.cycles-pp.errseq_sample
      0.07 ±  5%      -0.0        0.04 ± 71%  perf-profile.children.cycles-pp.find_lock_entries
      0.24 ±  5%      -0.0        0.21 ±  8%  perf-profile.children.cycles-pp.entry_SYSRETQ_unsafe_stack
      0.17 ±  9%      -0.0        0.13 ±  7%  perf-profile.children.cycles-pp.__check_heap_object
      0.22 ±  4%      -0.0        0.19 ±  5%  perf-profile.children.cycles-pp.lookup_fast
      0.14 ±  5%      -0.0        0.11 ±  6%  perf-profile.children.cycles-pp.lookup_one_qstr_excl
      0.23 ±  4%      -0.0        0.20        perf-profile.children.cycles-pp.shmem_get_partial_folio
      0.14 ± 14%      -0.0        0.12 ±  6%  perf-profile.children.cycles-pp.__srcu_read_lock
      0.12 ±  4%      -0.0        0.10 ± 10%  perf-profile.children.cycles-pp.__d_lookup
      0.13 ±  8%      -0.0        0.10 ± 10%  perf-profile.children.cycles-pp.lookup_dcache
      0.16 ±  6%      -0.0        0.14 ±  5%  perf-profile.children.cycles-pp.down_write
      0.11 ± 10%      -0.0        0.08 ±  7%  perf-profile.children.cycles-pp.set_cached_acl
      0.14 ±  3%      -0.0        0.12 ±  3%  perf-profile.children.cycles-pp.fput
      0.15 ±  5%      -0.0        0.12 ±  4%  perf-profile.children.cycles-pp.filemap_get_entry
      0.13 ±  8%      -0.0        0.11 ±  6%  perf-profile.children.cycles-pp.find_get_entries
      0.10 ±  5%      -0.0        0.08 ±  6%  perf-profile.children.cycles-pp.xas_find
      0.09 ±  5%      -0.0        0.07 ±  9%  perf-profile.children.cycles-pp.__d_lookup_unhash
      0.07 ±  9%      +0.0        0.09 ± 12%  perf-profile.children.cycles-pp.rebalance_domains
      0.16 ±  3%      +0.0        0.18 ±  5%  perf-profile.children.cycles-pp.d_instantiate
      0.10 ±  6%      +0.0        0.13 ±  5%  perf-profile.children.cycles-pp.__d_instantiate
      0.07 ± 15%      +0.0        0.10 ±  8%  perf-profile.children.cycles-pp.perf_rotate_context
      0.06 ± 13%      +0.0        0.09 ± 12%  perf-profile.children.cycles-pp.irqtime_account_irq
      0.04 ± 45%      +0.0        0.08 ± 10%  perf-profile.children.cycles-pp.arch_scale_freq_tick
      0.06 ± 11%      +0.0        0.10 ± 11%  perf-profile.children.cycles-pp.load_balance
      0.11 ± 16%      +0.0        0.16 ±  6%  perf-profile.children.cycles-pp.perf_mux_hrtimer_handler
      0.02 ± 99%      +0.0        0.07 ± 11%  perf-profile.children.cycles-pp.native_sched_clock
      0.02 ± 99%      +0.0        0.08 ± 12%  perf-profile.children.cycles-pp.find_busiest_group
      0.00            +0.1        0.06 ± 11%  perf-profile.children.cycles-pp.update_sg_lb_stats
      0.02 ±141%      +0.1        0.08 ± 12%  perf-profile.children.cycles-pp.update_sd_lb_stats
      0.28 ±  5%      +0.1        0.34 ±  6%  perf-profile.children.cycles-pp.syscall_enter_from_user_mode
      0.00            +0.1        0.06 ± 14%  perf-profile.children.cycles-pp.update_load_avg
      0.00            +0.1        0.07 ±  7%  perf-profile.children.cycles-pp.xas_clear_mark
      0.00            +0.1        0.07 ± 11%  perf-profile.children.cycles-pp.rmqueue_bulk
      0.14 ±  8%      +0.1        0.21 ±  5%  perf-profile.children.cycles-pp.lockref_get
      0.00            +0.1        0.08 ±  8%  perf-profile.children.cycles-pp.__free_one_page
      0.01 ±223%      +0.1        0.10 ±  9%  perf-profile.children.cycles-pp.rmqueue
      0.06 ± 13%      +0.1        0.16 ±  4%  perf-profile.children.cycles-pp.__alloc_pages
      0.04 ± 45%      +0.1        0.15 ±  4%  perf-profile.children.cycles-pp.get_page_from_freelist
      0.09 ± 10%      +0.1        0.20 ±  6%  perf-profile.children.cycles-pp.xas_load
      0.04 ± 71%      +0.1        0.15 ± 14%  perf-profile.children.cycles-pp.task_tick_fair
      0.07            +0.1        0.21 ±  3%  perf-profile.children.cycles-pp.rcu_nocb_try_bypass
      0.00            +0.2        0.15 ±  3%  perf-profile.children.cycles-pp.i_callback
      0.22 ±  5%      +0.2        0.40 ±  4%  perf-profile.children.cycles-pp.scheduler_tick
      0.13 ± 10%      +0.2        0.32 ±  6%  perf-profile.children.cycles-pp._raw_spin_lock_irqsave
      0.07 ± 14%      +0.2        0.27 ±  2%  perf-profile.children.cycles-pp.free_pcppages_bulk
      0.39 ±  6%      +0.2        0.63 ±  4%  perf-profile.children.cycles-pp.update_process_times
      0.15 ±  8%      +0.2        0.40 ±  3%  perf-profile.children.cycles-pp.__unfreeze_partials
      0.09 ± 13%      +0.2        0.34 ±  2%  perf-profile.children.cycles-pp.free_unref_page
      0.41 ±  6%      +0.2        0.66 ±  5%  perf-profile.children.cycles-pp.tick_sched_handle
      0.47 ±  5%      +0.3        0.74 ±  4%  perf-profile.children.cycles-pp.tick_sched_timer
      0.14 ±  6%      +0.3        0.44        perf-profile.children.cycles-pp.rcu_segcblist_enqueue
      0.04 ± 71%      +0.3        0.33 ±  6%  perf-profile.children.cycles-pp.intel_idle
      0.00            +0.4        0.36 ±  4%  perf-profile.children.cycles-pp.xas_descend
      0.71 ±  6%      +0.4        1.08 ±  4%  perf-profile.children.cycles-pp.__hrtimer_run_queues
      0.85 ±  6%      +0.4        1.28 ±  5%  perf-profile.children.cycles-pp.hrtimer_interrupt
      0.87 ±  6%      +0.4        1.31 ±  5%  perf-profile.children.cycles-pp.__sysvec_apic_timer_interrupt
      0.66 ±  4%      +0.5        1.15 ±  2%  perf-profile.children.cycles-pp.rcu_cblist_dequeue
      0.00            +0.7        0.68        perf-profile.children.cycles-pp.radix_tree_node_rcu_free
      4.08 ± 12%      +0.7        4.77        perf-profile.children.cycles-pp.vfs_unlink
      0.92 ±  3%      +0.8        1.75 ±  2%  perf-profile.children.cycles-pp.kmem_cache_free
      0.54 ±  4%      +0.9        1.49 ±  2%  perf-profile.children.cycles-pp.__call_rcu_common
      1.41 ± 10%      +1.3        2.75        perf-profile.children.cycles-pp.shmem_unlink
      0.00            +1.6        1.56 ±  3%  perf-profile.children.cycles-pp.__xa_erase
      0.00            +1.6        1.61 ±  2%  perf-profile.children.cycles-pp.xa_erase
      0.00            +1.6        1.63 ±  2%  perf-profile.children.cycles-pp.simple_offset_remove
     19.65 ±  3%      +2.4       22.09        perf-profile.children.cycles-pp.open_last_lookups
      0.00            +3.0        3.00        perf-profile.children.cycles-pp.radix_tree_node_ctor
     17.21 ±  4%      +3.0       20.22        perf-profile.children.cycles-pp.lookup_open
      0.00            +3.2        3.16        perf-profile.children.cycles-pp.xas_expand
      0.71 ±  4%      +3.2        3.91        perf-profile.children.cycles-pp.shuffle_freelist
      0.82 ±  4%      +3.3        4.15        perf-profile.children.cycles-pp.allocate_slab
      3.59 ±  3%      +3.3        6.92 ±  2%  perf-profile.children.cycles-pp.__irq_exit_rcu
      0.23 ± 13%      +3.4        3.60 ±  4%  perf-profile.children.cycles-pp.run_ksoftirqd
      0.24 ± 13%      +3.4        3.62 ±  4%  perf-profile.children.cycles-pp.smpboot_thread_fn
      0.26 ± 11%      +3.4        3.66 ±  4%  perf-profile.children.cycles-pp.ret_from_fork
      0.26 ± 11%      +3.4        3.66 ±  4%  perf-profile.children.cycles-pp.kthread
      0.89 ±  3%      +3.4        4.29        perf-profile.children.cycles-pp.___slab_alloc
     13.57 ±  6%      +3.7       17.24        perf-profile.children.cycles-pp.shmem_mknod
      4.62 ±  2%      +3.8        8.40 ±  2%  perf-profile.children.cycles-pp.sysvec_apic_timer_interrupt
      4.74 ±  2%      +3.8        8.56 ±  2%  perf-profile.children.cycles-pp.asm_sysvec_apic_timer_interrupt
      2.43 ±  2%      +4.1        6.56        perf-profile.children.cycles-pp.kmem_cache_alloc_lru
      0.89 ±  4%      +4.3        5.21 ±  2%  perf-profile.children.cycles-pp.__slab_free
      0.00            +4.7        4.68        perf-profile.children.cycles-pp.xas_alloc
      0.00            +5.3        5.35        perf-profile.children.cycles-pp.xas_create
      0.00            +5.5        5.54        perf-profile.children.cycles-pp.__xa_alloc
      0.00            +5.6        5.57        perf-profile.children.cycles-pp.__xa_alloc_cyclic
      0.00            +5.7        5.66        perf-profile.children.cycles-pp.simple_offset_add
      3.66 ±  3%      +6.6       10.29        perf-profile.children.cycles-pp.rcu_do_batch
      3.66 ±  3%      +6.7       10.32        perf-profile.children.cycles-pp.rcu_core
      3.79 ±  3%      +6.7       10.48        perf-profile.children.cycles-pp.__do_softirq
      0.00            +7.0        6.97        perf-profile.children.cycles-pp.xas_store
     13.91 ± 14%      -2.4       11.56 ±  4%  perf-profile.self.cycles-pp.native_queued_spin_lock_slowpath
     28.01 ±  2%      -1.3       26.71        perf-profile.self.cycles-pp.intel_idle_ibrs
      4.33 ±  4%      -0.8        3.54        perf-profile.self.cycles-pp._raw_spin_lock
      2.52 ± 11%      -0.7        1.80 ±  3%  perf-profile.self.cycles-pp.generic_permission
      3.43 ±  4%      -0.6        2.85        perf-profile.self.cycles-pp.syscall_return_via_sysret
      3.18 ±  2%      -0.6        2.61        perf-profile.self.cycles-pp.syscall_exit_to_user_mode
      1.31 ± 24%      -0.4        0.94 ±  4%  perf-profile.self.cycles-pp.__mnt_want_write
      1.96 ±  2%      -0.3        1.64        perf-profile.self.cycles-pp.entry_SYSCALL_64_after_hwframe
      0.95 ±  9%      -0.3        0.64 ±  4%  perf-profile.self.cycles-pp.__destroy_inode
      0.75 ±  2%      -0.2        0.50 ±  2%  perf-profile.self.cycles-pp.file_free_rcu
      1.44 ±  3%      -0.2        1.21        perf-profile.self.cycles-pp.__entry_text_start
      0.64 ± 12%      -0.2        0.40 ±  3%  perf-profile.self.cycles-pp.vfs_unlink
      0.62 ±  7%      -0.2        0.40 ±  5%  perf-profile.self.cycles-pp.do_dentry_open
      0.55 ±  9%      -0.2        0.34 ±  3%  perf-profile.self.cycles-pp.inode_init_owner
      0.64 ±  7%      -0.2        0.43 ±  5%  perf-profile.self.cycles-pp.__fput
      0.45 ±  9%      -0.2        0.26 ±  5%  perf-profile.self.cycles-pp.open_last_lookups
      0.62 ± 17%      -0.2        0.43 ±  4%  perf-profile.self.cycles-pp.lookup_open
      0.80 ±  8%      -0.2        0.62 ±  3%  perf-profile.self.cycles-pp.shmem_get_inode
      0.64 ± 12%      -0.2        0.46 ±  2%  perf-profile.self.cycles-pp.apparmor_path_unlink
      0.50 ± 11%      -0.2        0.32 ±  6%  perf-profile.self.cycles-pp.drop_nlink
      0.70 ±  7%      -0.2        0.53 ±  3%  perf-profile.self.cycles-pp.link_path_walk
      0.86 ±  4%      -0.2        0.70 ±  4%  perf-profile.self.cycles-pp.inode_maybe_inc_iversion
      0.36 ± 11%      -0.1        0.24 ± 16%  perf-profile.self.cycles-pp.locks_remove_posix
      0.30 ± 19%      -0.1        0.18 ± 22%  perf-profile.self.cycles-pp.alloc_fd
      0.34 ± 15%      -0.1        0.23 ±  8%  perf-profile.self.cycles-pp.fsnotify_perm
      0.50 ± 11%      -0.1        0.40 ±  4%  perf-profile.self.cycles-pp.apparmor_file_open
      0.32 ±  9%      -0.1        0.22 ±  8%  perf-profile.self.cycles-pp.apparmor_path_mknod
      0.44 ± 16%      -0.1        0.34 ±  4%  perf-profile.self.cycles-pp.shmem_evict_inode
      0.30 ±  5%      -0.1        0.20 ±  7%  perf-profile.self.cycles-pp.may_delete
      0.52 ± 11%      -0.1        0.42 ±  6%  perf-profile.self.cycles-pp.apparmor_file_alloc_security
      0.21 ± 34%      -0.1        0.12 ±  6%  perf-profile.self.cycles-pp.dput
      0.59 ±  5%      -0.1        0.51 ±  3%  perf-profile.self.cycles-pp.inode_init_once
      0.48 ±  6%      -0.1        0.40 ±  5%  perf-profile.self.cycles-pp.__list_add_valid
      0.47 ±  4%      -0.1        0.39 ±  7%  perf-profile.self.cycles-pp._raw_spin_lock_irq
      0.23 ± 24%      -0.1        0.15 ±  6%  perf-profile.self.cycles-pp.__legitimize_mnt
      0.28 ±  3%      -0.1        0.21 ±  2%  perf-profile.self.cycles-pp.lockref_put_or_lock
      0.41 ±  4%      -0.1        0.35 ±  3%  perf-profile.self.cycles-pp.mod_objcg_state
      0.28 ± 15%      -0.1        0.23 ±  8%  perf-profile.self.cycles-pp.mnt_drop_write
      0.18 ± 17%      -0.1        0.12 ± 14%  perf-profile.self.cycles-pp.xa_load
      0.31 ±  7%      -0.1        0.26 ±  5%  perf-profile.self.cycles-pp.strncpy_from_user
      0.20 ± 15%      -0.1        0.15 ±  8%  perf-profile.self.cycles-pp.simple_lookup
      0.19 ±  3%      -0.1        0.14 ±  9%  perf-profile.self.cycles-pp._atomic_dec_and_lock
      0.44 ±  5%      -0.0        0.39 ±  5%  perf-profile.self.cycles-pp.kmem_cache_alloc
      0.34 ±  3%      -0.0        0.30 ±  4%  perf-profile.self.cycles-pp.memcg_list_lru_alloc
      0.17 ± 13%      -0.0        0.12 ±  4%  perf-profile.self.cycles-pp.errseq_sample
      0.54 ±  3%      -0.0        0.50 ±  5%  perf-profile.self.cycles-pp.init_file
      0.16 ± 10%      -0.0        0.12 ±  8%  perf-profile.self.cycles-pp.__check_heap_object
      0.25 ±  4%      -0.0        0.21 ±  5%  perf-profile.self.cycles-pp.__virt_addr_valid
      0.36            -0.0        0.33 ±  4%  perf-profile.self.cycles-pp.apparmor_file_free_security
      0.19 ±  6%      -0.0        0.15 ±  4%  perf-profile.self.cycles-pp.get_obj_cgroup_from_current
      0.13 ±  6%      -0.0        0.10 ±  8%  perf-profile.self.cycles-pp.down_write
      0.21 ±  4%      -0.0        0.18 ±  7%  perf-profile.self.cycles-pp.entry_SYSRETQ_unsafe_stack
      0.15 ±  7%      -0.0        0.13 ±  2%  perf-profile.self.cycles-pp.up_write
      0.10 ± 11%      -0.0        0.08 ±  6%  perf-profile.self.cycles-pp.set_cached_acl
      0.12 ±  8%      -0.0        0.09 ±  7%  perf-profile.self.cycles-pp.d_alloc_parallel
      0.12 ±  6%      -0.0        0.10 ±  4%  perf-profile.self.cycles-pp.syscall_enter_from_user_mode
      0.10 ±  7%      -0.0        0.08 ±  8%  perf-profile.self.cycles-pp.do_filp_open
      0.12 ±  5%      -0.0        0.10 ±  4%  perf-profile.self.cycles-pp.entry_SYSCALL_64_safe_stack
      0.12 ±  5%      -0.0        0.10 ±  3%  perf-profile.self.cycles-pp.inode_sb_list_add
      0.10 ±  5%      -0.0        0.08 ±  7%  perf-profile.self.cycles-pp.filemap_get_entry
      0.08 ±  5%      -0.0        0.07 ± 11%  perf-profile.self.cycles-pp.__d_lookup_unhash
      0.08 ±  5%      -0.0        0.06 ±  7%  perf-profile.self.cycles-pp.___d_drop
      0.09 ±  7%      -0.0        0.07 ±  5%  perf-profile.self.cycles-pp.exit_to_user_mode_loop
      0.07 ± 11%      -0.0        0.05 ±  7%  perf-profile.self.cycles-pp.mntput_no_expire
      0.16 ±  4%      +0.0        0.18 ±  3%  perf-profile.self.cycles-pp.__open64
      0.04 ± 45%      +0.0        0.08 ± 10%  perf-profile.self.cycles-pp.arch_scale_freq_tick
      0.04 ± 44%      +0.0        0.07 ±  6%  perf-profile.self.cycles-pp.__d_instantiate
      0.13 ±  5%      +0.0        0.18 ±  7%  perf-profile.self.cycles-pp.__filename_parentat
      0.00            +0.1        0.05 ±  8%  perf-profile.self.cycles-pp.xas_clear_mark
      0.00            +0.1        0.06 ±  9%  perf-profile.self.cycles-pp.__hrtimer_run_queues
      0.14 ±  7%      +0.1        0.20 ±  6%  perf-profile.self.cycles-pp.lockref_get
      0.00            +0.1        0.06 ± 11%  perf-profile.self.cycles-pp.irqtime_account_irq
      0.01 ±223%      +0.1        0.07 ± 13%  perf-profile.self.cycles-pp.native_sched_clock
      0.09 ±  9%      +0.1        0.16 ± 10%  perf-profile.self.cycles-pp._raw_spin_lock_irqsave
      0.01 ±223%      +0.1        0.08 ±  7%  perf-profile.self.cycles-pp.xas_load
      0.00            +0.1        0.08 ± 10%  perf-profile.self.cycles-pp.___slab_alloc
      0.00            +0.1        0.11 ±  5%  perf-profile.self.cycles-pp.xas_create
      0.05 ±  8%      +0.1        0.17 ±  6%  perf-profile.self.cycles-pp.rcu_nocb_try_bypass
      0.15 ± 10%      +0.1        0.27 ±  3%  perf-profile.self.cycles-pp.slab_pre_alloc_hook
      0.00            +0.2        0.15 ±  7%  perf-profile.self.cycles-pp.rcu_do_batch
      0.00            +0.2        0.15 ±  3%  perf-profile.self.cycles-pp.i_callback
      0.14 ±  7%      +0.2        0.39 ±  3%  perf-profile.self.cycles-pp.rcu_segcblist_enqueue
      0.00            +0.2        0.25 ±  5%  perf-profile.self.cycles-pp.xas_alloc
      0.04 ± 45%      +0.3        0.32 ±  6%  perf-profile.self.cycles-pp.shuffle_freelist
      0.00            +0.3        0.29 ±  3%  perf-profile.self.cycles-pp.xas_expand
      0.04 ± 71%      +0.3        0.33 ±  6%  perf-profile.self.cycles-pp.intel_idle
      0.00            +0.3        0.31 ±  4%  perf-profile.self.cycles-pp.xas_descend
      0.00            +0.3        0.34 ±  6%  perf-profile.self.cycles-pp.xas_store
      0.65 ±  4%      +0.5        1.12 ±  2%  perf-profile.self.cycles-pp.rcu_cblist_dequeue
      0.31 ±  3%      +0.5        0.80        perf-profile.self.cycles-pp.__call_rcu_common
      0.15 ±  3%      +0.6        0.75 ±  3%  perf-profile.self.cycles-pp.kmem_cache_alloc_lru
      0.00            +0.7        0.68        perf-profile.self.cycles-pp.radix_tree_node_rcu_free
      0.50 ±  3%      +0.9        1.36 ±  2%  perf-profile.self.cycles-pp.kmem_cache_free
      0.00            +2.9        2.87        perf-profile.self.cycles-pp.radix_tree_node_ctor
      0.89 ±  4%      +4.2        5.11 ±  2%  perf-profile.self.cycles-pp.__slab_free


***************************************************************************************************
lkp-csl-d02: 36 threads 1 sockets Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz (Cascade Lake) with 128G memory
=========================================================================================
compiler/cpufreq_governor/kconfig/mode/nr_task/rootfs/tbox_group/test/testcase:
  gcc-12/performance/x86_64-rhel-8.3/thread/16/debian-11.1-x86_64-20220510.cgz/lkp-csl-d02/unlink2/will-it-scale

commit: 
  c2c3172c26 ("shmem: Refactor shmem_symlink()")
  a1a690e009 ("shmem: stable directory offsets")

c2c3172c269f808d a1a690e009744e4526526b2f838 
---------------- --------------------------- 
         %stddev     %change         %stddev
             \          |                \  
    152086           -26.3%     112046 ± 20%  meminfo.SUnreclaim
      5272 ± 10%     -43.9%       2957 ± 70%  perf-c2c.HITM.local
     38031           -26.3%      28019 ± 20%  proc-vmstat.nr_slab_unreclaimable
      8013           +85.9%      14896 ± 58%  vmstat.system.cs
     16670 ±  3%     -38.5%      10253 ± 70%  turbostat.C1
      0.16 ±  7%     -42.9%       0.09 ± 71%  turbostat.CPU%c6
      0.11 ±  7%     -42.4%       0.06 ± 71%  turbostat.Pkg%pc2
     58.94 ±  8%     +84.3%     108.61 ± 38%  sched_debug.cfs_rq:/.load_avg.avg
     17.37 ± 29%    +154.4%      44.17 ± 66%  sched_debug.cfs_rq:/.removed.load_avg.avg
     88662 ± 16%     -81.3%      16537 ±119%  sched_debug.cfs_rq:/.spread0.max
      0.00 ±  8%     -14.7%       0.00 ±  7%  sched_debug.cpu.next_balance.stddev
   1330086 ±  4%     -40.0%     798527 ± 70%  will-it-scale.16.threads
     54.99           -36.2%      35.11 ± 70%  will-it-scale.16.threads_idle
     83130 ±  4%     -40.0%      49907 ± 70%  will-it-scale.per_thread_ops
   1330086 ±  4%     -40.0%     798527 ± 70%  will-it-scale.workload
      0.00           -50.0%       0.00 ± 70%  perf-sched.sch_delay.max.ms.wait_for_partner.fifo_open.do_dentry_open.do_open
      0.01 ± 18%     +97.2%       0.02 ± 21%  perf-sched.total_sch_delay.average.ms
    258.53 ±  3%     -39.4%     156.56 ± 70%  perf-sched.wait_and_delay.avg.ms.do_task_dead.do_exit.do_group_exit.__x64_sys_exit_group.do_syscall_64
      3.47 ±  3%     -33.3%       2.31 ± 30%  perf-sched.wait_and_delay.avg.ms.schedule_timeout.rcu_gp_fqs_loop.rcu_gp_kthread.kthread
    275.59 ±  6%     -66.4%      92.71 ± 69%  perf-sched.wait_and_delay.avg.ms.smpboot_thread_fn.kthread.ret_from_fork
    258.53 ±  3%     -39.4%     156.56 ± 70%  perf-sched.wait_time.avg.ms.do_task_dead.do_exit.do_group_exit.__x64_sys_exit_group.do_syscall_64
      3.47 ±  3%     -33.3%       2.31 ± 30%  perf-sched.wait_time.avg.ms.schedule_timeout.rcu_gp_fqs_loop.rcu_gp_kthread.kthread
    274.83 ±  6%     -66.5%      92.08 ± 69%  perf-sched.wait_time.avg.ms.smpboot_thread_fn.kthread.ret_from_fork
     23.44 ±  9%     -41.5%      13.71 ± 70%  perf-stat.i.MPKI
      2.57 ±  5%     -43.2%       1.46 ± 70%  perf-stat.i.cpi
      1466 ±  4%     -81.4%     272.44 ± 70%  perf-stat.i.cycles-between-cache-misses
    431526           -39.2%     262449 ± 70%  perf-stat.i.iTLB-loads
    505.11 ±  4%     -56.3%     220.66 ± 70%  perf-stat.i.metric.K/sec
     23.41 ±  9%     -41.5%      13.70 ± 70%  perf-stat.overall.MPKI
      2.57 ±  5%     -43.2%       1.46 ± 70%  perf-stat.overall.cpi
      1463 ±  4%     -81.5%     271.05 ± 70%  perf-stat.overall.cycles-between-cache-misses
    430084           -39.2%     261537 ± 70%  perf-stat.ps.iTLB-loads
     41.47           -14.1       27.35 ± 70%  perf-profile.calltrace.cycles-pp.__open64
     40.98           -14.0       26.94 ± 70%  perf-profile.calltrace.cycles-pp.__x64_sys_openat.do_syscall_64.entry_SYSCALL_64_after_hwframe.__open64
     41.10           -14.0       27.07 ± 70%  perf-profile.calltrace.cycles-pp.entry_SYSCALL_64_after_hwframe.__open64
     41.04           -14.0       27.02 ± 70%  perf-profile.calltrace.cycles-pp.do_syscall_64.entry_SYSCALL_64_after_hwframe.__open64
     40.93           -14.0       26.91 ± 70%  perf-profile.calltrace.cycles-pp.do_sys_openat2.__x64_sys_openat.do_syscall_64.entry_SYSCALL_64_after_hwframe.__open64
     31.17 ±  2%     -11.7       19.49 ± 70%  perf-profile.calltrace.cycles-pp.unlink
     30.89 ±  2%     -11.6       19.32 ± 70%  perf-profile.calltrace.cycles-pp.entry_SYSCALL_64_after_hwframe.unlink
     30.81 ±  2%     -11.6       19.25 ± 70%  perf-profile.calltrace.cycles-pp.__x64_sys_unlink.do_syscall_64.entry_SYSCALL_64_after_hwframe.unlink
     30.86 ±  2%     -11.6       19.30 ± 70%  perf-profile.calltrace.cycles-pp.do_syscall_64.entry_SYSCALL_64_after_hwframe.unlink
     30.28 ±  2%     -11.4       18.93 ± 70%  perf-profile.calltrace.cycles-pp.do_unlinkat.__x64_sys_unlink.do_syscall_64.entry_SYSCALL_64_after_hwframe.unlink
      3.95 ± 21%      -2.5        1.45 ± 70%  perf-profile.calltrace.cycles-pp.do_open.path_openat.do_filp_open.do_sys_openat2.__x64_sys_openat
      3.23 ± 21%      -2.1        1.18 ± 70%  perf-profile.calltrace.cycles-pp.do_dentry_open.do_open.path_openat.do_filp_open.do_sys_openat2
      3.21 ± 10%      -1.5        1.67 ± 70%  perf-profile.calltrace.cycles-pp.shmem_evict_inode.evict.do_unlinkat.__x64_sys_unlink.do_syscall_64
      2.91 ± 13%      -1.5        1.40 ± 70%  perf-profile.calltrace.cycles-pp.alloc_inode.new_inode.shmem_get_inode.shmem_mknod.lookup_open
      1.86 ± 24%      -1.2        0.66 ± 70%  perf-profile.calltrace.cycles-pp.destroy_inode.do_unlinkat.__x64_sys_unlink.do_syscall_64.entry_SYSCALL_64_after_hwframe
      2.28 ± 13%      -1.2        1.08 ± 70%  perf-profile.calltrace.cycles-pp.__filename_parentat.do_unlinkat.__x64_sys_unlink.do_syscall_64.entry_SYSCALL_64_after_hwframe
      2.17 ± 14%      -1.1        1.02 ± 70%  perf-profile.calltrace.cycles-pp.path_parentat.__filename_parentat.do_unlinkat.__x64_sys_unlink.do_syscall_64
      1.90 ± 18%      -1.1        0.77 ± 70%  perf-profile.calltrace.cycles-pp.link_path_walk.path_openat.do_filp_open.do_sys_openat2.__x64_sys_openat
      1.89 ± 16%      -1.1        0.77 ± 70%  perf-profile.calltrace.cycles-pp.link_path_walk.path_parentat.__filename_parentat.do_unlinkat.__x64_sys_unlink
      2.00 ± 12%      -1.1        0.91 ± 70%  perf-profile.calltrace.cycles-pp.syscall_exit_to_user_mode.do_syscall_64.entry_SYSCALL_64_after_hwframe.__close
      1.99 ± 12%      -1.1        0.91 ± 70%  perf-profile.calltrace.cycles-pp.exit_to_user_mode_prepare.syscall_exit_to_user_mode.do_syscall_64.entry_SYSCALL_64_after_hwframe.__close
      1.90 ± 13%      -1.0        0.85 ± 70%  perf-profile.calltrace.cycles-pp.task_work_run.exit_to_user_mode_loop.exit_to_user_mode_prepare.syscall_exit_to_user_mode.do_syscall_64
      1.60 ± 19%      -1.0        0.61 ± 70%  perf-profile.calltrace.cycles-pp.__destroy_inode.destroy_inode.do_unlinkat.__x64_sys_unlink.do_syscall_64
      1.61 ± 16%      -0.9        0.67 ± 70%  perf-profile.calltrace.cycles-pp.__fput.task_work_run.exit_to_user_mode_loop.exit_to_user_mode_prepare.syscall_exit_to_user_mode
      1.40 ± 19%      -0.9        0.50 ± 71%  perf-profile.calltrace.cycles-pp.inode_permission.link_path_walk.path_parentat.__filename_parentat.do_unlinkat
      1.36 ± 20%      -0.9        0.48 ± 71%  perf-profile.calltrace.cycles-pp.generic_permission.inode_permission.link_path_walk.path_parentat.__filename_parentat
      2.05 ±  7%      -0.9        1.19 ± 70%  perf-profile.calltrace.cycles-pp.shmem_reserve_inode.shmem_get_inode.shmem_mknod.lookup_open.open_last_lookups
      1.34 ± 23%      -0.9        0.48 ± 70%  perf-profile.calltrace.cycles-pp.inode_permission.link_path_walk.path_openat.do_filp_open.do_sys_openat2
      1.29 ± 24%      -0.8        0.46 ± 70%  perf-profile.calltrace.cycles-pp.generic_permission.inode_permission.link_path_walk.path_openat.do_filp_open
      1.73            -0.7        1.06 ± 70%  perf-profile.calltrace.cycles-pp.alloc_empty_file.path_openat.do_filp_open.do_sys_openat2.__x64_sys_openat
      1.45 ± 10%      -0.6        0.88 ± 70%  perf-profile.calltrace.cycles-pp._raw_spin_lock.shmem_evict_inode.evict.do_unlinkat.__x64_sys_unlink
      1.30 ±  8%      -0.6        0.75 ± 71%  perf-profile.calltrace.cycles-pp.native_queued_spin_lock_slowpath._raw_spin_lock.shmem_reserve_inode.shmem_get_inode.shmem_mknod
      1.30 ±  4%      -0.5        0.76 ± 70%  perf-profile.calltrace.cycles-pp.d_alloc_parallel.lookup_open.open_last_lookups.path_openat.do_filp_open
      1.19 ±  5%      -0.5        0.70 ± 70%  perf-profile.calltrace.cycles-pp.d_alloc.d_alloc_parallel.lookup_open.open_last_lookups.path_openat
      1.10 ±  2%      -0.4        0.65 ± 70%  perf-profile.calltrace.cycles-pp.init_file.alloc_empty_file.path_openat.do_filp_open.do_sys_openat2
      0.80 ± 18%      -0.4        0.35 ± 70%  perf-profile.calltrace.cycles-pp.iput.do_unlinkat.__x64_sys_unlink.do_syscall_64.entry_SYSCALL_64_after_hwframe
      1.16 ±  2%      -0.4        0.75 ± 70%  perf-profile.calltrace.cycles-pp.inode_wait_for_writeback.evict.do_unlinkat.__x64_sys_unlink.do_syscall_64
      1.01 ±  8%      -0.4        0.59 ± 70%  perf-profile.calltrace.cycles-pp.__d_alloc.d_alloc.d_alloc_parallel.lookup_open.open_last_lookups
      1.13 ±  2%      -0.4        0.73 ± 70%  perf-profile.calltrace.cycles-pp._raw_spin_lock.inode_wait_for_writeback.evict.do_unlinkat.__x64_sys_unlink
      0.91            -0.4        0.51 ± 70%  perf-profile.calltrace.cycles-pp.try_to_unlazy.open_last_lookups.path_openat.do_filp_open.do_sys_openat2
      0.88 ±  2%      -0.4        0.50 ± 70%  perf-profile.calltrace.cycles-pp.__legitimize_path.try_to_unlazy.open_last_lookups.path_openat.do_filp_open
      0.62            -0.3        0.35 ± 70%  perf-profile.calltrace.cycles-pp.security_file_alloc.init_file.alloc_empty_file.path_openat.do_filp_open
      0.61            -0.2        0.38 ± 70%  perf-profile.calltrace.cycles-pp._raw_spin_lock.new_inode.shmem_get_inode.shmem_mknod.lookup_open
      0.59 ±  2%      -0.2        0.37 ± 70%  perf-profile.calltrace.cycles-pp.__list_add_valid.inode_sb_list_add.new_inode.shmem_get_inode.shmem_mknod
     77.22           -15.2       62.07 ± 27%  perf-profile.children.cycles-pp.entry_SYSCALL_64_after_hwframe
     77.08           -15.1       61.98 ± 27%  perf-profile.children.cycles-pp.do_syscall_64
     41.54           -14.2       27.39 ± 70%  perf-profile.children.cycles-pp.__open64
     40.98           -12.4       28.53 ± 58%  perf-profile.children.cycles-pp.__x64_sys_openat
     40.94           -12.4       28.51 ± 58%  perf-profile.children.cycles-pp.do_sys_openat2
     31.25 ±  2%     -11.7       19.52 ± 70%  perf-profile.children.cycles-pp.unlink
     30.81 ±  2%     -11.6       19.25 ± 70%  perf-profile.children.cycles-pp.__x64_sys_unlink
     30.30 ±  2%     -11.4       18.94 ± 70%  perf-profile.children.cycles-pp.do_unlinkat
      4.34 ± 26%      -2.9        1.42 ± 70%  perf-profile.children.cycles-pp.inode_permission
      3.96 ± 21%      -2.5        1.46 ± 70%  perf-profile.children.cycles-pp.do_open
      3.80 ± 17%      -2.3        1.55 ± 70%  perf-profile.children.cycles-pp.link_path_walk
      3.44 ± 22%      -2.2        1.21 ± 70%  perf-profile.children.cycles-pp.generic_permission
      3.24 ± 21%      -2.1        1.18 ± 70%  perf-profile.children.cycles-pp.do_dentry_open
      3.21 ± 10%      -1.5        1.67 ± 70%  perf-profile.children.cycles-pp.shmem_evict_inode
      2.91 ± 13%      -1.5        1.40 ± 70%  perf-profile.children.cycles-pp.alloc_inode
      1.95 ± 34%      -1.5        0.49 ± 70%  perf-profile.children.cycles-pp.__mnt_want_write
      1.91 ± 31%      -1.4        0.56 ± 70%  perf-profile.children.cycles-pp.mnt_want_write
      1.87 ± 23%      -1.2        0.66 ± 70%  perf-profile.children.cycles-pp.destroy_inode
      2.28 ± 13%      -1.2        1.09 ± 70%  perf-profile.children.cycles-pp.__filename_parentat
      2.17 ± 14%      -1.2        1.02 ± 70%  perf-profile.children.cycles-pp.path_parentat
      1.60 ± 20%      -1.0        0.62 ± 70%  perf-profile.children.cycles-pp.__destroy_inode
      2.05 ±  7%      -0.9        1.19 ± 71%  perf-profile.children.cycles-pp.shmem_reserve_inode
      1.07 ± 29%      -0.8        0.31 ± 70%  perf-profile.children.cycles-pp.inode_init_always
      1.74            -0.7        1.06 ± 70%  perf-profile.children.cycles-pp.alloc_empty_file
      1.18 ±  8%      -0.6        0.57 ± 70%  perf-profile.children.cycles-pp.inode_maybe_inc_iversion
      0.87 ± 33%      -0.6        0.26 ± 70%  perf-profile.children.cycles-pp.may_delete
      0.84 ± 25%      -0.6        0.29 ± 70%  perf-profile.children.cycles-pp.inode_init_owner
      1.30 ±  4%      -0.5        0.76 ± 70%  perf-profile.children.cycles-pp.d_alloc_parallel
      0.76 ± 25%      -0.5        0.26 ± 70%  perf-profile.children.cycles-pp.security_path_unlink
      0.92 ± 16%      -0.5        0.42 ± 70%  perf-profile.children.cycles-pp.iput
      0.71 ± 30%      -0.5        0.21 ± 70%  perf-profile.children.cycles-pp.__fsnotify_parent
      0.75 ± 24%      -0.5        0.25 ± 70%  perf-profile.children.cycles-pp.apparmor_path_unlink
      1.20 ±  5%      -0.5        0.70 ± 70%  perf-profile.children.cycles-pp.d_alloc
      0.85 ±  2%      -0.5        0.35 ± 71%  perf-profile.children.cycles-pp.file_free_rcu
      0.81 ± 28%      -0.5        0.33 ± 70%  perf-profile.children.cycles-pp.fsnotify
      1.11 ±  2%      -0.5        0.66 ± 70%  perf-profile.children.cycles-pp.init_file
      1.05            -0.4        0.61 ± 70%  perf-profile.children.cycles-pp.try_to_unlazy
      0.68 ± 23%      -0.4        0.25 ± 70%  perf-profile.children.cycles-pp.may_open
      1.00            -0.4        0.57 ± 70%  perf-profile.children.cycles-pp.__legitimize_path
      1.17 ±  2%      -0.4        0.75 ± 70%  perf-profile.children.cycles-pp.inode_wait_for_writeback
      1.01 ±  8%      -0.4        0.59 ± 70%  perf-profile.children.cycles-pp.__d_alloc
      0.55 ± 32%      -0.4        0.15 ± 71%  perf-profile.children.cycles-pp.fsnotify_perm
      0.81            -0.4        0.42 ± 72%  perf-profile.children.cycles-pp.lockref_get_not_dead
      0.65 ±  7%      -0.3        0.31 ± 70%  perf-profile.children.cycles-pp.simple_lookup
      0.84 ±  3%      -0.3        0.54 ± 70%  perf-profile.children.cycles-pp.memcg_list_lru_alloc
      0.62            -0.3        0.35 ± 70%  perf-profile.children.cycles-pp.security_file_alloc
      0.38 ± 23%      -0.3        0.11 ± 70%  perf-profile.children.cycles-pp.drop_nlink
      0.68 ±  2%      -0.3        0.42 ± 70%  perf-profile.children.cycles-pp.strncpy_from_user
      0.57 ±  4%      -0.3        0.31 ± 70%  perf-profile.children.cycles-pp._raw_spin_lock_irq
      0.51 ±  4%      -0.2        0.28 ± 70%  perf-profile.children.cycles-pp.clear_inode
      0.54 ±  5%      -0.2        0.32 ± 71%  perf-profile.children.cycles-pp.security_file_open
      0.51 ±  8%      -0.2        0.29 ± 70%  perf-profile.children.cycles-pp.filp_close
      0.51            -0.2        0.29 ± 70%  perf-profile.children.cycles-pp.apparmor_file_alloc_security
      0.53 ±  5%      -0.2        0.31 ± 71%  perf-profile.children.cycles-pp.apparmor_file_open
      0.37 ± 23%      -0.2        0.15 ± 70%  perf-profile.children.cycles-pp.mnt_drop_write
      0.41 ± 16%      -0.2        0.19 ± 70%  perf-profile.children.cycles-pp.fsnotify_destroy_marks
      0.39 ± 16%      -0.2        0.18 ± 70%  perf-profile.children.cycles-pp.fsnotify_grab_connector
      0.28 ± 22%      -0.2        0.10 ± 71%  perf-profile.children.cycles-pp.simple_acl_create
      0.40 ±  6%      -0.2        0.23 ± 71%  perf-profile.children.cycles-pp.__entry_text_start
      0.26 ± 29%      -0.2        0.10 ± 70%  perf-profile.children.cycles-pp.__srcu_read_lock
      0.41            -0.2        0.25 ± 70%  perf-profile.children.cycles-pp.__check_object_size
      0.36 ±  5%      -0.2        0.21 ± 70%  perf-profile.children.cycles-pp.__d_add
      0.34 ± 11%      -0.1        0.20 ± 70%  perf-profile.children.cycles-pp.locks_remove_posix
      0.33            -0.1        0.18 ± 71%  perf-profile.children.cycles-pp.security_file_free
      0.32 ±  2%      -0.1        0.18 ± 71%  perf-profile.children.cycles-pp.apparmor_file_free_security
      0.36            -0.1        0.22 ± 70%  perf-profile.children.cycles-pp.lockref_put_or_lock
      0.22 ± 23%      -0.1        0.08 ± 71%  perf-profile.children.cycles-pp.errseq_sample
      0.33 ±  8%      -0.1        0.20 ± 70%  perf-profile.children.cycles-pp.security_path_mknod
      0.31 ±  8%      -0.1        0.19 ± 70%  perf-profile.children.cycles-pp.apparmor_path_mknod
      0.18 ± 15%      -0.1        0.09 ± 70%  perf-profile.children.cycles-pp.file_ra_state_init
      0.17 ± 15%      -0.1        0.07 ± 70%  perf-profile.children.cycles-pp.inode_to_bdi
      0.20 ±  4%      -0.1        0.11 ± 70%  perf-profile.children.cycles-pp.step_into
      0.19 ±  2%      -0.1        0.11 ± 70%  perf-profile.children.cycles-pp.check_heap_object
      0.17 ±  5%      -0.1        0.09 ± 70%  perf-profile.children.cycles-pp.down_write
      0.14 ±  5%      -0.1        0.08 ± 71%  perf-profile.children.cycles-pp.fput
      0.12 ± 17%      -0.1        0.06 ± 71%  perf-profile.children.cycles-pp._find_next_zero_bit
      0.11 ± 11%      -0.1        0.05 ± 72%  perf-profile.children.cycles-pp.set_cached_acl
      0.10 ± 12%      -0.0        0.05 ± 72%  perf-profile.children.cycles-pp.shmem_get_partial_folio
      0.12 ±  4%      -0.0        0.07 ± 70%  perf-profile.children.cycles-pp.refill_obj_stock
      0.11 ±  4%      -0.0        0.06 ± 70%  perf-profile.children.cycles-pp.rcu_all_qs
      0.09 ±  5%      -0.0        0.04 ± 71%  perf-profile.children.cycles-pp.__d_lookup_unhash
      0.09 ± 10%      -0.0        0.05 ± 72%  perf-profile.children.cycles-pp.task_work_add
      0.09            -0.0        0.05 ± 71%  perf-profile.children.cycles-pp.ihold
      0.07 ±  7%      -0.0        0.04 ± 71%  perf-profile.children.cycles-pp.xas_find
      0.26 ±  4%      +1.5        1.81 ±115%  perf-profile.children.cycles-pp.__list_del_entry_valid
      0.05 ±  8%      +1.6        1.66 ±132%  perf-profile.children.cycles-pp.scheduler_tick
      0.08 ± 10%      +1.6        1.70 ±127%  perf-profile.children.cycles-pp.tick_sched_timer
      0.06 ±  7%      +1.6        1.69 ±128%  perf-profile.children.cycles-pp.update_process_times
      0.07 ±  7%      +1.6        1.69 ±128%  perf-profile.children.cycles-pp.tick_sched_handle
      0.14 ±  3%      +1.6        1.77 ±119%  perf-profile.children.cycles-pp.__sysvec_apic_timer_interrupt
      0.00            +1.6        1.63 ±135%  perf-profile.children.cycles-pp.task_tick_fair
      0.13 ±  7%      +1.6        1.77 ±119%  perf-profile.children.cycles-pp.hrtimer_interrupt
      0.06            +1.6        1.70 ±127%  perf-profile.children.cycles-pp.rcu_nocb_try_bypass
      0.11 ±  4%      +1.6        1.75 ±121%  perf-profile.children.cycles-pp.__hrtimer_run_queues
      0.49 ±  5%      +2.0        2.50 ± 63%  perf-profile.children.cycles-pp.__call_rcu_common
      3.14 ±  4%      +3.7        6.86 ± 21%  perf-profile.children.cycles-pp.sysvec_apic_timer_interrupt
      3.15 ±  5%      +3.7        6.87 ± 21%  perf-profile.children.cycles-pp.asm_sysvec_apic_timer_interrupt
      0.54 ±  4%      +3.9        4.41 ±  6%  perf-profile.children.cycles-pp.__slab_free
      3.30 ± 23%      -2.2        1.11 ± 70%  perf-profile.self.cycles-pp.generic_permission
      5.20            -2.0        3.22 ± 70%  perf-profile.self.cycles-pp._raw_spin_lock
      1.91 ± 34%      -1.4        0.46 ± 70%  perf-profile.self.cycles-pp.__mnt_want_write
      1.30 ± 37%      -0.9        0.40 ± 70%  perf-profile.self.cycles-pp.current_time
      1.26 ± 20%      -0.8        0.44 ± 70%  perf-profile.self.cycles-pp.__destroy_inode
      0.97 ± 31%      -0.7        0.25 ± 70%  perf-profile.self.cycles-pp.inode_init_always
      0.81 ± 44%      -0.7        0.15 ± 70%  perf-profile.self.cycles-pp.inode_permission
      1.15 ±  8%      -0.6        0.54 ± 70%  perf-profile.self.cycles-pp.inode_maybe_inc_iversion
      0.83 ± 29%      -0.6        0.24 ± 70%  perf-profile.self.cycles-pp.vfs_unlink
      0.83 ± 30%      -0.6        0.24 ± 71%  perf-profile.self.cycles-pp.lookup_open
      0.81 ± 26%      -0.5        0.27 ± 70%  perf-profile.self.cycles-pp.inode_init_owner
      0.81 ± 23%      -0.5        0.27 ± 70%  perf-profile.self.cycles-pp.__fput
      0.75 ± 21%      -0.5        0.24 ± 71%  perf-profile.self.cycles-pp.do_dentry_open
      0.73 ± 25%      -0.5        0.23 ± 70%  perf-profile.self.cycles-pp.apparmor_path_unlink
      0.70 ± 31%      -0.5        0.20 ± 70%  perf-profile.self.cycles-pp.__fsnotify_parent
      0.84            -0.5        0.35 ± 71%  perf-profile.self.cycles-pp.file_free_rcu
      0.79 ± 29%      -0.5        0.31 ± 70%  perf-profile.self.cycles-pp.fsnotify
      0.62 ± 31%      -0.5        0.16 ± 71%  perf-profile.self.cycles-pp.open_last_lookups
      0.54 ± 32%      -0.4        0.14 ± 71%  perf-profile.self.cycles-pp.fsnotify_perm
      0.80            -0.4        0.40 ± 71%  perf-profile.self.cycles-pp.lockref_get_not_dead
      0.52 ± 25%      -0.3        0.18 ± 71%  perf-profile.self.cycles-pp.shmem_evict_inode
      0.44 ± 30%      -0.3        0.11 ± 70%  perf-profile.self.cycles-pp.iput
      0.56 ± 13%      -0.3        0.26 ± 70%  perf-profile.self.cycles-pp.link_path_walk
      0.57 ±  4%      -0.3        0.31 ± 70%  perf-profile.self.cycles-pp._raw_spin_lock_irq
      0.37 ± 24%      -0.3        0.11 ± 70%  perf-profile.self.cycles-pp.drop_nlink
      0.39 ± 27%      -0.3        0.13 ± 71%  perf-profile.self.cycles-pp.may_delete
      0.31 ± 43%      -0.2        0.07 ± 70%  perf-profile.self.cycles-pp.dentry_unlink_inode
      0.38 ± 18%      -0.2        0.15 ± 70%  perf-profile.self.cycles-pp.mnt_want_write
      0.52 ±  7%      -0.2        0.30 ± 71%  perf-profile.self.cycles-pp.apparmor_file_open
      0.36 ± 22%      -0.2        0.14 ± 70%  perf-profile.self.cycles-pp.mnt_drop_write
      0.34 ± 21%      -0.2        0.13 ± 71%  perf-profile.self.cycles-pp.__d_alloc
      0.49            -0.2        0.27 ± 70%  perf-profile.self.cycles-pp.apparmor_file_alloc_security
      0.29 ± 31%      -0.2        0.09 ± 71%  perf-profile.self.cycles-pp.shmem_reserve_inode
      0.48 ±  5%      -0.2        0.29 ± 70%  perf-profile.self.cycles-pp.init_file
      0.26 ± 26%      -0.2        0.08 ± 70%  perf-profile.self.cycles-pp.simple_lookup
      0.41 ±  7%      -0.2        0.24 ± 70%  perf-profile.self.cycles-pp.memcg_list_lru_alloc
      0.26 ± 29%      -0.2        0.09 ± 70%  perf-profile.self.cycles-pp.__srcu_read_lock
      0.34 ± 10%      -0.2        0.18 ± 71%  perf-profile.self.cycles-pp.locks_remove_posix
      0.31            -0.1        0.17 ± 71%  perf-profile.self.cycles-pp.apparmor_file_free_security
      0.21 ± 24%      -0.1        0.07 ± 70%  perf-profile.self.cycles-pp.errseq_sample
      0.35 ±  3%      -0.1        0.22 ± 70%  perf-profile.self.cycles-pp.mod_objcg_state
      0.30 ±  4%      -0.1        0.17 ± 70%  perf-profile.self.cycles-pp.lockref_put_or_lock
      0.30 ±  7%      -0.1        0.17 ± 70%  perf-profile.self.cycles-pp.apparmor_path_mknod
      0.27            -0.1        0.15 ± 70%  perf-profile.self.cycles-pp.strncpy_from_user
      0.16 ± 18%      -0.1        0.07 ± 70%  perf-profile.self.cycles-pp.inode_to_bdi
      0.19 ±  8%      -0.1        0.10 ± 70%  perf-profile.self.cycles-pp.__entry_text_start
      0.14 ±  3%      -0.1        0.08 ± 70%  perf-profile.self.cycles-pp.step_into
      0.13 ±  6%      -0.1        0.07 ± 70%  perf-profile.self.cycles-pp.down_write
      0.11 ± 19%      -0.1        0.05 ± 72%  perf-profile.self.cycles-pp._find_next_zero_bit
      0.10 ±  8%      -0.0        0.05 ± 71%  perf-profile.self.cycles-pp.set_cached_acl
      0.11 ±  4%      -0.0        0.07 ± 70%  perf-profile.self.cycles-pp.refill_obj_stock
      0.10 ±  4%      -0.0        0.06 ± 70%  perf-profile.self.cycles-pp.__filename_parentat
      0.09 ±  5%      -0.0        0.05 ± 72%  perf-profile.self.cycles-pp.rcu_all_qs
      0.09 ±  5%      -0.0        0.05 ± 72%  perf-profile.self.cycles-pp.task_work_add
      0.09 ±  5%      -0.0        0.05 ± 70%  perf-profile.self.cycles-pp.ihold
      0.08 ±  5%      -0.0        0.04 ± 71%  perf-profile.self.cycles-pp.__d_lookup_unhash
      0.06 ±  7%      -0.0        0.03 ± 70%  perf-profile.self.cycles-pp.exit_to_user_mode_loop
      0.06            -0.0        0.03 ± 70%  perf-profile.self.cycles-pp.getname_flags
      0.26 ±  3%      +1.5        1.80 ±116%  perf-profile.self.cycles-pp.__list_del_entry_valid
      0.02 ±141%      +1.7        1.68 ±129%  perf-profile.self.cycles-pp.rcu_nocb_try_bypass
      0.53 ±  5%      +3.8        4.36 ±  7%  perf-profile.self.cycles-pp.__slab_free



***************************************************************************************************
lkp-skl-d08: 36 threads 1 sockets Intel(R) Core(TM) i9-9980XE CPU @ 3.00GHz (Skylake) with 32G memory
=========================================================================================
class/compiler/cpufreq_governor/disk/fs/kconfig/nr_threads/rootfs/tbox_group/test/testcase/testtime:
  filesystem/gcc-12/performance/1SSD/ext4/x86_64-rhel-8.3/10%/debian-11.1-x86_64-20220510.cgz/lkp-skl-d08/handle/stress-ng/60s

commit: 
  c2c3172c26 ("shmem: Refactor shmem_symlink()")
  a1a690e009 ("shmem: stable directory offsets")

c2c3172c269f808d a1a690e009744e4526526b2f838 
---------------- --------------------------- 
         %stddev     %change         %stddev
             \          |                \  
      5.92            -1.2%       5.85        iostat.cpu.system
     88.56 ± 21%     -29.4%      62.51 ± 39%  sched_debug.cfs_rq:/.util_est_enqueued.avg
      1294 ± 10%     -21.2%       1020 ± 13%  sched_debug.cpu.curr->pid.avg
  11210546            +3.0%   11542342        stress-ng.handle.ops
    186841            +3.0%     192371        stress-ng.handle.ops_per_sec
    100.40 ± 53%     -51.2%      49.00 ± 74%  proc-vmstat.nr_dirtied
    488851 ±  5%      -8.4%     447861        proc-vmstat.numa_hit
    480285 ±  5%      -6.8%     447764        proc-vmstat.numa_local
    750318 ±  6%      -8.4%     687348        proc-vmstat.pgalloc_normal
    710717 ±  7%      -9.0%     646894        proc-vmstat.pgfree
  1.43e+09            -1.7%  1.406e+09        perf-stat.i.branch-instructions
      1.25            -0.0        1.20        perf-stat.i.branch-miss-rate%
  19188631            -5.1%   18207135        perf-stat.i.branch-misses
      0.15            +0.0        0.15        perf-stat.i.dTLB-load-miss-rate%
   2781970            +2.6%    2854604        perf-stat.i.dTLB-load-misses
 1.904e+09            -1.5%  1.875e+09        perf-stat.i.dTLB-loads
 1.139e+09            -1.7%   1.12e+09        perf-stat.i.dTLB-stores
     67.67 ± 10%      +7.1       74.72        perf-stat.i.iTLB-load-miss-rate%
   1568674 ± 46%     -32.7%    1055235 ± 14%  perf-stat.i.iTLB-loads
 7.352e+09            -1.4%  7.247e+09        perf-stat.i.instructions
    124.64            -1.9%     122.30        perf-stat.i.metric.M/sec
     41311 ±  5%     -11.0%      36758 ±  2%  perf-stat.i.node-stores
      1.34            -0.0        1.29        perf-stat.overall.branch-miss-rate%
      0.15            +0.0        0.15        perf-stat.overall.dTLB-load-miss-rate%
     68.34 ± 10%      +7.3       75.68 ±  2%  perf-stat.overall.iTLB-load-miss-rate%
 1.407e+09            -1.7%  1.383e+09        perf-stat.ps.branch-instructions
  18876614            -5.1%   17906237        perf-stat.ps.branch-misses
   2737798            +2.6%    2809345        perf-stat.ps.dTLB-load-misses
 1.873e+09            -1.5%  1.845e+09        perf-stat.ps.dTLB-loads
 1.121e+09            -1.7%  1.102e+09        perf-stat.ps.dTLB-stores
   1543675 ± 46%     -32.8%    1037667 ± 14%  perf-stat.ps.iTLB-loads
 7.235e+09            -1.4%   7.13e+09        perf-stat.ps.instructions
     40647 ±  5%     -11.1%      36139 ±  3%  perf-stat.ps.node-stores
 4.557e+11            -1.2%  4.502e+11        perf-stat.total.instructions
      0.01           -20.0%       0.00        perf-sched.sch_delay.avg.ms.smpboot_thread_fn.kthread.ret_from_fork
      0.08 ± 10%     -30.6%       0.06 ±  9%  perf-sched.wait_and_delay.avg.ms.exit_to_user_mode_loop.exit_to_user_mode_prepare.irqentry_exit_to_user_mode.asm_sysvec_apic_timer_interrupt
      0.19 ±  5%     -26.8%       0.14 ±  3%  perf-sched.wait_and_delay.max.ms.exit_to_user_mode_loop.exit_to_user_mode_prepare.irqentry_exit_to_user_mode.asm_sysvec_apic_timer_interrupt
      0.08 ± 20%     -30.2%       0.06 ± 15%  perf-sched.wait_time.avg.ms.__cond_resched.__fput.task_work_run.exit_to_user_mode_loop.exit_to_user_mode_prepare
      0.08 ± 10%     -31.9%       0.06 ±  9%  perf-sched.wait_time.avg.ms.__cond_resched.__kmem_cache_alloc_node.__kmalloc.do_sys_name_to_handle.__x64_sys_name_to_handle_at
      0.08 ± 12%     -31.4%       0.06 ±  8%  perf-sched.wait_time.avg.ms.__cond_resched.__kmem_cache_alloc_node.__kmalloc.handle_to_path.do_handle_open
      0.08 ±  9%     -29.4%       0.06 ± 14%  perf-sched.wait_time.avg.ms.__cond_resched.apparmor_file_alloc_security.security_file_alloc.init_file.alloc_empty_file
      0.08 ±  9%    -100.0%       0.00        perf-sched.wait_time.avg.ms.__cond_resched.dentry_kill.dput.dcache_dir_close.__fput
      0.08 ± 10%     -30.5%       0.06 ±  7%  perf-sched.wait_time.avg.ms.__cond_resched.dput.__fput.task_work_run.exit_to_user_mode_loop
      0.08 ± 11%    -100.0%       0.00        perf-sched.wait_time.avg.ms.__cond_resched.dput.dcache_dir_close.__fput.task_work_run
      0.08 ± 10%     -30.3%       0.06 ±  8%  perf-sched.wait_time.avg.ms.__cond_resched.dput.path_put.__x64_sys_name_to_handle_at.do_syscall_64
      0.08 ±  9%     -34.9%       0.05 ±  8%  perf-sched.wait_time.avg.ms.__cond_resched.dput.path_put.do_handle_open.do_syscall_64
      0.08 ± 10%     -29.6%       0.05 ± 10%  perf-sched.wait_time.avg.ms.__cond_resched.dput.terminate_walk.path_openat.do_file_open_root
      0.08 ±  9%     -30.6%       0.06 ± 10%  perf-sched.wait_time.avg.ms.__cond_resched.dput.terminate_walk.path_openat.do_filp_open
      0.08 ± 10%     -29.1%       0.06 ±  9%  perf-sched.wait_time.avg.ms.__cond_resched.ilookup5.shmem_fh_to_dentry.exportfs_decode_fh_raw.exportfs_decode_fh
      0.08 ± 18%     -30.9%       0.05 ± 10%  perf-sched.wait_time.avg.ms.__cond_resched.kmem_cache_alloc.alloc_empty_file.path_openat.do_file_open_root
      0.07 ±  7%     -23.8%       0.06 ± 10%  perf-sched.wait_time.avg.ms.__cond_resched.kmem_cache_alloc.alloc_empty_file.path_openat.do_filp_open
      0.08 ± 10%     -32.2%       0.05 ±  9%  perf-sched.wait_time.avg.ms.__cond_resched.kmem_cache_alloc.getname_flags.part.0
      0.08 ± 11%     -33.4%       0.05 ± 12%  perf-sched.wait_time.avg.ms.__cond_resched.kmem_cache_alloc.getname_kernel.do_file_open_root.file_open_root
      0.08 ± 11%     -30.8%       0.06 ±  8%  perf-sched.wait_time.avg.ms.__cond_resched.kmem_cache_alloc.security_file_alloc.init_file.alloc_empty_file
      0.08 ± 10%    -100.0%       0.00        perf-sched.wait_time.avg.ms.__cond_resched.slab_pre_alloc_hook.constprop.0.kmem_cache_alloc_lru
      0.08 ± 11%     -34.6%       0.05 ± 11%  perf-sched.wait_time.avg.ms.__cond_resched.task_work_run.exit_to_user_mode_loop.exit_to_user_mode_prepare.syscall_exit_to_user_mode
      0.08 ± 10%     -30.6%       0.06 ±  9%  perf-sched.wait_time.avg.ms.exit_to_user_mode_loop.exit_to_user_mode_prepare.irqentry_exit_to_user_mode.asm_sysvec_apic_timer_interrupt
      0.08 ± 11%     -32.3%       0.05 ±  9%  perf-sched.wait_time.avg.ms.exit_to_user_mode_loop.exit_to_user_mode_prepare.syscall_exit_to_user_mode.do_syscall_64
      0.13 ± 29%     -43.2%       0.07 ± 22%  perf-sched.wait_time.max.ms.__cond_resched.__fput.task_work_run.exit_to_user_mode_loop.exit_to_user_mode_prepare
      0.18 ± 11%     -22.3%       0.14 ± 12%  perf-sched.wait_time.max.ms.__cond_resched.__kmem_cache_alloc_node.__kmalloc.do_sys_name_to_handle.__x64_sys_name_to_handle_at
      0.17 ±  9%     -28.2%       0.12 ±  6%  perf-sched.wait_time.max.ms.__cond_resched.__kmem_cache_alloc_node.__kmalloc.handle_to_path.do_handle_open
      0.13 ± 14%    -100.0%       0.00        perf-sched.wait_time.max.ms.__cond_resched.dentry_kill.dput.dcache_dir_close.__fput
      0.15 ±  9%     -26.8%       0.11 ± 16%  perf-sched.wait_time.max.ms.__cond_resched.dput.__fput.task_work_run.exit_to_user_mode_loop
      0.11 ± 32%    -100.0%       0.00        perf-sched.wait_time.max.ms.__cond_resched.dput.dcache_dir_close.__fput.task_work_run
      0.17 ±  2%     -26.6%       0.12 ±  7%  perf-sched.wait_time.max.ms.__cond_resched.dput.path_put.__x64_sys_name_to_handle_at.do_syscall_64
      0.16 ± 10%     -42.7%       0.09 ± 18%  perf-sched.wait_time.max.ms.__cond_resched.dput.path_put.do_handle_open.do_syscall_64
      0.16 ± 10%     -27.0%       0.12 ±  6%  perf-sched.wait_time.max.ms.__cond_resched.dput.terminate_walk.path_openat.do_file_open_root
      0.16 ±  6%     -24.1%       0.12 ±  9%  perf-sched.wait_time.max.ms.__cond_resched.ilookup5.shmem_fh_to_dentry.exportfs_decode_fh_raw.exportfs_decode_fh
      0.13 ± 27%     -36.6%       0.08 ± 24%  perf-sched.wait_time.max.ms.__cond_resched.kmem_cache_alloc.alloc_empty_file.path_openat.do_file_open_root
      0.13 ± 16%     -27.5%       0.10 ± 20%  perf-sched.wait_time.max.ms.__cond_resched.kmem_cache_alloc.alloc_empty_file.path_openat.do_filp_open
      0.18 ±  9%     -23.2%       0.14 ± 13%  perf-sched.wait_time.max.ms.__cond_resched.kmem_cache_alloc.getname_flags.part.0
      0.15 ± 11%     -35.8%       0.10 ± 15%  perf-sched.wait_time.max.ms.__cond_resched.kmem_cache_alloc.getname_kernel.do_file_open_root.file_open_root
      0.16 ±  7%     -35.5%       0.10 ± 24%  perf-sched.wait_time.max.ms.__cond_resched.kmem_cache_alloc.security_file_alloc.init_file.alloc_empty_file
      0.17 ± 10%    -100.0%       0.00        perf-sched.wait_time.max.ms.__cond_resched.slab_pre_alloc_hook.constprop.0.kmem_cache_alloc_lru
      0.16 ±  8%     -36.2%       0.10 ± 21%  perf-sched.wait_time.max.ms.__cond_resched.task_work_run.exit_to_user_mode_loop.exit_to_user_mode_prepare.syscall_exit_to_user_mode
      0.19 ±  5%     -26.8%       0.14 ±  3%  perf-sched.wait_time.max.ms.exit_to_user_mode_loop.exit_to_user_mode_prepare.irqentry_exit_to_user_mode.asm_sysvec_apic_timer_interrupt
      0.18 ±  4%     -31.4%       0.12 ±  8%  perf-sched.wait_time.max.ms.exit_to_user_mode_loop.exit_to_user_mode_prepare.syscall_exit_to_user_mode.do_syscall_64
      1.42 ±  6%      -0.8        0.59 ±  3%  perf-profile.calltrace.cycles-pp.do_dentry_open.do_open.path_openat.do_filp_open.do_sys_openat2
      1.89 ±  4%      -0.8        1.07 ±  5%  perf-profile.calltrace.cycles-pp.do_open.path_openat.do_filp_open.do_sys_openat2.__x64_sys_openat
      4.24 ±  3%      -0.8        3.43 ±  2%  perf-profile.calltrace.cycles-pp.__x64_sys_openat.do_syscall_64.entry_SYSCALL_64_after_hwframe.open64
      4.19 ±  3%      -0.8        3.38 ±  2%  perf-profile.calltrace.cycles-pp.do_sys_openat2.__x64_sys_openat.do_syscall_64.entry_SYSCALL_64_after_hwframe.open64
      3.46 ±  3%      -0.8        2.68 ±  3%  perf-profile.calltrace.cycles-pp.do_filp_open.do_sys_openat2.__x64_sys_openat.do_syscall_64.entry_SYSCALL_64_after_hwframe
      3.36 ±  3%      -0.8        2.59 ±  3%  perf-profile.calltrace.cycles-pp.path_openat.do_filp_open.do_sys_openat2.__x64_sys_openat.do_syscall_64
      6.64 ±  3%      -0.8        5.88 ±  2%  perf-profile.calltrace.cycles-pp.open64
      5.39 ±  3%      -0.8        4.63 ±  2%  perf-profile.calltrace.cycles-pp.entry_SYSCALL_64_after_hwframe.open64
      4.95 ±  3%      -0.8        4.19 ±  2%  perf-profile.calltrace.cycles-pp.do_syscall_64.entry_SYSCALL_64_after_hwframe.open64
      5.48 ±  4%      -0.7        4.80 ±  3%  perf-profile.calltrace.cycles-pp.do_syscall_64.entry_SYSCALL_64_after_hwframe.__close
      6.32 ±  3%      -0.6        5.68 ±  2%  perf-profile.calltrace.cycles-pp.entry_SYSCALL_64_after_hwframe.__close
      8.83 ±  3%      -0.5        8.29        perf-profile.calltrace.cycles-pp.__close
      4.33 ±  4%      -0.5        3.82 ±  5%  perf-profile.calltrace.cycles-pp.syscall_exit_to_user_mode.do_syscall_64.entry_SYSCALL_64_after_hwframe.__close
      2.36 ±  5%      -0.5        1.86 ±  8%  perf-profile.calltrace.cycles-pp.__fput.task_work_run.exit_to_user_mode_loop.exit_to_user_mode_prepare.syscall_exit_to_user_mode
      2.93 ±  5%      -0.5        2.45 ±  6%  perf-profile.calltrace.cycles-pp.exit_to_user_mode_loop.exit_to_user_mode_prepare.syscall_exit_to_user_mode.do_syscall_64.entry_SYSCALL_64_after_hwframe
      2.76 ±  5%      -0.5        2.28 ±  7%  perf-profile.calltrace.cycles-pp.task_work_run.exit_to_user_mode_loop.exit_to_user_mode_prepare.syscall_exit_to_user_mode.do_syscall_64
      2.98 ±  5%      -0.5        2.50 ±  6%  perf-profile.calltrace.cycles-pp.exit_to_user_mode_prepare.syscall_exit_to_user_mode.do_syscall_64.entry_SYSCALL_64_after_hwframe.__close
      1.17 ± 13%      -0.3        0.85 ± 17%  perf-profile.calltrace.cycles-pp.rcu_core.__do_softirq.run_ksoftirqd.smpboot_thread_fn.kthread
      1.00 ±  3%      -0.1        0.85 ±  9%  perf-profile.calltrace.cycles-pp.__x64_sys_close.do_syscall_64.entry_SYSCALL_64_after_hwframe.__close
      0.77 ±  4%      -0.1        0.66 ± 13%  perf-profile.calltrace.cycles-pp.filp_close.__x64_sys_close.do_syscall_64.entry_SYSCALL_64_after_hwframe.__close
      1.17 ±  6%      +0.1        1.29 ±  5%  perf-profile.calltrace.cycles-pp.do_dentry_open.do_open.path_openat.do_file_open_root.file_open_root
      3.33 ±  5%      +0.2        3.56 ±  2%  perf-profile.calltrace.cycles-pp.do_file_open_root.file_open_root.do_handle_open.do_syscall_64.entry_SYSCALL_64_after_hwframe
      3.38 ±  5%      +0.2        3.61 ±  2%  perf-profile.calltrace.cycles-pp.file_open_root.do_handle_open.do_syscall_64.entry_SYSCALL_64_after_hwframe.open_by_handle_at
      4.26 ±  3%      -0.8        3.44 ±  2%  perf-profile.children.cycles-pp.__x64_sys_openat
      4.21 ±  3%      -0.8        3.40 ±  2%  perf-profile.children.cycles-pp.do_sys_openat2
      3.47 ±  3%      -0.8        2.69 ±  3%  perf-profile.children.cycles-pp.do_filp_open
      6.69 ±  3%      -0.8        5.93 ±  2%  perf-profile.children.cycles-pp.open64
      2.59 ±  5%      -0.7        1.89 ±  3%  perf-profile.children.cycles-pp.do_dentry_open
      3.35 ±  4%      -0.7        2.66 ±  3%  perf-profile.children.cycles-pp.do_open
      6.40 ±  4%      -0.6        5.80 ±  2%  perf-profile.children.cycles-pp.path_openat
      8.94 ±  3%      -0.5        8.41 ±  2%  perf-profile.children.cycles-pp.__close
      2.38 ±  5%      -0.5        1.88 ±  8%  perf-profile.children.cycles-pp.__fput
      2.95 ±  5%      -0.5        2.46 ±  6%  perf-profile.children.cycles-pp.exit_to_user_mode_loop
      2.77 ±  5%      -0.5        2.30 ±  7%  perf-profile.children.cycles-pp.task_work_run
      3.20 ±  5%      -0.4        2.75 ±  5%  perf-profile.children.cycles-pp.exit_to_user_mode_prepare
      2.89 ±  7%      -0.4        2.44 ±  4%  perf-profile.children.cycles-pp.dput
      0.36 ±  8%      -0.2        0.14 ± 16%  perf-profile.children.cycles-pp.lockref_put_return
      1.00 ±  3%      -0.1        0.86 ± 10%  perf-profile.children.cycles-pp.__x64_sys_close
      0.77 ±  4%      -0.1        0.66 ± 13%  perf-profile.children.cycles-pp.filp_close
      0.35 ± 15%      -0.1        0.24 ± 16%  perf-profile.children.cycles-pp.__slab_free
      0.44 ± 13%      -0.1        0.33 ±  8%  perf-profile.children.cycles-pp.shmem_encode_fh
      0.49 ±  6%      -0.1        0.39 ±  3%  perf-profile.children.cycles-pp.memcg_slab_post_alloc_hook
      0.70 ±  7%      -0.1        0.60 ±  9%  perf-profile.children.cycles-pp.lockref_get
      0.39 ±  8%      -0.1        0.30 ± 24%  perf-profile.children.cycles-pp.locks_remove_posix
      0.34 ±  7%      -0.1        0.26 ±  5%  perf-profile.children.cycles-pp.__call_rcu_common
      0.13 ±  8%      -0.1        0.07 ±  9%  perf-profile.children.cycles-pp.___slab_alloc
      0.25 ± 12%      -0.0        0.20 ± 11%  perf-profile.children.cycles-pp.get_obj_cgroup_from_current
      0.37 ± 10%      -0.0        0.33 ±  9%  perf-profile.children.cycles-pp.rep_movs_alternative
      0.08 ± 16%      -0.0        0.05 ± 54%  perf-profile.children.cycles-pp.obj_cgroup_charge
      0.16 ± 12%      -0.0        0.13 ±  4%  perf-profile.children.cycles-pp.close_fd
      0.13 ±  4%      -0.0        0.10 ± 14%  perf-profile.children.cycles-pp.is_vmalloc_addr
      3.34 ±  5%      +0.2        3.57 ±  2%  perf-profile.children.cycles-pp.do_file_open_root
      0.36 ±  8%      -0.2        0.13 ± 19%  perf-profile.self.cycles-pp.lockref_put_return
      0.34 ± 15%      -0.1        0.23 ± 16%  perf-profile.self.cycles-pp.__slab_free
      0.44 ± 13%      -0.1        0.33 ± 10%  perf-profile.self.cycles-pp.shmem_encode_fh
      0.69 ±  7%      -0.1        0.60 ±  9%  perf-profile.self.cycles-pp.lockref_get
      0.39 ±  7%      -0.1        0.30 ± 25%  perf-profile.self.cycles-pp.locks_remove_posix
      0.37 ±  7%      -0.1        0.29 ±  5%  perf-profile.self.cycles-pp.memcg_slab_post_alloc_hook
      0.20 ± 10%      -0.1        0.15 ±  9%  perf-profile.self.cycles-pp.__call_rcu_common
      0.13 ± 21%      -0.0        0.09 ± 28%  perf-profile.self.cycles-pp.__legitimize_path
      0.13 ±  6%      -0.0        0.10 ± 14%  perf-profile.self.cycles-pp.is_vmalloc_addr
      0.10 ±  7%      -0.0        0.08 ±  7%  perf-profile.self.cycles-pp.do_handle_open
      0.15 ± 11%      +0.0        0.18 ±  5%  perf-profile.self.cycles-pp.init_file





Disclaimer:
Results have been estimated based on internal Intel analysis and are provided
for informational purposes only. Any difference in system hardware or software
design or configuration may affect actual performance.
Chuck Lever July 22, 2023, 8:33 p.m. UTC | #3
> On Jul 17, 2023, at 2:46 AM, kernel test robot <oliver.sang@intel.com> wrote:
> 
> 
> hi, Chuck Lever,
> 
> we reported a 3.0% improvement of stress-ng.handle.ops_per_sec for this commit
> on
> https://lore.kernel.org/oe-lkp/202307132153.a52cdb2d-oliver.sang@intel.com/
> 
> but now we noticed a regression, detail as below, FYI
> 
> Hello,
> 
> kernel test robot noticed a -15.5% regression of will-it-scale.per_thread_ops on:
> 
> 
> commit: a1a690e009744e4526526b2f838beec5ef9233cc ("[PATCH v7 3/3] shmem: stable directory offsets")
> url: https://github.com/intel-lab-lkp/linux/commits/Chuck-Lever/libfs-Add-directory-operations-for-stable-offsets/20230701-014925
> base: https://git.kernel.org/cgit/linux/kernel/git/akpm/mm.git mm-everything
> patch link: https://lore.kernel.org/all/168814734331.530310.3911190551060453102.stgit@manet.1015granger.net/
> patch subject: [PATCH v7 3/3] shmem: stable directory offsets
> 
> testcase: will-it-scale
> test machine: 104 threads 2 sockets (Skylake) with 192G memory
> parameters:
> 
> nr_task: 16
> mode: thread
> test: unlink2
> cpufreq_governor: performance
> 
> 
> In addition to that, the commit also has significant impact on the following tests:
> 
> +------------------+-------------------------------------------------------------------------------------------------+
> | testcase: change | will-it-scale: will-it-scale.per_thread_ops -40.0% regression                                   |
> | test machine     | 36 threads 1 sockets Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz (Cascade Lake) with 128G memory |
> | test parameters  | cpufreq_governor=performance                                                                    |
> |                  | mode=thread                                                                                     |
> |                  | nr_task=16                                                                                      |
> |                  | test=unlink2                                                                                    |
> +------------------+-------------------------------------------------------------------------------------------------+
> | testcase: change | stress-ng: stress-ng.handle.ops_per_sec 3.0% improvement                                        |
> | test machine     | 36 threads 1 sockets Intel(R) Core(TM) i9-9980XE CPU @ 3.00GHz (Skylake) with 32G memory        |
> | test parameters  | class=filesystem                                                                                |
> |                  | cpufreq_governor=performance                                                                    |
> |                  | disk=1SSD                                                                                       |
> |                  | fs=ext4                                                                                         |
> |                  | nr_threads=10%                                                                                  |
> |                  | test=handle                                                                                     |
> |                  | testtime=60s                                                                                    |
> +------------------+-------------------------------------------------------------------------------------------------+
> 
> 
> If you fix the issue in a separate patch/commit (i.e. not just a new version of
> the same patch/commit), kindly add following tags
> | Reported-by: kernel test robot <oliver.sang@intel.com>
> | Closes: https://lore.kernel.org/oe-lkp/202307171436.29248fcf-oliver.sang@intel.com
> 
> 
> Details are as below:
> -------------------------------------------------------------------------------------------------->
> 
> 
> To reproduce:
> 
>        git clone https://github.com/intel/lkp-tests.git
>        cd lkp-tests
>        sudo bin/lkp install job.yaml           # job file is attached in this email

I'm trying to reproduce the regression here, but the reproducer fails
at this step with:

==> Installing package will-it-scale with /export/xfs/lkp-tests/sbin/pacman-LKP -U...
warning: source_date_epoch_from_changelog set but %changelog is missing
Executing(%install): /bin/sh -e /var/tmp/rpm-tmp.Py4eQi
+ umask 022
+ cd /export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILD
+ '[' /export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64 '!=' / ']'
+ rm -rf /export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64
++ dirname /export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64
+ mkdir -p /export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT
+ mkdir /export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64
+ CFLAGS='-march=x86-64 -mtune=generic -O2 -pipe -fstack-protector-strong --param=ssp-buffer-size=4'
+ export CFLAGS
+ CXXFLAGS='-march=x86-64 -mtune=generic -O2 -pipe -fstack-protector-strong --param=ssp-buffer-size=4'
+ export CXXFLAGS
+ FFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -m64  -mtune=generic -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection -I/usr/lib64/gfortran/modules'
+ export FFLAGS
+ FCFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -m64  -mtune=generic -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection -I/usr/lib64/gfortran/modules'
+ export FCFLAGS
+ LDFLAGS=-Wl,-O1,--sort-common,--as-needed,-z,relro
+ export LDFLAGS
+ LT_SYS_LIBRARY_PATH=/usr/lib64:
+ export LT_SYS_LIBRARY_PATH
+ CC=gcc
+ export CC
+ CXX=g++
+ export CXX
+ cp -a /export/xfs/lkp-tests/programs/will-it-scale/pkg/will-it-scale-lkp/lkp /export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64
+ /usr/lib/rpm/check-buildroot
+ /usr/lib/rpm/redhat/brp-ldconfig
+ /usr/lib/rpm/brp-compress
+ /usr/lib/rpm/brp-strip /usr/bin/strip
+ /usr/lib/rpm/brp-strip-comment-note /usr/bin/strip /usr/bin/objdump
/usr/bin/objdump: '/export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64/lkp/benchmarks/will-it-scale/writeseek2/export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64/lkp/benchmarks/will-it-scale/dup1_threads': No such file
/usr/bin/strip: '/export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64/lkp/benchmarks/will-it-scale/writeseek2/export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64/lkp/benchmarks/will-it-scale/dup1_threads': No such file
/usr/bin/objdump: '/export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x8/export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64/lkp/benchmarks/will-it-scale/brk1_processes': No such file
/usr/bin/strip: '/export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x8/export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64/lkp/benchmarks/will-it-scale/brk1_processes': No such file
/usr/bin/objdump: '/export/xfs/lkp-tests/programs/will-it-sc_processes': No such file
/usr/bin/strip: '/export/xfs/lkp-tests/programs/will-it-sc_processes': No such file
/usr/bin/objdump: '6_64/lkp/benchmarks/will-it-scale/pread2_threads': No such file
/usr/bin/strip: '6_64/lkp/benchmarks/will-it-scale/pread2_threads': No such file
/usr/bin/objdump: 'ale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64/lkp/benchmarks/will-it-scale/poll2_processes': No such file
/usr/bin/strip: 'ale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64/lkp/benchmarks/will-it-scale/poll2_processes': No such file
+ /usr/lib/rpm/redhat/brp-strip-lto /usr/bin/strip
+ /usr/lib/rpm/brp-strip-static-archive /usr/bin/strip
+ /usr/lib/rpm/check-rpaths
+ /usr/lib/rpm/redhat/brp-mangle-shebangs
mangling shebang in /lkp/benchmarks/python3/bin/python3.8-config from /bin/sh to #!/usr/bin/sh
*** ERROR: ambiguous python shebang in /lkp/benchmarks/python3/lib/python3.8/encodings/rot_13.py: #!/usr/bin/env python. Change it to python3 (or python2) explicitly.
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/lib2to3/pgen2/token.py from /usr/bin/env python3 to #!/usr/bin/python3
*** ERROR: ambiguous python shebang in /lkp/benchmarks/python3/lib/python3.8/lib2to3/tests/data/different_encoding.py: #!/usr/bin/env python. Change it to python3 (or python2) explicitly.
*** ERROR: ambiguous python shebang in /lkp/benchmarks/python3/lib/python3.8/lib2to3/tests/data/false_encoding.py: #!/usr/bin/env python. Change it to python3 (or python2) explicitly.
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/lib2to3/tests/pytree_idempotency.py from /usr/bin/env python3 to #!/usr/bin/python3
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/config-3.8-x86_64-linux-gnu/makesetup from /bin/sh to #!/usr/bin/sh
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/config-3.8-x86_64-linux-gnu/install-sh from /bin/sh to #!/usr/bin/sh
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/ctypes/macholib/fetch_macholib from /bin/sh to #!/usr/bin/sh
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/bytedesign.py from /usr/bin/env python3 to #!/usr/bin/python3
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/clock.py from /usr/bin/env python3 to #!/usr/bin/python3
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/forest.py from /usr/bin/env python3 to #!/usr/bin/python3
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/fractalcurves.py from /usr/bin/env python3 to #!/usr/bin/python3
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/lindenmayer.py from /usr/bin/env python3 to #!/usr/bin/python3
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/minimal_hanoi.py from /usr/bin/env python3 to #!/usr/bin/python3
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/paint.py from /usr/bin/env python3 to #!/usr/bin/python3
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/peace.py from /usr/bin/env python3 to #!/usr/bin/python3
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/penrose.py from /usr/bin/env python3 to #!/usr/bin/python3
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/planet_and_moon.py from /usr/bin/env python3 to #!/usr/bin/python3
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/tree.py from /usr/bin/env python3 to #!/usr/bin/python3
*** WARNING: ./lkp/benchmarks/python3/lib/python3.8/turtledemo/two_canvases.py is executable but has no shebang, removing executable bit
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/yinyang.py from /usr/bin/env python3 to #!/usr/bin/python3
*** WARNING: ./lkp/benchmarks/python3/lib/python3.8/idlelib/idle.bat is executable but has no shebang, removing executable bit
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/idlelib/pyshell.py from /usr/bin/env python3 to #!/usr/bin/python3
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/test/ziptestdata/exe_with_z64 from /bin/bash to #!/usr/bin/bash
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/test/ziptestdata/exe_with_zip from /bin/bash to #!/usr/bin/bash
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/test/ziptestdata/header.sh from /bin/bash to #!/usr/bin/bash
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/test/bisect_cmd.py from /usr/bin/env python3 to #!/usr/bin/python3
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/test/curses_tests.py from /usr/bin/env python3 to #!/usr/bin/python3
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/test/regrtest.py from /usr/bin/env python3 to #!/usr/bin/python3
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/test/re_tests.py from /usr/bin/env python3 to #!/usr/bin/python3
*** WARNING: ./lkp/benchmarks/python3/lib/python3.8/test/test_dataclasses.py is executable but has no shebang, removing executable bit
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/base64.py from /usr/bin/env python3 to #!/usr/bin/python3
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/cProfile.py from /usr/bin/env python3 to #!/usr/bin/python3
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/pdb.py from /usr/bin/env python3 to #!/usr/bin/python3
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/platform.py from /usr/bin/env python3 to #!/usr/bin/python3
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/profile.py from /usr/bin/env python3 to #!/usr/bin/python3
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/quopri.py from /usr/bin/env python3 to #!/usr/bin/python3
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/smtpd.py from /usr/bin/env python3 to #!/usr/bin/python3
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/smtplib.py from /usr/bin/env python3 to #!/usr/bin/python3
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/tabnanny.py from /usr/bin/env python3 to #!/usr/bin/python3
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/tarfile.py from /usr/bin/env python3 to #!/usr/bin/python3
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/timeit.py from /usr/bin/env python3 to #!/usr/bin/python3
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/trace.py from /usr/bin/env python3 to #!/usr/bin/python3
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/uu.py from /usr/bin/env python3 to #!/usr/bin/python3
mangling shebang in /lkp/benchmarks/python3/lib/python3.8/webbrowser.py from /usr/bin/env python3 to #!/usr/bin/python3
mangling shebang in /lkp/benchmarks/will-it-scale/runalltests from /bin/sh to #!/usr/bin/sh
error: Bad exit status from /var/tmp/rpm-tmp.Py4eQi (%install)

RPM build warnings:
    source_date_epoch_from_changelog set but %changelog is missing

RPM build errors:
    Bad exit status from /var/tmp/rpm-tmp.Py4eQi (%install)
error: open of /export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/RPMS/will-it-scale-LKP.rpm failed: No such file or directory
==> WARNING: Failed to install built package(s).
[cel@manet lkp-tests]$

I'm on Fedora 38 x86_64.


--
Chuck Lever
Chuck Lever July 25, 2023, 3:12 p.m. UTC | #4
> On Jul 22, 2023, at 4:33 PM, Chuck Lever III <chuck.lever@oracle.com> wrote:
> 
> 
> 
>> On Jul 17, 2023, at 2:46 AM, kernel test robot <oliver.sang@intel.com> wrote:
>> 
>> 
>> hi, Chuck Lever,
>> 
>> we reported a 3.0% improvement of stress-ng.handle.ops_per_sec for this commit
>> on
>> https://lore.kernel.org/oe-lkp/202307132153.a52cdb2d-oliver.sang@intel.com/
>> 
>> but now we noticed a regression, detail as below, FYI
>> 
>> Hello,
>> 
>> kernel test robot noticed a -15.5% regression of will-it-scale.per_thread_ops on:
>> 
>> 
>> commit: a1a690e009744e4526526b2f838beec5ef9233cc ("[PATCH v7 3/3] shmem: stable directory offsets")
>> url: https://github.com/intel-lab-lkp/linux/commits/Chuck-Lever/libfs-Add-directory-operations-for-stable-offsets/20230701-014925
>> base: https://git.kernel.org/cgit/linux/kernel/git/akpm/mm.git mm-everything
>> patch link: https://lore.kernel.org/all/168814734331.530310.3911190551060453102.stgit@manet.1015granger.net/
>> patch subject: [PATCH v7 3/3] shmem: stable directory offsets
>> 
>> testcase: will-it-scale
>> test machine: 104 threads 2 sockets (Skylake) with 192G memory
>> parameters:
>> 
>> nr_task: 16
>> mode: thread
>> test: unlink2
>> cpufreq_governor: performance
>> 
>> 
>> In addition to that, the commit also has significant impact on the following tests:
>> 
>> +------------------+-------------------------------------------------------------------------------------------------+
>> | testcase: change | will-it-scale: will-it-scale.per_thread_ops -40.0% regression                                   |
>> | test machine     | 36 threads 1 sockets Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz (Cascade Lake) with 128G memory |
>> | test parameters  | cpufreq_governor=performance                                                                    |
>> |                  | mode=thread                                                                                     |
>> |                  | nr_task=16                                                                                      |
>> |                  | test=unlink2                                                                                    |
>> +------------------+-------------------------------------------------------------------------------------------------+
>> | testcase: change | stress-ng: stress-ng.handle.ops_per_sec 3.0% improvement                                        |
>> | test machine     | 36 threads 1 sockets Intel(R) Core(TM) i9-9980XE CPU @ 3.00GHz (Skylake) with 32G memory        |
>> | test parameters  | class=filesystem                                                                                |
>> |                  | cpufreq_governor=performance                                                                    |
>> |                  | disk=1SSD                                                                                       |
>> |                  | fs=ext4                                                                                         |
>> |                  | nr_threads=10%                                                                                  |
>> |                  | test=handle                                                                                     |
>> |                  | testtime=60s                                                                                    |
>> +------------------+-------------------------------------------------------------------------------------------------+
>> 
>> 
>> If you fix the issue in a separate patch/commit (i.e. not just a new version of
>> the same patch/commit), kindly add following tags
>> | Reported-by: kernel test robot <oliver.sang@intel.com>
>> | Closes: https://lore.kernel.org/oe-lkp/202307171436.29248fcf-oliver.sang@intel.com
>> 
>> 
>> Details are as below:
>> -------------------------------------------------------------------------------------------------->
>> 
>> 
>> To reproduce:
>> 
>>       git clone https://github.com/intel/lkp-tests.git
>>       cd lkp-tests
>>       sudo bin/lkp install job.yaml           # job file is attached in this email

Has anyone from the lkp or ltp teams had a chance to look at this?
I'm stuck without this reproducer.


> I'm trying to reproduce the regression here, but the reproducer fails
> at this step with:
> 
> ==> Installing package will-it-scale with /export/xfs/lkp-tests/sbin/pacman-LKP -U...
> warning: source_date_epoch_from_changelog set but %changelog is missing
> Executing(%install): /bin/sh -e /var/tmp/rpm-tmp.Py4eQi
> + umask 022
> + cd /export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILD
> + '[' /export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64 '!=' / ']'
> + rm -rf /export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64
> ++ dirname /export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64
> + mkdir -p /export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT
> + mkdir /export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64
> + CFLAGS='-march=x86-64 -mtune=generic -O2 -pipe -fstack-protector-strong --param=ssp-buffer-size=4'
> + export CFLAGS
> + CXXFLAGS='-march=x86-64 -mtune=generic -O2 -pipe -fstack-protector-strong --param=ssp-buffer-size=4'
> + export CXXFLAGS
> + FFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -m64  -mtune=generic -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection -I/usr/lib64/gfortran/modules'
> + export FFLAGS
> + FCFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -m64  -mtune=generic -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection -I/usr/lib64/gfortran/modules'
> + export FCFLAGS
> + LDFLAGS=-Wl,-O1,--sort-common,--as-needed,-z,relro
> + export LDFLAGS
> + LT_SYS_LIBRARY_PATH=/usr/lib64:
> + export LT_SYS_LIBRARY_PATH
> + CC=gcc
> + export CC
> + CXX=g++
> + export CXX
> + cp -a /export/xfs/lkp-tests/programs/will-it-scale/pkg/will-it-scale-lkp/lkp /export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64
> + /usr/lib/rpm/check-buildroot
> + /usr/lib/rpm/redhat/brp-ldconfig
> + /usr/lib/rpm/brp-compress
> + /usr/lib/rpm/brp-strip /usr/bin/strip
> + /usr/lib/rpm/brp-strip-comment-note /usr/bin/strip /usr/bin/objdump
> /usr/bin/objdump: '/export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64/lkp/benchmarks/will-it-scale/writeseek2/export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64/lkp/benchmarks/will-it-scale/dup1_threads': No such file
> /usr/bin/strip: '/export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64/lkp/benchmarks/will-it-scale/writeseek2/export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64/lkp/benchmarks/will-it-scale/dup1_threads': No such file
> /usr/bin/objdump: '/export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x8/export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64/lkp/benchmarks/will-it-scale/brk1_processes': No such file
> /usr/bin/strip: '/export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x8/export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64/lkp/benchmarks/will-it-scale/brk1_processes': No such file
> /usr/bin/objdump: '/export/xfs/lkp-tests/programs/will-it-sc_processes': No such file
> /usr/bin/strip: '/export/xfs/lkp-tests/programs/will-it-sc_processes': No such file
> /usr/bin/objdump: '6_64/lkp/benchmarks/will-it-scale/pread2_threads': No such file
> /usr/bin/strip: '6_64/lkp/benchmarks/will-it-scale/pread2_threads': No such file
> /usr/bin/objdump: 'ale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64/lkp/benchmarks/will-it-scale/poll2_processes': No such file
> /usr/bin/strip: 'ale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64/lkp/benchmarks/will-it-scale/poll2_processes': No such file
> + /usr/lib/rpm/redhat/brp-strip-lto /usr/bin/strip
> + /usr/lib/rpm/brp-strip-static-archive /usr/bin/strip
> + /usr/lib/rpm/check-rpaths
> + /usr/lib/rpm/redhat/brp-mangle-shebangs
> mangling shebang in /lkp/benchmarks/python3/bin/python3.8-config from /bin/sh to #!/usr/bin/sh
> *** ERROR: ambiguous python shebang in /lkp/benchmarks/python3/lib/python3.8/encodings/rot_13.py: #!/usr/bin/env python. Change it to python3 (or python2) explicitly.
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/lib2to3/pgen2/token.py from /usr/bin/env python3 to #!/usr/bin/python3
> *** ERROR: ambiguous python shebang in /lkp/benchmarks/python3/lib/python3.8/lib2to3/tests/data/different_encoding.py: #!/usr/bin/env python. Change it to python3 (or python2) explicitly.
> *** ERROR: ambiguous python shebang in /lkp/benchmarks/python3/lib/python3.8/lib2to3/tests/data/false_encoding.py: #!/usr/bin/env python. Change it to python3 (or python2) explicitly.
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/lib2to3/tests/pytree_idempotency.py from /usr/bin/env python3 to #!/usr/bin/python3
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/config-3.8-x86_64-linux-gnu/makesetup from /bin/sh to #!/usr/bin/sh
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/config-3.8-x86_64-linux-gnu/install-sh from /bin/sh to #!/usr/bin/sh
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/ctypes/macholib/fetch_macholib from /bin/sh to #!/usr/bin/sh
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/bytedesign.py from /usr/bin/env python3 to #!/usr/bin/python3
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/clock.py from /usr/bin/env python3 to #!/usr/bin/python3
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/forest.py from /usr/bin/env python3 to #!/usr/bin/python3
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/fractalcurves.py from /usr/bin/env python3 to #!/usr/bin/python3
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/lindenmayer.py from /usr/bin/env python3 to #!/usr/bin/python3
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/minimal_hanoi.py from /usr/bin/env python3 to #!/usr/bin/python3
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/paint.py from /usr/bin/env python3 to #!/usr/bin/python3
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/peace.py from /usr/bin/env python3 to #!/usr/bin/python3
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/penrose.py from /usr/bin/env python3 to #!/usr/bin/python3
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/planet_and_moon.py from /usr/bin/env python3 to #!/usr/bin/python3
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/tree.py from /usr/bin/env python3 to #!/usr/bin/python3
> *** WARNING: ./lkp/benchmarks/python3/lib/python3.8/turtledemo/two_canvases.py is executable but has no shebang, removing executable bit
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/yinyang.py from /usr/bin/env python3 to #!/usr/bin/python3
> *** WARNING: ./lkp/benchmarks/python3/lib/python3.8/idlelib/idle.bat is executable but has no shebang, removing executable bit
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/idlelib/pyshell.py from /usr/bin/env python3 to #!/usr/bin/python3
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/test/ziptestdata/exe_with_z64 from /bin/bash to #!/usr/bin/bash
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/test/ziptestdata/exe_with_zip from /bin/bash to #!/usr/bin/bash
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/test/ziptestdata/header.sh from /bin/bash to #!/usr/bin/bash
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/test/bisect_cmd.py from /usr/bin/env python3 to #!/usr/bin/python3
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/test/curses_tests.py from /usr/bin/env python3 to #!/usr/bin/python3
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/test/regrtest.py from /usr/bin/env python3 to #!/usr/bin/python3
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/test/re_tests.py from /usr/bin/env python3 to #!/usr/bin/python3
> *** WARNING: ./lkp/benchmarks/python3/lib/python3.8/test/test_dataclasses.py is executable but has no shebang, removing executable bit
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/base64.py from /usr/bin/env python3 to #!/usr/bin/python3
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/cProfile.py from /usr/bin/env python3 to #!/usr/bin/python3
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/pdb.py from /usr/bin/env python3 to #!/usr/bin/python3
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/platform.py from /usr/bin/env python3 to #!/usr/bin/python3
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/profile.py from /usr/bin/env python3 to #!/usr/bin/python3
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/quopri.py from /usr/bin/env python3 to #!/usr/bin/python3
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/smtpd.py from /usr/bin/env python3 to #!/usr/bin/python3
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/smtplib.py from /usr/bin/env python3 to #!/usr/bin/python3
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/tabnanny.py from /usr/bin/env python3 to #!/usr/bin/python3
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/tarfile.py from /usr/bin/env python3 to #!/usr/bin/python3
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/timeit.py from /usr/bin/env python3 to #!/usr/bin/python3
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/trace.py from /usr/bin/env python3 to #!/usr/bin/python3
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/uu.py from /usr/bin/env python3 to #!/usr/bin/python3
> mangling shebang in /lkp/benchmarks/python3/lib/python3.8/webbrowser.py from /usr/bin/env python3 to #!/usr/bin/python3
> mangling shebang in /lkp/benchmarks/will-it-scale/runalltests from /bin/sh to #!/usr/bin/sh
> error: Bad exit status from /var/tmp/rpm-tmp.Py4eQi (%install)
> 
> RPM build warnings:
>    source_date_epoch_from_changelog set but %changelog is missing
> 
> RPM build errors:
>    Bad exit status from /var/tmp/rpm-tmp.Py4eQi (%install)
> error: open of /export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/RPMS/will-it-scale-LKP.rpm failed: No such file or directory
> ==> WARNING: Failed to install built package(s).
> [cel@manet lkp-tests]$
> 
> I'm on Fedora 38 x86_64.
> 
> 
> --
> Chuck Lever


--
Chuck Lever
Philip Li July 25, 2023, 3:54 p.m. UTC | #5
On Tue, Jul 25, 2023 at 03:12:22PM +0000, Chuck Lever III wrote:
> 
> 
> > On Jul 22, 2023, at 4:33 PM, Chuck Lever III <chuck.lever@oracle.com> wrote:
> > 
> > 
> > 
> >> On Jul 17, 2023, at 2:46 AM, kernel test robot <oliver.sang@intel.com> wrote:
> >> 
> >> 
> >> hi, Chuck Lever,
> >> 
> >> we reported a 3.0% improvement of stress-ng.handle.ops_per_sec for this commit
> >> on
> >> https://lore.kernel.org/oe-lkp/202307132153.a52cdb2d-oliver.sang@intel.com/
> >> 
> >> but now we noticed a regression, detail as below, FYI
> >> 
> >> Hello,
> >> 
> >> kernel test robot noticed a -15.5% regression of will-it-scale.per_thread_ops on:
> >> 
> >> 
> >> commit: a1a690e009744e4526526b2f838beec5ef9233cc ("[PATCH v7 3/3] shmem: stable directory offsets")
> >> url: https://github.com/intel-lab-lkp/linux/commits/Chuck-Lever/libfs-Add-directory-operations-for-stable-offsets/20230701-014925
> >> base: https://git.kernel.org/cgit/linux/kernel/git/akpm/mm.git mm-everything
> >> patch link: https://lore.kernel.org/all/168814734331.530310.3911190551060453102.stgit@manet.1015granger.net/
> >> patch subject: [PATCH v7 3/3] shmem: stable directory offsets
> >> 
> >> testcase: will-it-scale
> >> test machine: 104 threads 2 sockets (Skylake) with 192G memory
> >> parameters:
> >> 
> >> nr_task: 16
> >> mode: thread
> >> test: unlink2
> >> cpufreq_governor: performance
> >> 
> >> 
> >> In addition to that, the commit also has significant impact on the following tests:
> >> 
> >> +------------------+-------------------------------------------------------------------------------------------------+
> >> | testcase: change | will-it-scale: will-it-scale.per_thread_ops -40.0% regression                                   |
> >> | test machine     | 36 threads 1 sockets Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz (Cascade Lake) with 128G memory |
> >> | test parameters  | cpufreq_governor=performance                                                                    |
> >> |                  | mode=thread                                                                                     |
> >> |                  | nr_task=16                                                                                      |
> >> |                  | test=unlink2                                                                                    |
> >> +------------------+-------------------------------------------------------------------------------------------------+
> >> | testcase: change | stress-ng: stress-ng.handle.ops_per_sec 3.0% improvement                                        |
> >> | test machine     | 36 threads 1 sockets Intel(R) Core(TM) i9-9980XE CPU @ 3.00GHz (Skylake) with 32G memory        |
> >> | test parameters  | class=filesystem                                                                                |
> >> |                  | cpufreq_governor=performance                                                                    |
> >> |                  | disk=1SSD                                                                                       |
> >> |                  | fs=ext4                                                                                         |
> >> |                  | nr_threads=10%                                                                                  |
> >> |                  | test=handle                                                                                     |
> >> |                  | testtime=60s                                                                                    |
> >> +------------------+-------------------------------------------------------------------------------------------------+
> >> 
> >> 
> >> If you fix the issue in a separate patch/commit (i.e. not just a new version of
> >> the same patch/commit), kindly add following tags
> >> | Reported-by: kernel test robot <oliver.sang@intel.com>
> >> | Closes: https://lore.kernel.org/oe-lkp/202307171436.29248fcf-oliver.sang@intel.com
> >> 
> >> 
> >> Details are as below:
> >> -------------------------------------------------------------------------------------------------->
> >> 
> >> 
> >> To reproduce:
> >> 
> >>       git clone https://github.com/intel/lkp-tests.git
> >>       cd lkp-tests
> >>       sudo bin/lkp install job.yaml           # job file is attached in this email
> 
> Has anyone from the lkp or ltp teams had a chance to look at this?
> I'm stuck without this reproducer.

Sorry about this that fedora is not fully supported now [1]. A possible way
is to run the test inside docker [2]. But we haven't fully tested the
reproduce steps in docker yet, which is in our TODO list. Also a concern is
that docker environment probably can't reproduce the performance regression.

For now, not sure whether it is convenient for you to have a ubuntu or debian
environment to give a try? Another alternative is if you have new patch, we
can assist to verify it on our machines.

[1] https://github.com/intel/lkp-tests#supported-distributions
[2] https://github.com/intel/lkp-tests/tree/master/docker

> 
> 
> > I'm trying to reproduce the regression here, but the reproducer fails
> > at this step with:
> > 
> > ==> Installing package will-it-scale with /export/xfs/lkp-tests/sbin/pacman-LKP -U...
> > warning: source_date_epoch_from_changelog set but %changelog is missing
> > Executing(%install): /bin/sh -e /var/tmp/rpm-tmp.Py4eQi
> > + umask 022
> > + cd /export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILD
> > + '[' /export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64 '!=' / ']'
> > + rm -rf /export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64
> > ++ dirname /export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64
> > + mkdir -p /export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT
> > + mkdir /export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64
> > + CFLAGS='-march=x86-64 -mtune=generic -O2 -pipe -fstack-protector-strong --param=ssp-buffer-size=4'
> > + export CFLAGS
> > + CXXFLAGS='-march=x86-64 -mtune=generic -O2 -pipe -fstack-protector-strong --param=ssp-buffer-size=4'
> > + export CXXFLAGS
> > + FFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -m64  -mtune=generic -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection -I/usr/lib64/gfortran/modules'
> > + export FFLAGS
> > + FCFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -m64  -mtune=generic -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection -I/usr/lib64/gfortran/modules'
> > + export FCFLAGS
> > + LDFLAGS=-Wl,-O1,--sort-common,--as-needed,-z,relro
> > + export LDFLAGS
> > + LT_SYS_LIBRARY_PATH=/usr/lib64:
> > + export LT_SYS_LIBRARY_PATH
> > + CC=gcc
> > + export CC
> > + CXX=g++
> > + export CXX
> > + cp -a /export/xfs/lkp-tests/programs/will-it-scale/pkg/will-it-scale-lkp/lkp /export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64
> > + /usr/lib/rpm/check-buildroot
> > + /usr/lib/rpm/redhat/brp-ldconfig
> > + /usr/lib/rpm/brp-compress
> > + /usr/lib/rpm/brp-strip /usr/bin/strip
> > + /usr/lib/rpm/brp-strip-comment-note /usr/bin/strip /usr/bin/objdump
> > /usr/bin/objdump: '/export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64/lkp/benchmarks/will-it-scale/writeseek2/export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64/lkp/benchmarks/will-it-scale/dup1_threads': No such file
> > /usr/bin/strip: '/export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64/lkp/benchmarks/will-it-scale/writeseek2/export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64/lkp/benchmarks/will-it-scale/dup1_threads': No such file
> > /usr/bin/objdump: '/export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x8/export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64/lkp/benchmarks/will-it-scale/brk1_processes': No such file
> > /usr/bin/strip: '/export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x8/export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64/lkp/benchmarks/will-it-scale/brk1_processes': No such file
> > /usr/bin/objdump: '/export/xfs/lkp-tests/programs/will-it-sc_processes': No such file
> > /usr/bin/strip: '/export/xfs/lkp-tests/programs/will-it-sc_processes': No such file
> > /usr/bin/objdump: '6_64/lkp/benchmarks/will-it-scale/pread2_threads': No such file
> > /usr/bin/strip: '6_64/lkp/benchmarks/will-it-scale/pread2_threads': No such file
> > /usr/bin/objdump: 'ale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64/lkp/benchmarks/will-it-scale/poll2_processes': No such file
> > /usr/bin/strip: 'ale/pkg/rpm_build/BUILDROOT/will-it-scale-LKP-1-1.x86_64/lkp/benchmarks/will-it-scale/poll2_processes': No such file
> > + /usr/lib/rpm/redhat/brp-strip-lto /usr/bin/strip
> > + /usr/lib/rpm/brp-strip-static-archive /usr/bin/strip
> > + /usr/lib/rpm/check-rpaths
> > + /usr/lib/rpm/redhat/brp-mangle-shebangs
> > mangling shebang in /lkp/benchmarks/python3/bin/python3.8-config from /bin/sh to #!/usr/bin/sh
> > *** ERROR: ambiguous python shebang in /lkp/benchmarks/python3/lib/python3.8/encodings/rot_13.py: #!/usr/bin/env python. Change it to python3 (or python2) explicitly.
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/lib2to3/pgen2/token.py from /usr/bin/env python3 to #!/usr/bin/python3
> > *** ERROR: ambiguous python shebang in /lkp/benchmarks/python3/lib/python3.8/lib2to3/tests/data/different_encoding.py: #!/usr/bin/env python. Change it to python3 (or python2) explicitly.
> > *** ERROR: ambiguous python shebang in /lkp/benchmarks/python3/lib/python3.8/lib2to3/tests/data/false_encoding.py: #!/usr/bin/env python. Change it to python3 (or python2) explicitly.
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/lib2to3/tests/pytree_idempotency.py from /usr/bin/env python3 to #!/usr/bin/python3
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/config-3.8-x86_64-linux-gnu/makesetup from /bin/sh to #!/usr/bin/sh
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/config-3.8-x86_64-linux-gnu/install-sh from /bin/sh to #!/usr/bin/sh
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/ctypes/macholib/fetch_macholib from /bin/sh to #!/usr/bin/sh
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/bytedesign.py from /usr/bin/env python3 to #!/usr/bin/python3
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/clock.py from /usr/bin/env python3 to #!/usr/bin/python3
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/forest.py from /usr/bin/env python3 to #!/usr/bin/python3
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/fractalcurves.py from /usr/bin/env python3 to #!/usr/bin/python3
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/lindenmayer.py from /usr/bin/env python3 to #!/usr/bin/python3
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/minimal_hanoi.py from /usr/bin/env python3 to #!/usr/bin/python3
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/paint.py from /usr/bin/env python3 to #!/usr/bin/python3
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/peace.py from /usr/bin/env python3 to #!/usr/bin/python3
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/penrose.py from /usr/bin/env python3 to #!/usr/bin/python3
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/planet_and_moon.py from /usr/bin/env python3 to #!/usr/bin/python3
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/tree.py from /usr/bin/env python3 to #!/usr/bin/python3
> > *** WARNING: ./lkp/benchmarks/python3/lib/python3.8/turtledemo/two_canvases.py is executable but has no shebang, removing executable bit
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/turtledemo/yinyang.py from /usr/bin/env python3 to #!/usr/bin/python3
> > *** WARNING: ./lkp/benchmarks/python3/lib/python3.8/idlelib/idle.bat is executable but has no shebang, removing executable bit
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/idlelib/pyshell.py from /usr/bin/env python3 to #!/usr/bin/python3
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/test/ziptestdata/exe_with_z64 from /bin/bash to #!/usr/bin/bash
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/test/ziptestdata/exe_with_zip from /bin/bash to #!/usr/bin/bash
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/test/ziptestdata/header.sh from /bin/bash to #!/usr/bin/bash
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/test/bisect_cmd.py from /usr/bin/env python3 to #!/usr/bin/python3
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/test/curses_tests.py from /usr/bin/env python3 to #!/usr/bin/python3
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/test/regrtest.py from /usr/bin/env python3 to #!/usr/bin/python3
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/test/re_tests.py from /usr/bin/env python3 to #!/usr/bin/python3
> > *** WARNING: ./lkp/benchmarks/python3/lib/python3.8/test/test_dataclasses.py is executable but has no shebang, removing executable bit
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/base64.py from /usr/bin/env python3 to #!/usr/bin/python3
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/cProfile.py from /usr/bin/env python3 to #!/usr/bin/python3
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/pdb.py from /usr/bin/env python3 to #!/usr/bin/python3
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/platform.py from /usr/bin/env python3 to #!/usr/bin/python3
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/profile.py from /usr/bin/env python3 to #!/usr/bin/python3
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/quopri.py from /usr/bin/env python3 to #!/usr/bin/python3
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/smtpd.py from /usr/bin/env python3 to #!/usr/bin/python3
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/smtplib.py from /usr/bin/env python3 to #!/usr/bin/python3
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/tabnanny.py from /usr/bin/env python3 to #!/usr/bin/python3
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/tarfile.py from /usr/bin/env python3 to #!/usr/bin/python3
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/timeit.py from /usr/bin/env python3 to #!/usr/bin/python3
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/trace.py from /usr/bin/env python3 to #!/usr/bin/python3
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/uu.py from /usr/bin/env python3 to #!/usr/bin/python3
> > mangling shebang in /lkp/benchmarks/python3/lib/python3.8/webbrowser.py from /usr/bin/env python3 to #!/usr/bin/python3
> > mangling shebang in /lkp/benchmarks/will-it-scale/runalltests from /bin/sh to #!/usr/bin/sh
> > error: Bad exit status from /var/tmp/rpm-tmp.Py4eQi (%install)
> > 
> > RPM build warnings:
> >    source_date_epoch_from_changelog set but %changelog is missing
> > 
> > RPM build errors:
> >    Bad exit status from /var/tmp/rpm-tmp.Py4eQi (%install)
> > error: open of /export/xfs/lkp-tests/programs/will-it-scale/pkg/rpm_build/RPMS/will-it-scale-LKP.rpm failed: No such file or directory
> > ==> WARNING: Failed to install built package(s).
> > [cel@manet lkp-tests]$
> > 
> > I'm on Fedora 38 x86_64.
> > 
> > 
> > --
> > Chuck Lever
> 
> 
> --
> Chuck Lever
> 
> 
>
Christian Brauner July 25, 2023, 3:59 p.m. UTC | #6
On Tue, Jul 25, 2023 at 11:54:26PM +0800, Philip Li wrote:
> On Tue, Jul 25, 2023 at 03:12:22PM +0000, Chuck Lever III wrote:
> > 
> > 
> > > On Jul 22, 2023, at 4:33 PM, Chuck Lever III <chuck.lever@oracle.com> wrote:
> > > 
> > > 
> > > 
> > >> On Jul 17, 2023, at 2:46 AM, kernel test robot <oliver.sang@intel.com> wrote:
> > >> 
> > >> 
> > >> hi, Chuck Lever,
> > >> 
> > >> we reported a 3.0% improvement of stress-ng.handle.ops_per_sec for this commit
> > >> on
> > >> https://lore.kernel.org/oe-lkp/202307132153.a52cdb2d-oliver.sang@intel.com/
> > >> 
> > >> but now we noticed a regression, detail as below, FYI
> > >> 
> > >> Hello,
> > >> 
> > >> kernel test robot noticed a -15.5% regression of will-it-scale.per_thread_ops on:
> > >> 
> > >> 
> > >> commit: a1a690e009744e4526526b2f838beec5ef9233cc ("[PATCH v7 3/3] shmem: stable directory offsets")
> > >> url: https://github.com/intel-lab-lkp/linux/commits/Chuck-Lever/libfs-Add-directory-operations-for-stable-offsets/20230701-014925
> > >> base: https://git.kernel.org/cgit/linux/kernel/git/akpm/mm.git mm-everything
> > >> patch link: https://lore.kernel.org/all/168814734331.530310.3911190551060453102.stgit@manet.1015granger.net/
> > >> patch subject: [PATCH v7 3/3] shmem: stable directory offsets
> > >> 
> > >> testcase: will-it-scale
> > >> test machine: 104 threads 2 sockets (Skylake) with 192G memory
> > >> parameters:
> > >> 
> > >> nr_task: 16
> > >> mode: thread
> > >> test: unlink2
> > >> cpufreq_governor: performance
> > >> 
> > >> 
> > >> In addition to that, the commit also has significant impact on the following tests:
> > >> 
> > >> +------------------+-------------------------------------------------------------------------------------------------+
> > >> | testcase: change | will-it-scale: will-it-scale.per_thread_ops -40.0% regression                                   |
> > >> | test machine     | 36 threads 1 sockets Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz (Cascade Lake) with 128G memory |
> > >> | test parameters  | cpufreq_governor=performance                                                                    |
> > >> |                  | mode=thread                                                                                     |
> > >> |                  | nr_task=16                                                                                      |
> > >> |                  | test=unlink2                                                                                    |
> > >> +------------------+-------------------------------------------------------------------------------------------------+
> > >> | testcase: change | stress-ng: stress-ng.handle.ops_per_sec 3.0% improvement                                        |
> > >> | test machine     | 36 threads 1 sockets Intel(R) Core(TM) i9-9980XE CPU @ 3.00GHz (Skylake) with 32G memory        |
> > >> | test parameters  | class=filesystem                                                                                |
> > >> |                  | cpufreq_governor=performance                                                                    |
> > >> |                  | disk=1SSD                                                                                       |
> > >> |                  | fs=ext4                                                                                         |
> > >> |                  | nr_threads=10%                                                                                  |
> > >> |                  | test=handle                                                                                     |
> > >> |                  | testtime=60s                                                                                    |
> > >> +------------------+-------------------------------------------------------------------------------------------------+
> > >> 
> > >> 
> > >> If you fix the issue in a separate patch/commit (i.e. not just a new version of
> > >> the same patch/commit), kindly add following tags
> > >> | Reported-by: kernel test robot <oliver.sang@intel.com>
> > >> | Closes: https://lore.kernel.org/oe-lkp/202307171436.29248fcf-oliver.sang@intel.com
> > >> 
> > >> 
> > >> Details are as below:
> > >> -------------------------------------------------------------------------------------------------->
> > >> 
> > >> 
> > >> To reproduce:
> > >> 
> > >>       git clone https://github.com/intel/lkp-tests.git
> > >>       cd lkp-tests
> > >>       sudo bin/lkp install job.yaml           # job file is attached in this email
> > 
> > Has anyone from the lkp or ltp teams had a chance to look at this?
> > I'm stuck without this reproducer.
> 
> Sorry about this that fedora is not fully supported now [1]. A possible way
> is to run the test inside docker [2]. But we haven't fully tested the
> reproduce steps in docker yet, which is in our TODO list. Also a concern is
> that docker environment probably can't reproduce the performance regression.
> 
> For now, not sure whether it is convenient for you to have a ubuntu or debian
> environment to give a try? Another alternative is if you have new patch, we
> can assist to verify it on our machines.

So while we have your attention here. I've asked this a while ago in
another mail: It would be really really helpful if there was a way for
us to ask/trigger a perf test run for specific branches/patches we
suspect of being performance sensitive.

It's a bit of a shame that we have no simple way of submitting a custom
job and get performance results reported. I know that resources for this
are probably scarce but some way to at least request it would be really
really nice.
Philip Li July 26, 2023, 2:55 a.m. UTC | #7
On Tue, Jul 25, 2023 at 05:59:22PM +0200, Christian Brauner wrote:
> On Tue, Jul 25, 2023 at 11:54:26PM +0800, Philip Li wrote:
> > On Tue, Jul 25, 2023 at 03:12:22PM +0000, Chuck Lever III wrote:
> > > 
> > > 
> > > > On Jul 22, 2023, at 4:33 PM, Chuck Lever III <chuck.lever@oracle.com> wrote:
> > > > 
> > > > 
> > > > 
> > > >> On Jul 17, 2023, at 2:46 AM, kernel test robot <oliver.sang@intel.com> wrote:
> > > >> 
> > > >> 
> > > >> hi, Chuck Lever,
> > > >> 
> > > >> we reported a 3.0% improvement of stress-ng.handle.ops_per_sec for this commit
> > > >> on
> > > >> https://lore.kernel.org/oe-lkp/202307132153.a52cdb2d-oliver.sang@intel.com/
> > > >> 
> > > >> but now we noticed a regression, detail as below, FYI
> > > >> 
> > > >> Hello,
> > > >> 
> > > >> kernel test robot noticed a -15.5% regression of will-it-scale.per_thread_ops on:
> > > >> 
> > > >> 
> > > >> commit: a1a690e009744e4526526b2f838beec5ef9233cc ("[PATCH v7 3/3] shmem: stable directory offsets")
> > > >> url: https://github.com/intel-lab-lkp/linux/commits/Chuck-Lever/libfs-Add-directory-operations-for-stable-offsets/20230701-014925
> > > >> base: https://git.kernel.org/cgit/linux/kernel/git/akpm/mm.git mm-everything
> > > >> patch link: https://lore.kernel.org/all/168814734331.530310.3911190551060453102.stgit@manet.1015granger.net/
> > > >> patch subject: [PATCH v7 3/3] shmem: stable directory offsets
> > > >> 
> > > >> testcase: will-it-scale
> > > >> test machine: 104 threads 2 sockets (Skylake) with 192G memory
> > > >> parameters:
> > > >> 
> > > >> nr_task: 16
> > > >> mode: thread
> > > >> test: unlink2
> > > >> cpufreq_governor: performance
> > > >> 
> > > >> 
> > > >> In addition to that, the commit also has significant impact on the following tests:
> > > >> 
> > > >> +------------------+-------------------------------------------------------------------------------------------------+
> > > >> | testcase: change | will-it-scale: will-it-scale.per_thread_ops -40.0% regression                                   |
> > > >> | test machine     | 36 threads 1 sockets Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz (Cascade Lake) with 128G memory |
> > > >> | test parameters  | cpufreq_governor=performance                                                                    |
> > > >> |                  | mode=thread                                                                                     |
> > > >> |                  | nr_task=16                                                                                      |
> > > >> |                  | test=unlink2                                                                                    |
> > > >> +------------------+-------------------------------------------------------------------------------------------------+
> > > >> | testcase: change | stress-ng: stress-ng.handle.ops_per_sec 3.0% improvement                                        |
> > > >> | test machine     | 36 threads 1 sockets Intel(R) Core(TM) i9-9980XE CPU @ 3.00GHz (Skylake) with 32G memory        |
> > > >> | test parameters  | class=filesystem                                                                                |
> > > >> |                  | cpufreq_governor=performance                                                                    |
> > > >> |                  | disk=1SSD                                                                                       |
> > > >> |                  | fs=ext4                                                                                         |
> > > >> |                  | nr_threads=10%                                                                                  |
> > > >> |                  | test=handle                                                                                     |
> > > >> |                  | testtime=60s                                                                                    |
> > > >> +------------------+-------------------------------------------------------------------------------------------------+
> > > >> 
> > > >> 
> > > >> If you fix the issue in a separate patch/commit (i.e. not just a new version of
> > > >> the same patch/commit), kindly add following tags
> > > >> | Reported-by: kernel test robot <oliver.sang@intel.com>
> > > >> | Closes: https://lore.kernel.org/oe-lkp/202307171436.29248fcf-oliver.sang@intel.com
> > > >> 
> > > >> 
> > > >> Details are as below:
> > > >> -------------------------------------------------------------------------------------------------->
> > > >> 
> > > >> 
> > > >> To reproduce:
> > > >> 
> > > >>       git clone https://github.com/intel/lkp-tests.git
> > > >>       cd lkp-tests
> > > >>       sudo bin/lkp install job.yaml           # job file is attached in this email
> > > 
> > > Has anyone from the lkp or ltp teams had a chance to look at this?
> > > I'm stuck without this reproducer.
> > 
> > Sorry about this that fedora is not fully supported now [1]. A possible way
> > is to run the test inside docker [2]. But we haven't fully tested the
> > reproduce steps in docker yet, which is in our TODO list. Also a concern is
> > that docker environment probably can't reproduce the performance regression.
> > 
> > For now, not sure whether it is convenient for you to have a ubuntu or debian
> > environment to give a try? Another alternative is if you have new patch, we
> > can assist to verify it on our machines.
> 
> So while we have your attention here. I've asked this a while ago in
> another mail: It would be really really helpful if there was a way for
> us to ask/trigger a perf test run for specific branches/patches we
> suspect of being performance sensitive.
> 
> It's a bit of a shame that we have no simple way of submitting a custom
> job and get performance results reported. I know that resources for this
> are probably scarce but some way to at least request it would be really
> really nice.

Apologize for this limitation. We have some mid-term TODO list to allow
the verification of reported issue (start from build report) for fix patch.

We will consider runtime side as well per this input to provide a better
experience. And we can start with a controlled scope like to queue the
test in the report, so test suite/parameters/platform is in a manageable
manner.

Thanks for the input.
Tavian Barnes Nov. 13, 2023, 6:06 p.m. UTC | #8
On Fri, 30 Jun 2023 at 13:49:03 -0400, Chuck Lever wrote:
> From: Chuck Lever <chuck.lever@oracle.com>
>
> The current cursor-based directory offset mechanism doesn't work
> when a tmpfs filesystem is exported via NFS. This is because NFS
> clients do not open directories. Each server-side READDIR operation
> has to open the directory, read it, then close it. The cursor state
> for that directory, being associated strictly with the opened
> struct file, is thus discarded after each NFS READDIR operation.
>
> Directory offsets are cached not only by NFS clients, but also by
> user space libraries on those clients. Essentially there is no way
> to invalidate those caches when directory offsets have changed on
> an NFS server after the offset-to-dentry mapping changes. Thus the
> whole application stack depends on unchanging directory offsets.
>
> The solution we've come up with is to make the directory offset for
> each file in a tmpfs filesystem stable for the life of the directory
> entry it represents.
>
> shmem_readdir() and shmem_dir_llseek() now use an xarray to map each
> directory offset (an loff_t integer) to the memory address of a
> struct dentry.

I believe this patch is responsible for a tmpfs behaviour change when
a directory is modified while being read.  The following test program

#include <dirent.h>
#include <err.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <unistd.h>

int main(int argc, char *argv[]) {
	const char *tmp = "/tmp";
	if (argc >= 2)
		tmp = argv[1];

	char *dir_path;
	if (asprintf(&dir_path, "%s/foo.XXXXXX", tmp) < 0)
		err(EXIT_FAILURE, "asprintf()");

	if (!mkdtemp(dir_path))
		err(EXIT_FAILURE, "mkdtemp(%s)", dir_path);

	char *file_path;
	if (asprintf(&file_path, "%s/bar", dir_path) < 0)
		err(EXIT_FAILURE, "asprintf()");

	if (creat(file_path, 0644) < 0)
		err(EXIT_FAILURE, "creat(%s)", file_path);

	DIR *dir = opendir(dir_path);
	if (!dir)
		err(EXIT_FAILURE, "opendir(%s)", dir_path);

	struct dirent *de;
	while ((de = readdir(dir))) {
		printf("readdir(): %s/%s\n", dir_path, de->d_name);
		if (de->d_name[0] == '.')
			continue;

		if (unlink(file_path) != 0)
			err(EXIT_FAILURE, "unlink(%s)", file_path);

		if (creat(file_path, 0644) < 0)
			err(EXIT_FAILURE, "creat(%s)", file_path);
	}

	return EXIT_SUCCESS;
}

when run on Linux 6.5, doesn't print the new directory entry:

tavianator@graphene $ uname -a
Linux graphene 6.5.9-arch2-1 #1 SMP PREEMPT_DYNAMIC Thu, 26 Oct 2023 00:52:20 +0000 x86_64 GNU/Linux
tavianator@graphene $ gcc -Wall foo.c -o foo
tavianator@graphene $ ./foo
readdir(): /tmp/foo.wgmdmm/.
readdir(): /tmp/foo.wgmdmm/..
readdir(): /tmp/foo.wgmdmm/bar

But on Linux 6.6, readdir() never stops:

tavianator@tachyon $ uname -a
Linux tachyon 6.6.1-arch1-1 #1 SMP PREEMPT_DYNAMIC Wed, 08 Nov 2023 16:05:38 +0000 x86_64 GNU/Linux
tavianator@tachyon $ gcc foo.c -o foo
tavianator@tachyon $ ./foo
readdir(): /tmp/foo.XnIRqj/.
readdir(): /tmp/foo.XnIRqj/..
readdir(): /tmp/foo.XnIRqj/bar
readdir(): /tmp/foo.XnIRqj/bar
readdir(): /tmp/foo.XnIRqj/bar
readdir(): /tmp/foo.XnIRqj/bar
readdir(): /tmp/foo.XnIRqj/bar
readdir(): /tmp/foo.XnIRqj/bar
readdir(): /tmp/foo.XnIRqj/bar
readdir(): /tmp/foo.XnIRqj/bar
...
foo: creat(/tmp/foo.TTL6Fg/bar): Too many open files

POSIX says[1]

> If a file is removed from or added to the directory after the most recent
> call to opendir() or rewinddir(), whether a subsequent call to readdir()
> returns an entry for that file is unspecified.

so this isn't necessarily a *bug*, but I just wanted to point out the
behaviour change.  I only noticed it because it broke one of my tests in
bfs[2] (in a non-default build configuration).

[1]: https://pubs.opengroup.org/onlinepubs/9699919799/functions/readdir.html
[2]: https://github.com/tavianator/bfs/blob/main/tests/gnu/ignore_readdir_race_notdir.sh
Chuck Lever Nov. 13, 2023, 6:58 p.m. UTC | #9
On Mon, Nov 13, 2023 at 01:06:16PM -0500, tavianator@tavianator.com wrote:
> On Fri, 30 Jun 2023 at 13:49:03 -0400, Chuck Lever wrote:
> > From: Chuck Lever <chuck.lever@oracle.com>
> >
> > The current cursor-based directory offset mechanism doesn't work
> > when a tmpfs filesystem is exported via NFS. This is because NFS
> > clients do not open directories. Each server-side READDIR operation
> > has to open the directory, read it, then close it. The cursor state
> > for that directory, being associated strictly with the opened
> > struct file, is thus discarded after each NFS READDIR operation.
> >
> > Directory offsets are cached not only by NFS clients, but also by
> > user space libraries on those clients. Essentially there is no way
> > to invalidate those caches when directory offsets have changed on
> > an NFS server after the offset-to-dentry mapping changes. Thus the
> > whole application stack depends on unchanging directory offsets.
> >
> > The solution we've come up with is to make the directory offset for
> > each file in a tmpfs filesystem stable for the life of the directory
> > entry it represents.
> >
> > shmem_readdir() and shmem_dir_llseek() now use an xarray to map each
> > directory offset (an loff_t integer) to the memory address of a
> > struct dentry.
> 
> I believe this patch is responsible for a tmpfs behaviour change when
> a directory is modified while being read.  The following test program
> 
> #include <dirent.h>
> #include <err.h>
> #include <fcntl.h>
> #include <stdio.h>
> #include <stdlib.h>
> #include <sys/stat.h>
> #include <unistd.h>
> 
> int main(int argc, char *argv[]) {
> 	const char *tmp = "/tmp";
> 	if (argc >= 2)
> 		tmp = argv[1];
> 
> 	char *dir_path;
> 	if (asprintf(&dir_path, "%s/foo.XXXXXX", tmp) < 0)
> 		err(EXIT_FAILURE, "asprintf()");
> 
> 	if (!mkdtemp(dir_path))
> 		err(EXIT_FAILURE, "mkdtemp(%s)", dir_path);
> 
> 	char *file_path;
> 	if (asprintf(&file_path, "%s/bar", dir_path) < 0)
> 		err(EXIT_FAILURE, "asprintf()");
> 
> 	if (creat(file_path, 0644) < 0)
> 		err(EXIT_FAILURE, "creat(%s)", file_path);
> 
> 	DIR *dir = opendir(dir_path);
> 	if (!dir)
> 		err(EXIT_FAILURE, "opendir(%s)", dir_path);
> 
> 	struct dirent *de;
> 	while ((de = readdir(dir))) {
> 		printf("readdir(): %s/%s\n", dir_path, de->d_name);
> 		if (de->d_name[0] == '.')
> 			continue;
> 
> 		if (unlink(file_path) != 0)
> 			err(EXIT_FAILURE, "unlink(%s)", file_path);
> 
> 		if (creat(file_path, 0644) < 0)
> 			err(EXIT_FAILURE, "creat(%s)", file_path);
> 	}
> 
> 	return EXIT_SUCCESS;
> }
> 
> when run on Linux 6.5, doesn't print the new directory entry:
> 
> tavianator@graphene $ uname -a
> Linux graphene 6.5.9-arch2-1 #1 SMP PREEMPT_DYNAMIC Thu, 26 Oct 2023 00:52:20 +0000 x86_64 GNU/Linux
> tavianator@graphene $ gcc -Wall foo.c -o foo
> tavianator@graphene $ ./foo
> readdir(): /tmp/foo.wgmdmm/.
> readdir(): /tmp/foo.wgmdmm/..
> readdir(): /tmp/foo.wgmdmm/bar
> 
> But on Linux 6.6, readdir() never stops:
> 
> tavianator@tachyon $ uname -a
> Linux tachyon 6.6.1-arch1-1 #1 SMP PREEMPT_DYNAMIC Wed, 08 Nov 2023 16:05:38 +0000 x86_64 GNU/Linux
> tavianator@tachyon $ gcc foo.c -o foo
> tavianator@tachyon $ ./foo
> readdir(): /tmp/foo.XnIRqj/.
> readdir(): /tmp/foo.XnIRqj/..
> readdir(): /tmp/foo.XnIRqj/bar
> readdir(): /tmp/foo.XnIRqj/bar
> readdir(): /tmp/foo.XnIRqj/bar
> readdir(): /tmp/foo.XnIRqj/bar
> readdir(): /tmp/foo.XnIRqj/bar
> readdir(): /tmp/foo.XnIRqj/bar
> readdir(): /tmp/foo.XnIRqj/bar
> readdir(): /tmp/foo.XnIRqj/bar
> ...
> foo: creat(/tmp/foo.TTL6Fg/bar): Too many open files
> 
> POSIX says[1]
> 
> > If a file is removed from or added to the directory after the most recent
> > call to opendir() or rewinddir(), whether a subsequent call to readdir()
> > returns an entry for that file is unspecified.
> 
> so this isn't necessarily a *bug*, but I just wanted to point out the
> behaviour change.

I'm betting dollars to donuts that the v6.6 tmpfs behavior doesn't
match the behavior of other filesystems either.

Thanks for the reproducer, I'll look into it.


> I only noticed it because it broke one of my tests in
> bfs[2] (in a non-default build configuration).
> 
> [1]: https://pubs.opengroup.org/onlinepubs/9699919799/functions/readdir.html
> [2]: https://github.com/tavianator/bfs/blob/main/tests/gnu/ignore_readdir_race_notdir.sh
diff mbox series

Patch

diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h
index 9029abd29b1c..a5454a80ab30 100644
--- a/include/linux/shmem_fs.h
+++ b/include/linux/shmem_fs.h
@@ -27,6 +27,7 @@  struct shmem_inode_info {
 	atomic_t		stop_eviction;	/* hold when working on inode */
 	struct timespec64	i_crtime;	/* file creation time */
 	unsigned int		fsflags;	/* flags for FS_IOC_[SG]ETFLAGS */
+	struct offset_ctx	dir_offsets;	/* stable entry offsets */
 	struct inode		vfs_inode;
 };
 
diff --git a/mm/shmem.c b/mm/shmem.c
index ba3d7db90c9d..c9e6c1dd4fe8 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -2355,6 +2355,11 @@  static void shmem_set_inode_flags(struct inode *inode, unsigned int fsflags)
 #define shmem_initxattrs NULL
 #endif
 
+static struct offset_ctx *shmem_get_offset_ctx(struct inode *inode)
+{
+	return &SHMEM_I(inode)->dir_offsets;
+}
+
 static struct inode *shmem_get_inode(struct mnt_idmap *idmap, struct super_block *sb,
 				     struct inode *dir, umode_t mode, dev_t dev,
 				     unsigned long flags)
@@ -2410,7 +2415,8 @@  static struct inode *shmem_get_inode(struct mnt_idmap *idmap, struct super_block
 			/* Some things misbehave if size == 0 on a directory */
 			inode->i_size = 2 * BOGO_DIRENT_SIZE;
 			inode->i_op = &shmem_dir_inode_operations;
-			inode->i_fop = &simple_dir_operations;
+			inode->i_fop = &simple_offset_dir_operations;
+			simple_offset_init(shmem_get_offset_ctx(inode));
 			break;
 		case S_IFLNK:
 			/*
@@ -3082,7 +3088,10 @@  shmem_mknod(struct mnt_idmap *idmap, struct inode *dir,
 		if (error && error != -EOPNOTSUPP)
 			goto out_iput;
 
-		error = 0;
+		error = simple_offset_add(shmem_get_offset_ctx(dir), dentry);
+		if (error)
+			goto out_iput;
+
 		dir->i_size += BOGO_DIRENT_SIZE;
 		dir->i_ctime = dir->i_mtime = current_time(dir);
 		inode_inc_iversion(dir);
@@ -3159,6 +3168,13 @@  static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentr
 			goto out;
 	}
 
+	ret = simple_offset_add(shmem_get_offset_ctx(dir), dentry);
+	if (ret) {
+		if (inode->i_nlink)
+			shmem_free_inode(inode->i_sb);
+		goto out;
+	}
+
 	dir->i_size += BOGO_DIRENT_SIZE;
 	inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
 	inode_inc_iversion(dir);
@@ -3177,6 +3193,8 @@  static int shmem_unlink(struct inode *dir, struct dentry *dentry)
 	if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
 		shmem_free_inode(inode->i_sb);
 
+	simple_offset_remove(shmem_get_offset_ctx(dir), dentry);
+
 	dir->i_size -= BOGO_DIRENT_SIZE;
 	inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
 	inode_inc_iversion(dir);
@@ -3235,24 +3253,29 @@  static int shmem_rename2(struct mnt_idmap *idmap,
 {
 	struct inode *inode = d_inode(old_dentry);
 	int they_are_dirs = S_ISDIR(inode->i_mode);
+	int error;
 
 	if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
 		return -EINVAL;
 
 	if (flags & RENAME_EXCHANGE)
-		return simple_rename_exchange(old_dir, old_dentry, new_dir, new_dentry);
+		return simple_offset_rename_exchange(old_dir, old_dentry,
+						     new_dir, new_dentry);
 
 	if (!simple_empty(new_dentry))
 		return -ENOTEMPTY;
 
 	if (flags & RENAME_WHITEOUT) {
-		int error;
-
 		error = shmem_whiteout(idmap, old_dir, old_dentry);
 		if (error)
 			return error;
 	}
 
+	simple_offset_remove(shmem_get_offset_ctx(old_dir), old_dentry);
+	error = simple_offset_add(shmem_get_offset_ctx(new_dir), old_dentry);
+	if (error)
+		return error;
+
 	if (d_really_is_positive(new_dentry)) {
 		(void) shmem_unlink(new_dir, new_dentry);
 		if (they_are_dirs) {
@@ -3296,19 +3319,23 @@  static int shmem_symlink(struct mnt_idmap *idmap, struct inode *dir,
 	if (error && error != -EOPNOTSUPP)
 		goto out_iput;
 
+	error = simple_offset_add(shmem_get_offset_ctx(dir), dentry);
+	if (error)
+		goto out_iput;
+
 	inode->i_size = len-1;
 	if (len <= SHORT_SYMLINK_LEN) {
 		inode->i_link = kmemdup(symname, len, GFP_KERNEL);
 		if (!inode->i_link) {
 			error = -ENOMEM;
-			goto out_iput;
+			goto out_remove_offset;
 		}
 		inode->i_op = &shmem_short_symlink_operations;
 	} else {
 		inode_nohighmem(inode);
 		error = shmem_get_folio(inode, 0, &folio, SGP_WRITE);
 		if (error)
-			goto out_iput;
+			goto out_remove_offset;
 		inode->i_mapping->a_ops = &shmem_aops;
 		inode->i_op = &shmem_symlink_inode_operations;
 		memcpy(folio_address(folio), symname, len);
@@ -3323,6 +3350,9 @@  static int shmem_symlink(struct mnt_idmap *idmap, struct inode *dir,
 	d_instantiate(dentry, inode);
 	dget(dentry);
 	return 0;
+
+out_remove_offset:
+	simple_offset_remove(shmem_get_offset_ctx(dir), dentry);
 out_iput:
 	iput(inode);
 	return error;
@@ -4055,6 +4085,8 @@  static void shmem_destroy_inode(struct inode *inode)
 {
 	if (S_ISREG(inode->i_mode))
 		mpol_free_shared_policy(&SHMEM_I(inode)->policy);
+	if (S_ISDIR(inode->i_mode))
+		simple_offset_destroy(shmem_get_offset_ctx(inode));
 }
 
 static void shmem_init_inode(void *foo)
@@ -4135,6 +4167,7 @@  static const struct inode_operations shmem_dir_inode_operations = {
 	.mknod		= shmem_mknod,
 	.rename		= shmem_rename2,
 	.tmpfile	= shmem_tmpfile,
+	.get_offset_ctx	= shmem_get_offset_ctx,
 #endif
 #ifdef CONFIG_TMPFS_XATTR
 	.listxattr	= shmem_listxattr,