From patchwork Mon Oct 19 13:27:48 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Luis Henriques X-Patchwork-Id: 11844371 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id BD5B314B2 for ; Mon, 19 Oct 2020 13:27:54 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by mail.kernel.org (Postfix) with ESMTP id 9031E21D41 for ; Mon, 19 Oct 2020 13:27:54 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1728047AbgJSN1y (ORCPT ); Mon, 19 Oct 2020 09:27:54 -0400 Received: from mx2.suse.de ([195.135.220.15]:55640 "EHLO mx2.suse.de" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1727811AbgJSN1x (ORCPT ); Mon, 19 Oct 2020 09:27:53 -0400 X-Virus-Scanned: by amavisd-new at test-mx.suse.de Received: from relay2.suse.de (unknown [195.135.221.27]) by mx2.suse.de (Postfix) with ESMTP id EE637AD68; Mon, 19 Oct 2020 13:27:49 +0000 (UTC) Received: from localhost (brahms [local]) by brahms (OpenSMTPD) with ESMTPA id 001ec119; Mon, 19 Oct 2020 13:27:53 +0000 (UTC) From: Luis Henriques To: fstests@vger.kernel.org Cc: Eryu Guan , Jeff Layton , ceph-devel@vger.kernel.org, Luis Henriques Subject: [PATCH v2 1/3] ceph: add copy_file_range (remote copy operation) testing Date: Mon, 19 Oct 2020 14:27:48 +0100 Message-Id: <20201019132750.29293-2-lhenriques@suse.de> In-Reply-To: <20201019132750.29293-1-lhenriques@suse.de> References: <20201019132750.29293-1-lhenriques@suse.de> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: fstests@vger.kernel.org From: Luis Henriques Test remote copy operation (CEPH_OSD_OP_COPY_FROM) with different combinations of both object sizes and copy sizes. Test remote copy operation (CEPH_OSD_OP_COPY_FROM) with several combinations of both object sizes and copy sizes. It also uses several combinations of copy ranges. For example, copying the 1st object in the src file into: 1) the beginning (1st object) of dst file, 2) the end (last object) of dst file and 3) the middle of the dst file. Signed-off-by: Luis Henriques --- common/ceph | 23 +++++ common/rc | 1 + tests/ceph/001 | 233 +++++++++++++++++++++++++++++++++++++++++++++ tests/ceph/001.out | 129 +++++++++++++++++++++++++ tests/ceph/group | 1 + 5 files changed, 387 insertions(+) create mode 100644 common/ceph create mode 100755 tests/ceph/001 create mode 100644 tests/ceph/001.out create mode 100644 tests/ceph/group diff --git a/common/ceph b/common/ceph new file mode 100644 index 000000000000..f80209f98b23 --- /dev/null +++ b/common/ceph @@ -0,0 +1,23 @@ +# +# CephFS specific common functions. +# + +# _ceph_create_file_layout +# This function creates a new empty file and sets the file layout according to +# parameters. It will exit if the file already exists. +_ceph_create_file_layout() +{ + local fname=$1 + local stripe_unit=$2 + local stripe_count=$3 + local obj_size=$4 + + if [ -e $fname ]; then + echo "File $fname already exists." + exit 1 + fi + touch $fname + $SETFATTR_PROG -n ceph.file.layout \ + -v "stripe_unit=$objsz stripe_count=1 object_size=$objsz" \ + $fname +} diff --git a/common/rc b/common/rc index 27a27ea36f75..65ebfe201139 100644 --- a/common/rc +++ b/common/rc @@ -138,6 +138,7 @@ case "$FSTYP" in 9p) ;; ceph) + . ./common/ceph ;; glusterfs) ;; diff --git a/tests/ceph/001 b/tests/ceph/001 new file mode 100755 index 000000000000..0515fff81262 --- /dev/null +++ b/tests/ceph/001 @@ -0,0 +1,233 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2020 SUSE Linux Products GmbH. All Rights Reserved. +# +# FS QA Test No. ceph/001 +# +# Test remote copy operation (CEPH_OSD_OP_COPY_FROM) with several combinations +# of both object sizes and copy sizes. It also uses several combinations of +# copy ranges. For example, copying the 1st object in the src file into +# 1) the beginning (1st object) of dst file, 2) the end (last object) of dst +# file and 3) the middle of the dst file. +# +seq=`basename $0` +seqres=$RESULT_DIR/$seq +echo "QA output created by $seq" + +here=`pwd` +tmp=/tmp/$$ +status=1 # failure is the default! +trap "_cleanup; exit \$status" 0 1 2 3 15 + +_cleanup() +{ + cd / + rm -rf $tmp.* +} + +# get standard environment +. common/rc +. common/filter +. common/attr +. common/reflink + +# real QA test starts here +_supported_fs ceph + +_require_xfs_io_command "copy_range" +_require_attrs +_require_test + +workdir=$TEST_DIR/test-$seq +rm -rf $workdir +mkdir $workdir +rm -f $seqres.full + +check_range() +{ + local file=$1 + local off0=$2 + local off1=$3 + local val=$4 + _read_range $file $off0 $off1 | grep -v -q $val + [ $? -eq 0 ] && echo "file $file is not '$val' in [ $off0 $off1 ]" +} + +run_copy_range_tests() +{ + objsz=$1 + halfobj=$(($objsz / 2)) + file="$workdir/file-$objsz" + copy="$workdir/copy-$objsz" + dest="$workdir/dest-$objsz" + + # create files and set file layout, which needs to be done before + # writing any data + _ceph_create_file_layout $file $objsz 1 $objsz + _ceph_create_file_layout $copy $objsz 1 $objsz + _ceph_create_file_layout $dest $objsz 1 $objsz + + # file containing 3 objects with 'aaaa|bbbb|cccc' + $XFS_IO_PROG -c "pwrite -S 0x61 0 $objsz" $file >> $seqres.full 2>&1 + $XFS_IO_PROG -c "pwrite -S 0x62 $objsz $objsz" $file >> $seqres.full 2>&1 + $XFS_IO_PROG -c "pwrite -S 0x63 $(($objsz * 2)) $objsz" $file >> $seqres.full 2>&1 + + echo " Copy whole file (3 objects):" + echo " aaaa|bbbb|cccc => aaaa|bbbb|cccc" + $XFS_IO_PROG -c "copy_range -s 0 -d 0 -l $(($objsz * 3)) $file" "$copy" + cmp $file $copy + + echo " Copy single object to beginning:" + # dest file with 3 objects with 'dddd|dddd|dddd' + $XFS_IO_PROG -c "pwrite -S 0x64 0 $(($objsz * 3))" $dest >> $seqres.full 2>&1 + + echo " dddd|dddd|dddd => aaaa|dddd|dddd" + $XFS_IO_PROG -c "copy_range -s 0 -d 0 -l $objsz $file" "$dest" + check_range $dest 0 $objsz 61 + check_range $dest $objsz $(($objsz * 2)) 64 + + echo " aaaa|dddd|dddd => bbbb|dddd|dddd" + $XFS_IO_PROG -c "copy_range -s $objsz -d 0 -l $objsz $file" "$dest" + check_range $dest 0 $objsz 62 + check_range $dest $objsz $(($objsz * 2)) 64 + + echo " bbbb|dddd|dddd => cccc|dddd|dddd" + $XFS_IO_PROG -c "copy_range -s $(($objsz * 2)) -d 0 -l $objsz $file" "$dest" + check_range $dest 0 $objsz 63 + check_range $dest $objsz $(($objsz * 2)) 64 + + echo " Copy single object to middle:" + + echo " cccc|dddd|dddd => cccc|aaaa|dddd" + $XFS_IO_PROG -c "copy_range -s 0 -d $objsz -l $objsz $file" "$dest" + check_range $dest 0 $objsz 63 + check_range $dest $objsz $objsz 61 + check_range $dest $(($objsz * 2)) $objsz 64 + + echo " cccc|aaaa|dddd => cccc|bbbb|dddd" + $XFS_IO_PROG -c "copy_range -s $objsz -d $objsz -l $objsz $file" "$dest" + check_range $dest 0 $objsz 63 + check_range $dest $objsz $objsz 62 + check_range $dest $(($objsz * 2)) $objsz 64 + + echo " cccc|bbbb|dddd => cccc|cccc|dddd" + $XFS_IO_PROG -c "copy_range -s $((objsz * 2)) -d $objsz -l $objsz $file" "$dest" + check_range $dest 0 $objsz 63 + check_range $dest $objsz $objsz 63 + check_range $dest $(($objsz * 2)) $objsz 64 + + echo " Copy single object to end:" + + echo " cccc|cccc|dddd => cccc|cccc|aaaa" + $XFS_IO_PROG -c "copy_range -s 0 -d $(($objsz * 2)) -l $objsz $file" "$dest" + check_range $dest 0 $(($objsz * 2)) 63 + check_range $dest $(($objsz * 2)) $objsz 61 + + echo " cccc|cccc|aaaa => cccc|cccc|bbbb" + $XFS_IO_PROG -c "copy_range -s $objsz -d $(($objsz * 2)) -l $objsz $file" "$dest" + check_range $dest 0 $(($objsz * 2)) 63 + check_range $dest $(($objsz * 2)) $objsz 62 + + echo " cccc|cccc|aaaa => cccc|cccc|cccc" + $XFS_IO_PROG -c "copy_range -s $(($objsz * 2)) -d $(($objsz * 2)) -l $objsz $file" "$dest" + check_range $dest 0 $(($objsz * 3)) 63 + + echo " Copy 2 objects to beginning:" + + echo " cccc|cccc|cccc => aaaa|bbbb|cccc" + $XFS_IO_PROG -c "copy_range -s 0 -d 0 -l $(($objsz * 2)) $file" "$dest" + cmp $file $dest + + echo " aaaa|bbbb|cccc => bbbb|cccc|cccc" + $XFS_IO_PROG -c "copy_range -s $objsz -d 0 -l $(($objsz * 2)) $file" "$dest" + check_range $dest 0 $objsz 62 + check_range $dest $objsz $(($objsz * 2)) 63 + + echo " Copy 2 objects to end:" + + echo " bbbb|cccc|cccc => bbbb|aaaa|bbbb" + $XFS_IO_PROG -c "copy_range -s 0 -d $objsz -l $(($objsz * 2)) $file" "$dest" + check_range $dest 0 $objsz 62 + check_range $dest $objsz $objsz 61 + check_range $dest $(($objsz * 2)) $objsz 62 + + echo " bbbb|aaaa|bbbb => bbbb|bbbb|cccc" + $XFS_IO_PROG -c "copy_range -s $objsz -d $objsz -l $(($objsz * 2)) $file" "$dest" + check_range $dest 0 $(($objsz * 2)) 62 + check_range $dest $(($objsz * 2)) $objsz 63 + + echo " Append 1 object:" + + echo " bbbb|bbbb|cccc => bbbb|bbbb|cccc|aaaa" + $XFS_IO_PROG -c "copy_range -s 0 -d $(($objsz * 3)) -l $objsz $file" "$dest" + check_range $dest 0 $(($objsz * 2)) 62 + check_range $dest $(($objsz * 2)) $objsz 63 + check_range $dest $(($objsz * 3)) $objsz 61 + + echo " Cross object boundary (no full object copy)" + echo " dddd|dddd|dddd|dddd => ddaa|aadd|dddd|dddd" + $XFS_IO_PROG -c "pwrite -S 0x64 0 $(($objsz * 4))" $dest >> $seqres.full 2>&1 + $XFS_IO_PROG -c "copy_range -s 0 -d $halfobj -l $objsz $file" "$dest" + check_range $dest 0 $halfobj 64 + check_range $dest $halfobj $objsz 61 + check_range $dest $(($objsz + $halfobj)) $(($objsz * 2 + $halfobj)) 64 + + echo " dddd|dddd|dddd|dddd => ddaa|bbdd|dddd|dddd" + $XFS_IO_PROG -c "pwrite -S 0x64 0 $(($objsz * 4))" $dest >> $seqres.full 2>&1 + $XFS_IO_PROG -c "copy_range -s $halfobj -d $halfobj -l $objsz $file" "$dest" + check_range $dest 0 $halfobj 64 + check_range $dest $halfobj $halfobj 61 + check_range $dest $objsz $halfobj 62 + check_range $dest $(($objsz + $halfobj)) $(($objsz * 2 + $halfobj)) 64 + + echo " Cross object boundaries (with full object copy)" + echo " dddd|dddd|dddd|dddd => ddaa|bbbb|dddd|dddd" + $XFS_IO_PROG -c "pwrite -S 0x64 0 $(($objsz * 4))" $dest >> $seqres.full 2>&1 + $XFS_IO_PROG -c "copy_range -s $halfobj -d $halfobj -l $(($objsz + $halfobj)) $file" "$dest" + check_range $dest 0 $halfobj 64 + check_range $dest $halfobj $halfobj 61 + check_range $dest $objsz $objsz 62 + check_range $dest $(($objsz * 2)) $(($objsz * 2)) 64 + + echo " dddd|dddd|dddd|dddd => ddaa|bbbb|ccdd|dddd" + $XFS_IO_PROG -c "pwrite -S 0x64 0 $(($objsz * 4))" $dest >> $seqres.full 2>&1 + $XFS_IO_PROG -c "copy_range -s $halfobj -d $halfobj -l $(($objsz * 2)) $file" "$dest" + check_range $dest 0 $halfobj 64 + check_range $dest $halfobj $halfobj 61 + check_range $dest $objsz $objsz 62 + check_range $dest $(($objsz * 2)) $halfobj 63 + check_range $dest $(($objsz * 2 + $halfobj)) $(($objsz + $halfobj)) 64 + + echo " dddd|dddd|dddd|dddd => dddd|aaaa|bbdd|dddd" + $XFS_IO_PROG -c "pwrite -S 0x64 0 $(($objsz * 4))" $dest >> $seqres.full 2>&1 + $XFS_IO_PROG -c "copy_range -s 0 -d $objsz -l $(($objsz + $halfobj)) $file" "$dest" + check_range $dest 0 $objsz 64 + check_range $dest $objsz $objsz 61 + check_range $dest $(($objsz * 2)) $halfobj 62 + check_range $dest $(($objsz * 2 + $halfobj)) $(($objsz + $halfobj)) 64 + + echo " Cross object boundaries (with 2 full object copies)" + echo " dddd|dddd|dddd|dddd => ddaa|aabb|bbcc|ccdd" + $XFS_IO_PROG -c "pwrite -S 0x64 0 $(($objsz * 4))" $dest >> $seqres.full 2>&1 + $XFS_IO_PROG -c "copy_range -s 0 -d $halfobj -l $(($objsz * 3)) $file" "$dest" + check_range $dest 0 $halfobj 64 + check_range $dest $halfobj $objsz 61 + check_range $dest $(($objsz + $halfobj)) $objsz 62 + check_range $dest $(($objsz * 2 + $halfobj)) $objsz 63 + check_range $dest $(($objsz * 3 + $halfobj)) $halfobj 64 + +} + +echo "Object size: 65536" # CEPH_MIN_STRIPE_UNIT +run_copy_range_tests 65536 +echo "Object size: 1M" +run_copy_range_tests 1048576 +echo "Object size: 4M" +run_copy_range_tests 4194304 +# the max object size is 1TB, but by default OSDs only accept a max of 128M objects +echo "Object size: 128M" +run_copy_range_tests 134217728 + +# success, all done +status=0 +exit diff --git a/tests/ceph/001.out b/tests/ceph/001.out new file mode 100644 index 000000000000..3cc7837a595d --- /dev/null +++ b/tests/ceph/001.out @@ -0,0 +1,129 @@ +QA output created by 001 +Object size: 65536 + Copy whole file (3 objects): + aaaa|bbbb|cccc => aaaa|bbbb|cccc + Copy single object to beginning: + dddd|dddd|dddd => aaaa|dddd|dddd + aaaa|dddd|dddd => bbbb|dddd|dddd + bbbb|dddd|dddd => cccc|dddd|dddd + Copy single object to middle: + cccc|dddd|dddd => cccc|aaaa|dddd + cccc|aaaa|dddd => cccc|bbbb|dddd + cccc|bbbb|dddd => cccc|cccc|dddd + Copy single object to end: + cccc|cccc|dddd => cccc|cccc|aaaa + cccc|cccc|aaaa => cccc|cccc|bbbb + cccc|cccc|aaaa => cccc|cccc|cccc + Copy 2 objects to beginning: + cccc|cccc|cccc => aaaa|bbbb|cccc + aaaa|bbbb|cccc => bbbb|cccc|cccc + Copy 2 objects to end: + bbbb|cccc|cccc => bbbb|aaaa|bbbb + bbbb|aaaa|bbbb => bbbb|bbbb|cccc + Append 1 object: + bbbb|bbbb|cccc => bbbb|bbbb|cccc|aaaa + Cross object boundary (no full object copy) + dddd|dddd|dddd|dddd => ddaa|aadd|dddd|dddd + dddd|dddd|dddd|dddd => ddaa|bbdd|dddd|dddd + Cross object boundaries (with full object copy) + dddd|dddd|dddd|dddd => ddaa|bbbb|dddd|dddd + dddd|dddd|dddd|dddd => ddaa|bbbb|ccdd|dddd + dddd|dddd|dddd|dddd => dddd|aaaa|bbdd|dddd + Cross object boundaries (with 2 full object copies) + dddd|dddd|dddd|dddd => ddaa|aabb|bbcc|ccdd +Object size: 1M + Copy whole file (3 objects): + aaaa|bbbb|cccc => aaaa|bbbb|cccc + Copy single object to beginning: + dddd|dddd|dddd => aaaa|dddd|dddd + aaaa|dddd|dddd => bbbb|dddd|dddd + bbbb|dddd|dddd => cccc|dddd|dddd + Copy single object to middle: + cccc|dddd|dddd => cccc|aaaa|dddd + cccc|aaaa|dddd => cccc|bbbb|dddd + cccc|bbbb|dddd => cccc|cccc|dddd + Copy single object to end: + cccc|cccc|dddd => cccc|cccc|aaaa + cccc|cccc|aaaa => cccc|cccc|bbbb + cccc|cccc|aaaa => cccc|cccc|cccc + Copy 2 objects to beginning: + cccc|cccc|cccc => aaaa|bbbb|cccc + aaaa|bbbb|cccc => bbbb|cccc|cccc + Copy 2 objects to end: + bbbb|cccc|cccc => bbbb|aaaa|bbbb + bbbb|aaaa|bbbb => bbbb|bbbb|cccc + Append 1 object: + bbbb|bbbb|cccc => bbbb|bbbb|cccc|aaaa + Cross object boundary (no full object copy) + dddd|dddd|dddd|dddd => ddaa|aadd|dddd|dddd + dddd|dddd|dddd|dddd => ddaa|bbdd|dddd|dddd + Cross object boundaries (with full object copy) + dddd|dddd|dddd|dddd => ddaa|bbbb|dddd|dddd + dddd|dddd|dddd|dddd => ddaa|bbbb|ccdd|dddd + dddd|dddd|dddd|dddd => dddd|aaaa|bbdd|dddd + Cross object boundaries (with 2 full object copies) + dddd|dddd|dddd|dddd => ddaa|aabb|bbcc|ccdd +Object size: 4M + Copy whole file (3 objects): + aaaa|bbbb|cccc => aaaa|bbbb|cccc + Copy single object to beginning: + dddd|dddd|dddd => aaaa|dddd|dddd + aaaa|dddd|dddd => bbbb|dddd|dddd + bbbb|dddd|dddd => cccc|dddd|dddd + Copy single object to middle: + cccc|dddd|dddd => cccc|aaaa|dddd + cccc|aaaa|dddd => cccc|bbbb|dddd + cccc|bbbb|dddd => cccc|cccc|dddd + Copy single object to end: + cccc|cccc|dddd => cccc|cccc|aaaa + cccc|cccc|aaaa => cccc|cccc|bbbb + cccc|cccc|aaaa => cccc|cccc|cccc + Copy 2 objects to beginning: + cccc|cccc|cccc => aaaa|bbbb|cccc + aaaa|bbbb|cccc => bbbb|cccc|cccc + Copy 2 objects to end: + bbbb|cccc|cccc => bbbb|aaaa|bbbb + bbbb|aaaa|bbbb => bbbb|bbbb|cccc + Append 1 object: + bbbb|bbbb|cccc => bbbb|bbbb|cccc|aaaa + Cross object boundary (no full object copy) + dddd|dddd|dddd|dddd => ddaa|aadd|dddd|dddd + dddd|dddd|dddd|dddd => ddaa|bbdd|dddd|dddd + Cross object boundaries (with full object copy) + dddd|dddd|dddd|dddd => ddaa|bbbb|dddd|dddd + dddd|dddd|dddd|dddd => ddaa|bbbb|ccdd|dddd + dddd|dddd|dddd|dddd => dddd|aaaa|bbdd|dddd + Cross object boundaries (with 2 full object copies) + dddd|dddd|dddd|dddd => ddaa|aabb|bbcc|ccdd +Object size: 128M + Copy whole file (3 objects): + aaaa|bbbb|cccc => aaaa|bbbb|cccc + Copy single object to beginning: + dddd|dddd|dddd => aaaa|dddd|dddd + aaaa|dddd|dddd => bbbb|dddd|dddd + bbbb|dddd|dddd => cccc|dddd|dddd + Copy single object to middle: + cccc|dddd|dddd => cccc|aaaa|dddd + cccc|aaaa|dddd => cccc|bbbb|dddd + cccc|bbbb|dddd => cccc|cccc|dddd + Copy single object to end: + cccc|cccc|dddd => cccc|cccc|aaaa + cccc|cccc|aaaa => cccc|cccc|bbbb + cccc|cccc|aaaa => cccc|cccc|cccc + Copy 2 objects to beginning: + cccc|cccc|cccc => aaaa|bbbb|cccc + aaaa|bbbb|cccc => bbbb|cccc|cccc + Copy 2 objects to end: + bbbb|cccc|cccc => bbbb|aaaa|bbbb + bbbb|aaaa|bbbb => bbbb|bbbb|cccc + Append 1 object: + bbbb|bbbb|cccc => bbbb|bbbb|cccc|aaaa + Cross object boundary (no full object copy) + dddd|dddd|dddd|dddd => ddaa|aadd|dddd|dddd + dddd|dddd|dddd|dddd => ddaa|bbdd|dddd|dddd + Cross object boundaries (with full object copy) + dddd|dddd|dddd|dddd => ddaa|bbbb|dddd|dddd + dddd|dddd|dddd|dddd => ddaa|bbbb|ccdd|dddd + dddd|dddd|dddd|dddd => dddd|aaaa|bbdd|dddd + Cross object boundaries (with 2 full object copies) + dddd|dddd|dddd|dddd => ddaa|aabb|bbcc|ccdd diff --git a/tests/ceph/group b/tests/ceph/group new file mode 100644 index 000000000000..11f0b9ad03d3 --- /dev/null +++ b/tests/ceph/group @@ -0,0 +1 @@ +001 auto quick copy From patchwork Mon Oct 19 13:27:49 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Luis Henriques X-Patchwork-Id: 11844373 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id F16D817C9 for ; Mon, 19 Oct 2020 13:27:55 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by mail.kernel.org (Postfix) with ESMTP id DC5E4222BA for ; Mon, 19 Oct 2020 13:27:55 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1728214AbgJSN1x (ORCPT ); Mon, 19 Oct 2020 09:27:53 -0400 Received: from mx2.suse.de ([195.135.220.15]:55656 "EHLO mx2.suse.de" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1728203AbgJSN1w (ORCPT ); Mon, 19 Oct 2020 09:27:52 -0400 X-Virus-Scanned: by amavisd-new at test-mx.suse.de Received: from relay2.suse.de (unknown [195.135.221.27]) by mx2.suse.de (Postfix) with ESMTP id 6E5B2ACDF; Mon, 19 Oct 2020 13:27:50 +0000 (UTC) Received: from localhost (brahms [local]) by brahms (OpenSMTPD) with ESMTPA id 39482c5b; Mon, 19 Oct 2020 13:27:54 +0000 (UTC) From: Luis Henriques To: fstests@vger.kernel.org Cc: Eryu Guan , Jeff Layton , ceph-devel@vger.kernel.org, Luis Henriques Subject: [PATCH v2 2/3] ceph: test combination of copy_file_range with truncate Date: Mon, 19 Oct 2020 14:27:49 +0100 Message-Id: <20201019132750.29293-3-lhenriques@suse.de> In-Reply-To: <20201019132750.29293-1-lhenriques@suse.de> References: <20201019132750.29293-1-lhenriques@suse.de> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: fstests@vger.kernel.org From: Luis Henriques This tests a bug found while testing copy_file_range. This bug was an issue with how the OSDs handled the truncate_seq value, which was being copied from the original object into the destination object. This test ensures the kernel client correctly handles fixed/non-fixed OSDs. Link: https://tracker.ceph.com/issues/37378 Signed-off-by: Luis Henriques --- tests/ceph/002 | 79 ++++++++++++++++++++++++++++++++++++++++++++++ tests/ceph/002.out | 8 +++++ tests/ceph/group | 1 + 3 files changed, 88 insertions(+) create mode 100755 tests/ceph/002 create mode 100644 tests/ceph/002.out diff --git a/tests/ceph/002 b/tests/ceph/002 new file mode 100755 index 000000000000..f0fd28a14991 --- /dev/null +++ b/tests/ceph/002 @@ -0,0 +1,79 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2020 SUSE Linux Products GmbH. All Rights Reserved. +# +# FS QA Test No. ceph/002 +# +# Test bug found while testing copy_file_range. +# +# This bug was an issue with how the OSDs handled the truncate_seq, copying it +# from the original object into the destination object. This test ensures the +# kernel client correctly handles fixed/non-fixed OSDs. +# +# The bug was tracked here: +# +# https://tracker.ceph.com/issues/37378 +# +# The most relevant commits are: +# +# ceph OSD: dcd6a99ef9f5 ("osd: add new 'copy-from2' operation") +# linux kernel: 78beb0ff2fec ("ceph: use copy-from2 op in copy_file_range") +# +seq=`basename $0` +seqres=$RESULT_DIR/$seq +echo "QA output created by $seq" + +here=`pwd` +tmp=/tmp/$$ +status=1 # failure is the default! +trap "_cleanup; exit \$status" 0 1 2 3 15 + +_cleanup() +{ + cd / + rm -rf $tmp.* +} + +# get standard environment +. common/rc +. common/filter +. common/attr + +# real QA test starts here +_supported_fs ceph + +_require_xfs_io_command "copy_range" +_require_attrs +_require_test + +workdir=$TEST_DIR/test-$seq +rm -rf $workdir +mkdir $workdir +rm -f $seqres.full + +# Use 4M object size +objsz=4194304 +file="$workdir/file-$objsz" +dest="$workdir/dest-$objsz" + +# object_size has to be a multiple of stripe_unit +_ceph_create_file_layout $file $objsz 1 $objsz +_ceph_create_file_layout $dest $objsz 1 $objsz + +# Create a 3 objects size files +$XFS_IO_PROG -c "pwrite -S 0x61 0 $objsz" $file >> $seqres.full 2>&1 +$XFS_IO_PROG -c "pwrite -S 0x62 $objsz $objsz" $file >> $seqres.full 2>&1 +$XFS_IO_PROG -c "pwrite -S 0x63 $(($objsz * 2)) $objsz" $file >> $seqres.full 2>&1 + +$XFS_IO_PROG -c "pwrite -S 0x64 0 $(($objsz * 3))" $dest >> $seqres.full 2>&1 +# Truncate the destination file (messing up with the truncate_seq) +$XFS_IO_PROG -c "truncate 0" $dest >> $seqres.full 2>&1 + +# copy the whole file over +$XFS_IO_PROG -c "copy_range -s 0 -d 0 -l $(($objsz * 3)) $file" "$dest" + +hexdump $dest + +#success, all done +status=0 +exit diff --git a/tests/ceph/002.out b/tests/ceph/002.out new file mode 100644 index 000000000000..6f067250afff --- /dev/null +++ b/tests/ceph/002.out @@ -0,0 +1,8 @@ +QA output created by 002 +0000000 6161 6161 6161 6161 6161 6161 6161 6161 +* +0400000 6262 6262 6262 6262 6262 6262 6262 6262 +* +0800000 6363 6363 6363 6363 6363 6363 6363 6363 +* +0c00000 diff --git a/tests/ceph/group b/tests/ceph/group index 11f0b9ad03d3..c28fe473c1a4 100644 --- a/tests/ceph/group +++ b/tests/ceph/group @@ -1 +1,2 @@ 001 auto quick copy +002 auto quick copy From patchwork Mon Oct 19 13:27:50 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Luis Henriques X-Patchwork-Id: 11844369 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 2F5C014B2 for ; Mon, 19 Oct 2020 13:27:53 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by mail.kernel.org (Postfix) with ESMTP id 184E6222E8 for ; Mon, 19 Oct 2020 13:27:53 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1728240AbgJSN1w (ORCPT ); Mon, 19 Oct 2020 09:27:52 -0400 Received: from mx2.suse.de ([195.135.220.15]:55672 "EHLO mx2.suse.de" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1728214AbgJSN1w (ORCPT ); Mon, 19 Oct 2020 09:27:52 -0400 X-Virus-Scanned: by amavisd-new at test-mx.suse.de Received: from relay2.suse.de (unknown [195.135.221.27]) by mx2.suse.de (Postfix) with ESMTP id DCAE4AD6B; Mon, 19 Oct 2020 13:27:50 +0000 (UTC) Received: from localhost (brahms [local]) by brahms (OpenSMTPD) with ESMTPA id ab6b6be4; Mon, 19 Oct 2020 13:27:54 +0000 (UTC) From: Luis Henriques To: fstests@vger.kernel.org Cc: Eryu Guan , Jeff Layton , ceph-devel@vger.kernel.org, Luis Henriques Subject: [PATCH v2 3/3] ceph: test copy_file_range with infile = outfile Date: Mon, 19 Oct 2020 14:27:50 +0100 Message-Id: <20201019132750.29293-4-lhenriques@suse.de> In-Reply-To: <20201019132750.29293-1-lhenriques@suse.de> References: <20201019132750.29293-1-lhenriques@suse.de> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: fstests@vger.kernel.org From: Luis Henriques This runs a set of simple tests where the infile file is the same as the outfile. Signed-off-by: Luis Henriques --- tests/ceph/003 | 116 +++++++++++++++++++++++++++++++++++++++++++++ tests/ceph/003.out | 11 +++++ tests/ceph/group | 1 + 3 files changed, 128 insertions(+) create mode 100755 tests/ceph/003 create mode 100644 tests/ceph/003.out diff --git a/tests/ceph/003 b/tests/ceph/003 new file mode 100755 index 000000000000..6a5a4ea429f8 --- /dev/null +++ b/tests/ceph/003 @@ -0,0 +1,116 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2020 SUSE Linux Products GmbH. All Rights Reserved. +# +# FS QA Test No. ceph/005 +# +# Test copy_file_range with infile = outfile +# +seq=`basename $0` +seqres=$RESULT_DIR/$seq +echo "QA output created by $seq" + +here=`pwd` +tmp=/tmp/$$ +status=1 # failure is the default! +trap "_cleanup; exit \$status" 0 1 2 3 15 + +_cleanup() +{ + cd / + rm -rf $tmp.* +} + +# get standard environment +. common/rc +. common/filter +. common/attr +. common/reflink + +# real QA test starts here +_supported_fs ceph + +_require_xfs_io_command "copy_range" +_require_attrs +_require_test + +workdir=$TEST_DIR/test-$seq +rm -rf $workdir +mkdir $workdir +rm -f $seqres.full + +check_range() +{ + local file=$1 + local off0=$2 + local off1=$3 + local val=$4 + _read_range $file $off0 $off1 | grep -v -q $val + [ $? -eq 0 ] && echo "file $file is not '$val' in [ $off0 $off1 ]" +} + +objsz=4194304 +halfobj=$(($objsz / 2)) +file="$workdir/file-$objsz" +copy="$workdir/copy-$objsz" +dest="$workdir/dest-$objsz" +backup="$file.backup" + +# object_size has to be a multiple of stripe_unit +_ceph_create_file_layout $file $objsz 1 $objsz +_ceph_create_file_layout $backup $objsz 1 $objsz + +$XFS_IO_PROG -c "pwrite -S 0x61 0 $objsz" $file >> $seqres.full 2>&1 +$XFS_IO_PROG -c "pwrite -S 0x62 $objsz $objsz" $file >> $seqres.full 2>&1 +$XFS_IO_PROG -c "pwrite -S 0x63 $(($objsz * 2)) $objsz" $file >> $seqres.full 2>&1 + +cp $file $backup + +echo " Copy single object to the end:" +echo " aaaa|bbbb|cccc => aaaa|bbbb|aaaa" +$XFS_IO_PROG -c "copy_range -s 0 -d $(($objsz * 2)) -l $objsz $file" "$file" +check_range $file 0 $objsz 61 +check_range $file $objsz $objsz 62 +check_range $file $(($objsz * 2)) $objsz 61 + +echo " Copy single object to the beginning:" +echo " aaaa|bbbb|aaaa => bbbb|bbbb|aaaa" +$XFS_IO_PROG -c "copy_range -s $objsz -d 0 -l $objsz $file" "$file" +check_range $file 0 $(($objsz * 2)) 62 +check_range $file $(($objsz * 2)) $objsz 61 + +echo " Copy single object to the middle:" +echo " bbbb|bbbb|aaaa => bbbb|aaaa|aaaa" +$XFS_IO_PROG -c "copy_range -s $(($objsz * 2)) -d $objsz -l $objsz $file" "$file" +check_range $file 0 $objsz 62 +check_range $file $objsz $(($objsz * 2)) 61 + +cp $backup $file +echo " Cross object boundary (no full object copy)" +echo " aaaa|bbbb|cccc => aaaa|bbaa|aacc" +$XFS_IO_PROG -c "copy_range -s 0 -d $(($objsz + $halfobj)) -l $objsz $file" "$file" +check_range $file 0 $objsz 61 +check_range $file $objsz $halfobj 62 +check_range $file $(($objsz + $halfobj)) $objsz 61 +check_range $file $(($objsz * 2 + $halfobj)) $halfobj 63 + +cp $backup $file +echo " aaaa|bbbb|cccc => aaaa|bbaa|bbcc" +$XFS_IO_PROG -c "copy_range -s $halfobj -d $(($objsz + $halfobj)) -l $objsz $file" "$file" +check_range $file 0 $objsz 61 +check_range $file $objsz $halfobj 62 +check_range $file $(($objsz + $halfobj)) $halfobj 61 +check_range $file $(($objsz * 2)) $halfobj 62 +check_range $file $(($objsz * 2 + $halfobj)) $halfobj 63 + +cp $backup $file +echo " aaaa|bbbb|cccc => aaaa|bbbb|aabb" +$XFS_IO_PROG -c "copy_range -s $halfobj -d $(($objsz * 2)) -l $objsz $file" "$file" +check_range $file 0 $objsz 61 +check_range $file $objsz $objsz 62 +check_range $file $(($objsz * 2)) $halfobj 61 +check_range $file $(($objsz * 2 + $halfobj)) $halfobj 62 + +#success, all done +status=0 +exit diff --git a/tests/ceph/003.out b/tests/ceph/003.out new file mode 100644 index 000000000000..76c83b265253 --- /dev/null +++ b/tests/ceph/003.out @@ -0,0 +1,11 @@ +QA output created by 003 + Copy single object to the end: + aaaa|bbbb|cccc => aaaa|bbbb|aaaa + Copy single object to the beginning: + aaaa|bbbb|aaaa => bbbb|bbbb|aaaa + Copy single object to the middle: + bbbb|bbbb|aaaa => bbbb|aaaa|aaaa + Cross object boundary (no full object copy) + aaaa|bbbb|cccc => aaaa|bbaa|aacc + aaaa|bbbb|cccc => aaaa|bbaa|bbcc + aaaa|bbbb|cccc => aaaa|bbbb|aabb diff --git a/tests/ceph/group b/tests/ceph/group index c28fe473c1a4..adbf61547766 100644 --- a/tests/ceph/group +++ b/tests/ceph/group @@ -1,2 +1,3 @@ 001 auto quick copy 002 auto quick copy +003 auto quick copy