From patchwork Mon Dec 23 21:57:26 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Darrick J. Wong" X-Patchwork-Id: 13919310 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 172C918BBAC for ; Mon, 23 Dec 2024 21:57:27 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1734991047; cv=none; b=LNS/1iZ+y44FQ2kOwDvX37I3T1x1v7bfbW6mggngiPmlvUz1eHg2jcmyW+oVFmf73HA+4hY9bhNADFqPiZlHEYcmOjGDi4g2IuDzwlupw+jBPqueXGNSxrPrEM7pl4wQT4Dssokcv11l5vr3vKjQ6pn+Ur6TOjcGjIfgVWvtbbw= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1734991047; c=relaxed/simple; bh=J6+SqxA7kZkn0HINHyeZN1HJ3CekGL7YYX8UUynpvMY=; h=Date:Subject:From:To:Cc:Message-ID:In-Reply-To:References: MIME-Version:Content-Type; b=RUlucGtp4eLM5WvL/gtaOFVDpwysJnvtyxE0dCHiYdNV96QUy/+wOadSdy14XUTsr2wia8P2nkxhnR/jpqolXSRVdGKG8kqz9yCZjNTH62CxubA70maKsxwuyJV+rbavnjCb+EBykdNK2vN+RkNPXI34NKLFQF8Nkm4CTlok+qM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=qfB5tThW; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="qfB5tThW" Received: by smtp.kernel.org (Postfix) with ESMTPSA id D8AB6C4CED3; Mon, 23 Dec 2024 21:57:26 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1734991046; bh=J6+SqxA7kZkn0HINHyeZN1HJ3CekGL7YYX8UUynpvMY=; h=Date:Subject:From:To:Cc:In-Reply-To:References:From; b=qfB5tThWy3aNn6cVWJhcE/Z7KB5R16pnevg8OoQtlxi5aJi89/X6Nb2GkTSz5S9BV miVzT3L2sz7E5xPjTbb3YYI5CMD1IPndu41esedwb+tNdNS/ItqjXpKD+TKcxlQVYo T2XWas8+x53PdDdpzpWa9BdRXh2+H3ox2/EgWDF/qhe//sdeqyohRq0TFv7dXiBoa+ 1XC3vxMaW/v9cCuaJMfrsVXBg29munvEQ5eRH4wOHi5Bnl8GjaIOwsFHMtWY+16j/F yIbj4yw8+ZpQzF+ikInIrWHK1madbSqaDONWmv7tTD/XBHUunIXdXFHgTxvsGdLoMV r9ceHHjvTATeA== Date: Mon, 23 Dec 2024 13:57:26 -0800 Subject: [PATCH 1/4] libxfs: resync libxfs_alloc_file_space interface with the kernel From: "Darrick J. Wong" To: djwong@kernel.org, aalbersh@kernel.org Cc: hch@lst.de, linux-xfs@vger.kernel.org Message-ID: <173498941944.2295644.931587730201619009.stgit@frogsfrogsfrogs> In-Reply-To: <173498941923.2295644.12590815257028938964.stgit@frogsfrogsfrogs> References: <173498941923.2295644.12590815257028938964.stgit@frogsfrogsfrogs> Precedence: bulk X-Mailing-List: linux-xfs@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 From: Darrick J. Wong Make the userspace xfs_alloc_file_space behave (more or less) like the kernel version, at least as far as the interface goes. Signed-off-by: "Darrick J. Wong" Reviewed-by: Christoph Hellwig --- include/libxfs.h | 4 +- libxfs/util.c | 140 ++++++++++++++++++++++++++++++++++++------------------ mkfs/proto.c | 2 - 3 files changed, 96 insertions(+), 50 deletions(-) diff --git a/include/libxfs.h b/include/libxfs.h index 348d36be192966..878fefbbde7524 100644 --- a/include/libxfs.h +++ b/include/libxfs.h @@ -176,8 +176,8 @@ extern int libxfs_log_header(char *, uuid_t *, int, int, int, xfs_lsn_t, /* Shared utility routines */ -extern int libxfs_alloc_file_space (struct xfs_inode *, xfs_off_t, - xfs_off_t, int, int); +int libxfs_alloc_file_space(struct xfs_inode *ip, xfs_off_t offset, + xfs_off_t len, uint32_t bmapi_flags); /* XXX: this is messy and needs fixing */ #ifndef __LIBXFS_INTERNAL_XFS_H__ diff --git a/libxfs/util.c b/libxfs/util.c index 97da94506aee01..e5892fc86c3e92 100644 --- a/libxfs/util.c +++ b/libxfs/util.c @@ -179,78 +179,124 @@ libxfs_mod_incore_sb( */ int libxfs_alloc_file_space( - xfs_inode_t *ip, - xfs_off_t offset, - xfs_off_t len, - int alloc_type, - int attr_flags) + struct xfs_inode *ip, + xfs_off_t offset, + xfs_off_t len, + uint32_t bmapi_flags) { - xfs_mount_t *mp; - xfs_off_t count; - xfs_filblks_t datablocks; - xfs_filblks_t allocated_fsb; - xfs_filblks_t allocatesize_fsb; - xfs_bmbt_irec_t *imapp; - xfs_bmbt_irec_t imaps[1]; - int reccount; - uint resblks; - xfs_fileoff_t startoffset_fsb; - xfs_trans_t *tp; - int xfs_bmapi_flags; - int error; + xfs_mount_t *mp = ip->i_mount; + xfs_off_t count; + xfs_filblks_t allocatesize_fsb; + xfs_extlen_t extsz, temp; + xfs_fileoff_t startoffset_fsb; + xfs_fileoff_t endoffset_fsb; + int rt; + xfs_trans_t *tp; + xfs_bmbt_irec_t imaps[1], *imapp; + int error; if (len <= 0) return -EINVAL; + rt = XFS_IS_REALTIME_INODE(ip); + extsz = xfs_get_extsz_hint(ip); + count = len; - error = 0; imapp = &imaps[0]; - reccount = 1; - xfs_bmapi_flags = alloc_type ? XFS_BMAPI_PREALLOC : 0; - mp = ip->i_mount; - startoffset_fsb = XFS_B_TO_FSBT(mp, offset); - allocatesize_fsb = XFS_B_TO_FSB(mp, count); + startoffset_fsb = XFS_B_TO_FSBT(mp, offset); + endoffset_fsb = XFS_B_TO_FSB(mp, offset + count); + allocatesize_fsb = endoffset_fsb - startoffset_fsb; - /* allocate file space until done or until there is an error */ + /* + * Allocate file space until done or until there is an error + */ while (allocatesize_fsb && !error) { - datablocks = allocatesize_fsb; + xfs_fileoff_t s, e; + unsigned int dblocks, rblocks, resblks; + int nimaps = 1; - resblks = (uint)XFS_DIOSTRAT_SPACE_RES(mp, datablocks); - error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, - 0, 0, &tp); /* - * Check for running out of space + * Determine space reservations for data/realtime. */ - if (error) { - ASSERT(error == -ENOSPC); + if (unlikely(extsz)) { + s = startoffset_fsb; + do_div(s, extsz); + s *= extsz; + e = startoffset_fsb + allocatesize_fsb; + div_u64_rem(startoffset_fsb, extsz, &temp); + if (temp) + e += temp; + div_u64_rem(e, extsz, &temp); + if (temp) + e += extsz - temp; + } else { + s = 0; + e = allocatesize_fsb; + } + + /* + * The transaction reservation is limited to a 32-bit block + * count, hence we need to limit the number of blocks we are + * trying to reserve to avoid an overflow. We can't allocate + * more than @nimaps extents, and an extent is limited on disk + * to XFS_BMBT_MAX_EXTLEN (21 bits), so use that to enforce the + * limit. + */ + resblks = min_t(xfs_fileoff_t, (e - s), + (XFS_MAX_BMBT_EXTLEN * nimaps)); + if (unlikely(rt)) { + dblocks = XFS_DIOSTRAT_SPACE_RES(mp, 0); + rblocks = resblks; + } else { + dblocks = XFS_DIOSTRAT_SPACE_RES(mp, resblks); + rblocks = 0; + } + + error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, + dblocks, rblocks, false, &tp); + if (error) break; - } - xfs_trans_ijoin(tp, ip, 0); - error = xfs_bmapi_write(tp, ip, startoffset_fsb, allocatesize_fsb, - xfs_bmapi_flags, 0, imapp, &reccount); + error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK, + XFS_IEXT_ADD_NOSPLIT_CNT); + if (error) + goto error; + error = xfs_bmapi_write(tp, ip, startoffset_fsb, + allocatesize_fsb, bmapi_flags, 0, imapp, + &nimaps); if (error) - goto error0; + goto error; + + ip->i_diflags |= XFS_DIFLAG_PREALLOC; + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); - /* - * Complete the transaction - */ error = xfs_trans_commit(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); if (error) break; - allocated_fsb = imapp->br_blockcount; - if (reccount == 0) - return -ENOSPC; - - startoffset_fsb += allocated_fsb; - allocatesize_fsb -= allocated_fsb; + /* + * If xfs_bmapi_write finds a delalloc extent at the requested + * range, it tries to convert the entire delalloc extent to a + * real allocation. + * If the allocator cannot find a single free extent large + * enough to cover the start block of the requested range, + * xfs_bmapi_write will return 0 but leave *nimaps set to 0. + * In that case we simply need to keep looping with the same + * startoffset_fsb. + */ + if (nimaps) { + startoffset_fsb += imapp->br_blockcount; + allocatesize_fsb -= imapp->br_blockcount; + } } + return error; -error0: /* Cancel bmap, cancel trans */ +error: xfs_trans_cancel(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); return error; } diff --git a/mkfs/proto.c b/mkfs/proto.c index 05c2621f8a0b13..0764064e043e97 100644 --- a/mkfs/proto.c +++ b/mkfs/proto.c @@ -203,7 +203,7 @@ rsvfile( int error; xfs_trans_t *tp; - error = -libxfs_alloc_file_space(ip, 0, llen, 1, 0); + error = -libxfs_alloc_file_space(ip, 0, llen, XFS_BMAPI_PREALLOC); if (error) { fail(_("error reserving space for a file"), error); From patchwork Mon Dec 23 21:57:42 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Darrick J. Wong" X-Patchwork-Id: 13919311 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id AF72A1422AB for ; Mon, 23 Dec 2024 21:57:42 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1734991062; cv=none; b=bc08wjd157TOb9B4aznBeL82T1xT9AZR3OOKfI/vYrTptP42SsTYpM/gNfGBgl1RQd9wo0KmJYUISEOluuh7vr6SgsnWTNIc1/VZG2xPMEsvtxb0ms9f17+6hgy8aOuPTmIXB/radTuDbkbVfLQUE2/17voZJ02Wff+Tb6taXKY= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1734991062; c=relaxed/simple; bh=t7BV/BvzMP2navKnjz89aOqx6t2KqTA6RP4yrzYSisk=; h=Date:Subject:From:To:Cc:Message-ID:In-Reply-To:References: MIME-Version:Content-Type; b=V+sv/lkAzEJyMa8ktDbO4igheJBndF0QALKSPWq2gHchMK1+AA1uxKlGDxnpqimrOqrR5WnTVf1VAoPPRiUI2RKDbkkOQutVVcV32UdGAXTh+hnKQ+f4huPoWfNJSCKdTMxt1cJHviGPCp02aa1UtSrxJvb+nmtpOOwQq8osbEY= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=E3/pMFdf; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="E3/pMFdf" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 8561BC4CED3; Mon, 23 Dec 2024 21:57:42 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1734991062; bh=t7BV/BvzMP2navKnjz89aOqx6t2KqTA6RP4yrzYSisk=; h=Date:Subject:From:To:Cc:In-Reply-To:References:From; b=E3/pMFdfi8wuH3KlfwqdcY4l9xuuf95uQrcX6reOCeCtyhKpAvCTs0WVJDr6kTJTC mgcO7flv8aVXW/eu6UvxRonaYxIjqjjmv8zgUvyNtOF+qyKY2AkWWJ11qjU/BE/XYW 5d2OJlXsTVCIfL3WFphjIva/26U7wwvPzksLlvgL6fnBB+Eb33+UOOuL+weBquXnBl Jjf78zvzBPSjx3tAbtehMLzmdBrK3+XNnuuimMryZQoTGCkzarHZ0BTUcnt2dRd1P9 a6h44Ao6Sitw6ha+US6JApLjXdOFftKoS3p1GMj7pQmzw5D5v+4VzjzMip4686FIXd KsJUZX0+1wrOA== Date: Mon, 23 Dec 2024 13:57:42 -0800 Subject: [PATCH 2/4] mkfs: support copying in large or sparse files From: "Darrick J. Wong" To: djwong@kernel.org, aalbersh@kernel.org Cc: hch@lst.de, linux-xfs@vger.kernel.org Message-ID: <173498941959.2295644.1190041955142131540.stgit@frogsfrogsfrogs> In-Reply-To: <173498941923.2295644.12590815257028938964.stgit@frogsfrogsfrogs> References: <173498941923.2295644.12590815257028938964.stgit@frogsfrogsfrogs> Precedence: bulk X-Mailing-List: linux-xfs@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 From: Darrick J. Wong Restructure the protofile code to handle sparse files and files that are larger than the program's address space. Signed-off-by: "Darrick J. Wong" Reviewed-by: Christoph Hellwig --- include/libxfs.h | 2 + libxfs/util.c | 67 +++++++++++++++++++++ mkfs/proto.c | 175 ++++++++++++++++++++++++++++++++---------------------- 3 files changed, 174 insertions(+), 70 deletions(-) diff --git a/include/libxfs.h b/include/libxfs.h index 878fefbbde7524..985646e6ad89d1 100644 --- a/include/libxfs.h +++ b/include/libxfs.h @@ -178,6 +178,8 @@ extern int libxfs_log_header(char *, uuid_t *, int, int, int, xfs_lsn_t, int libxfs_alloc_file_space(struct xfs_inode *ip, xfs_off_t offset, xfs_off_t len, uint32_t bmapi_flags); +int libxfs_file_write(struct xfs_inode *ip, void *buf, off_t pos, + size_t len); /* XXX: this is messy and needs fixing */ #ifndef __LIBXFS_INTERNAL_XFS_H__ diff --git a/libxfs/util.c b/libxfs/util.c index e5892fc86c3e92..8c2ecff5855775 100644 --- a/libxfs/util.c +++ b/libxfs/util.c @@ -527,3 +527,70 @@ get_random_u32(void) return ret; } #endif + +/* + * Write a buffer to a file on the data device. There must not be sparse holes + * or unwritten extents. + */ +int +libxfs_file_write( + struct xfs_inode *ip, + void *buf, + off_t pos, + size_t len) +{ + struct xfs_bmbt_irec map; + struct xfs_mount *mp = ip->i_mount; + struct xfs_buf *bp; + xfs_fileoff_t bno = XFS_B_TO_FSBT(mp, pos); + xfs_fileoff_t end_bno = XFS_B_TO_FSB(mp, pos + len); + unsigned int block_off = pos % mp->m_sb.sb_blocksize; + size_t count; + size_t bcount; + int nmap; + int error = 0; + + /* Write up to 1MB at a time. */ + while (bno < end_bno) { + xfs_filblks_t maplen; + + maplen = min(end_bno - bno, XFS_B_TO_FSBT(mp, 1048576)); + nmap = 1; + error = libxfs_bmapi_read(ip, bno, maplen, &map, &nmap, 0); + if (error) + return error; + if (nmap != 1) + return -ENOSPC; + + if (map.br_startblock == HOLESTARTBLOCK || + map.br_state == XFS_EXT_UNWRITTEN) + return -EINVAL; + + error = libxfs_buf_get(mp->m_dev, + XFS_FSB_TO_DADDR(mp, map.br_startblock), + XFS_FSB_TO_BB(mp, map.br_blockcount), + &bp); + if (error) + break; + bp->b_ops = NULL; + + if (block_off > 0) + memset((char *)bp->b_addr, 0, block_off); + count = min(len, XFS_FSB_TO_B(mp, map.br_blockcount)); + memmove(bp->b_addr, buf + block_off, count); + bcount = BBTOB(bp->b_length); + if (count < bcount) + memset((char *)bp->b_addr + block_off + count, 0, + bcount - (block_off + count)); + + libxfs_buf_mark_dirty(bp); + libxfs_buf_relse(bp); + + buf += count; + len -= count; + bno += map.br_blockcount; + block_off = 0; + } + + return error; +} diff --git a/mkfs/proto.c b/mkfs/proto.c index 0764064e043e97..6946c22ff14d2a 100644 --- a/mkfs/proto.c +++ b/mkfs/proto.c @@ -16,7 +16,7 @@ static char *getstr(char **pp); static void fail(char *msg, int i); static struct xfs_trans * getres(struct xfs_mount *mp, uint blocks); static void rsvfile(xfs_mount_t *mp, xfs_inode_t *ip, long long len); -static char *newregfile(char **pp, int *len); +static int newregfile(char **pp, char **fname); static void rtinit(xfs_mount_t *mp); static long filesize(int fd); static int slashes_are_spaces; @@ -261,88 +261,120 @@ writesymlink( } static void -writefile( - struct xfs_trans *tp, +writefile_range( struct xfs_inode *ip, - char *buf, - int len) + const char *fname, + int fd, + off_t pos, + uint64_t len) { - struct xfs_bmbt_irec map; - struct xfs_mount *mp; - struct xfs_buf *bp; - xfs_daddr_t d; - xfs_extlen_t nb; - int nmap; + static char buf[131072]; int error; - mp = ip->i_mount; - if (len > 0) { - int bcount; + if (XFS_IS_REALTIME_INODE(ip)) { + fprintf(stderr, + _("%s: creating realtime files from proto file not supported.\n"), + progname); + exit(1); + } - nb = XFS_B_TO_FSB(mp, len); - nmap = 1; - error = -libxfs_bmapi_write(tp, ip, 0, nb, 0, nb, &map, &nmap); - if (error == ENOSYS && XFS_IS_REALTIME_INODE(ip)) { - fprintf(stderr, - _("%s: creating realtime files from proto file not supported.\n"), - progname); + while (len > 0) { + ssize_t read_len; + + read_len = pread(fd, buf, min(len, sizeof(buf)), pos); + if (read_len < 0) { + fprintf(stderr, _("%s: read failed on %s: %s\n"), + progname, fname, strerror(errno)); exit(1); } - if (error) { + + error = -libxfs_alloc_file_space(ip, pos, read_len, 0); + if (error) fail(_("error allocating space for a file"), error); + + error = -libxfs_file_write(ip, buf, pos, read_len); + if (error) + fail(_("error writing file"), error); + + pos += read_len; + len -= read_len; + } +} + +static void +writefile( + struct xfs_inode *ip, + const char *fname, + int fd) +{ + struct xfs_trans *tp; + struct xfs_mount *mp = ip->i_mount; + struct stat statbuf; + off_t data_pos; + off_t eof = 0; + int error; + + /* do not try to read from non-regular files */ + error = fstat(fd, &statbuf); + if (error < 0) + fail(_("unable to stat file to copyin"), errno); + + if (!S_ISREG(statbuf.st_mode)) + return; + + data_pos = lseek(fd, 0, SEEK_DATA); + while (data_pos >= 0) { + off_t hole_pos; + + hole_pos = lseek(fd, data_pos, SEEK_HOLE); + if (hole_pos < 0) { + /* save error, break */ + data_pos = hole_pos; + break; } - if (nmap != 1) { - fprintf(stderr, - _("%s: cannot allocate space for file\n"), - progname); - exit(1); - } - d = XFS_FSB_TO_DADDR(mp, map.br_startblock); - error = -libxfs_trans_get_buf(NULL, mp->m_dev, d, - nb << mp->m_blkbb_log, 0, &bp); - if (error) { - fprintf(stderr, - _("%s: cannot allocate buffer for file\n"), - progname); - exit(1); + if (hole_pos <= data_pos) { + /* shouldn't happen??? */ + break; } - memmove(bp->b_addr, buf, len); - bcount = BBTOB(bp->b_length); - if (len < bcount) - memset((char *)bp->b_addr + len, 0, bcount - len); - libxfs_buf_mark_dirty(bp); - libxfs_buf_relse(bp); + + writefile_range(ip, fname, fd, data_pos, hole_pos - data_pos); + eof = hole_pos; + + data_pos = lseek(fd, hole_pos, SEEK_DATA); } - ip->i_disk_size = len; + if (data_pos < 0 && errno != ENXIO) + fail(_("error finding file data to import"), errno); + + /* extend EOF only after writing all the file data */ + error = -libxfs_trans_alloc_inode(ip, &M_RES(mp)->tr_ichange, 0, 0, + false, &tp); + if (error) + fail(_("error creating isize transaction"), error); + + libxfs_trans_ijoin(tp, ip, 0); + ip->i_disk_size = eof; + libxfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + error = -libxfs_trans_commit(tp); + if (error) + fail(_("error committing isize transaction"), error); } -static char * +static int newregfile( char **pp, - int *len) + char **fname) { - char *buf; int fd; - char *fname; - long size; + off_t size; - fname = getstr(pp); - if ((fd = open(fname, O_RDONLY)) < 0 || (size = filesize(fd)) < 0) { + *fname = getstr(pp); + if ((fd = open(*fname, O_RDONLY)) < 0 || (size = filesize(fd)) < 0) { fprintf(stderr, _("%s: cannot open %s: %s\n"), - progname, fname, strerror(errno)); + progname, *fname, strerror(errno)); exit(1); } - if ((*len = (int)size)) { - buf = malloc(size); - if (read(fd, buf, size) < size) { - fprintf(stderr, _("%s: read failed on %s: %s\n"), - progname, fname, strerror(errno)); - exit(1); - } - } else - buf = NULL; - close(fd); - return buf; + + return fd; } static void @@ -552,7 +584,8 @@ parseproto( int fmt; int i; xfs_inode_t *ip; - int len; + int fd = -1; + off_t len; long long llen; int majdev; int mindev; @@ -563,6 +596,7 @@ parseproto( int isroot = 0; struct cred creds; char *value; + char *fname = NULL; struct xfs_name xname; struct xfs_parent_args *ppargs = NULL; @@ -636,16 +670,13 @@ parseproto( flags = XFS_ILOG_CORE; switch (fmt) { case IF_REGULAR: - buf = newregfile(pp, &len); - tp = getres(mp, XFS_B_TO_FSB(mp, len)); + fd = newregfile(pp, &fname); + tp = getres(mp, 0); ppargs = newpptr(mp); error = creatproto(&tp, pip, mode | S_IFREG, 0, &creds, fsxp, &ip); if (error) fail(_("Inode allocation failed"), error); - writefile(tp, ip, buf, len); - if (buf) - free(buf); libxfs_trans_ijoin(tp, pip, 0); xname.type = XFS_DIR3_FT_REG_FILE; newdirent(mp, tp, pip, &xname, ip, ppargs); @@ -800,6 +831,10 @@ parseproto( } libxfs_parent_finish(mp, ppargs); + if (fmt == IF_REGULAR) { + writefile(ip, fname, fd); + close(fd); + } libxfs_irele(ip); } @@ -950,7 +985,7 @@ rtinit( rtfreesp_init(mp); } -static long +static off_t filesize( int fd) { @@ -958,5 +993,5 @@ filesize( if (fstat(fd, &stb) < 0) return -1; - return (long)stb.st_size; + return stb.st_size; } From patchwork Mon Dec 23 21:57:57 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Darrick J. Wong" X-Patchwork-Id: 13919312 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 481141422AB for ; Mon, 23 Dec 2024 21:57:58 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1734991078; cv=none; b=PrQGD07Qv9BU8o8PfYe+Z8jTzohtkYIwGk6E4krKLWxLTDoyU6TOgSeWpnFf6ceCzV96z3jE6N2C9sZerU7wdpHwXqYJpC9NnCG/f3JLZxxdYLzS1oNYHbREJDTQgyzFx+TAZ/5Hwm5dspNLSs+/nRrKUXHNsU0EqvrAOLPO4aI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1734991078; c=relaxed/simple; bh=K8855pYoTCEOQJRj81NPifMfru2FuypGPABkLtneq0Q=; h=Date:Subject:From:To:Cc:Message-ID:In-Reply-To:References: MIME-Version:Content-Type; b=VBbirb+pebz+cGp/TK4TMqE4KlPBmT/MBUlk4/o4GOQs0DAEPWFJQLqlEcY9wFD2R0U7MJ3d6z+vYuQbJxbqL0S1OSHjZL/yLnUi33smE+gpfcKbO5Gt66cVJN6yIXvjuS59xAE8FKWQzjpfqjOYA9ZaZHtnKgGfHphdhJMxpJQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=rTNGXKow; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="rTNGXKow" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 1F16AC4CED3; Mon, 23 Dec 2024 21:57:58 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1734991078; bh=K8855pYoTCEOQJRj81NPifMfru2FuypGPABkLtneq0Q=; h=Date:Subject:From:To:Cc:In-Reply-To:References:From; b=rTNGXKow15J4nNHdLXxGTTegaw7oZoMGPfMtjxaCeHRY9Nj2vxcJZ8Lf3YRmHDk0D kN6Djy+ODLEEusE2mIyfYAWZrt0ghJjCHg40TqadmsSMt32CRnSaqYutn2Fi2KSuc6 Xmzrjw7NUqhPNP7lMA5DSYXKYE168YGg9h1pW7gk4m5FNKzYKFMBd222miJ/dPstlD Sragu5NVirjIQOPHmwaERU0lqFHSEoQgsbqoGPLc7DgYEw1mi8lwMjuh0K+gV8QvdG CB9s0ORAjRKZzm0mL8mwp6HQNjNSkYjXhA8Wuc6isdveF8ILt0LkrA+r9Dyyj/VHlI 2CMtVlpd0ByyA== Date: Mon, 23 Dec 2024 13:57:57 -0800 Subject: [PATCH 3/4] mkfs: support copying in xattrs From: "Darrick J. Wong" To: djwong@kernel.org, aalbersh@kernel.org Cc: hch@lst.de, linux-xfs@vger.kernel.org Message-ID: <173498941974.2295644.15019113821847240813.stgit@frogsfrogsfrogs> In-Reply-To: <173498941923.2295644.12590815257028938964.stgit@frogsfrogsfrogs> References: <173498941923.2295644.12590815257028938964.stgit@frogsfrogsfrogs> Precedence: bulk X-Mailing-List: linux-xfs@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 From: Darrick J. Wong Update the protofile code to import extended attributes from the source files. Signed-off-by: "Darrick J. Wong" Reviewed-by: Christoph Hellwig --- mkfs/proto.c | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/mkfs/proto.c b/mkfs/proto.c index 6946c22ff14d2a..7a0493ec71cfd9 100644 --- a/mkfs/proto.c +++ b/mkfs/proto.c @@ -6,6 +6,8 @@ #include "libxfs.h" #include +#include +#include #include "libfrog/convert.h" #include "proto.h" @@ -359,6 +361,97 @@ writefile( fail(_("error committing isize transaction"), error); } +static void +writeattr( + struct xfs_inode *ip, + const char *fname, + int fd, + const char *attrname, + char *valuebuf, + size_t valuelen) +{ + struct xfs_da_args args = { + .dp = ip, + .geo = ip->i_mount->m_attr_geo, + .owner = ip->i_ino, + .whichfork = XFS_ATTR_FORK, + .op_flags = XFS_DA_OP_OKNOENT, + .value = valuebuf, + }; + ssize_t ret; + int error; + + ret = fgetxattr(fd, attrname, valuebuf, valuelen); + if (ret < 0) { + if (errno == EOPNOTSUPP) + return; + fail(_("error collecting xattr value"), errno); + } + if (ret == 0) + return; + + if (!strncmp(attrname, XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN)) { + args.name = (unsigned char *)attrname + XATTR_TRUSTED_PREFIX_LEN; + args.attr_filter = LIBXFS_ATTR_ROOT; + } else if (!strncmp(attrname, XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN)) { + args.name = (unsigned char *)attrname + XATTR_SECURITY_PREFIX_LEN; + args.attr_filter = LIBXFS_ATTR_SECURE; + } else if (!strncmp(attrname, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN)) { + args.name = (unsigned char *)attrname + XATTR_USER_PREFIX_LEN; + args.attr_filter = 0; + } else { + args.name = (unsigned char *)attrname; + args.attr_filter = 0; + } + args.namelen = strlen((char *)args.name); + + args.valuelen = ret; + libxfs_attr_sethash(&args); + + error = -libxfs_attr_set(&args, XFS_ATTRUPDATE_UPSERT, false); + if (error) + fail(_("setting xattr value"), error); +} + +static void +writeattrs( + struct xfs_inode *ip, + const char *fname, + int fd) +{ + char *namebuf, *p, *end; + char *valuebuf = NULL; + ssize_t ret; + + namebuf = malloc(XATTR_LIST_MAX); + if (!namebuf) + fail(_("error allocating xattr name buffer"), errno); + + ret = flistxattr(fd, namebuf, XATTR_LIST_MAX); + if (ret < 0) { + if (errno == EOPNOTSUPP) + goto out_namebuf; + fail(_("error collecting xattr names"), errno); + } + + p = namebuf; + end = namebuf + ret; + for (p = namebuf; p < end; p += strlen(p) + 1) { + if (!valuebuf) { + valuebuf = malloc(ATTR_MAX_VALUELEN); + if (!valuebuf) + fail(_("error allocating xattr value buffer"), + errno); + } + + writeattr(ip, fname, fd, p, valuebuf, ATTR_MAX_VALUELEN); + } + + free(valuebuf); +out_namebuf: + free(namebuf); +} + static int newregfile( char **pp, @@ -833,6 +926,7 @@ parseproto( libxfs_parent_finish(mp, ppargs); if (fmt == IF_REGULAR) { writefile(ip, fname, fd); + writeattrs(ip, fname, fd); close(fd); } libxfs_irele(ip); From patchwork Mon Dec 23 21:58:13 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Darrick J. Wong" X-Patchwork-Id: 13919313 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id DA95918BBAC for ; Mon, 23 Dec 2024 21:58:13 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1734991093; cv=none; b=mfCsjKzURyQ2Q0Ocgk7E65oMLHMWQLLtBqvc555IwtMaSXra3jtKhjSOdfv47GW0W4O+4xPsAHfhjAeEgKifFshMxaNabIv5os7YGbPE0J5+X3/UU5weNn+RP54Dk/wwWCfBCcb+0tWKZay2eQeFQpN4huko/4TfrF+jOQ5BKzs= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1734991093; c=relaxed/simple; bh=zsJIsJcSFDJGarZb3+weqEjM+wb2PWvrWq9bEYNucQA=; h=Date:Subject:From:To:Cc:Message-ID:In-Reply-To:References: MIME-Version:Content-Type; b=iL4VUhkvuMuBEPUORLfLA5KpsgGRqfBzbgrR/aL4DnJzgKgAIC7e6HnG+Xtwm6UQHyJFt0Gb8/mQR8PJyIatBsWNYlWyVCXP3cIymV0zSK0NHrEGsl40UymrLsKZxI8kLoTJW0YctwYfhXJ+gHZDrk6U2yfxoLUiwR5GF0qjVMw= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=s7/T/rIa; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="s7/T/rIa" Received: by smtp.kernel.org (Postfix) with ESMTPSA id B3DD1C4CED3; Mon, 23 Dec 2024 21:58:13 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1734991093; bh=zsJIsJcSFDJGarZb3+weqEjM+wb2PWvrWq9bEYNucQA=; h=Date:Subject:From:To:Cc:In-Reply-To:References:From; b=s7/T/rIaHWt0Y9RZx/ONHteWh05B3Twys2pcrtzUA08mORVGFoNOxmyF3wBBfmPCP azLwa0yMrZoiAAKTbfe8dgAq2ojj2SbJR4UKojr+cj/p26FrKczmqmP0uQiXpo7WNr xauqh2aIu/TApnErS6TRn4wm2qtxJI2JXh3mRqy3jCDfcbK5vmR74rI6R1cELcwsGa 5pCAtB1KLlkJO6xW963NsfS+iulEwTVrMf7LtESGudeNYEpwspN/jcWHg8IhD5TwBx JkKy6t2zzQQRJHV7sYcncwHzkU2LLBjuZ3AveQD/K6Oom62qp+ZiFF5M4VhkeX/1Qi JO7e2TI3Vrc2g== Date: Mon, 23 Dec 2024 13:58:13 -0800 Subject: [PATCH 4/4] mkfs: add a utility to generate protofiles From: "Darrick J. Wong" To: djwong@kernel.org, aalbersh@kernel.org Cc: hch@lst.de, linux-xfs@vger.kernel.org Message-ID: <173498941990.2295644.470215387134229607.stgit@frogsfrogsfrogs> In-Reply-To: <173498941923.2295644.12590815257028938964.stgit@frogsfrogsfrogs> References: <173498941923.2295644.12590815257028938964.stgit@frogsfrogsfrogs> Precedence: bulk X-Mailing-List: linux-xfs@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 From: Darrick J. Wong Add a new utility to generate mkfs protofiles from a directory tree. Signed-off-by: "Darrick J. Wong" Reviewed-by: Christoph Hellwig --- man/man8/xfs_protofile.8 | 33 ++++++++++ mkfs/Makefile | 10 +++ mkfs/xfs_protofile.in | 152 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 man/man8/xfs_protofile.8 create mode 100644 mkfs/xfs_protofile.in diff --git a/man/man8/xfs_protofile.8 b/man/man8/xfs_protofile.8 new file mode 100644 index 00000000000000..75090c138f3393 --- /dev/null +++ b/man/man8/xfs_protofile.8 @@ -0,0 +1,33 @@ +.TH xfs_protofile 8 +.SH NAME +xfs_protofile \- create a protofile for use with mkfs.xfs +.SH SYNOPSIS +.B xfs_protofile +.I path +[ +.I paths... +] +.br +.B xfs_protofile \-V +.SH DESCRIPTION +.B xfs_protofile +walks a directory tree to generate a protofile. +The protofile format is specified in the +.BR mkfs.xfs (8) +manual page and is derived from 3rd edition Unix. +.SH OPTIONS +.TP 1.0i +.I path +Create protofile directives to copy this path into the root directory. +If the path is a directory, protofile directives will be emitted to +replicate the entire subtree as a subtree of the root directory. +If the path is a not a directory, protofile directives will be emitted +to create the file as an entry in the root directory. +The first path must resolve to a directory. + +.SH BUGS +Filenames cannot contain spaces. +Extended attributes are not copied into the filesystem. + +.PD +.RE diff --git a/mkfs/Makefile b/mkfs/Makefile index 754a17ea5137b7..3d3f08ad54f844 100644 --- a/mkfs/Makefile +++ b/mkfs/Makefile @@ -6,6 +6,7 @@ TOPDIR = .. include $(TOPDIR)/include/builddefs LTCOMMAND = mkfs.xfs +XFS_PROTOFILE = xfs_protofile HFILES = CFILES = proto.c xfs_mkfs.c @@ -23,14 +24,21 @@ LLDLIBS += $(LIBXFS) $(LIBXCMD) $(LIBFROG) $(LIBRT) $(LIBBLKID) \ $(LIBUUID) $(LIBINIH) $(LIBURCU) $(LIBPTHREAD) LTDEPENDENCIES += $(LIBXFS) $(LIBXCMD) $(LIBFROG) LLDFLAGS = -static-libtool-libs +DIRT = $(XFS_PROTOFILE) -default: depend $(LTCOMMAND) $(CFGFILES) +default: depend $(LTCOMMAND) $(CFGFILES) $(XFS_PROTOFILE) include $(BUILDRULES) +$(XFS_PROTOFILE): $(XFS_PROTOFILE).in + @echo " [SED] $@" + $(Q)$(SED) -e "s|@pkg_version@|$(PKG_VERSION)|g" < $< > $@ + $(Q)chmod a+x $@ + install: default $(INSTALL) -m 755 -d $(PKG_SBIN_DIR) $(LTINSTALL) -m 755 $(LTCOMMAND) $(PKG_SBIN_DIR) + $(INSTALL) -m 755 $(XFS_PROTOFILE) $(PKG_SBIN_DIR) $(INSTALL) -m 755 -d $(MKFS_CFG_DIR) $(INSTALL) -m 644 $(CFGFILES) $(MKFS_CFG_DIR) diff --git a/mkfs/xfs_protofile.in b/mkfs/xfs_protofile.in new file mode 100644 index 00000000000000..9aee4336888523 --- /dev/null +++ b/mkfs/xfs_protofile.in @@ -0,0 +1,152 @@ +#!/usr/bin/python3 + +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright (c) 2018-2024 Oracle. All rights reserved. +# +# Author: Darrick J. Wong + +# Walk a filesystem tree to generate a protofile for mkfs. + +import os +import argparse +import sys +import stat + +def emit_proto_header(): + '''Emit the protofile header.''' + print('/') + print('0 0') + +def stat_to_str(statbuf): + '''Convert a stat buffer to a proto string.''' + + if stat.S_ISREG(statbuf.st_mode): + type = '-' + elif stat.S_ISCHR(statbuf.st_mode): + type = 'c' + elif stat.S_ISBLK(statbuf.st_mode): + type = 'b' + elif stat.S_ISFIFO(statbuf.st_mode): + type = 'p' + elif stat.S_ISDIR(statbuf.st_mode): + type = 'd' + elif stat.S_ISLNK(statbuf.st_mode): + type = 'l' + + if statbuf.st_mode & stat.S_ISUID: + suid = 'u' + else: + suid = '-' + + if statbuf.st_mode & stat.S_ISGID: + sgid = 'g' + else: + sgid = '-' + + perms = stat.S_IMODE(statbuf.st_mode) + + return '%s%s%s%o %d %d' % (type, suid, sgid, perms, statbuf.st_uid, \ + statbuf.st_gid) + +def stat_to_extra(statbuf, fullpath): + '''Compute the extras column for a protofile.''' + + if stat.S_ISREG(statbuf.st_mode): + return ' %s' % fullpath + elif stat.S_ISCHR(statbuf.st_mode) or stat.S_ISBLK(statbuf.st_mode): + return ' %d %d' % (statbuf.st_rdev, statbuf.st_rdev) + elif stat.S_ISLNK(statbuf.st_mode): + return ' %s' % os.readlink(fullpath) + return '' + +def max_fname_len(s1): + '''Return the length of the longest string in s1.''' + ret = 0 + for s in s1: + if len(s) > ret: + ret = len(s) + return ret + +def walk_tree(path, depth): + '''Walk the directory tree rooted by path.''' + dirs = [] + files = [] + + for fname in os.listdir(path): + fullpath = os.path.join(path, fname) + sb = os.lstat(fullpath) + + if stat.S_ISDIR(sb.st_mode): + dirs.append(fname) + continue + elif stat.S_ISSOCK(sb.st_mode): + continue + else: + files.append(fname) + + for fname in files: + if ' ' in fname: + raise ValueError( \ + f'{fname}: Spaces not allowed in file names.') + for fname in dirs: + if ' ' in fname: + raise Exception( \ + f'{fname}: Spaces not allowed in file names.') + + fname_width = max_fname_len(files) + for fname in files: + fullpath = os.path.join(path, fname) + sb = os.lstat(fullpath) + extra = stat_to_extra(sb, fullpath) + print('%*s%-*s %s%s' % (depth, ' ', fname_width, fname, \ + stat_to_str(sb), extra)) + + for fname in dirs: + fullpath = os.path.join(path, fname) + sb = os.lstat(fullpath) + extra = stat_to_extra(sb, fullpath) + print('%*s%s %s' % (depth, ' ', fname, \ + stat_to_str(sb))) + walk_tree(fullpath, depth + 1) + + if depth > 1: + print('%*s$' % (depth - 1, ' ')) + +def main(): + parser = argparse.ArgumentParser( \ + description = "Generate mkfs.xfs protofile for a directory tree.") + parser.add_argument('paths', metavar = 'paths', type = str, \ + nargs = '*', help = 'Directory paths to walk.') + parser.add_argument("-V", help = "Report version and exit.", \ + action = "store_true") + args = parser.parse_args() + + if args.V: + print("xfs_protofile version @pkg_version@") + sys.exit(0) + + emit_proto_header() + if len(args.paths) == 0: + print('d--755 0 0') + print('$') + else: + # Copy the first argument's stat to the rootdir + statbuf = os.stat(args.paths[0]) + if not stat.S_ISDIR(statbuf.st_mode): + raise NotADirectoryError(path) + print(stat_to_str(statbuf)) + + # All files under each path go in the root dir, recursively + for path in args.paths: + print(': Descending path %s' % path) + try: + walk_tree(path, 1) + except Exception as e: + print(e, file = sys.stderr) + return 1 + + print('$') + return 0 + +if __name__ == '__main__': + sys.exit(main())