diff mbox series

[bpf-next,v5,4/8] libbpf: Support BTF.ext loading and output in either endianness

Message ID 596b03b8e225de62e6877346fbed19502a205c88.1725347944.git.tony.ambardar@gmail.com (mailing list archive)
State Superseded
Delegated to: BPF
Headers show
Series libbpf, selftests/bpf: Support cross-endian usage | expand

Checks

Context Check Description
bpf/vmtest-bpf-next-VM_Test-12 success Logs for s390x-gcc / build-release
bpf/vmtest-bpf-next-VM_Test-16 success Logs for s390x-gcc / veristat
bpf/vmtest-bpf-next-VM_Test-17 success Logs for set-matrix
bpf/vmtest-bpf-next-VM_Test-19 success Logs for x86_64-gcc / build-release
bpf/vmtest-bpf-next-VM_Test-18 success Logs for x86_64-gcc / build / build for x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-4 success Logs for aarch64-gcc / build / build for aarch64 with gcc
bpf/vmtest-bpf-next-VM_Test-11 success Logs for s390x-gcc / build / build for s390x with gcc
bpf/vmtest-bpf-next-VM_Test-27 success Logs for x86_64-llvm-17 / build / build for x86_64 with llvm-17
bpf/vmtest-bpf-next-VM_Test-33 success Logs for x86_64-llvm-17 / veristat
bpf/vmtest-bpf-next-VM_Test-13 fail Logs for s390x-gcc / test (test_progs, false, 360) / test_progs on s390x with gcc
bpf/vmtest-bpf-next-VM_Test-15 success Logs for s390x-gcc / test (test_verifier, false, 360) / test_verifier on s390x with gcc
bpf/vmtest-bpf-next-VM_Test-9 success Logs for aarch64-gcc / test (test_verifier, false, 360) / test_verifier on aarch64 with gcc
bpf/vmtest-bpf-next-VM_Test-28 success Logs for x86_64-llvm-17 / build-release / build for x86_64 with llvm-17-O2
bpf/vmtest-bpf-next-VM_Test-34 success Logs for x86_64-llvm-18 / build / build for x86_64 with llvm-18
bpf/vmtest-bpf-next-VM_Test-35 success Logs for x86_64-llvm-18 / build-release / build for x86_64 with llvm-18-O2
bpf/vmtest-bpf-next-VM_Test-41 success Logs for x86_64-llvm-18 / veristat
bpf/vmtest-bpf-next-VM_Test-14 success Logs for s390x-gcc / test (test_progs_no_alu32, false, 360) / test_progs_no_alu32 on s390x with gcc
bpf/vmtest-bpf-next-VM_Test-20 success Logs for x86_64-gcc / test (test_maps, false, 360) / test_maps on x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-6 success Logs for aarch64-gcc / test (test_maps, false, 360) / test_maps on aarch64 with gcc
bpf/vmtest-bpf-next-VM_Test-25 success Logs for x86_64-gcc / test (test_verifier, false, 360) / test_verifier on x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-29 success Logs for x86_64-llvm-17 / test (test_maps, false, 360) / test_maps on x86_64 with llvm-17
bpf/vmtest-bpf-next-VM_Test-32 success Logs for x86_64-llvm-17 / test (test_verifier, false, 360) / test_verifier on x86_64 with llvm-17
bpf/vmtest-bpf-next-VM_Test-40 success Logs for x86_64-llvm-18 / test (test_verifier, false, 360) / test_verifier on x86_64 with llvm-18
bpf/vmtest-bpf-next-VM_Test-21 success Logs for x86_64-gcc / test (test_progs, false, 360) / test_progs on x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-22 success Logs for x86_64-gcc / test (test_progs_no_alu32, false, 360) / test_progs_no_alu32 on x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-23 success Logs for x86_64-gcc / test (test_progs_no_alu32_parallel, true, 30) / test_progs_no_alu32_parallel on x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-24 success Logs for x86_64-gcc / test (test_progs_parallel, true, 30) / test_progs_parallel on x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-26 success Logs for x86_64-gcc / veristat / veristat on x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-30 success Logs for x86_64-llvm-17 / test (test_progs, false, 360) / test_progs on x86_64 with llvm-17
bpf/vmtest-bpf-next-VM_Test-31 success Logs for x86_64-llvm-17 / test (test_progs_no_alu32, false, 360) / test_progs_no_alu32 on x86_64 with llvm-17
bpf/vmtest-bpf-next-VM_Test-37 success Logs for x86_64-llvm-18 / test (test_progs, false, 360) / test_progs on x86_64 with llvm-18
bpf/vmtest-bpf-next-VM_Test-36 success Logs for x86_64-llvm-18 / test (test_maps, false, 360) / test_maps on x86_64 with llvm-18
bpf/vmtest-bpf-next-VM_Test-38 success Logs for x86_64-llvm-18 / test (test_progs_cpuv4, false, 360) / test_progs_cpuv4 on x86_64 with llvm-18
bpf/vmtest-bpf-next-VM_Test-39 success Logs for x86_64-llvm-18 / test (test_progs_no_alu32, false, 360) / test_progs_no_alu32 on x86_64 with llvm-18
bpf/vmtest-bpf-next-PR success PR summary
bpf/vmtest-bpf-next-VM_Test-1 success Logs for ShellCheck
bpf/vmtest-bpf-next-VM_Test-0 success Logs for Lint
bpf/vmtest-bpf-next-VM_Test-5 success Logs for aarch64-gcc / build-release
bpf/vmtest-bpf-next-VM_Test-2 success Logs for Unittests
bpf/vmtest-bpf-next-VM_Test-3 success Logs for Validate matrix.py
bpf/vmtest-bpf-next-VM_Test-7 success Logs for s390x-gcc / build-release
bpf/vmtest-bpf-next-VM_Test-8 success Logs for set-matrix
bpf/vmtest-bpf-next-VM_Test-10 success Logs for x86_64-gcc / build-release
netdev/series_format success Posting correctly formatted
netdev/tree_selection success Clearly marked for bpf-next, async
netdev/ynl success Generated files up to date; no warnings/errors; no diff in generated;
netdev/fixes_present success Fixes tag not required for -next series
netdev/header_inline success No static functions without inline keyword in header files
netdev/build_32bit success Errors and warnings before: 7 this patch: 7
netdev/build_tools fail Errors and warnings before: 4 this patch: 172
netdev/cc_maintainers success CCed 13 of 13 maintainers
netdev/build_clang success Errors and warnings before: 7 this patch: 7
netdev/verify_signedoff success Signed-off-by tag matches author and committer
netdev/deprecated_api success None detected
netdev/check_selftest success No net selftest shell script
netdev/verify_fixes success No Fixes tag
netdev/build_allmodconfig_warn success Errors and warnings before: 7 this patch: 7
netdev/checkpatch warning CHECK: Macro argument 'btf_ext' may be better as '(btf_ext)' to avoid precedence issues CHECK: Macro argument 'info_t' may be better as '(info_t)' to avoid precedence issues CHECK: Please use a blank line after function/struct/union/enum declarations WARNING: Macros with flow control statements should be avoided WARNING: Missing a blank line after declarations WARNING: line length of 82 exceeds 80 columns WARNING: line length of 83 exceeds 80 columns WARNING: line length of 84 exceeds 80 columns WARNING: line length of 85 exceeds 80 columns WARNING: line length of 99 exceeds 80 columns
netdev/build_clang_rust success No Rust files in patch. Skipping build
netdev/kdoc success Errors and warnings before: 32 this patch: 32
netdev/source_inline success Was 0 now: 0

Commit Message

Tony Ambardar Sept. 3, 2024, 7:33 a.m. UTC 
Support for handling BTF data of either endianness was added in [1], but
did not include BTF.ext data for lack of use cases. Later, support for
static linking [2] provided a use case, but this feature and later ones
were restricted to native-endian usage.

Add support for BTF.ext handling in either endianness. Convert BTF.ext data
to native endianness when read into memory for further processing, and
support raw data access that restores the original byte-order for output.
Add internal header functions for byte-swapping func, line, and core info
records.

Add new API functions btf_ext__endianness() and btf_ext__set_endianness()
for query and setting byte-order, as already exist for BTF data.

[1] 3289959b97ca ("libbpf: Support BTF loading and raw data output in both endianness")
[2] 8fd27bf69b86 ("libbpf: Add BPF static linker BTF and BTF.ext support")

Signed-off-by: Tony Ambardar <tony.ambardar@gmail.com>
---
 tools/lib/bpf/btf.c             | 238 +++++++++++++++++++++++++++++---
 tools/lib/bpf/btf.h             |   3 +
 tools/lib/bpf/libbpf.map        |   2 +
 tools/lib/bpf/libbpf_internal.h |  28 ++++
 4 files changed, 255 insertions(+), 16 deletions(-)

Comments

Andrii Nakryiko Sept. 4, 2024, 7:48 p.m. UTC | #1 
On Tue, Sep 3, 2024 at 12:33 AM Tony Ambardar <tony.ambardar@gmail.com> wrote:
>
> Support for handling BTF data of either endianness was added in [1], but
> did not include BTF.ext data for lack of use cases. Later, support for
> static linking [2] provided a use case, but this feature and later ones
> were restricted to native-endian usage.
>
> Add support for BTF.ext handling in either endianness. Convert BTF.ext data
> to native endianness when read into memory for further processing, and
> support raw data access that restores the original byte-order for output.
> Add internal header functions for byte-swapping func, line, and core info
> records.
>
> Add new API functions btf_ext__endianness() and btf_ext__set_endianness()
> for query and setting byte-order, as already exist for BTF data.
>
> [1] 3289959b97ca ("libbpf: Support BTF loading and raw data output in both endianness")
> [2] 8fd27bf69b86 ("libbpf: Add BPF static linker BTF and BTF.ext support")
>
> Signed-off-by: Tony Ambardar <tony.ambardar@gmail.com>
> ---
>  tools/lib/bpf/btf.c             | 238 +++++++++++++++++++++++++++++---
>  tools/lib/bpf/btf.h             |   3 +
>  tools/lib/bpf/libbpf.map        |   2 +
>  tools/lib/bpf/libbpf_internal.h |  28 ++++
>  4 files changed, 255 insertions(+), 16 deletions(-)
>
> diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c
> index 5f094c1f4388..c11dfc81d007 100644
> --- a/tools/lib/bpf/btf.c
> +++ b/tools/lib/bpf/btf.c
> @@ -3023,25 +3023,140 @@ static int btf_ext_setup_core_relos(struct btf_ext *btf_ext)
>         return btf_ext_setup_info(btf_ext, &param);
>  }
>
> -static int btf_ext_parse_hdr(__u8 *data, __u32 data_size)
> +/* Swap byte-order of BTF.ext header with any endianness */
> +static void btf_ext_bswap_hdr(struct btf_ext *btf_ext, __u32 hdr_len)
>  {
> -       const struct btf_ext_header *hdr = (struct btf_ext_header *)data;
> +       struct btf_ext_header *h = btf_ext->hdr;
>
> -       if (data_size < offsetofend(struct btf_ext_header, hdr_len) ||
> -           data_size < hdr->hdr_len) {
> -               pr_debug("BTF.ext header not found\n");
> +       h->magic = bswap_16(h->magic);
> +       h->hdr_len = bswap_32(h->hdr_len);
> +       h->func_info_off = bswap_32(h->func_info_off);
> +       h->func_info_len = bswap_32(h->func_info_len);
> +       h->line_info_off = bswap_32(h->line_info_off);
> +       h->line_info_len = bswap_32(h->line_info_len);
> +
> +       if (hdr_len < offsetofend(struct btf_ext_header, core_relo_len))
> +               return;
> +
> +       h->core_relo_off = bswap_32(h->core_relo_off);
> +       h->core_relo_len = bswap_32(h->core_relo_len);
> +}
> +
> +/* Swap metadata byte-order of generic info subsection */
> +static int info_subsec_bswap_metadata(const struct btf_ext *btf_ext, struct btf_ext_info *ext_info)
> +{
> +       const bool is_native = btf_ext->swapped_endian;
> +       __u32 left, *rs, rec_size, num_info;
> +       struct btf_ext_info_sec *si;
> +
> +       if (ext_info->len == 0)
> +               return 0;
> +
> +       rs = ext_info->info - sizeof(__u32);    /* back up to record size */
> +       rec_size = is_native ? *rs : bswap_32(*rs);
> +       if (rec_size != ext_info->rec_size)
>                 return -EINVAL;
> +       *rs = bswap_32(*rs);
> +
> +       si = ext_info->info;                    /* sec info #1 */
> +       left = ext_info->len;
> +       while (left > 0) {
> +               num_info = is_native ? si->num_info : bswap_32(si->num_info);
> +               si->sec_name_off = bswap_32(si->sec_name_off);
> +               si->num_info = bswap_32(si->num_info);
> +               si = (void *)si->data + rec_size * num_info;
> +               left -= offsetof(struct btf_ext_info_sec, data) +
> +                       rec_size * num_info;
>         }
>
> +       return 0;
> +}
> +
> +/* Swap byte order of info subsection metadata and records in the correct
> + * order depending on whether or not data is in native endianness.
> + */
> +#define ORDER_INFO_BSWAP(btf_ext, ext_info, info_t, swap_fn)           \
> +{                                                                      \
> +       const bool is_native = btf_ext->swapped_endian;                 \
> +       struct btf_ext_info_sec *si;                                    \
> +       int c, err;                                                     \
> +       info_t *i;                                                      \
> +       if (is_native)                                                  \
> +               for_each_btf_ext_sec(ext_info, si)                      \
> +                       for_each_btf_ext_rec(ext_info, si, c, i)        \
> +                               swap_fn(i);                             \
> +       err = info_subsec_bswap_metadata(btf_ext, ext_info);            \
> +       if (err) {                                                      \
> +               pr_warn(#info_t " record size mismatch!\n");            \
> +               return err;                                             \
> +       }                                                               \
> +       if (!is_native)                                                 \
> +               for_each_btf_ext_sec(ext_info, si)                      \
> +                       for_each_btf_ext_rec(ext_info, si, c, i)        \
> +                               swap_fn(i);                             \
> +}
> +

Nope-nope-nope, I'm never landing something like this :)

Ok, if there is no clean swapping and setup separation we can come up
with, let's still do a decent job at trying to keep all this coherent
and simple.

How about this? btf_ext_bswap_hdr() stays, it seems fine.

btf_ext_setup_func_info(), btf_ext_setup_line_info(), and
btf_ext_setup_core_relos() all call into generic btf_ext_setup_info,
right? We teach btf_ext_setup_info() to work with both endianness (but
not swap data at all). The only two fields that we need to byte-swap
on the fly are record_size and sinfo->num_info, so it's minimal
changes to btf_ext_setup_info().

Oh, and while we are at it, if data is in non-native endianness,
btf_ext_setup_info() should return failure if any of the record sizes
are not *exactly* matching our expectations. This should be a trivial
addition as we have never extended those records, I believe.

This will take care of correctness checking regardless of endianness.

Now, forget about for_each_btf_ext_{sec,rec}(), it will be too fragile
to make them work for inverted endianness. But we know now that the
data is correct, right? So, fine, let's have a bit of duplication (but
without any error checking) to go over raw .BTF.ext data and swap all
the records and all metadata fields. This logic now will be used after
btf_ext_setup_*() steps, and in btf_ext_raw_data(). For
btf_ext_raw_data() you'll also btf_ext_bswap_hdr() at the very end,
right?

How does that sound? Am I missing something big again?

And fine, let's use callbacks for different record types to keep this
simple. I dislike callbacks, in principle, but sometimes they are the
simplest way forward, unfortunately (a proper iterator would be
better, but that's another story and I don't want to get into that
implementation task just yet).

> +/*
> + * Swap endianness of the whole info segment in a BTF.ext data section:
> + *   - requires BTF.ext header data in native byte order
> + *   - only support info structs from BTF version 1
> + */
> +static int btf_ext_bswap_info(struct btf_ext *btf_ext)
> +{
> +       const struct btf_ext_header *h = btf_ext->hdr;
> +       struct btf_ext_info ext = {};
> +
> +       /* Swap func_info subsection byte-order */
> +       ext.info = (void *)h + h->hdr_len + h->func_info_off + sizeof(__u32);
> +       ext.len = h->func_info_len - (h->func_info_len ? sizeof(__u32) : 0);
> +       ext.rec_size = sizeof(struct bpf_func_info);
> +
> +       ORDER_INFO_BSWAP(btf_ext, &ext, struct bpf_func_info, bpf_func_info_bswap);

You shouldn't have bent over backwards just to use
for_each_btf_ext_{sec,rec}() macros. Just because I proposed it
(initially, without actually coding anything) doesn't mean it's the
best and final solution :)

> +
> +       /* Swap line_info subsection byte-order */
> +       ext.info = (void *)h + h->hdr_len + h->line_info_off + sizeof(__u32);
> +       ext.len = h->line_info_len - (h->line_info_len ? sizeof(__u32) : 0);
> +       ext.rec_size = sizeof(struct bpf_line_info);
> +
> +       ORDER_INFO_BSWAP(btf_ext, &ext, struct bpf_line_info, bpf_line_info_bswap);
> +
> +       /* Swap core_relo subsection byte-order (if present) */
> +       if (h->hdr_len < offsetofend(struct btf_ext_header, core_relo_len))
> +               return 0;
> +
> +       ext.info = (void *)h + h->hdr_len + h->core_relo_off + sizeof(__u32);
> +       ext.len = h->core_relo_len - (h->core_relo_len ? sizeof(__u32) : 0);
> +       ext.rec_size = sizeof(struct bpf_core_relo);
> +
> +       ORDER_INFO_BSWAP(btf_ext, &ext, struct bpf_core_relo, bpf_core_relo_bswap);
> +
> +       return 0;
> +}
> +#undef ORDER_INFO_BSWAP

[...]
Tony Ambardar Sept. 16, 2024, 8:21 a.m. UTC | #2 
On Wed, Sep 04, 2024 at 12:48:36PM -0700, Andrii Nakryiko wrote:
> On Tue, Sep 3, 2024 at 12:33 AM Tony Ambardar <tony.ambardar@gmail.com> wrote:
> >
> > Support for handling BTF data of either endianness was added in [1], but
> > did not include BTF.ext data for lack of use cases. Later, support for
> > static linking [2] provided a use case, but this feature and later ones
> > were restricted to native-endian usage.
> >
> > Add support for BTF.ext handling in either endianness. Convert BTF.ext data
> > to native endianness when read into memory for further processing, and
> > support raw data access that restores the original byte-order for output.
> > Add internal header functions for byte-swapping func, line, and core info
> > records.
> >
> > Add new API functions btf_ext__endianness() and btf_ext__set_endianness()
> > for query and setting byte-order, as already exist for BTF data.
> >
> > [1] 3289959b97ca ("libbpf: Support BTF loading and raw data output in both endianness")
> > [2] 8fd27bf69b86 ("libbpf: Add BPF static linker BTF and BTF.ext support")
> >
> > Signed-off-by: Tony Ambardar <tony.ambardar@gmail.com>
> > ---
> >  tools/lib/bpf/btf.c             | 238 +++++++++++++++++++++++++++++---
> >  tools/lib/bpf/btf.h             |   3 +
> >  tools/lib/bpf/libbpf.map        |   2 +
> >  tools/lib/bpf/libbpf_internal.h |  28 ++++
> >  4 files changed, 255 insertions(+), 16 deletions(-)
> >
> > diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c
> > index 5f094c1f4388..c11dfc81d007 100644
> > --- a/tools/lib/bpf/btf.c
> > +++ b/tools/lib/bpf/btf.c
> > @@ -3023,25 +3023,140 @@ static int btf_ext_setup_core_relos(struct btf_ext *btf_ext)
> >         return btf_ext_setup_info(btf_ext, &param);
> >  }
> >
> > -static int btf_ext_parse_hdr(__u8 *data, __u32 data_size)
> > +/* Swap byte-order of BTF.ext header with any endianness */
> > +static void btf_ext_bswap_hdr(struct btf_ext *btf_ext, __u32 hdr_len)
> >  {
> > -       const struct btf_ext_header *hdr = (struct btf_ext_header *)data;
> > +       struct btf_ext_header *h = btf_ext->hdr;
> >
> > -       if (data_size < offsetofend(struct btf_ext_header, hdr_len) ||
> > -           data_size < hdr->hdr_len) {
> > -               pr_debug("BTF.ext header not found\n");
> > +       h->magic = bswap_16(h->magic);
> > +       h->hdr_len = bswap_32(h->hdr_len);
> > +       h->func_info_off = bswap_32(h->func_info_off);
> > +       h->func_info_len = bswap_32(h->func_info_len);
> > +       h->line_info_off = bswap_32(h->line_info_off);
> > +       h->line_info_len = bswap_32(h->line_info_len);
> > +
> > +       if (hdr_len < offsetofend(struct btf_ext_header, core_relo_len))
> > +               return;
> > +
> > +       h->core_relo_off = bswap_32(h->core_relo_off);
> > +       h->core_relo_len = bswap_32(h->core_relo_len);
> > +}
> > +
> > +/* Swap metadata byte-order of generic info subsection */
> > +static int info_subsec_bswap_metadata(const struct btf_ext *btf_ext, struct btf_ext_info *ext_info)
> > +{
> > +       const bool is_native = btf_ext->swapped_endian;
> > +       __u32 left, *rs, rec_size, num_info;
> > +       struct btf_ext_info_sec *si;
> > +
> > +       if (ext_info->len == 0)
> > +               return 0;
> > +
> > +       rs = ext_info->info - sizeof(__u32);    /* back up to record size */
> > +       rec_size = is_native ? *rs : bswap_32(*rs);
> > +       if (rec_size != ext_info->rec_size)
> >                 return -EINVAL;
> > +       *rs = bswap_32(*rs);
> > +
> > +       si = ext_info->info;                    /* sec info #1 */
> > +       left = ext_info->len;
> > +       while (left > 0) {
> > +               num_info = is_native ? si->num_info : bswap_32(si->num_info);
> > +               si->sec_name_off = bswap_32(si->sec_name_off);
> > +               si->num_info = bswap_32(si->num_info);
> > +               si = (void *)si->data + rec_size * num_info;
> > +               left -= offsetof(struct btf_ext_info_sec, data) +
> > +                       rec_size * num_info;
> >         }
> >
> > +       return 0;
> > +}
> > +
> > +/* Swap byte order of info subsection metadata and records in the correct
> > + * order depending on whether or not data is in native endianness.
> > + */
> > +#define ORDER_INFO_BSWAP(btf_ext, ext_info, info_t, swap_fn)           \
> > +{                                                                      \
> > +       const bool is_native = btf_ext->swapped_endian;                 \
> > +       struct btf_ext_info_sec *si;                                    \
> > +       int c, err;                                                     \
> > +       info_t *i;                                                      \
> > +       if (is_native)                                                  \
> > +               for_each_btf_ext_sec(ext_info, si)                      \
> > +                       for_each_btf_ext_rec(ext_info, si, c, i)        \
> > +                               swap_fn(i);                             \
> > +       err = info_subsec_bswap_metadata(btf_ext, ext_info);            \
> > +       if (err) {                                                      \
> > +               pr_warn(#info_t " record size mismatch!\n");            \
> > +               return err;                                             \
> > +       }                                                               \
> > +       if (!is_native)                                                 \
> > +               for_each_btf_ext_sec(ext_info, si)                      \
> > +                       for_each_btf_ext_rec(ext_info, si, c, i)        \
> > +                               swap_fn(i);                             \
> > +}
> > +
> 
> Nope-nope-nope, I'm never landing something like this :)
> 
> Ok, if there is no clean swapping and setup separation we can come up
> with, let's still do a decent job at trying to keep all this coherent
> and simple.
> 
> How about this? btf_ext_bswap_hdr() stays, it seems fine.
> 
> btf_ext_setup_func_info(), btf_ext_setup_line_info(), and
> btf_ext_setup_core_relos() all call into generic btf_ext_setup_info,
> right? We teach btf_ext_setup_info() to work with both endianness (but
> not swap data at all). The only two fields that we need to byte-swap
> on the fly are record_size and sinfo->num_info, so it's minimal
> changes to btf_ext_setup_info().
> 
> Oh, and while we are at it, if data is in non-native endianness,
> btf_ext_setup_info() should return failure if any of the record sizes
> are not *exactly* matching our expectations. This should be a trivial
> addition as we have never extended those records, I believe.
> 
> This will take care of correctness checking regardless of endianness.
> 
> Now, forget about for_each_btf_ext_{sec,rec}(), it will be too fragile
> to make them work for inverted endianness. But we know now that the
> data is correct, right? So, fine, let's have a bit of duplication (but
> without any error checking) to go over raw .BTF.ext data and swap all
> the records and all metadata fields. This logic now will be used after
> btf_ext_setup_*() steps, and in btf_ext_raw_data(). For
> btf_ext_raw_data() you'll also btf_ext_bswap_hdr() at the very end,
> right?
> 
> How does that sound? Am I missing something big again?
> 
> And fine, let's use callbacks for different record types to keep this
> simple. I dislike callbacks, in principle, but sometimes they are the
> simplest way forward, unfortunately (a proper iterator would be
> better, but that's another story and I don't want to get into that
> implementation task just yet).

Hi Andrii,

Sorry, this took longer than expected to get back to, but hopefully we're
nearly there.

In principle this all works, but so did first swapping info metadata and
data together per my v4 patch. Calling various "setup" functions before
byte-swapping seems confusing too, and warrants some reorganization, also
to cleanly pass the current byte-order state into info parsing functions.
On balance, I do agree there's an advantage to reusing the existing info
validation checks and dropping the extra ones I had to add, so this seems
a worthwhile approach to try. I'll follow up with a v6 shortly.

> 
> > +/*
> > + * Swap endianness of the whole info segment in a BTF.ext data section:
> > + *   - requires BTF.ext header data in native byte order
> > + *   - only support info structs from BTF version 1
> > + */
> > +static int btf_ext_bswap_info(struct btf_ext *btf_ext)
> > +{
> > +       const struct btf_ext_header *h = btf_ext->hdr;
> > +       struct btf_ext_info ext = {};
> > +
> > +       /* Swap func_info subsection byte-order */
> > +       ext.info = (void *)h + h->hdr_len + h->func_info_off + sizeof(__u32);
> > +       ext.len = h->func_info_len - (h->func_info_len ? sizeof(__u32) : 0);
> > +       ext.rec_size = sizeof(struct bpf_func_info);
> > +
> > +       ORDER_INFO_BSWAP(btf_ext, &ext, struct bpf_func_info, bpf_func_info_bswap);
> 
> You shouldn't have bent over backwards just to use
> for_each_btf_ext_{sec,rec}() macros. Just because I proposed it
> (initially, without actually coding anything) doesn't mean it's the
> best and final solution :)
> 
> > +
> > +       /* Swap line_info subsection byte-order */
> > +       ext.info = (void *)h + h->hdr_len + h->line_info_off + sizeof(__u32);
> > +       ext.len = h->line_info_len - (h->line_info_len ? sizeof(__u32) : 0);
> > +       ext.rec_size = sizeof(struct bpf_line_info);
> > +
> > +       ORDER_INFO_BSWAP(btf_ext, &ext, struct bpf_line_info, bpf_line_info_bswap);
> > +
> > +       /* Swap core_relo subsection byte-order (if present) */
> > +       if (h->hdr_len < offsetofend(struct btf_ext_header, core_relo_len))
> > +               return 0;
> > +
> > +       ext.info = (void *)h + h->hdr_len + h->core_relo_off + sizeof(__u32);
> > +       ext.len = h->core_relo_len - (h->core_relo_len ? sizeof(__u32) : 0);
> > +       ext.rec_size = sizeof(struct bpf_core_relo);
> > +
> > +       ORDER_INFO_BSWAP(btf_ext, &ext, struct bpf_core_relo, bpf_core_relo_bswap);
> > +
> > +       return 0;
> > +}
> > +#undef ORDER_INFO_BSWAP
> 
> [...]
diff mbox series

Patch

diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c
index 5f094c1f4388..c11dfc81d007 100644
--- a/tools/lib/bpf/btf.c
+++ b/tools/lib/bpf/btf.c
@@ -3023,25 +3023,140 @@  static int btf_ext_setup_core_relos(struct btf_ext *btf_ext)
 	return btf_ext_setup_info(btf_ext, &param);
 }
 
-static int btf_ext_parse_hdr(__u8 *data, __u32 data_size)
+/* Swap byte-order of BTF.ext header with any endianness */
+static void btf_ext_bswap_hdr(struct btf_ext *btf_ext, __u32 hdr_len)
 {
-	const struct btf_ext_header *hdr = (struct btf_ext_header *)data;
+	struct btf_ext_header *h = btf_ext->hdr;
 
-	if (data_size < offsetofend(struct btf_ext_header, hdr_len) ||
-	    data_size < hdr->hdr_len) {
-		pr_debug("BTF.ext header not found\n");
+	h->magic = bswap_16(h->magic);
+	h->hdr_len = bswap_32(h->hdr_len);
+	h->func_info_off = bswap_32(h->func_info_off);
+	h->func_info_len = bswap_32(h->func_info_len);
+	h->line_info_off = bswap_32(h->line_info_off);
+	h->line_info_len = bswap_32(h->line_info_len);
+
+	if (hdr_len < offsetofend(struct btf_ext_header, core_relo_len))
+		return;
+
+	h->core_relo_off = bswap_32(h->core_relo_off);
+	h->core_relo_len = bswap_32(h->core_relo_len);
+}
+
+/* Swap metadata byte-order of generic info subsection */
+static int info_subsec_bswap_metadata(const struct btf_ext *btf_ext, struct btf_ext_info *ext_info)
+{
+	const bool is_native = btf_ext->swapped_endian;
+	__u32 left, *rs, rec_size, num_info;
+	struct btf_ext_info_sec *si;
+
+	if (ext_info->len == 0)
+		return 0;
+
+	rs = ext_info->info - sizeof(__u32);	/* back up to record size */
+	rec_size = is_native ? *rs : bswap_32(*rs);
+	if (rec_size != ext_info->rec_size)
 		return -EINVAL;
+	*rs = bswap_32(*rs);
+
+	si = ext_info->info;			/* sec info #1 */
+	left = ext_info->len;
+	while (left > 0) {
+		num_info = is_native ? si->num_info : bswap_32(si->num_info);
+		si->sec_name_off = bswap_32(si->sec_name_off);
+		si->num_info = bswap_32(si->num_info);
+		si = (void *)si->data + rec_size * num_info;
+		left -= offsetof(struct btf_ext_info_sec, data) +
+			rec_size * num_info;
 	}
 
+	return 0;
+}
+
+/* Swap byte order of info subsection metadata and records in the correct
+ * order depending on whether or not data is in native endianness.
+ */
+#define ORDER_INFO_BSWAP(btf_ext, ext_info, info_t, swap_fn)		\
+{									\
+	const bool is_native = btf_ext->swapped_endian;			\
+	struct btf_ext_info_sec *si;					\
+	int c, err;							\
+	info_t *i;							\
+	if (is_native)							\
+		for_each_btf_ext_sec(ext_info, si)			\
+			for_each_btf_ext_rec(ext_info, si, c, i)	\
+				swap_fn(i);				\
+	err = info_subsec_bswap_metadata(btf_ext, ext_info);		\
+	if (err) {							\
+		pr_warn(#info_t " record size mismatch!\n");		\
+		return err;						\
+	}								\
+	if (!is_native)							\
+		for_each_btf_ext_sec(ext_info, si)			\
+			for_each_btf_ext_rec(ext_info, si, c, i)	\
+				swap_fn(i);				\
+}
+
+/*
+ * Swap endianness of the whole info segment in a BTF.ext data section:
+ *   - requires BTF.ext header data in native byte order
+ *   - only support info structs from BTF version 1
+ */
+static int btf_ext_bswap_info(struct btf_ext *btf_ext)
+{
+	const struct btf_ext_header *h = btf_ext->hdr;
+	struct btf_ext_info ext = {};
+
+	/* Swap func_info subsection byte-order */
+	ext.info = (void *)h + h->hdr_len + h->func_info_off + sizeof(__u32);
+	ext.len = h->func_info_len - (h->func_info_len ? sizeof(__u32) : 0);
+	ext.rec_size = sizeof(struct bpf_func_info);
+
+	ORDER_INFO_BSWAP(btf_ext, &ext, struct bpf_func_info, bpf_func_info_bswap);
+
+	/* Swap line_info subsection byte-order */
+	ext.info = (void *)h + h->hdr_len + h->line_info_off + sizeof(__u32);
+	ext.len = h->line_info_len - (h->line_info_len ? sizeof(__u32) : 0);
+	ext.rec_size = sizeof(struct bpf_line_info);
+
+	ORDER_INFO_BSWAP(btf_ext, &ext, struct bpf_line_info, bpf_line_info_bswap);
+
+	/* Swap core_relo subsection byte-order (if present) */
+	if (h->hdr_len < offsetofend(struct btf_ext_header, core_relo_len))
+		return 0;
+
+	ext.info = (void *)h + h->hdr_len + h->core_relo_off + sizeof(__u32);
+	ext.len = h->core_relo_len - (h->core_relo_len ? sizeof(__u32) : 0);
+	ext.rec_size = sizeof(struct bpf_core_relo);
+
+	ORDER_INFO_BSWAP(btf_ext, &ext, struct bpf_core_relo, bpf_core_relo_bswap);
+
+	return 0;
+}
+#undef ORDER_INFO_BSWAP
+
+/* Validate hdr data & info sections, convert to native endianness */
+static int btf_ext_parse(struct btf_ext *btf_ext)
+{
+	__u32 hdr_len, info_size, data_size = btf_ext->data_size;
+	struct btf_ext_header *hdr = btf_ext->hdr;
+	bool swapped_endian = false;
+
+	if (data_size < offsetofend(struct btf_ext_header, hdr_len)) {
+		pr_debug("BTF.ext header too short\n");
+		return -EINVAL;
+	}
+
+	hdr_len = hdr->hdr_len;
 	if (hdr->magic == bswap_16(BTF_MAGIC)) {
-		pr_warn("BTF.ext in non-native endianness is not supported\n");
-		return -ENOTSUP;
+		swapped_endian = true;
+		hdr_len = bswap_32(hdr_len);
 	} else if (hdr->magic != BTF_MAGIC) {
 		pr_debug("Invalid BTF.ext magic:%x\n", hdr->magic);
 		return -EINVAL;
 	}
 
-	if (hdr->version != BTF_VERSION) {
+	/* Ensure known version of structs, current BTF_VERSION == 1 */
+	if (hdr->version != 1) {
 		pr_debug("Unsupported BTF.ext version:%u\n", hdr->version);
 		return -ENOTSUP;
 	}
@@ -3051,11 +3166,50 @@  static int btf_ext_parse_hdr(__u8 *data, __u32 data_size)
 		return -ENOTSUP;
 	}
 
-	if (data_size == hdr->hdr_len) {
+	if (data_size < hdr_len) {
+		pr_debug("BTF.ext header not found\n");
+		return -EINVAL;
+	} else if (data_size == hdr_len) {
 		pr_debug("BTF.ext has no data\n");
 		return -EINVAL;
 	}
 
+	/* Verify mandatory hdr info details present */
+	if (hdr_len < offsetofend(struct btf_ext_header, line_info_len)) {
+		pr_warn("BTF.ext header missing func_info, line_info\n");
+		return -EINVAL;
+	}
+
+	/* Keep hdr native byte-order in memory for introspection */
+	if (swapped_endian)
+		btf_ext_bswap_hdr(btf_ext, hdr_len);
+
+	/* Basic info section consistency checks*/
+	info_size = btf_ext->data_size - hdr_len;
+	if (info_size & 0x03) {
+		pr_warn("BTF.ext info size not 4-byte multiple\n");
+		return -EINVAL;
+	}
+	info_size -= hdr->func_info_len + hdr->line_info_len;
+	if (hdr_len >= offsetofend(struct btf_ext_header, core_relo_len))
+		info_size -= hdr->core_relo_len;
+	if (info_size) {
+		pr_warn("BTF.ext info size mismatch with header data\n");
+		return -EINVAL;
+	}
+
+	/* Keep infos native byte-order in memory for introspection */
+	if (swapped_endian) {
+		int err = btf_ext_bswap_info(btf_ext);
+		if (err)
+			return err;
+	}
+	/*
+	 * Set btf_ext->swapped_endian only after all header and info data has
+	 * been swapped, helping btf_ext_bswap_info() determine if its data
+	 * is in native byte order when called.
+	 */
+	btf_ext->swapped_endian = swapped_endian;
 	return 0;
 }
 
@@ -3067,6 +3221,7 @@  void btf_ext__free(struct btf_ext *btf_ext)
 	free(btf_ext->line_info.sec_idxs);
 	free(btf_ext->core_relo_info.sec_idxs);
 	free(btf_ext->data);
+	free(btf_ext->data_swapped);
 	free(btf_ext);
 }
 
@@ -3087,15 +3242,10 @@  struct btf_ext *btf_ext__new(const __u8 *data, __u32 size)
 	}
 	memcpy(btf_ext->data, data, size);
 
-	err = btf_ext_parse_hdr(btf_ext->data, size);
+	err = btf_ext_parse(btf_ext);
 	if (err)
 		goto done;
 
-	if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, line_info_len)) {
-		err = -EINVAL;
-		goto done;
-	}
-
 	err = btf_ext_setup_func_info(btf_ext);
 	if (err)
 		goto done;
@@ -3120,15 +3270,71 @@  struct btf_ext *btf_ext__new(const __u8 *data, __u32 size)
 	return btf_ext;
 }
 
+static void *btf_ext_raw_data(const struct btf_ext *btf_ext_ro, __u32 *size,
+			      bool swap_endian)
+{
+	struct btf_ext *btf_ext = (struct btf_ext *)btf_ext_ro;
+	const __u32 data_sz = btf_ext->data_size;
+	void *data;
+
+	data = swap_endian ? btf_ext->data_swapped : btf_ext->data;
+	if (data) {
+		*size = data_sz;
+		return data;
+	}
+
+	data = calloc(1, data_sz);
+	if (!data)
+		return NULL;
+	memcpy(data, btf_ext->data, data_sz);
+
+	if (swap_endian) {
+		btf_ext_bswap_info(btf_ext);
+		btf_ext_bswap_hdr(btf_ext, btf_ext->hdr->hdr_len);
+		btf_ext->data_swapped = data;
+	}
+
+	*size = data_sz;
+	return data;
+}
+
 const void *btf_ext__raw_data(const struct btf_ext *btf_ext, __u32 *size)
 {
+	__u32 data_sz;
+	void *data;
+
+	data = btf_ext_raw_data(btf_ext, &data_sz, btf_ext->swapped_endian);
+	if (!data)
+		return errno = ENOMEM, NULL;
+
 	*size = btf_ext->data_size;
-	return btf_ext->data;
+	return data;
 }
 
 __attribute__((alias("btf_ext__raw_data")))
 const void *btf_ext__get_raw_data(const struct btf_ext *btf_ext, __u32 *size);
 
+enum btf_endianness btf_ext__endianness(const struct btf_ext *btf_ext)
+{
+	if (is_host_big_endian())
+		return btf_ext->swapped_endian ? BTF_LITTLE_ENDIAN : BTF_BIG_ENDIAN;
+	else
+		return btf_ext->swapped_endian ? BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN;
+}
+
+int btf_ext__set_endianness(struct btf_ext *btf_ext, enum btf_endianness endian)
+{
+	if (endian != BTF_LITTLE_ENDIAN && endian != BTF_BIG_ENDIAN)
+		return libbpf_err(-EINVAL);
+
+	btf_ext->swapped_endian = is_host_big_endian() != (endian == BTF_BIG_ENDIAN);
+
+	if (!btf_ext->swapped_endian) {
+		free(btf_ext->data_swapped);
+		btf_ext->data_swapped = NULL;
+	}
+	return 0;
+}
 
 struct btf_dedup;
 
diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h
index b68d216837a9..e3cf91687c78 100644
--- a/tools/lib/bpf/btf.h
+++ b/tools/lib/bpf/btf.h
@@ -167,6 +167,9 @@  LIBBPF_API const char *btf__str_by_offset(const struct btf *btf, __u32 offset);
 LIBBPF_API struct btf_ext *btf_ext__new(const __u8 *data, __u32 size);
 LIBBPF_API void btf_ext__free(struct btf_ext *btf_ext);
 LIBBPF_API const void *btf_ext__raw_data(const struct btf_ext *btf_ext, __u32 *size);
+LIBBPF_API enum btf_endianness btf_ext__endianness(const struct btf_ext *btf_ext);
+LIBBPF_API int btf_ext__set_endianness(struct btf_ext *btf_ext,
+				       enum btf_endianness endian);
 
 LIBBPF_API int btf__find_str(struct btf *btf, const char *s);
 LIBBPF_API int btf__add_str(struct btf *btf, const char *s);
diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map
index 8f0d9ea3b1b4..5c17632807b6 100644
--- a/tools/lib/bpf/libbpf.map
+++ b/tools/lib/bpf/libbpf.map
@@ -421,6 +421,8 @@  LIBBPF_1.5.0 {
 	global:
 		btf__distill_base;
 		btf__relocate;
+		btf_ext__endianness;
+		btf_ext__set_endianness;
 		bpf_map__autoattach;
 		bpf_map__set_autoattach;
 		bpf_program__attach_sockmap;
diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h
index 8cda511a1982..a8531195acd4 100644
--- a/tools/lib/bpf/libbpf_internal.h
+++ b/tools/lib/bpf/libbpf_internal.h
@@ -484,6 +484,8 @@  struct btf_ext {
 		struct btf_ext_header *hdr;
 		void *data;
 	};
+	void *data_swapped;
+	bool swapped_endian;
 	struct btf_ext_info func_info;
 	struct btf_ext_info line_info;
 	struct btf_ext_info core_relo_info;
@@ -511,6 +513,32 @@  struct bpf_line_info_min {
 	__u32	line_col;
 };
 
+/* Functions to byte-swap info records */
+
+static inline void bpf_func_info_bswap(struct bpf_func_info *i)
+{
+	i->insn_off = bswap_32(i->insn_off);
+	i->type_id = bswap_32(i->type_id);
+}
+
+static inline void bpf_line_info_bswap(struct bpf_line_info *i)
+{
+	i->insn_off = bswap_32(i->insn_off);
+	i->file_name_off = bswap_32(i->file_name_off);
+	i->line_off = bswap_32(i->line_off);
+	i->line_col = bswap_32(i->line_col);
+}
+
+static inline void bpf_core_relo_bswap(struct bpf_core_relo *i)
+{
+	_Static_assert(sizeof(i->kind) == sizeof(__u32),
+		       "enum bpf_core_relo_kind is not 32-bit\n");
+	i->insn_off = bswap_32(i->insn_off);
+	i->type_id = bswap_32(i->type_id);
+	i->access_str_off = bswap_32(i->access_str_off);
+	i->kind = bswap_32(i->kind);
+}
+
 enum btf_field_iter_kind {
 	BTF_FIELD_ITER_IDS,
 	BTF_FIELD_ITER_STRS,