Message ID | 20221115194552.338640-1-plautrba@redhat.com (mailing list archive) |
---|---|
State | Superseded |
Headers | show |
Series | [v2,1/3] python/sepolicy: Fix sepolicy manpage -w ... | expand |
On Tue, Nov 15, 2022 at 2:58 PM Petr Lautrbach <plautrba@redhat.com> wrote: > > From: Petr Lautrbach <lautrbach@redhat.com> > > Commit 7494bb1298b3 ("sepolicy: generate man pages in parallel") > improved sepolicy performance but broke `sepolicy manpage -w ...` as it > didn't collect data about domains and roles from ManPage() and so > HTMLManPages() generated only empty page. This is fixed now, domains > and roles are being collected and used for HTML pages. > > Signed-off-by: Petr Lautrbach <lautrbach@redhat.com> I ran `sepolicy manpage -w -d unconfined_t` and received the following error: """ Traceback (most recent call last): File "/usr/lib64/python3.10/multiprocessing/pool.py", line 125, in worker result = (True, func(*args, **kwds)) File "/home/jim/local/usr/bin/./sepolicy", line 335, in manpage_work return (m.manpage_domains, m.manpage_roles) AttributeError: 'ManPage' object has no attribute 'manpage_domains' """ I don't get the error from the master branch, but I do confirm the bug you reported above. Thanks, Jim > --- > python/sepolicy/sepolicy.py | 13 +++++++++++-- > python/sepolicy/sepolicy/manpage.py | 12 +++++------- > 2 files changed, 16 insertions(+), 9 deletions(-) > > diff --git a/python/sepolicy/sepolicy.py b/python/sepolicy/sepolicy.py > index 733d40484709..2ca02ee9a0cf 100755 > --- a/python/sepolicy/sepolicy.py > +++ b/python/sepolicy/sepolicy.py > @@ -332,9 +332,10 @@ def manpage_work(domain, path, root, source_files, web): > from sepolicy.manpage import ManPage > m = ManPage(domain, path, root, source_files, web) > print(m.get_man_page_path()) > + return (m.manpage_domains, m.manpage_roles) > > def manpage(args): > - from sepolicy.manpage import HTMLManPages, manpage_domains, manpage_roles, gen_domains > + from sepolicy.manpage import HTMLManPages, gen_domains > > path = args.path > if not args.policy and args.root != "/": > @@ -347,9 +348,17 @@ def manpage(args): > else: > test_domains = args.domain > > + manpage_domains = set() > + manpage_roles = set() > p = Pool() > + async_results = [] > for domain in test_domains: > - p.apply_async(manpage_work, [domain, path, args.root, args.source_files, args.web]) > + async_results.append(p.apply_async(manpage_work, [domain, path, args.root, args.source_files, args.web])) > + results = map(lambda x: x.get(), async_results) > + for result in results: > + manpage_domains.update(set(result[0])) > + manpage_roles.update(set(result[1])) > + > p.close() > p.join() > > diff --git a/python/sepolicy/sepolicy/manpage.py b/python/sepolicy/sepolicy/manpage.py > index 3e61e333193f..de72cb6cda5f 100755 > --- a/python/sepolicy/sepolicy/manpage.py > +++ b/python/sepolicy/sepolicy/manpage.py > @@ -21,7 +21,7 @@ > # 02111-1307 USA > # > # > -__all__ = ['ManPage', 'HTMLManPages', 'manpage_domains', 'manpage_roles', 'gen_domains'] > +__all__ = ['ManPage', 'HTMLManPages', 'gen_domains'] > > import string > import selinux > @@ -147,10 +147,6 @@ def _gen_types(): > def prettyprint(f, trim): > return " ".join(f[:-len(trim)].split("_")) > > -# for HTML man pages > -manpage_domains = [] > -manpage_roles = [] > - > fedora_releases = ["Fedora17", "Fedora18"] > rhel_releases = ["RHEL6", "RHEL7"] > > @@ -408,6 +404,8 @@ class ManPage: > """ > modules_dict = None > enabled_str = ["Disabled", "Enabled"] > + manpage_domains = [] > + manpage_roles = [] > > def __init__(self, domainname, path="/tmp", root="/", source_files=False, html=False): > self.html = html > @@ -453,10 +451,10 @@ class ManPage: > if self.domainname + "_r" in self.all_roles: > self.__gen_user_man_page() > if self.html: > - manpage_roles.append(self.man_page_path) > + self.manpage_roles.append(self.man_page_path) > else: > if self.html: > - manpage_domains.append(self.man_page_path) > + self.manpage_domains.append(self.man_page_path) > self.__gen_man_page() > self.fd.close() > > -- > 2.38.1 >
James Carter <jwcart2@gmail.com> writes: > On Tue, Nov 15, 2022 at 2:58 PM Petr Lautrbach <plautrba@redhat.com> wrote: >> >> From: Petr Lautrbach <lautrbach@redhat.com> >> >> Commit 7494bb1298b3 ("sepolicy: generate man pages in parallel") >> improved sepolicy performance but broke `sepolicy manpage -w ...` as it >> didn't collect data about domains and roles from ManPage() and so >> HTMLManPages() generated only empty page. This is fixed now, domains >> and roles are being collected and used for HTML pages. >> >> Signed-off-by: Petr Lautrbach <lautrbach@redhat.com> > > I ran `sepolicy manpage -w -d unconfined_t` and received the following error: > """ > Traceback (most recent call last): > File "/usr/lib64/python3.10/multiprocessing/pool.py", line 125, in worker > result = (True, func(*args, **kwds)) > File "/home/jim/local/usr/bin/./sepolicy", line 335, in manpage_work > return (m.manpage_domains, m.manpage_roles) > AttributeError: 'ManPage' object has no attribute 'manpage_domains' > """ > I think you nee to remove "-Es" command line options from /home/jim/local/usr/bin/./sepolicy sed -i '1s/ -Es//' /home/jim/local/usr/bin/./sepolicy > I don't get the error from the master branch, but I do confirm the bug > you reported above. > > Thanks, > Jim > >> --- >> python/sepolicy/sepolicy.py | 13 +++++++++++-- >> python/sepolicy/sepolicy/manpage.py | 12 +++++------- >> 2 files changed, 16 insertions(+), 9 deletions(-) >> >> diff --git a/python/sepolicy/sepolicy.py b/python/sepolicy/sepolicy.py >> index 733d40484709..2ca02ee9a0cf 100755 >> --- a/python/sepolicy/sepolicy.py >> +++ b/python/sepolicy/sepolicy.py >> @@ -332,9 +332,10 @@ def manpage_work(domain, path, root, source_files, web): >> from sepolicy.manpage import ManPage >> m = ManPage(domain, path, root, source_files, web) >> print(m.get_man_page_path()) >> + return (m.manpage_domains, m.manpage_roles) >> >> def manpage(args): >> - from sepolicy.manpage import HTMLManPages, manpage_domains, manpage_roles, gen_domains >> + from sepolicy.manpage import HTMLManPages, gen_domains >> >> path = args.path >> if not args.policy and args.root != "/": >> @@ -347,9 +348,17 @@ def manpage(args): >> else: >> test_domains = args.domain >> >> + manpage_domains = set() >> + manpage_roles = set() >> p = Pool() >> + async_results = [] >> for domain in test_domains: >> - p.apply_async(manpage_work, [domain, path, args.root, args.source_files, args.web]) >> + async_results.append(p.apply_async(manpage_work, [domain, path, args.root, args.source_files, args.web])) >> + results = map(lambda x: x.get(), async_results) >> + for result in results: >> + manpage_domains.update(set(result[0])) >> + manpage_roles.update(set(result[1])) >> + >> p.close() >> p.join() >> >> diff --git a/python/sepolicy/sepolicy/manpage.py b/python/sepolicy/sepolicy/manpage.py >> index 3e61e333193f..de72cb6cda5f 100755 >> --- a/python/sepolicy/sepolicy/manpage.py >> +++ b/python/sepolicy/sepolicy/manpage.py >> @@ -21,7 +21,7 @@ >> # 02111-1307 USA >> # >> # >> -__all__ = ['ManPage', 'HTMLManPages', 'manpage_domains', 'manpage_roles', 'gen_domains'] >> +__all__ = ['ManPage', 'HTMLManPages', 'gen_domains'] >> >> import string >> import selinux >> @@ -147,10 +147,6 @@ def _gen_types(): >> def prettyprint(f, trim): >> return " ".join(f[:-len(trim)].split("_")) >> >> -# for HTML man pages >> -manpage_domains = [] >> -manpage_roles = [] >> - >> fedora_releases = ["Fedora17", "Fedora18"] >> rhel_releases = ["RHEL6", "RHEL7"] >> >> @@ -408,6 +404,8 @@ class ManPage: >> """ >> modules_dict = None >> enabled_str = ["Disabled", "Enabled"] >> + manpage_domains = [] >> + manpage_roles = [] >> >> def __init__(self, domainname, path="/tmp", root="/", source_files=False, html=False): >> self.html = html >> @@ -453,10 +451,10 @@ class ManPage: >> if self.domainname + "_r" in self.all_roles: >> self.__gen_user_man_page() >> if self.html: >> - manpage_roles.append(self.man_page_path) >> + self.manpage_roles.append(self.man_page_path) >> else: >> if self.html: >> - manpage_domains.append(self.man_page_path) >> + self.manpage_domains.append(self.man_page_path) >> self.__gen_man_page() >> self.fd.close() >> >> -- >> 2.38.1 >>
On Thu, Nov 17, 2022 at 7:39 AM Petr Lautrbach <plautrba@redhat.com> wrote: > > James Carter <jwcart2@gmail.com> writes: > > > On Tue, Nov 15, 2022 at 2:58 PM Petr Lautrbach <plautrba@redhat.com> wrote: > >> > >> From: Petr Lautrbach <lautrbach@redhat.com> > >> > >> Commit 7494bb1298b3 ("sepolicy: generate man pages in parallel") > >> improved sepolicy performance but broke `sepolicy manpage -w ...` as it > >> didn't collect data about domains and roles from ManPage() and so > >> HTMLManPages() generated only empty page. This is fixed now, domains > >> and roles are being collected and used for HTML pages. > >> > >> Signed-off-by: Petr Lautrbach <lautrbach@redhat.com> > > > > I ran `sepolicy manpage -w -d unconfined_t` and received the following error: > > """ > > Traceback (most recent call last): > > File "/usr/lib64/python3.10/multiprocessing/pool.py", line 125, in worker > > result = (True, func(*args, **kwds)) > > File "/home/jim/local/usr/bin/./sepolicy", line 335, in manpage_work > > return (m.manpage_domains, m.manpage_roles) > > AttributeError: 'ManPage' object has no attribute 'manpage_domains' > > """ > > > > I think you nee to remove "-Es" command line options from /home/jim/local/usr/bin/./sepolicy > > sed -i '1s/ -Es//' /home/jim/local/usr/bin/./sepolicy > That worked. Thanks, Jim > > > I don't get the error from the master branch, but I do confirm the bug > > you reported above. > > > > Thanks, > > Jim > > > >> --- > >> python/sepolicy/sepolicy.py | 13 +++++++++++-- > >> python/sepolicy/sepolicy/manpage.py | 12 +++++------- > >> 2 files changed, 16 insertions(+), 9 deletions(-) > >> > >> diff --git a/python/sepolicy/sepolicy.py b/python/sepolicy/sepolicy.py > >> index 733d40484709..2ca02ee9a0cf 100755 > >> --- a/python/sepolicy/sepolicy.py > >> +++ b/python/sepolicy/sepolicy.py > >> @@ -332,9 +332,10 @@ def manpage_work(domain, path, root, source_files, web): > >> from sepolicy.manpage import ManPage > >> m = ManPage(domain, path, root, source_files, web) > >> print(m.get_man_page_path()) > >> + return (m.manpage_domains, m.manpage_roles) > >> > >> def manpage(args): > >> - from sepolicy.manpage import HTMLManPages, manpage_domains, manpage_roles, gen_domains > >> + from sepolicy.manpage import HTMLManPages, gen_domains > >> > >> path = args.path > >> if not args.policy and args.root != "/": > >> @@ -347,9 +348,17 @@ def manpage(args): > >> else: > >> test_domains = args.domain > >> > >> + manpage_domains = set() > >> + manpage_roles = set() > >> p = Pool() > >> + async_results = [] > >> for domain in test_domains: > >> - p.apply_async(manpage_work, [domain, path, args.root, args.source_files, args.web]) > >> + async_results.append(p.apply_async(manpage_work, [domain, path, args.root, args.source_files, args.web])) > >> + results = map(lambda x: x.get(), async_results) > >> + for result in results: > >> + manpage_domains.update(set(result[0])) > >> + manpage_roles.update(set(result[1])) > >> + > >> p.close() > >> p.join() > >> > >> diff --git a/python/sepolicy/sepolicy/manpage.py b/python/sepolicy/sepolicy/manpage.py > >> index 3e61e333193f..de72cb6cda5f 100755 > >> --- a/python/sepolicy/sepolicy/manpage.py > >> +++ b/python/sepolicy/sepolicy/manpage.py > >> @@ -21,7 +21,7 @@ > >> # 02111-1307 USA > >> # > >> # > >> -__all__ = ['ManPage', 'HTMLManPages', 'manpage_domains', 'manpage_roles', 'gen_domains'] > >> +__all__ = ['ManPage', 'HTMLManPages', 'gen_domains'] > >> > >> import string > >> import selinux > >> @@ -147,10 +147,6 @@ def _gen_types(): > >> def prettyprint(f, trim): > >> return " ".join(f[:-len(trim)].split("_")) > >> > >> -# for HTML man pages > >> -manpage_domains = [] > >> -manpage_roles = [] > >> - > >> fedora_releases = ["Fedora17", "Fedora18"] > >> rhel_releases = ["RHEL6", "RHEL7"] > >> > >> @@ -408,6 +404,8 @@ class ManPage: > >> """ > >> modules_dict = None > >> enabled_str = ["Disabled", "Enabled"] > >> + manpage_domains = [] > >> + manpage_roles = [] > >> > >> def __init__(self, domainname, path="/tmp", root="/", source_files=False, html=False): > >> self.html = html > >> @@ -453,10 +451,10 @@ class ManPage: > >> if self.domainname + "_r" in self.all_roles: > >> self.__gen_user_man_page() > >> if self.html: > >> - manpage_roles.append(self.man_page_path) > >> + self.manpage_roles.append(self.man_page_path) > >> else: > >> if self.html: > >> - manpage_domains.append(self.man_page_path) > >> + self.manpage_domains.append(self.man_page_path) > >> self.__gen_man_page() > >> self.fd.close() > >> > >> -- > >> 2.38.1 > >> >
On Tue, Nov 15, 2022 at 2:58 PM Petr Lautrbach <plautrba@redhat.com> wrote: > > From: Petr Lautrbach <lautrbach@redhat.com> > > Commit 7494bb1298b3 ("sepolicy: generate man pages in parallel") > improved sepolicy performance but broke `sepolicy manpage -w ...` as it > didn't collect data about domains and roles from ManPage() and so > HTMLManPages() generated only empty page. This is fixed now, domains > and roles are being collected and used for HTML pages. > > Signed-off-by: Petr Lautrbach <lautrbach@redhat.com> For these three patches: Acked-by: James Carter <jwcart2@gmail.com> > --- > python/sepolicy/sepolicy.py | 13 +++++++++++-- > python/sepolicy/sepolicy/manpage.py | 12 +++++------- > 2 files changed, 16 insertions(+), 9 deletions(-) > > diff --git a/python/sepolicy/sepolicy.py b/python/sepolicy/sepolicy.py > index 733d40484709..2ca02ee9a0cf 100755 > --- a/python/sepolicy/sepolicy.py > +++ b/python/sepolicy/sepolicy.py > @@ -332,9 +332,10 @@ def manpage_work(domain, path, root, source_files, web): > from sepolicy.manpage import ManPage > m = ManPage(domain, path, root, source_files, web) > print(m.get_man_page_path()) > + return (m.manpage_domains, m.manpage_roles) > > def manpage(args): > - from sepolicy.manpage import HTMLManPages, manpage_domains, manpage_roles, gen_domains > + from sepolicy.manpage import HTMLManPages, gen_domains > > path = args.path > if not args.policy and args.root != "/": > @@ -347,9 +348,17 @@ def manpage(args): > else: > test_domains = args.domain > > + manpage_domains = set() > + manpage_roles = set() > p = Pool() > + async_results = [] > for domain in test_domains: > - p.apply_async(manpage_work, [domain, path, args.root, args.source_files, args.web]) > + async_results.append(p.apply_async(manpage_work, [domain, path, args.root, args.source_files, args.web])) > + results = map(lambda x: x.get(), async_results) > + for result in results: > + manpage_domains.update(set(result[0])) > + manpage_roles.update(set(result[1])) > + > p.close() > p.join() > > diff --git a/python/sepolicy/sepolicy/manpage.py b/python/sepolicy/sepolicy/manpage.py > index 3e61e333193f..de72cb6cda5f 100755 > --- a/python/sepolicy/sepolicy/manpage.py > +++ b/python/sepolicy/sepolicy/manpage.py > @@ -21,7 +21,7 @@ > # 02111-1307 USA > # > # > -__all__ = ['ManPage', 'HTMLManPages', 'manpage_domains', 'manpage_roles', 'gen_domains'] > +__all__ = ['ManPage', 'HTMLManPages', 'gen_domains'] > > import string > import selinux > @@ -147,10 +147,6 @@ def _gen_types(): > def prettyprint(f, trim): > return " ".join(f[:-len(trim)].split("_")) > > -# for HTML man pages > -manpage_domains = [] > -manpage_roles = [] > - > fedora_releases = ["Fedora17", "Fedora18"] > rhel_releases = ["RHEL6", "RHEL7"] > > @@ -408,6 +404,8 @@ class ManPage: > """ > modules_dict = None > enabled_str = ["Disabled", "Enabled"] > + manpage_domains = [] > + manpage_roles = [] > > def __init__(self, domainname, path="/tmp", root="/", source_files=False, html=False): > self.html = html > @@ -453,10 +451,10 @@ class ManPage: > if self.domainname + "_r" in self.all_roles: > self.__gen_user_man_page() > if self.html: > - manpage_roles.append(self.man_page_path) > + self.manpage_roles.append(self.man_page_path) > else: > if self.html: > - manpage_domains.append(self.man_page_path) > + self.manpage_domains.append(self.man_page_path) > self.__gen_man_page() > self.fd.close() > > -- > 2.38.1 >
On Tue, Nov 15, 2022 at 8:49 PM Petr Lautrbach <plautrba@redhat.com> wrote: > > From: Petr Lautrbach <lautrbach@redhat.com> > > Commit 7494bb1298b3 ("sepolicy: generate man pages in parallel") > improved sepolicy performance but broke `sepolicy manpage -w ...` as it > didn't collect data about domains and roles from ManPage() and so > HTMLManPages() generated only empty page. This is fixed now, domains > and roles are being collected and used for HTML pages. > > Signed-off-by: Petr Lautrbach <lautrbach@redhat.com> > --- > python/sepolicy/sepolicy.py | 13 +++++++++++-- > python/sepolicy/sepolicy/manpage.py | 12 +++++------- > 2 files changed, 16 insertions(+), 9 deletions(-) > > diff --git a/python/sepolicy/sepolicy.py b/python/sepolicy/sepolicy.py > index 733d40484709..2ca02ee9a0cf 100755 > --- a/python/sepolicy/sepolicy.py > +++ b/python/sepolicy/sepolicy.py [...] > @@ -347,9 +348,17 @@ def manpage(args): > else: > test_domains = args.domain > > + manpage_domains = set() > + manpage_roles = set() > p = Pool() > + async_results = [] > for domain in test_domains: > - p.apply_async(manpage_work, [domain, path, args.root, args.source_files, args.web]) > + async_results.append(p.apply_async(manpage_work, [domain, path, args.root, args.source_files, args.web])) > + results = map(lambda x: x.get(), async_results) > + for result in results: > + manpage_domains.update(set(result[0])) > + manpage_roles.update(set(result[1])) The above four lines can be written a bit more nicely as follows: for result in async_results: domains, roles = result.get() manpage_domains.update(domains) manpage_roles.update(roles) Note that set.update() accepts any iterable, it doesn't need to be a set. > + > p.close() > p.join() > [...] -- Ondrej Mosnacek Senior Software Engineer, Linux Security - SELinux kernel Red Hat, Inc.
diff --git a/python/sepolicy/sepolicy.py b/python/sepolicy/sepolicy.py index 733d40484709..2ca02ee9a0cf 100755 --- a/python/sepolicy/sepolicy.py +++ b/python/sepolicy/sepolicy.py @@ -332,9 +332,10 @@ def manpage_work(domain, path, root, source_files, web): from sepolicy.manpage import ManPage m = ManPage(domain, path, root, source_files, web) print(m.get_man_page_path()) + return (m.manpage_domains, m.manpage_roles) def manpage(args): - from sepolicy.manpage import HTMLManPages, manpage_domains, manpage_roles, gen_domains + from sepolicy.manpage import HTMLManPages, gen_domains path = args.path if not args.policy and args.root != "/": @@ -347,9 +348,17 @@ def manpage(args): else: test_domains = args.domain + manpage_domains = set() + manpage_roles = set() p = Pool() + async_results = [] for domain in test_domains: - p.apply_async(manpage_work, [domain, path, args.root, args.source_files, args.web]) + async_results.append(p.apply_async(manpage_work, [domain, path, args.root, args.source_files, args.web])) + results = map(lambda x: x.get(), async_results) + for result in results: + manpage_domains.update(set(result[0])) + manpage_roles.update(set(result[1])) + p.close() p.join() diff --git a/python/sepolicy/sepolicy/manpage.py b/python/sepolicy/sepolicy/manpage.py index 3e61e333193f..de72cb6cda5f 100755 --- a/python/sepolicy/sepolicy/manpage.py +++ b/python/sepolicy/sepolicy/manpage.py @@ -21,7 +21,7 @@ # 02111-1307 USA # # -__all__ = ['ManPage', 'HTMLManPages', 'manpage_domains', 'manpage_roles', 'gen_domains'] +__all__ = ['ManPage', 'HTMLManPages', 'gen_domains'] import string import selinux @@ -147,10 +147,6 @@ def _gen_types(): def prettyprint(f, trim): return " ".join(f[:-len(trim)].split("_")) -# for HTML man pages -manpage_domains = [] -manpage_roles = [] - fedora_releases = ["Fedora17", "Fedora18"] rhel_releases = ["RHEL6", "RHEL7"] @@ -408,6 +404,8 @@ class ManPage: """ modules_dict = None enabled_str = ["Disabled", "Enabled"] + manpage_domains = [] + manpage_roles = [] def __init__(self, domainname, path="/tmp", root="/", source_files=False, html=False): self.html = html @@ -453,10 +451,10 @@ class ManPage: if self.domainname + "_r" in self.all_roles: self.__gen_user_man_page() if self.html: - manpage_roles.append(self.man_page_path) + self.manpage_roles.append(self.man_page_path) else: if self.html: - manpage_domains.append(self.man_page_path) + self.manpage_domains.append(self.man_page_path) self.__gen_man_page() self.fd.close()