Skip to content

Commit e587c0b

Browse files
authored
Merge pull request #11036 from jan-cerny/faster_guides
Add a faster alternative for generating HTML guides
2 parents d09e81a + 4c08686 commit e587c0b

File tree

5 files changed

+180
-8
lines changed

5 files changed

+180
-8
lines changed

.github/workflows/gh-pages.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ jobs:
1414
PAGES_DIR: __pages
1515
steps:
1616
- name: Install Deps
17-
run: dnf install -y cmake git ninja-build openscap-utils python3-pyyaml python3-jinja2 python3-pytest ansible-lint libxslt python3-pip rsync
17+
run: dnf install -y cmake git ninja-build openscap-utils python3-pyyaml python3-jinja2 python3-pytest ansible-lint libxslt python3-pip rsync python3-lxml
1818
- name: Install deps python
1919
run: pip3 install json2html
2020
- name: Checkout

CMakeLists.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ find_package(PythonInterp REQUIRED)
137137

138138
find_python_module(yaml REQUIRED)
139139
find_python_module(jinja2 REQUIRED)
140+
find_python_module(lxml)
140141
find_python_module(pytest)
141142
find_python_module(pytest_cov)
142143
find_python_module(json2html)
@@ -245,6 +246,8 @@ message(STATUS "python myst-parser module (optional): ${PY_MYST_PARSER}")
245246
message(STATUS "python openpyxl module (optional): ${PY_OPENPYXL}")
246247
message(STATUS "python pandas module (optional): ${PY_PANDAS}")
247248
message(STATUS "python pcre2 module (optional): ${PY_PCRE2}")
249+
message(STATUS "python lxml module (optional): ${PY_LXML}")
250+
248251
message(STATUS " ")
249252

250253
message(STATUS "Build options:")

build-scripts/generate_guides.py

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
#!/usr/bin/python3
2+
3+
import argparse
4+
import collections
5+
import lxml.etree as ET
6+
import os
7+
8+
from ssg.constants import OSCAP_PROFILE, PREFIX_TO_NS
9+
import ssg.build_guides
10+
11+
BenchmarkData = collections.namedtuple(
12+
"BenchmarkData", ["title", "profiles", "product"])
13+
XSLT_PATH = "/usr/share/openscap/xsl/xccdf-guide.xsl"
14+
15+
16+
def get_benchmarks(ds, product):
17+
benchmarks = {}
18+
benchmark_xpath = "./ds:component/xccdf-1.2:Benchmark"
19+
for benchmark_el in ds.xpath(benchmark_xpath, namespaces=PREFIX_TO_NS):
20+
benchmark_id = benchmark_el.get("id")
21+
title = benchmark_el.xpath(
22+
"./xccdf-1.2:title", namespaces=PREFIX_TO_NS)[0].text
23+
profiles = get_profiles(benchmark_el)
24+
benchmarks[benchmark_id] = BenchmarkData(title, profiles, product)
25+
return benchmarks
26+
27+
28+
def get_profiles(benchmark_el):
29+
profiles = {}
30+
for profile_el in benchmark_el.xpath(
31+
"./xccdf-1.2:Profile", namespaces=PREFIX_TO_NS):
32+
profile_id = profile_el.get("id")
33+
profile_title = profile_el.xpath(
34+
"./xccdf-1.2:title", namespaces=PREFIX_TO_NS)[0].text
35+
profiles[profile_id] = profile_title
36+
return profiles
37+
38+
39+
def parse_args():
40+
parser = argparse.ArgumentParser()
41+
parser.add_argument(
42+
"--data-stream", required=True,
43+
help="Path to a SCAP source data stream, eg. 'ssg-rhel9-ds.xml'")
44+
parser.add_argument(
45+
"--oscap-version", required=True,
46+
help=f"Version of OpenSCAP that owns {XSLT_PATH}, eg. 1.3.8")
47+
parser.add_argument(
48+
"--product", required=True,
49+
help="Product ID, eg. rhel9")
50+
parser.add_argument(
51+
"--output-dir", required=True,
52+
help="Path to the directory where to generate the output files"
53+
", eg. 'build/guides'")
54+
args = parser.parse_args()
55+
return args
56+
57+
58+
def make_params(oscap_version, benchmark_id, profile_id):
59+
params = {
60+
"oscap-version": ET.XSLT.strparam(oscap_version),
61+
"benchmark_id": ET.XSLT.strparam(benchmark_id),
62+
"profile_id": ET.XSLT.strparam(profile_id)
63+
}
64+
return params
65+
66+
67+
def make_output_file_name(profile_id, product):
68+
short_profile_id = profile_id.replace(OSCAP_PROFILE, "")
69+
output_file_name = "ssg-%s-guide-%s.html" % (product, short_profile_id)
70+
return output_file_name
71+
72+
73+
def make_output_file_path(profile_id, product, output_dir):
74+
output_file_name = make_output_file_name(profile_id, product)
75+
output_file_path = os.path.join(output_dir, output_file_name)
76+
return output_file_path
77+
78+
79+
def generate_html_guide(ds, transform, params, output_file_path):
80+
html = transform(ds, **params)
81+
html.write_output(output_file_path)
82+
83+
84+
def make_index_options(benchmarks):
85+
index_options = {}
86+
for benchmark_id, benchmark_data in benchmarks.items():
87+
options = []
88+
for profile_id, profile_title in benchmark_data.profiles.items():
89+
guide_file_name = make_output_file_name(
90+
profile_id, benchmark_data.product)
91+
data_benchmark_id = "" if len(benchmarks) == 1 else benchmark_id
92+
option = (
93+
f"<option value=\"{guide_file_name}\" data-benchmark-id=\""
94+
f"{data_benchmark_id}\" data-profile-id=\"{profile_id}\">"
95+
f"{profile_title}</option>")
96+
options.append(option)
97+
index_options[benchmark_id] = options
98+
return index_options
99+
100+
101+
def make_index_links(benchmarks):
102+
index_links = []
103+
for benchmark_id, benchmark_data in benchmarks.items():
104+
for profile_id, profile_title in benchmark_data.profiles.items():
105+
guide_file_name = make_output_file_name(
106+
profile_id, benchmark_data.product)
107+
a_target = (
108+
f"<a target=\"guide\" href=\"{guide_file_name}\">"
109+
f"{profile_title} in {benchmark_id}</a>")
110+
index_links.append(a_target)
111+
return index_links
112+
113+
114+
def make_index_initial_src(benchmarks):
115+
for benchmark_data in benchmarks.values():
116+
for profile_id in benchmark_data.profiles:
117+
return make_output_file_name(profile_id, benchmark_data.product)
118+
return None
119+
120+
121+
def generate_html_index(benchmarks, data_stream, output_dir):
122+
benchmark_titles = {id_: data.title for id_, data in benchmarks.items()}
123+
product = list(benchmarks.values())[0].product
124+
input_basename = os.path.basename(data_stream)
125+
index_links = make_index_links(benchmarks)
126+
index_options = make_index_options(benchmarks)
127+
index_initial_src = make_index_initial_src(benchmarks)
128+
index_source = ssg.build_guides.build_index(
129+
benchmark_titles, input_basename, index_links, index_options,
130+
index_initial_src)
131+
output_path = make_output_file_path("index", product, output_dir)
132+
with open(output_path, "wb") as f:
133+
f.write(index_source.encode("utf-8"))
134+
135+
136+
def generate_html_guides(ds, benchmarks, oscap_version, output_dir):
137+
xslt = ET.parse(XSLT_PATH)
138+
transform = ET.XSLT(xslt)
139+
for benchmark_id, benchmark_data in benchmarks.items():
140+
for profile_id in benchmark_data.profiles:
141+
params = make_params(oscap_version, benchmark_id, profile_id)
142+
output_file_path = make_output_file_path(
143+
profile_id, benchmark_data.product, output_dir)
144+
generate_html_guide(ds, transform, params, output_file_path)
145+
146+
147+
def main():
148+
args = parse_args()
149+
ds = ET.parse(args.data_stream)
150+
benchmarks = get_benchmarks(ds, args.product)
151+
if not os.path.exists(args.output_dir):
152+
os.mkdir(args.output_dir)
153+
generate_html_guides(ds, benchmarks, args.oscap_version, args.output_dir)
154+
generate_html_index(benchmarks, args.data_stream, args.output_dir)
155+
156+
157+
if __name__ == "__main__":
158+
main()

cmake/SSGCommon.cmake

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -598,13 +598,23 @@ endmacro()
598598
# Build per-product HTML guides to see the status of various profiles and
599599
# rules in the generated XCCDF guides.
600600
macro(ssg_build_html_guides PRODUCT)
601-
add_custom_command(
602-
OUTPUT "${CMAKE_BINARY_DIR}/guides/ssg-${PRODUCT}-guide-index.html"
603-
COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/guides"
604-
COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/build_all_guides.py" --input "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml" --output "${CMAKE_BINARY_DIR}/guides" build
605-
DEPENDS generate-ssg-${PRODUCT}-ds.xml "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml"
606-
COMMENT "[${PRODUCT}-guides] generating HTML guides for all profiles in ssg-${PRODUCT}-ds.xml"
607-
)
601+
if(PYTHON_VERSION_MAJOR GREATER 2 AND PY_LXML)
602+
add_custom_command(
603+
OUTPUT "${CMAKE_BINARY_DIR}/guides/ssg-${PRODUCT}-guide-index.html"
604+
COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/guides"
605+
COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/generate_guides.py" --data-stream "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml" --product "${PRODUCT}" --output-dir "${CMAKE_BINARY_DIR}/guides" --oscap-version "${OSCAP_VERSION}"
606+
DEPENDS generate-ssg-${PRODUCT}-ds.xml "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml"
607+
COMMENT "[${PRODUCT}-guides] generating HTML guides for all profiles in ssg-${PRODUCT}-ds.xml"
608+
)
609+
else()
610+
add_custom_command(
611+
OUTPUT "${CMAKE_BINARY_DIR}/guides/ssg-${PRODUCT}-guide-index.html"
612+
COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/guides"
613+
COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/build_all_guides.py" --input "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml" --output "${CMAKE_BINARY_DIR}/guides" build
614+
DEPENDS generate-ssg-${PRODUCT}-ds.xml "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml"
615+
COMMENT "[${PRODUCT}-guides] generating HTML guides for all profiles in ssg-${PRODUCT}-ds.xml"
616+
)
617+
endif()
608618
add_custom_target(
609619
generate-ssg-${PRODUCT}-guide-index.html
610620
DEPENDS "${CMAKE_BINARY_DIR}/guides/ssg-${PRODUCT}-guide-index.html"

docs/manual/developer/07_understanding_build_system.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,7 @@ refer to their help text for more information and usage:
112112
base product.
113113
- `expand_jinja.py` -- helper script used by the BATS (Bash unit test
114114
framework) to expand Jinja in test scripts.
115+
- `generate_guides.py` -- Generate HTML guides and HTML index for every profile in the built SCAP source data stream.
115116
- `generate_man_page.py` -- generates the ComplianceAsCode man page.
116117
- `generate_profile_remediations.py` -- Generate profile oriented Bash remediation scripts or profile oriented Ansible Playbooks from the built SCAP source data stream. The output is similar to the output of the `oscap xccdf generate fix` command, but the tool `generate_profile_remediations.py` generates the scripts or Playbooks for all profiles in the given SCAP source data stream at once.
117118
- `profile_tool.py` -- utility script to generate statistics about profiles

0 commit comments

Comments
 (0)