diff --git a/builds/testing.mk b/builds/testing.mk index c6f7d57fc..c4d9ec5fc 100644 --- a/builds/testing.mk +++ b/builds/testing.mk @@ -12,6 +12,7 @@ BASELINE = $(addprefix $(BASELINE_DIR), $(notdir $(FONTS:.ttf=.txt))) BENCHMARK = $(addprefix $(BENCHMARK_DIR), $(notdir $(FONTS:.ttf=.txt))) BASELINE_INFO = $(BASELINE_DIR)info.txt BENCHMARK_INFO = $(BENCHMARK_DIR)info.txt +HTMLCREATOR_SRC = $(FTBENCH_DIR)/src/tohtml.py HTMLCREATOR = $(OBJ_DIR)/tohtml.py HTMLFILE = $(OBJ_DIR)/benchmark.html @@ -82,6 +83,11 @@ $(FTBENCH_BIN): $(FTBENCH_OBJ) @$(LINK_CMD) $T$(subst /,$(COMPILER_SEP),$@ $<) $(LINK_LIBS) @echo "Built." +.PHONY: copy-html-script +copy-html-script: + @cp $(HTMLCREATOR_SRC) $(OBJ_DIR) + @echo "Copied tohtml.py to $(OBJ_DIR)" + # Create a baseline .PHONY: baseline baseline: $(FTBENCH_BIN) $(BASELINE_DIR) @@ -104,7 +110,7 @@ baseline: $(FTBENCH_BIN) $(BASELINE_DIR) # Benchmark and compare to baseline .PHONY: benchmark -benchmark: $(FTBENCH_BIN) $(BENCHMARK_DIR) +benchmark: $(FTBENCH_BIN) $(BENCHMARK_DIR) copy-html-script @$(RM) -f $(BENCHMARK) $(HTMLFILE) @echo "Creating benchmark..." @echo "$(FTBENCH_FLAG)" > $(BENCHMARK_INFO) @@ -127,5 +133,5 @@ benchmark: $(FTBENCH_BIN) $(BENCHMARK_DIR) clean-benchmark: @echo "Cleaning..." @$(RM) $(FTBENCH_BIN) $(FTBENCH_OBJ) - @$(RM) -rf $(BASELINE_DIR) $(BENCHMARK_DIR) $(HTMLFILE) + @$(RM) -rf $(BASELINE_DIR) $(BENCHMARK_DIR) $(HTMLFILE) $(HTMLCREATOR) @echo "Cleaned" diff --git a/src/tools/ftbench/src/tohtml.py b/src/tools/ftbench/src/tohtml.py new file mode 100644 index 000000000..bbc11a420 --- /dev/null +++ b/src/tools/ftbench/src/tohtml.py @@ -0,0 +1,310 @@ +"""This script generates a HTML file from the results of ftbench""" +import os +import re +import sys + +GITLAB_URL = "https://gitlab.freedesktop.org/freetype/freetype/-/commit/" +CSS_STYLE = """ + +""" +OBJ_DIR = sys.argv[1] +BASELINE_DIR = os.path.join(OBJ_DIR,"baseline") +BENCHMARK_DIR = os.path.join(OBJ_DIR,"benchmark") +BENCHMARK_HTML = os.path.join(OBJ_DIR,"benchmark.html") + +FONT_COUNT = 5 + +WARNING_SAME_COMMIT = "Warning: Baseline and Benchmark have the same commit ID!" +INFO_1 = "* Average time for single iteration. Smaller values are better." +INFO_2 = "* If a value in the 'Iterations' column is given as '*x* | *y*', values *x* and *y* give the number of iterations in the baseline and the benchmark test, respectively." + + + + +def main(): + """Entry point for theq script""" + with open(BENCHMARK_HTML, "w") as html_file: + write_to_html(html_file, "\n\n") + write_to_html(html_file, CSS_STYLE) + write_to_html(html_file, "\n\n") + write_to_html(html_file, "

Freetype Benchmark Results

\n") + + baseline_info = parse_info_file(os.path.join(BASELINE_DIR, "info.txt")) + benchmark_info = parse_info_file(os.path.join(BENCHMARK_DIR, "info.txt")) + + if baseline_info[1].strip() == benchmark_info[1].strip(): + write_to_html( + html_file, + f'

{WARNING_SAME_COMMIT}

\n', + ) + + generate_info_table(html_file, baseline_info, benchmark_info) + + # Generate total results table + generate_total_results_table(html_file, BASELINE_DIR, BENCHMARK_DIR) + + # Generate results tables + for filename in os.listdir(BASELINE_DIR): + if filename.endswith(".txt") and not filename == "info.txt": + baseline_results = read_file(os.path.join(BASELINE_DIR, filename)) + benchmark_results = read_file(os.path.join(BENCHMARK_DIR, filename)) + + generate_results_table( + html_file, baseline_results, benchmark_results, filename + ) + + + write_to_html(html_file, "
Freetype Benchmark
\n") + write_to_html(html_file, "\n\n") + +def write_to_html(html_file, content): + """Write content to html file""" + html_file.write(content) + + +def read_file(file_path): + """Read file and return list of lines""" + with open(file_path, "r") as f: + return f.readlines() + + +def parse_info_file(info_file): + """Get info from info.txt file and return as list""" + info = read_file(info_file) + info[1] = '{}\n'.format(GITLAB_URL, info[1].strip(), info[1][:8]) + return info + + +def generate_info_table(html_file, baseline_info, benchmark_info): + """Prepare info table for html""" + write_to_html(html_file, "

Info

\n") + write_to_html(html_file, '\n') + write_to_html( + html_file, "\n" + ) + info_list = ["Parameters", "Commit ID", "Commit Date", "Branch"] + for info, baseline_line, benchmark_line in zip( + info_list, baseline_info, benchmark_info + ): + write_to_html( + html_file, + '\n'.format( + info, baseline_line.strip(), benchmark_line.strip() + ), + ) + write_to_html(html_file, "
InfoBaselineBenchmark
{}{}{}

") + write_to_html(html_file, f"

{INFO_1}

") + write_to_html(html_file, f"

{INFO_2}

") + + +def generate_total_results_table(html_file, baseline_dir, benchmark_dir): + """Prepare total results table for html""" + + # This dictionary will store aggregated results. + test_results = {test: {"baseline": 0, "benchmark": 0, "n_baseline": 0, "n_benchmark": 0} for test in [ + "Load", "Load_Advances (Normal)", "Load_Advances (Fast)", "Load_Advances (Unscaled)", "Render", + "Get_Glyph", "Get_Char_Index", "Iterate CMap", "New_Face", "Embolden", "Stroke", "Get_BBox", + "Get_CBox", "New_Face & load glyph(s)" + ]} + + total_time = 0 + + for filename in os.listdir(baseline_dir): + if filename.endswith(".txt") and not filename == "info.txt": + + baseline_results = read_file(os.path.join(baseline_dir, filename)) + benchmark_results = read_file(os.path.join(benchmark_dir, filename)) + + for baseline_line, benchmark_line in zip(baseline_results, benchmark_results): + + if baseline_line.startswith("Total time:"): + baseline_match = re.match(r"Total time: (\d+)s", baseline_line) + benchmark_match = re.match(r"Total time: (\d+)s", benchmark_line) + + if baseline_match and benchmark_match: + total_time += int(baseline_match.group(1)) + total_time += int(benchmark_match.group(1)) + + + if baseline_line.startswith(" "): + baseline_match = re.match(r"\s+(.*?)\s+(\d+\.\d+)\s+microseconds\s+(\d+)\s", baseline_line) + benchmark_match = re.match(r"\s+(.*?)\s+(\d+\.\d+)\s+microseconds\s+(\d+)\s", benchmark_line) + + if baseline_match and benchmark_match: + test = baseline_match.group(1).strip() + baseline_value = float(baseline_match.group(2)) + benchmark_value = float(benchmark_match.group(2)) + baseline_n = int(baseline_match.group(3)) + benchmark_n = int(benchmark_match.group(3)) + + # Aggregate the results + if test in test_results: + test_results[test]["baseline"] += baseline_value + test_results[test]["benchmark"] += benchmark_value + test_results[test]["n_baseline"] += baseline_n + test_results[test]["n_benchmark"] += benchmark_n + + + + # Writing to HTML + write_to_html(html_file, "

Total Results

\n") + write_to_html(html_file, '\n') + write_to_html( + html_file, + '\ + \n' + ) + + total_baseline = total_benchmark = total_diff = total_n_baseline = total_n_benchmark = 0 + + for test, values in test_results.items(): + + baseline = values["baseline"] / FONT_COUNT + benchmark = values["benchmark"] / FONT_COUNT + n_baseline = values["n_baseline"] / FONT_COUNT + n_benchmark = values["n_benchmark"] / FONT_COUNT + + n_display = f"{n_baseline:.0f} | {n_benchmark:.0f}" if n_baseline != n_benchmark else int(n_baseline) + + diff = ((baseline - benchmark) / baseline) * 100 + + # Calculate for total row + total_baseline += baseline + total_benchmark += benchmark + total_n_baseline += n_baseline + total_n_benchmark += n_benchmark + + # Check which value is smaller for color highlighting + baseline_color = "highlight" if baseline <= benchmark else "" + benchmark_color = "highlight" if benchmark <= baseline else "" + + write_to_html( + html_file, + f'\ + \ + \n' + ) + + + + total_diff = ((total_baseline - total_benchmark) / total_baseline) * 100 + total_n_display = f"{total_n_baseline} | {total_n_benchmark}" if total_n_baseline != total_n_benchmark else str(total_n_baseline) + + write_to_html( + html_file, + f'' + ) + + write_to_html(html_file,'
TestIterations* Baseline (µs)* Benchmark (µs)Difference (%)
{test}{n_display}{baseline:.1f}{benchmark:.1f}{diff:.1f}
Total duration for all tests:{total_time:.0f} s
\n') + + + +def generate_results_table(html_file, baseline_results, benchmark_results, filename): + """Prepare results table for html""" + fontname = [ + line.split("/")[-1].strip("'")[:-2] + for line in baseline_results + if line.startswith("ftbench results for font") + ][0] + + write_to_html(html_file, "

Results for {}

\n".format(fontname)) + write_to_html(html_file, '\n') + write_to_html( + html_file, + '\ + \ + \ + \n'.format( + os.path.join("./baseline/", filename[:-4]), + os.path.join("./benchmark/", filename[:-4]), + ), + ) + + total_n = total_difference = total_time = 0 + + for baseline_line, benchmark_line in zip(baseline_results, benchmark_results): + + if baseline_line.startswith("Total time:"): + baseline_match = re.match(r"Total time: (\d+)s", baseline_line) + benchmark_match = re.match(r"Total time: (\d+)s", benchmark_line) + + if baseline_match and benchmark_match: + total_time += int(baseline_match.group(1)) + total_time += int(benchmark_match.group(1)) + + + if baseline_line.startswith(" "): + baseline_match = re.match( + r"\s+(.*?)\s+(\d+\.\d+)\s+microseconds\s+(\d+)\s", baseline_line + ) + benchmark_match = re.match( + r"\s+(.*?)\s+(\d+\.\d+)\s+microseconds\s+(\d+)\s", benchmark_line + ) + + if baseline_match and benchmark_match: + baseline_value = float(baseline_match.group(2)) + benchmark_value = float(benchmark_match.group(2)) + + percentage_diff = ( + (baseline_value - benchmark_value) / baseline_value + ) * 100 + + baseline_n = baseline_match.group(3) + benchmark_n = benchmark_match.group(3) + + + n = ( + baseline_n + if baseline_n == benchmark_n + else baseline_n + " | " + benchmark_n + ) + + + + total_n += int(baseline_n) + total_n += int(benchmark_n) + + + # Check which value is smaller for color highlighting + baseline_color = "highlight" if baseline_value <= benchmark_value else "" + benchmark_color = "highlight" if benchmark_value <= baseline_value else "" + + + write_to_html( + html_file, + f'\ + \n' + ) + + write_to_html( + html_file, + f'
TestIterations* Baseline (µs)* Benchmark (µs)Difference (%)
{baseline_match.group(1)}{n}{baseline_value:.1f}{benchmark_value:.1f}{percentage_diff:.1f}
Total duration for the font:{total_time:.0f} s
\n' + ) + + + +if __name__ == "__main__": + main()