diff --git a/builds/testing.mk b/builds/testing.mk index 6c3d2aba6..9a22d3fd4 100644 --- a/builds/testing.mk +++ b/builds/testing.mk @@ -3,7 +3,7 @@ FTBENCH_DIR = $(TOP_DIR)/src/tools/ftbench FTBENCH_SRC = $(FTBENCH_DIR)/ftbench.c FTBENCH_OBJ = $(OBJ_DIR)/bench.$(SO) FTBENCH_BIN = $(OBJ_DIR)/bench$E -FTBENCH_FLAG ?= -c 2000 +FTBENCH_FLAG ?= -c 500 INCLUDES = $(TOP_DIR)/include FONTS = $(wildcard $(FTBENCH_DIR)/fonts/*.ttf) BASELINE_DIR = $(OBJ_DIR)/baseline/ @@ -73,13 +73,13 @@ $(BASELINE_DIR) $(BENCHMARK_DIR): @mkdir -p $@ $(FTBENCH_OBJ): $(FTBENCH_SRC) - $(COMPILE) $T$(subst /,$(COMPILER_SEP),$@ $<) $(EXTRAFLAGS) - @echo "Object created." + @$(COMPILE) $T$(subst /,$(COMPILER_SEP),$@ $<) $(EXTRAFLAGS) + @echo "Object created." # Build ftbench $(FTBENCH_BIN): $(FTBENCH_OBJ) @echo "Linking ftbench..." - $(LINK_CMD) $T$(subst /,$(COMPILER_SEP),$@ $<) $(LINK_LIBS) + @$(LINK_CMD) $T$(subst /,$(COMPILER_SEP),$@ $<) $(LINK_LIBS) @echo "Built." # Create a baseline diff --git a/src/tools/ftbench/ftbench.c b/src/tools/ftbench/ftbench.c index 53215dcd4..c662a939e 100644 --- a/src/tools/ftbench/ftbench.c +++ b/src/tools/ftbench/ftbench.c @@ -300,7 +300,7 @@ if ( done ) printf( "%10.3f ms %10d done\n", - TIMER_GET( &timer ) / 1000, done ); + TIMER_GET( &timer ) / 1000000, done ); else printf( "no error-free calls\n" ); } diff --git a/src/tools/ftbench/src/tohtml.py b/src/tools/ftbench/src/tohtml.py index 80ad6f525..2f91b63f1 100644 --- a/src/tools/ftbench/src/tohtml.py +++ b/src/tools/ftbench/src/tohtml.py @@ -1,16 +1,13 @@ -"""This module converts the benchmark results into a HTML file.""" +"""This script generates a HTML file from the results of ftbench""" +# Ahmet Goksu ahmet@goksu.in ahmet.goksu.in + import os import re import sys -# Create the HTML file PROJECT_ROOT = sys.argv[1] BENCHMARK_HTML = os.path.join(PROJECT_ROOT, "benchmark.html") - -# GitLab URL GITLAB_URL = "https://gitlab.freedesktop.org/freetype/freetype/-/commit/" - -# CSS style CSS_STYLE = """ """ - -# Directories BASELINE_DIR = os.path.join(PROJECT_ROOT, "baseline") BENCHMARK_DIR = os.path.join(PROJECT_ROOT, "benchmark") -# Open HTML file for writing and write the header -with open(BENCHMARK_HTML, "w") as html_file: - html_file.write( - f"\n\ - \n\ - Benchmark Results\n\ - {CSS_STYLE}\ - \n\ - \n\ -

Benchmark Results

\n" +def main(): + """Entry point for the script""" + with open(BENCHMARK_HTML, "w") as html_file: + write_to_html(html_file, "\n\n") + write_to_html(html_file, CSS_STYLE) + write_to_html(html_file, "\n\n") + + baseline_info = parse_info_file(os.path.join(BASELINE_DIR, "info.txt")) + benchmark_info = parse_info_file(os.path.join(BENCHMARK_DIR, "info.txt")) + + if baseline_info[1].strip() == benchmark_info[1].strip(): + write_to_html( + html_file, + '

Warning: Baseline and Benchmark have the same commit ID

\n', + ) + + generate_info_table(html_file, baseline_info, benchmark_info) + + # Generate results tables + for filename in os.listdir(BASELINE_DIR): + if filename.endswith(".txt") and not filename == "info.txt": + baseline_results = read_file(os.path.join(BASELINE_DIR, filename)) + benchmark_results = read_file(os.path.join(BENCHMARK_DIR, filename)) + + generate_results_table( + html_file, baseline_results, benchmark_results, filename + ) + + write_to_html(html_file, "\n\n") + +def write_to_html(html_file, content): + """Write content to html file""" + html_file.write(content) + + +def read_file(file_path): + """Read file and return list of lines""" + with open(file_path, "r") as f: + return f.readlines() + + +def parse_info_file(info_file): + """Get info from info.txt file and return as list""" + info = read_file(info_file) + info[1] = '{}\n'.format(GITLAB_URL, info[1].strip(), info[1][:8]) + return info + + +def generate_info_table(html_file, baseline_info, benchmark_info): + """Prepare info table for html""" + write_to_html(html_file, "

Info

\n") + write_to_html(html_file, '\n') + write_to_html( + html_file, "\n" ) - - # If it's the info file, we want to handle it differently - with open(os.path.join(BASELINE_DIR, "info.txt"), "r") as f: - baseline_info = f.readlines() - with open(os.path.join(BENCHMARK_DIR, "info.txt"), "r") as f: - benchmark_info = f.readlines() - - # Check if commit ids are the same - if baseline_info[1].strip() == benchmark_info[1].strip(): - html_file.write( - '

Warning: Baseline and Benchmark have the same commit ID

\n' - ) - - baseline_info[1] = '{}\n'.format( - GITLAB_URL, baseline_info[1].strip(), baseline_info[1][:8] - ) - - benchmark_info[1] = '{}\n'.format( - GITLAB_URL, benchmark_info[1].strip(), benchmark_info[1][:8] - ) - - # Write info table - html_file.write("

Info

\n") - html_file.write('
InfoBaselineBenchmark
\n') - html_file.write("\n") info_list = ["Parameters", "Commit ID", "Commit Date", "Branch"] for info, baseline_line, benchmark_line in zip( info_list, baseline_info, benchmark_info ): - html_file.write( + write_to_html( + html_file, '\n'.format( info, baseline_line.strip(), benchmark_line.strip() - ) + ), ) - html_file.write("
InfoBaselineBenchmark
{}{}{}

") - html_file.write("*Smaller values mean faster operation
\n") - # Traverse through the 'baseline' directory - for filename in os.listdir(BASELINE_DIR): - if filename != "info.txt": - with open(os.path.join(BASELINE_DIR, filename), "r") as f: - baseline_results = f.readlines() - with open(os.path.join(BENCHMARK_DIR, filename), "r") as f: - benchmark_results = f.readlines() + write_to_html(html_file, "
") + write_to_html(html_file, "*Smaller values mean faster operation
\n") - # Get font name from within the .txt file - for line in baseline_results: - if line.startswith("ftbench results for font"): - fontname = line.split("/")[-1]\ - .strip("'")[:-2] - # Write test column headers - html_file.write("

Results for {}

\n".format(fontname)) - html_file.write('\n') - html_file.write( - '\ - \ - \ - \ - \ - \ - \n'.format( - os.path.join(BASELINE_DIR, fontname[:-4]), - os.path.join(BENCHMARK_DIR, fontname[:-4]), - ) +def generate_results_table(html_file, baseline_results, benchmark_results, filename): + """Prepare results table for html""" + fontname = [ + line.split("/")[-1].strip("'")[:-2] + for line in baseline_results + if line.startswith("ftbench results for font") + ][0] + + write_to_html(html_file, "

Results for {}

\n".format(fontname)) + write_to_html(html_file, '
TestNBaseline (ms)Benchmark (ms)Difference (%)
\n') + write_to_html( + html_file, + '\ + \ + \ + \n'.format( + os.path.join(BASELINE_DIR, filename[:-4]), + os.path.join(BENCHMARK_DIR, filename[:-4]), + ), + ) + + total_n = total_time_baseline = total_time_benchmark = total_difference = 0 + + for baseline_line, benchmark_line in zip(baseline_results, benchmark_results): + if baseline_line.startswith(" "): + baseline_match = re.match( + r"\s+(.*?)\s+(\d+\.\d+)\s+ms\s+(\d+)\s", baseline_line + ) + benchmark_match = re.match( + r"\s+(.*?)\s+(\d+\.\d+)\s+ms\s+(\d+)\s", benchmark_line ) - # Write test results - for baseline_line, benchmark_line in zip( - baseline_results, benchmark_results - ): - # Check if line is a test result - # Get results by Regex - if baseline_line.startswith(" "): - baseline_match = re.match( - r"\s+(.*?)\s+(\d+\.\d+)\s+ms\s+(\d+)\s", baseline_line + if baseline_match and benchmark_match: + baseline_value = float(baseline_match.group(2)) + benchmark_value = float(benchmark_match.group(2)) + + percentage_diff = ( + (baseline_value - benchmark_value) / baseline_value + ) * 100 + + baseline_n = baseline_match.group(3) + benchmark_n = benchmark_match.group(3) + + n = ( + baseline_n + if baseline_n == benchmark_n + else baseline_n + " / " + benchmark_n + ) + + total_n += int(baseline_n) + total_n += int(benchmark_n) + total_time_baseline += baseline_value + total_time_benchmark += benchmark_value + total_difference += percentage_diff + + if baseline_value > benchmark_value: + write_to_html( + html_file, + '\ + \n'.format( + baseline_match.group(1), + n, + baseline_value, + benchmark_value, + percentage_diff, + ), + ) + else: + write_to_html( + html_file, + '\ + \n'.format( + baseline_match.group(1), + n, + baseline_value, + benchmark_value, + percentage_diff, + ), ) - benchmark_match = re.match( - r"\s+(.*?)\s+(\d+\.\d+)\s+ms\s+(\d+)\s", benchmark_line - ) + write_to_html( + html_file, + '\ + \n'.format( + total_n, total_time_baseline, total_time_benchmark, total_difference / 14 + ), + ) + write_to_html(html_file, "
TestNBaseline (ms)Benchmark (ms)Difference (%)
{}{}{:.3f}{:.3f}{:.1f}
{}{}{:.3f}{:.3f}{:.1f}
TOTAL{}{:.3f}{:.3f}{:.1f}

\n") - if baseline_match and benchmark_match: - baseline_value = float(baseline_match.group(2)) - benchmark_value = float(benchmark_match.group(2)) - - # Calculate the percentage difference - percentage_diff = ( - (baseline_value - benchmark_value) / baseline_value - ) * 100 - - # Get itaration number - baseline_n = baseline_match.group(3) - benchmark_n = benchmark_match.group(3) - - # Check if iteration number is the same - if baseline_n == benchmark_n: - total_n = baseline_n - else: - total_n = baseline_n +\ - " / " + benchmark_n - - # Highlight the faster value - if baseline_value > benchmark_value: - html_file.write( - '\ - {}\ - {}\ - {:.3f}\ - {:.3f}\ - {:.1f}\ - \n'.format( - baseline_match.group(1), - total_n, - baseline_value, - benchmark_value, - percentage_diff, - ) - ) - else: - html_file.write( - '\ - {}\ - {}\ - {:.3f}\ - {:.3f}\ - {:.1f}\ - \n'.format( - baseline_match.group(1), - total_n, - baseline_value, - benchmark_value, - percentage_diff, - ) - ) - - html_file.write("
\n") - - html_file.write("
Freetype Benchmark
\n\n") +if __name__ == "__main__": + main()