[M] Change in coreboot[master]: [WIP] util/abuild: Add parser for statistics

[Kyösti Mälkki (Code Review)]

Attention is currently required from: Martin L Roth.

Kyösti Mälkki has uploaded this change for review. ( https://review.coreboot.org/c/coreboot/+/75804?usp=email )

Change subject: [WIP] util/abuild: Add parser for statistics
......................................................................

[WIP] util/abuild: Add parser for statistics

Change-Id: I01b976bfffc32d2b2835c697403a29f3630e6ad5
Signed-off-by: Kyösti Mälkki 
---
A util/abuild/showstats.py
1 file changed, 242 insertions(+), 0 deletions(-)

git pull ssh://review.coreboot.org:29418/coreboot refs/changes/04/75804/1

diff --git a/util/abuild/showstats.py b/util/abuild/showstats.py
new file mode 100755
index 0000000..4aa0c39
--- /dev/null
+++ b/util/abuild/showstats.py
@@ -0,0 +1,242 @@
+#! /usr/bin/python3
+
+import os
+import re
+import subprocess
+import tarfile
+import tempfile
+
+debug = 0
+
+abuild_timestamps_name  ='abuild.timestamps'
+abuild_timestamps_pattern = re.compile("TS_(\d+) (\d+)$")
+
+ccache_bin = os.getenv("CCACHE", default="ccache")
+ccache_statslog_name = 'ccache.stats'
+ccache_statslog_pattern = re.compile(" *(.*): +(\d*) / (\d*) (.*)$")
+
+def print_hit_ratio(a, b):
+	ratio = float(a) / float(b) * 100
+	print(f'{a:d} {b} ({ratio:2.2f})')
+
+def ccache_hit_rates(ccache):
+	totals = 0
+	rates = [[]] * len(ccache)
+	rates_str = [[]] * len(ccache)
+
+	for i in range(0, len(ccache)):
+		totals += ccache[i]
+	if totals == 0:
+		return -1
+
+	for i in range(0, len(ccache)):
+		rates[i] = float(ccache[i]) / float(totals) * 100
+		rates_str[i] = f'{rates[i]:2.2f}%'
+
+	return rates_str
+
+def parse_ccache_log_stats(content):
+	state = 0
+	for line in content.splitlines():
+
+		parsed = ccache_statslog_pattern.match(line)
+		if not parsed:
+			continue
+
+		if debug:
+			print(parsed.groups())
+
+		title = parsed.groups()[0]
+		value = (int(parsed.groups()[1]), int(parsed.groups()[2]))
+
+		if title == "Cacheable calls":
+			cacheable_1 = value[0]
+			total_calls_1 = value[1]
+			state = 1
+			continue
+
+		if title == "Hits":
+			if state == 1:
+				hits_1 = value[0]
+				cacheable_2 = value[1]
+				state = 2
+			continue
+
+		if title == "Direct":
+			if state == 2:
+				direct = value[0]
+				hits_2 = value[1]
+			continue
+
+		if title == "Preprocessed":
+			if state == 2:
+				preprocessed = value[0]
+				hits_3 = value[1]
+			continue
+
+		if title == "Misses":
+			misses = value[0]
+			cacheable_3 = value[1]
+			state = 3
+			continue
+
+		if title == "Uncacheable calls":
+			uncacheable = value[0]
+			total_calls_2 = value[1]
+			state = 4
+			continue
+
+	# Sanity check input
+	if cacheable_1 != cacheable_2 or cacheable_2 != cacheable_3:
+		return -1
+	if hits_1 != hits_2 or hits_2 != hits_3:
+		return -1
+	if total_calls_1 != total_calls_2:
+		return -1
+	if preprocessed + direct != hits_1:
+		return -1
+	if misses + hits_1 != cacheable_1:
+		return -1
+	if uncacheable + cacheable_1 != total_calls_1:
+		return -1
+
+	if debug:
+		print_hit_ratio(uncacheable, total_calls_1)
+		print_hit_ratio(cacheable_1, total_calls_1)
+		print_hit_ratio(misses, cacheable_1)
+		print_hit_ratio(preprocessed, hits_1)
+		print_hit_ratio(direct, hits_1)
+
+	return [uncacheable, misses, preprocessed, direct]
+
+
+def extract_ccache_stats(tar, orig_dir, board):
+	path = orig_dir + '/' + board + '/' + ccache_statslog_name
+
+	f = tar.extractfile(path)
+	ccache_tmp_statslog = tempfile.NamedTemporaryFile()
+	ccache_tmp_statslog.write(f.read())
+	f.close()
+
+	if debug:
+		print(ccache_tmp_statslog.name)
+
+	# ccache takes the filename in environment var
+	ccache_env = {**os.environ, 'CCACHE_STATSLOG': ccache_tmp_statslog.name}
+	log = subprocess.run([ccache_bin, "--show-log-stats"], capture_output=True, \
+			     text=True, env=ccache_env)
+
+	ccache_tmp_statslog.close()
+
+	return parse_ccache_log_stats(log.stdout)
+
+
+def print_elapsed(usecs):
+	minutes = usecs // (60 * 1000 * 1000)
+	usecs = usecs % (60 * 1000 * 1000)
+	seconds = usecs / (1000 * 1000)
+
+	if minutes > 0 :
+		elapsed=f'{minutes} min {seconds:2.0f} sec'
+	else:
+		elapsed=f'{seconds:2.3f} seconds'
+
+	return elapsed
+
+def parse_timestamps(content):
+	ts = [[]] * 3
+	elapsed_us = [[]] * 2
+
+	for line in content.splitlines():
+
+		parsed = abuild_timestamps_pattern.match(line)
+		if not parsed:
+			continue
+
+		idx = int(parsed.groups()[0])
+		ts[idx] = int(parsed.groups()[1])
+
+	elapsed_us[0] = ts[1] - ts[0]
+	elapsed_us[1] = ts[2] - ts[1]
+
+	if debug:
+		print(f'{elapsed_us}')
+
+	return elapsed_us
+
+
+def extract_timestamps(tar, orig_dir, board):
+	path = orig_dir + '/' + board + '/' + abuild_timestamps_name
+	f = tar.extractfile(path)
+	content = f.read().decode()
+	f.close()
+	if debug:
+		print(content)
+	return parse_timestamps(content)
+
+
+sum_of_ccache = None
+sum_of_ts = None
+
+def parse_stats_from_tar(tar, orig_dir, board):
+	global sum_of_ccache
+	global sum_of_ts
+
+	timestamps = extract_timestamps(tar, orig_dir, board)
+	ccache = extract_ccache_stats(tar, orig_dir, board)
+
+	if (timestamps != -1):
+		if sum_of_ts == None:
+			sum_of_ts = timestamps
+		else:
+			for i in range(0, len(sum_of_ts)):
+				sum_of_ts[i] += timestamps[i]
+
+	if (ccache != -1):
+		if sum_of_ccache == None:
+			sum_of_ccache = ccache
+		else:
+			for i in range(0, len(sum_of_ccache)):
+				sum_of_ccache[i] += ccache[i]
+
+	if (timestamps != -1) and (ccache != -1):
+		elapsed = print_elapsed(timestamps[1])
+		hits = ccache_hit_rates(ccache)
+		print(f'{board} ts: {timestamps} ccache: {ccache} {hits} {elapsed}')
+	elif (timestamps != -1):
+		elapsed = print_elapsed(timestamps[1])
+		print(f'{board} ts: {timestamps} ccache-disabled: {elapsed}')
+
+
+def collect_stats(statistics_tar):
+	boards = 0
+	stats_files_pattern = re.compile("(.+)/(.+)/(.*)$")
+
+	tar = tarfile.open(statistics_tar, 'r')
+	for line in tar.getnames():
+		parsed = stats_files_pattern.match(line);
+		if not parsed:
+			continue
+		if debug:
+			print(parsed.groups())
+
+		orig_dir = parsed.groups()[0]
+		board = parsed.groups()[1]
+		basename = parsed.groups()[2]
+		if basename == abuild_timestamps_name:
+			parse_stats_from_tar(tar, orig_dir, board)
+			boards += 1
+
+	tar.close()
+
+	cfg_total = print_elapsed(sum_of_ts[0])
+	cfg_avg = print_elapsed(sum_of_ts[0] / boards)
+	print(f'Total (config): {cfg_total} avg: {cfg_avg}')
+
+	ccache_total = print_elapsed(sum_of_ts[1])
+	ccache_avg = print_elapsed(sum_of_ts[1] / boards)
+	hits = ccache_hit_rates(sum_of_ccache)
+	print(f'Total (ccache): {sum_of_ccache} {hits} {ccache_total}')
+
+
+collect_stats('coreboot-builds/statistics.tar')