Browse Source

Introduce to combine log files from multiple bitcoinds.

This commit adds a tool for combining log files from multiple instances
of bitcoinds as well as the test_framework.log file. This gives a
combined view of what the test framework and all bitcoin instances were
doing during a qa test.
John Newbery 4 years ago
2 changed files with 151 additions and 0 deletions
  1. 111
  2. 40

+ 111
- 0
test/functional/ View File

@@ -0,0 +1,111 @@
#!/usr/bin/env python3
"""Combine logs from multiple bitcoin nodes as well as the test_framework log.

This streams the combined log output to stdout. Use > outputfile
to write to an outputfile."""

import argparse
from collections import defaultdict, namedtuple
import glob
import heapq
import os
import re
import sys

# Matches on the date format at the start of the log event
TIMESTAMP_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6}")

LogEvent = namedtuple('LogEvent', ['timestamp', 'source', 'event'])

def main():
"""Main function. Parses args, reads the log files and renders them as text or html."""

parser = argparse.ArgumentParser(usage='%(prog)s [options] <test temporary directory>', description=__doc__)
parser.add_argument('-c', '--color', dest='color', action='store_true', help='outputs the combined log with events colored by source (requires posix terminal colors. Use less -r for viewing)')
parser.add_argument('--html', dest='html', action='store_true', help='outputs the combined log as html. Requires jinja2. pip install jinja2')
args, unknown_args = parser.parse_known_args()

if args.color and != 'posix':
print("Color output requires posix terminal colors.")

if args.html and args.color:
print("Only one out of --color or --html should be specified")

# There should only be one unknown argument - the path of the temporary test directory
if len(unknown_args) != 1:
print("Unexpected arguments" + str(unknown_args))

log_events = read_logs(unknown_args[0])

print_logs(log_events, color=args.color, html=args.html)

def read_logs(tmp_dir):
"""Reads log files.

Delegates to generator function get_log_events() to provide individual log events
for each of the input log files."""

files = [("test", "%s/test_framework.log" % tmp_dir)]
for i, logfile in enumerate(glob.glob("%s/node*/regtest/debug.log" % tmp_dir)):
files.append(("node%d" % i, logfile))

return heapq.merge(*[get_log_events(source, f) for source, f in files])

def get_log_events(source, logfile):
"""Generator function that returns individual log events.

Log events may be split over multiple lines. We use the timestamp
regex match as the marker for a new log event."""
with open(logfile, 'r') as infile:
event = ''
timestamp = ''
for line in infile:
# skip blank lines
if line == '\n':
# if this line has a timestamp, it's the start of a new log event.
time_match = TIMESTAMP_PATTERN.match(line)
if time_match:
if event:
yield LogEvent(timestamp=timestamp, source=source, event=event.rstrip())
event = line
timestamp =
# if it doesn't have a timestamp, it's a continuation line of the previous log.
event += "\n" + line
# Flush the final event
yield LogEvent(timestamp=timestamp, source=source, event=event.rstrip())
except FileNotFoundError:
print("File %s could not be opened. Continuing without it." % logfile, file=sys.stderr)

def print_logs(log_events, color=False, html=False):
"""Renders the iterator of log events into text or html."""
if not html:
colors = defaultdict(lambda: '')
if color:
colors["test"] = "\033[0;36m" # CYAN
colors["node0"] = "\033[0;34m" # BLUE
colors["node1"] = "\033[0;32m" # GREEN
colors["node2"] = "\033[0;31m" # RED
colors["node3"] = "\033[0;33m" # YELLOW
colors["reset"] = "\033[0m" # Reset font color

for event in log_events:
print("{0} {1: <5} {2} {3}".format(colors[event.source.rstrip()], event.source, event.event, colors["reset"]))

import jinja2
except ImportError:
print("jinja2 not found. Try `pip install jinja2`")
.render(title="Combined Logs from testcase", log_events=[event._asdict() for event in log_events]))

if __name__ == '__main__':

+ 40
- 0
test/functional/combined_log_template.html View File

@@ -0,0 +1,40 @@
<html lang="en">
<title> {{ title }} </title>
ul {
list-style-type: none;
font-family: monospace;
li {
border: 1px solid slategray;
margin-bottom: 1px;
li:hover {
filter: brightness(85%);
li.log-test {
background-color: cyan;
li.log-node0 {
background-color: lightblue;
li.log-node1 {
background-color: lightgreen;
li.log-node2 {
background-color: lightsalmon;
li.log-node3 {
background-color: lightyellow;
{% for event in log_events %}
<li class="log-{{ event.source }}"> {{ event.source }} {{ event.timestamp }} {{event.event}}</li>
{% endfor %}