Skip to content

Commit ff4acc5

Browse files
christiansandberghardbyte
christiansandberg
authored and
hardbyte
committed
BLF logging support
Limited reading and writing BLF log files. Closes hardbyte#108 * Support for error frames and events * Uses global markers for events
1 parent ffbb2ec commit ff4acc5

8 files changed

Lines changed: 352 additions & 0 deletions

File tree

‎can/CAN.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,9 @@ def stop(self):
6060
"""
6161

6262

63+
from can.io.blf import BLFReader, BLFWriter
64+
65+
6366
class RedirectReader(Listener):
6467
"""
6568
A RedirectReader sends all received messages
@@ -109,6 +112,7 @@ class Logger(object):
109112
110113
The format is determined from the file format which can be one of:
111114
* .asc: :class:`can.ASCWriter`
115+
* .blf: :class:`can.BLFWriter`
112116
* .csv: :class:`can.CSVWriter`
113117
* .db: :class:`can.SqliteWriter`
114118
* other: :class:`can.Printer`
@@ -120,6 +124,8 @@ def __new__(cls, other, filename):
120124
return Printer()
121125
elif filename.endswith(".asc"):
122126
return ASCWriter(filename)
127+
elif filename.endswith(".blf"):
128+
return BLFWriter(filename)
123129
elif filename.endswith(".csv"):
124130
return CSVWriter(filename)
125131
elif filename.endswith(".db"):
@@ -134,6 +140,7 @@ class LogReader(object):
134140
135141
The format is determined from the file format which can be one of:
136142
* .asc
143+
* .blf
137144
* .csv
138145
* .db
139146
@@ -152,6 +159,8 @@ def __new__(cls, other, filename):
152159
if filename.endswith(".asc"):
153160
raise NotImplemented
154161
# return ASCReader(filename)
162+
if filename.endswith(".blf"):
163+
return BLFReader(filename)
155164
if filename.endswith(".csv"):
156165
raise NotImplemented
157166
# return CSVReader(filename)

‎can/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,4 @@ class CanError(IOError):
1616
from can.notifier import Notifier
1717
from can.broadcastmanager import send_periodic, CyclicSendTaskABC, MultiRateCyclicSendTaskABC
1818
from can.interfaces import interface
19+
from can.io.blf import BLFReader, BLFWriter

‎can/io/__init__.py‎

Whitespace-only changes.

‎can/io/blf.py‎

Lines changed: 273 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,273 @@
1+
"""
2+
Implements support for BLF (Binary Logging Format) which is a proprietary
3+
CAN log format from Vector Informatik GmbH.
4+
5+
No official specification of the binary logging format is available.
6+
This implementation is based on Toby Lorenz' C++ library "Vector BLF" which is
7+
licenced under GPLv3. https://bitbucket.org/tobylorenz/vector_blf.
8+
The file starts with a header. The rest is one or more "log containers"
9+
which consists of a header and some zlib compressed data, usually up to 128 kB
10+
of uncompressed data each. This data contains the actual CAN messages and other
11+
objects types.
12+
"""
13+
import struct
14+
import zlib
15+
import datetime
16+
import time
17+
18+
from can.message import Message
19+
from can.CAN import Listener
20+
21+
22+
# 0 = unknown, 2 = CANoe
23+
APPLICATION_ID = 5
24+
25+
# Header must be 144 bytes in total
26+
# signature ("LOGG"), header size,
27+
# application ID, application major, application minor, application build,
28+
# bin log major, bin log minor, bin log build, bin log patch,
29+
# file size, uncompressed size, count of objects, count of objects read,
30+
# time start (SYSTEMTIME), time stop (SYSTEMTIME)
31+
FILE_HEADER_STRUCT = struct.Struct("<4sLBBBBBBBBQQLL8H8H72x")
32+
33+
# signature ("LOBJ"), header size, header version (1), object size, object type,
34+
# flags, object version, size uncompressed or timestamp
35+
OBJ_HEADER_STRUCT = struct.Struct("<4sHHLLL2xHQ")
36+
37+
# channel, flags, dlc, arbitration id, data
38+
CAN_MSG_STRUCT = struct.Struct("<HBBL8s")
39+
40+
# channel, length
41+
CAN_ERROR_STRUCT = struct.Struct("<HH4x")
42+
43+
# commented event type, foreground color, background color, relocatable,
44+
# group name length, marker name length, description length
45+
GLOBAL_MARKER_STRUCT = struct.Struct("<LLL3xBLLL12x")
46+
47+
48+
CAN_MESSAGE = 1
49+
CAN_ERROR = 2
50+
LOG_CONTAINER = 10
51+
GLOBAL_MARKER = 96
52+
53+
CAN_MSG_EXT = 0x80000000
54+
REMOTE_FLAG = 0x80
55+
56+
57+
def timestamp_to_systemtime(timestamp):
58+
if timestamp is None or timestamp < 631152000:
59+
# Probably not a Unix timestamp
60+
return (0, 0, 0, 0, 0, 0, 0, 0)
61+
t = datetime.datetime.fromtimestamp(timestamp)
62+
return (t.year, t.month, t.isoweekday() % 7, t.day,
63+
t.hour, t.minute, t.second, int(round(t.microsecond / 1000.0)))
64+
65+
66+
def systemtime_to_timestamp(systemtime):
67+
try:
68+
t = datetime.datetime(
69+
systemtime[0], systemtime[1], systemtime[3],
70+
systemtime[4], systemtime[5], systemtime[6], systemtime[7] * 1000)
71+
return time.mktime(t.timetuple()) + systemtime[7] / 1000.0
72+
except ValueError:
73+
return 0
74+
75+
76+
class BLFReader(object):
77+
"""
78+
Iterator of CAN messages from a Binary Logging File.
79+
80+
Only CAN messages and error frames are supported. Other object types are
81+
silently ignored.
82+
"""
83+
84+
def __init__(self, filename):
85+
self.fp = open(filename, "rb")
86+
data = self.fp.read(FILE_HEADER_STRUCT.size)
87+
header = FILE_HEADER_STRUCT.unpack(data)
88+
#print(header)
89+
assert header[0] == b"LOGG", "Unknown file format"
90+
self.start_timestamp = systemtime_to_timestamp(header[14:22])
91+
92+
def __iter__(self):
93+
tail = b""
94+
while True:
95+
data = self.fp.read(OBJ_HEADER_STRUCT.size)
96+
if not data:
97+
# EOF
98+
break
99+
header = OBJ_HEADER_STRUCT.unpack(data)
100+
#print(header)
101+
assert header[0] == b"LOBJ", "Parse error"
102+
obj_type = header[4]
103+
obj_data_size = header[3] - OBJ_HEADER_STRUCT.size
104+
obj_data = self.fp.read(obj_data_size)
105+
# Read padding bytes
106+
self.fp.read(obj_data_size % 4)
107+
if obj_type == LOG_CONTAINER:
108+
uncompressed_size = header[7]
109+
data = zlib.decompress(obj_data, 15, uncompressed_size)
110+
if tail:
111+
data = tail + data
112+
pos = 0
113+
while pos + OBJ_HEADER_STRUCT.size < len(data):
114+
header = OBJ_HEADER_STRUCT.unpack(
115+
data[pos:pos + OBJ_HEADER_STRUCT.size])
116+
#print(header)
117+
assert header[0] == b"LOBJ", "Parse error"
118+
obj_size = header[3]
119+
if pos + obj_size > len(data):
120+
# Object continues in next log container
121+
break
122+
obj_data = data[pos + OBJ_HEADER_STRUCT.size:pos + obj_size]
123+
obj_type = header[4]
124+
timestamp = header[7] / 1000000000.0 + self.start_timestamp
125+
if obj_type == CAN_MESSAGE:
126+
(channel, flags, dlc, can_id,
127+
can_data) = CAN_MSG_STRUCT.unpack(obj_data)
128+
msg = Message(timestamp=timestamp,
129+
arbitration_id=can_id & 0x1FFFFFFF,
130+
extended_id=bool(can_id & CAN_MSG_EXT),
131+
is_remote_frame=bool(flags & REMOTE_FLAG),
132+
dlc=dlc,
133+
data=can_data[:dlc])
134+
msg.channel = channel
135+
yield msg
136+
elif obj_type == CAN_ERROR:
137+
channel, length = CAN_ERROR_STRUCT.unpack(obj_data)
138+
msg = Message(timestamp=timestamp, is_error_frame=True)
139+
msg.channel = channel
140+
yield msg
141+
pos += obj_size
142+
# Add padding bytes
143+
pos += obj_size % 4
144+
# Save remaing data that could not be processed
145+
tail = data[pos:]
146+
self.fp.close()
147+
148+
149+
class BLFWriter(Listener):
150+
"""
151+
Logs CAN data to a Binary Logging File compatible with Vector's tools.
152+
"""
153+
154+
#: Max log container size of uncompressed data
155+
MAX_CACHE_SIZE = 0x20000
156+
157+
#: ZLIB compression level
158+
COMPRESSION_LEVEL = 7
159+
160+
def __init__(self, filename, channel=1):
161+
self.fp = open(filename, "wb")
162+
self.channel = channel
163+
# Header will be written after log is done
164+
self.fp.write(b"\x00" * FILE_HEADER_STRUCT.size)
165+
self.cache = []
166+
self.cache_size = 0
167+
self.count_of_objects = 0
168+
self.uncompressed_size = FILE_HEADER_STRUCT.size
169+
self.start_timestamp = None
170+
self.stop_timestamp = None
171+
172+
def on_message_received(self, msg):
173+
if self.start_timestamp is None:
174+
self.start_timestamp = msg.timestamp
175+
self.stop_timestamp = msg.timestamp
176+
timestamp = int((msg.timestamp - self.start_timestamp) * 1000000000)
177+
if not msg.is_error_frame:
178+
obj_size = OBJ_HEADER_STRUCT.size + CAN_MSG_STRUCT.size
179+
header = OBJ_HEADER_STRUCT.pack(
180+
b"LOBJ", OBJ_HEADER_STRUCT.size, 1, obj_size, CAN_MESSAGE,
181+
2, 0, timestamp)
182+
flags = REMOTE_FLAG if msg.is_remote_frame else 0
183+
arb_id = msg.arbitration_id
184+
if msg.id_type:
185+
arb_id |= CAN_MSG_EXT
186+
data = CAN_MSG_STRUCT.pack(self.channel, flags, msg.dlc, arb_id,
187+
bytes(msg.data))
188+
else:
189+
obj_size = OBJ_HEADER_STRUCT.size + CAN_ERROR_STRUCT.size
190+
header = OBJ_HEADER_STRUCT.pack(
191+
b"LOBJ", OBJ_HEADER_STRUCT.size, 1, obj_size, CAN_ERROR,
192+
2, 0, timestamp)
193+
data = CAN_ERROR_STRUCT.pack(self.channel, 0)
194+
self._add_data(header + data)
195+
196+
def log_event(self, text, timestamp=None):
197+
"""Add an arbitrary message to the log file as a global marker.
198+
199+
:param str text:
200+
The group name of the marker.
201+
:param float timestamp:
202+
Absolute timestamp in Unix timestamp format. If not given, the
203+
marker will be placed along the last message.
204+
"""
205+
if timestamp is None:
206+
timestamp = self.stop_timestamp
207+
if self.start_timestamp is None:
208+
self.start_timestamp = timestamp
209+
self.stop_timestamp = timestamp
210+
try:
211+
# Only works on Windows
212+
text = text.encode("mbcs")
213+
except LookupError:
214+
text = text.encode("ascii")
215+
timestamp = int((timestamp - self.start_timestamp) * 1000000000)
216+
comment = b"Added by python-can"
217+
marker = b"python-can"
218+
obj_size = (OBJ_HEADER_STRUCT.size + GLOBAL_MARKER_STRUCT.size +
219+
len(text) + len(marker) + len(comment))
220+
header = OBJ_HEADER_STRUCT.pack(
221+
b"LOBJ", OBJ_HEADER_STRUCT.size, 1, obj_size, GLOBAL_MARKER,
222+
2, 0, timestamp)
223+
data = GLOBAL_MARKER_STRUCT.pack(
224+
0, 0xFFFFFF, 0xFF3300, 0, len(text), len(marker), len(comment))
225+
self._add_data(header + data + text + marker + comment)
226+
227+
def _add_data(self, data):
228+
if len(data) % 4:
229+
data = data + b"\x00" * (len(data) % 4)
230+
self.cache.append(data)
231+
self.cache_size += len(data)
232+
self.count_of_objects += 1
233+
if self.cache_size >= self.MAX_CACHE_SIZE:
234+
self._flush()
235+
236+
def _flush(self):
237+
"""Compresses and writes data in the cache to file."""
238+
cache = b"".join(self.cache)
239+
if not cache:
240+
# Nothing to write
241+
return
242+
uncompressed_data = cache[:self.MAX_CACHE_SIZE]
243+
# Save data that comes after max size to next round
244+
tail = cache[self.MAX_CACHE_SIZE:]
245+
self.cache = [tail]
246+
self.cache_size = len(tail)
247+
compressed_data = zlib.compress(uncompressed_data,
248+
self.COMPRESSION_LEVEL)
249+
obj_size = OBJ_HEADER_STRUCT.size + len(compressed_data)
250+
header = OBJ_HEADER_STRUCT.pack(
251+
b"LOBJ", 16, 1, obj_size, LOG_CONTAINER, 2, 0, len(uncompressed_data))
252+
self.fp.write(header)
253+
self.fp.write(compressed_data)
254+
# Write padding bytes
255+
self.fp.write(b"\x00" * (obj_size % 4))
256+
self.uncompressed_size += len(uncompressed_data) + OBJ_HEADER_STRUCT.size
257+
258+
def stop(self):
259+
"""Stops logging and closes the file."""
260+
self._flush()
261+
filesize = self.fp.tell()
262+
self.fp.close()
263+
264+
# Write header in the beginning of the file
265+
header = [b"LOGG", FILE_HEADER_STRUCT.size,
266+
APPLICATION_ID, 0, 0, 0, 2, 6, 8, 1]
267+
# The meaning of "count of objects read" is unknown
268+
header.extend([filesize, self.uncompressed_size,
269+
self.count_of_objects, 0])
270+
header.extend(timestamp_to_systemtime(self.start_timestamp))
271+
header.extend(timestamp_to_systemtime(self.stop_timestamp))
272+
with open(self.fp.name, "r+b") as f:
273+
f.write(FILE_HEADER_STRUCT.pack(*header))

‎doc/history.rst‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,10 @@ The usb2can interface was contributed by Joshua Villyard in 2015
2727
The IXXAT VCI interface was contributed by Giuseppe Corbelli and funded
2828
by `Weightpack <http://www.weightpack.com>`__ in 2016
2929

30+
The NI-CAN, remote and virtual interfaces plus the ASCII and BLF loggers were
31+
contributed by Christian Sandberg in 2016 and 2017. The BLF format is based on
32+
a C++ library by Toby Lorenz.
33+
3034

3135
Support for CAN within Python
3236
-----------------------------

‎doc/listeners.rst‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,3 +69,18 @@ engineered from existing log files. One description of the format can be found `
6969

7070
.. autoclass:: can.ASCWriter
7171
:members:
72+
73+
74+
BLF (Binary Logging Format)
75+
---------------------------
76+
77+
Implements support for BLF (Binary Logging Format) which is a proprietary
78+
CAN log format from Vector Informatik GmbH.
79+
80+
The data is stored in a compressed format which makes it very compact.
81+
82+
.. autoclass:: can.BLFWriter
83+
:members:
84+
85+
.. autoclass:: can.BLFReader
86+
:members:

‎test/data/logfile.blf‎

682 Bytes
Binary file not shown.

0 commit comments

Comments
 (0)