diff --git a/fitparse/base.py b/fitparse/base.py index 4bcd494..f5c4295 100644 --- a/fitparse/base.py +++ b/fitparse/base.py @@ -2,6 +2,8 @@ import os import struct +import sys + # Python 2 compat try: num_types = (int, float, long) @@ -18,11 +20,57 @@ ) from fitparse.utils import calc_crc, FitParseError, FitEOFError, FitCRCError, FitHeaderError +def get_field(message, is_dev, def_nums): + if type(def_nums) is not list: + def_nums = [def_nums] + for field_data in message: + fdef = field_data.field_def + if (fdef is not None and + hasattr(fdef, 'dev_data_index') == is_dev and + fdef.def_num in def_nums): + + return field_data + +def copy_field(src, dest): + if src is not None and dest is not None: + # src.value might have been postprocessed by unit/type/fild processors + # so re-compute the value + val = src.decode_raw_value() + dest.set_value(val) + # view postprocessed value in output + dest.value = src.value + +def copy_dev_to_native(message, dev_ids, n_id): + dev_field = get_field(message, True, dev_ids) + nat_field = get_field(message, False, n_id) + copy_field(dev_field, nat_field) + +def adjust_message(msg): + #print msg.mesg_num + #if msg.type == 'data' and msg.mesg_num == 20: # Record + # for field_data in msg: + # print field_data.field_def + + if msg.type == 'data' and msg.mesg_num == 20: # Record + copy_dev_to_native(msg, [0, 23], 5) # distance + + if msg.type == 'data' and msg.mesg_num == 19: # Lap + copy_dev_to_native(msg, 4, 9) # total_distance + + if msg.type == 'data' and msg.mesg_num == 18: # Session + copy_dev_to_native(msg, [7, 25], 9) # total_distance + copy_dev_to_native(msg, 21, 14) # avg_speed + + class FitFile(object): - def __init__(self, fileish, check_crc=True, data_processor=None): + def __init__(self, fileish, check_crc=True, data_processor=None, out=None): + self._verbose = False + print("HELLO") + if hasattr(fileish, 'read'): # BytesIO-like object self._file = fileish + self._out = out elif isinstance(fileish, str): # Python2 - file path, file contents in the case of a TypeError # Python3 - file path @@ -52,6 +100,9 @@ def close(self): if hasattr(self, "_file") and self._file and hasattr(self._file, "close"): self._file.close() self._file = None + if self._out and hasattr(self._out, "close"): + self._out.close() + self._out = None def __enter__(self): return self @@ -70,6 +121,11 @@ def _read(self, size): self._bytes_left -= len(data) return data + def _write(self, data): + if self._out and data: + self._out.write(data) + self._out_crc = calc_crc(data, self._out_crc) + def _read_struct(self, fmt, endian='<', data=None, always_tuple=False): fmt_with_endian = "%s%s" % (endian, fmt) size = struct.calcsize(fmt_with_endian) @@ -86,6 +142,21 @@ def _read_struct(self, fmt, endian='<', data=None, always_tuple=False): # Flatten tuple if it's got only one value return unpacked if (len(unpacked) > 1) or always_tuple else unpacked[0] + def _write_struct(self, data, fmt, endian='<'): + if self._out is None: + return + + fmt_with_endian = "%s%s" % (endian, fmt) + size = struct.calcsize(fmt_with_endian) + if size <= 0: + raise FitParseError("Invalid struct format: %s" % fmt_with_endian) + + if type(data)==tuple: + packed = struct.pack(fmt_with_endian, *data) + else: + packed = struct.pack(fmt_with_endian, data) + self._write(packed) + def _read_and_assert_crc(self, allow_zero=False): # CRC Calculation is little endian from SDK crc_expected, crc_actual = self._crc, self._read_struct('H') @@ -95,6 +166,9 @@ def _read_and_assert_crc(self, allow_zero=False): raise FitCRCError('CRC Mismatch [expected = 0x%04X, actual = 0x%04X]' % ( crc_expected, crc_actual)) + def _write_crc(self): + self._write_struct((self._out_crc,), 'H') + ########## # Private Data Parsing Methods @@ -106,6 +180,7 @@ def _parse_file_header(self): self._complete = False self._compressed_ts_accumulator = 0 self._crc = 0 + self._out_crc = 0 self._local_mesgs = {} self._messages = [] @@ -116,6 +191,8 @@ def _parse_file_header(self): # Larger fields are explicitly little endian from SDK header_size, protocol_ver_enc, profile_ver_enc, data_size = self._read_struct('2BHI4x', data=header_data) + self._write(header_data) + # Decode the same way the SDK does self.protocol_version = float("%d.%d" % (protocol_ver_enc >> 4, protocol_ver_enc & ((1 << 4) - 1))) self.profile_version = float("%d.%d" % (profile_ver_enc / 100, profile_ver_enc % 100)) @@ -129,10 +206,12 @@ def _parse_file_header(self): # Consume extra two bytes of header and check CRC self._read_and_assert_crc(allow_zero=True) + self._write_crc() # Consume any extra bytes, since header size "may be increased in # "future to add additional optional information" (from SDK) - self._read(extra_header_size - 2) + unknown = self._read(extra_header_size - 2) + self._write(unknown) # After we've consumed the header, set the bytes left to be read self._bytes_left = data_size @@ -142,6 +221,7 @@ def _parse_message(self): if self._bytes_left <= 0: if not self._complete: self._read_and_assert_crc() + self._write_crc() if self._file.tell() >= self._filesize: self._complete = True @@ -153,11 +233,15 @@ def _parse_message(self): return self._parse_message() header = self._parse_message_header() + self._write_message_header(header) if header.is_definition: message = self._parse_definition_message(header) + self._write_definition_message(message) else: message = self._parse_data_message(header) + adjust_message(message) + self._write_data_message(message) if message.mesg_type is not None: if message.mesg_type.name == 'developer_data_id': add_dev_data_id(message) @@ -185,6 +269,18 @@ def _parse_message_header(self): time_offset=None, ) + def _write_message_header(self, header): + data = 0 + if header.time_offset is not None: + data |= 0x80 + data |= header.local_mesg_num << 5 + data |= header.time_offset + else: + data |= 0x40 if header.is_definition else 0 + data |= 0x20 if header.is_developer_data else 0 + data |= header.local_mesg_num + self._write_struct((data,), 'B') + def _parse_definition_message(self, header): # Read reserved byte and architecture byte to resolve endian endian = '>' if self._read_struct('xB') else '<' @@ -242,8 +338,20 @@ def _parse_definition_message(self, header): dev_field_defs=dev_field_defs, ) self._local_mesgs[header.local_mesg_num] = def_mesg + if self._verbose: + print("DefinitionMessage", num_fields,len(dev_field_defs)) return def_mesg + def _write_definition_message(self, msg): + self._write_struct((0, msg.mesg_num, len(msg.field_defs)), 'HHB') + for fld in msg.field_defs: + self._write_struct((fld.def_num, fld.size, fld.base_type.identifier), '3B') + if msg.header.is_developer_data: + self._write_struct(len(msg.dev_field_defs), 'B') + for fld in msg.dev_field_defs: + self._write_struct((fld.def_num, fld.size, fld.dev_data_index), '3B') + + def _parse_raw_values_from_data_message(self, def_mesg): # Go through mesg's field defs and read them raw_values = [] @@ -271,8 +379,32 @@ def _parse_raw_values_from_data_message(self, def_mesg): raw_value = base_type.parse(raw_value) raw_values.append(raw_value) + if self._verbose: + print("read ", field_def, struct_fmt, raw_value) return raw_values + def _write_raw_values_from_data_message(self, def_mesg, raw_values): + for field_def, raw_value in zip(def_mesg.field_defs + def_mesg.dev_field_defs, raw_values): + base_type = field_def.base_type + is_byte = base_type.name == 'byte' + # Struct to read n base types (field def size / base type size) + struct_fmt = '%d%s' % ( + field_def.size / base_type.size, + base_type.fmt, + ) + if is_byte and raw_value is None: + raw_value = tuple(base_type.unparse(raw_value)*field_def.size) + else: + raw_value = base_type.unparse(raw_value) + if self._verbose: + print("write ", field_def, struct_fmt, raw_value) + try: + self._write_struct(raw_value, struct_fmt, endian=def_mesg.endian) + except: + print("Error in _write_struct:", raw_value, struct_fmt, def_mesg.endian) + print(sys.exc_info()[0]) + raise + @staticmethod def _resolve_subfield(field, def_mesg, raw_values): # Resolve into (field, parent) ie (subfield, field) or (field, None) @@ -408,12 +540,25 @@ def _parse_data_message(self, header): data_message = DataMessage(header=header, def_mesg=def_mesg, fields=field_datas) self._processor.run_message_processor(data_message) + if self._verbose: + print("DataMessage", len(field_datas)) return data_message + def _write_data_message(self, msg): + raw_values = [] + for fld in msg.fields: + if fld.field_def is not None: + raw_values.append(fld.raw_value) + + self._write_raw_values_from_data_message(msg.def_mesg, raw_values) + + + ########## # Public API - def get_messages(self, name=None, with_definitions=False, as_dict=False): + def get_messages(self, name=None, with_definitions=False, as_dict=False, verbose=False): + self._verbose=verbose if with_definitions: # with_definitions implies as_dict=False as_dict = False diff --git a/fitparse/records.py b/fitparse/records.py index 9924e68..4d084a4 100644 --- a/fitparse/records.py +++ b/fitparse/records.py @@ -6,6 +6,12 @@ int_types = (int, long,) except NameError: int_types = (int,) +try: + num_types = (int, float, long) + str = basestring +except NameError: + num_types = (int, float) + try: from itertools import zip_longest @@ -174,6 +180,38 @@ def __init__(self, *args, **kwargs): # NOTE:Not a property since you may want to override this in a data processor self.units = self.field.units + def _decode_raw_value(self, raw_value): + # Apply numeric transformations (scale+offset) + if isinstance(raw_value, tuple): + # Contains multiple values, apply transformations to all of them + return tuple(self._decode_raw_value(x) for x in raw_value) + elif isinstance(raw_value, num_types): + if self.field.scale: + raw_value = float(raw_value) / self.field.scale + if self.field.offset: + raw_value = raw_value - self.field.offset + return raw_value + + def decode_raw_value(self): + return self._decode_raw_value(self.raw_value) + + def _compute_raw_value(self, value): + # Apply numeric transformations (scale+offset) + if isinstance(value, tuple): + # Contains multiple values, apply transformations to all of them + return tuple(self._compute_raw_value(x) for x in value) + elif isinstance(value, num_types): + if self.field.scale: + value = float(value) * self.field.scale + if self.field.offset: + value = value + self.field.offset + return value + + def set_value(self, val): + self.value = val + self.raw_value = self._compute_raw_value(val) + + @property def name(self): return self.field.name if self.field else 'unknown_%d' % self.def_num @@ -236,7 +274,7 @@ def __str__(self): class BaseType(RecordBase): - __slots__ = ('name', 'identifier', 'fmt', 'parse') + __slots__ = ('name', 'identifier', 'fmt', 'parse', 'unparse') values = None # In case we're treated as a FieldType @property @@ -288,7 +326,6 @@ class Field(FieldAndSubFieldBase): __slots__ = ('name', 'type', 'def_num', 'scale', 'offset', 'units', 'components', 'subfields') field_type = 'field' - class SubField(FieldAndSubFieldBase): __slots__ = ('name', 'def_num', 'type', 'scale', 'offset', 'units', 'components', 'ref_fields') field_type = 'subfield' @@ -339,23 +376,30 @@ def parse_string(string): return string[:end].decode('utf-8', errors='replace') or None +def unparse_string(string): + if string is None: + return '\x00' + else: + return string.encode('utf-8') + '\x00' + + # The default base type -BASE_TYPE_BYTE = BaseType(name='byte', identifier=0x0D, fmt='B', parse=lambda x: None if all(b == 0xFF for b in x) else x) +BASE_TYPE_BYTE = BaseType(name='byte', identifier=0x0D, fmt='B', parse=lambda x: None if all(b == 0xFF for b in x) else x, unparse=lambda x: [0xFF] if x is None else x) BASE_TYPES = { - 0x00: BaseType(name='enum', identifier=0x00, fmt='B', parse=lambda x: None if x == 0xFF else x), - 0x01: BaseType(name='sint8', identifier=0x01, fmt='b', parse=lambda x: None if x == 0x7F else x), - 0x02: BaseType(name='uint8', identifier=0x02, fmt='B', parse=lambda x: None if x == 0xFF else x), - 0x83: BaseType(name='sint16', identifier=0x83, fmt='h', parse=lambda x: None if x == 0x7FFF else x), - 0x84: BaseType(name='uint16', identifier=0x84, fmt='H', parse=lambda x: None if x == 0xFFFF else x), - 0x85: BaseType(name='sint32', identifier=0x85, fmt='i', parse=lambda x: None if x == 0x7FFFFFFF else x), - 0x86: BaseType(name='uint32', identifier=0x86, fmt='I', parse=lambda x: None if x == 0xFFFFFFFF else x), - 0x07: BaseType(name='string', identifier=0x07, fmt='s', parse=parse_string), - 0x88: BaseType(name='float32', identifier=0x88, fmt='f', parse=lambda x: None if math.isnan(x) else x), - 0x89: BaseType(name='float64', identifier=0x89, fmt='d', parse=lambda x: None if math.isnan(x) else x), - 0x0A: BaseType(name='uint8z', identifier=0x0A, fmt='B', parse=lambda x: None if x == 0x0 else x), - 0x8B: BaseType(name='uint16z', identifier=0x8B, fmt='H', parse=lambda x: None if x == 0x0 else x), - 0x8C: BaseType(name='uint32z', identifier=0x8C, fmt='I', parse=lambda x: None if x == 0x0 else x), + 0x00: BaseType(name='enum', identifier=0x00, fmt='B', parse=lambda x: None if x == 0xFF else x, unparse=lambda x: 0xFF if x is None else x), + 0x01: BaseType(name='sint8', identifier=0x01, fmt='b', parse=lambda x: None if x == 0x7F else x, unparse=lambda x: 0x7F if x is None else x), + 0x02: BaseType(name='uint8', identifier=0x02, fmt='B', parse=lambda x: None if x == 0xFF else x, unparse=lambda x: 0xFF if x is None else x), + 0x83: BaseType(name='sint16', identifier=0x83, fmt='h', parse=lambda x: None if x == 0x7FFF else x, unparse=lambda x: 0x7FFF if x is None else x), + 0x84: BaseType(name='uint16', identifier=0x84, fmt='H', parse=lambda x: None if x == 0xFFFF else x, unparse=lambda x: 0xFFFF if x is None else x), + 0x85: BaseType(name='sint32', identifier=0x85, fmt='i', parse=lambda x: None if x == 0x7FFFFFFF else x, unparse=lambda x: 0x7FFFFFFF if x is None else x), + 0x86: BaseType(name='uint32', identifier=0x86, fmt='I', parse=lambda x: None if x == 0xFFFFFFFF else x, unparse=lambda x: 0xFFFFFFFF if x is None else x), + 0x07: BaseType(name='string', identifier=0x07, fmt='s', parse=parse_string, unparse=unparse_string), + 0x88: BaseType(name='float32', identifier=0x88, fmt='f', parse=lambda x: None if math.isnan(x) else x, unparse=lambda x: float('nan') if x is None else x), + 0x89: BaseType(name='float64', identifier=0x89, fmt='d', parse=lambda x: None if math.isnan(x) else x, unparse=lambda x: float('nan') if x is None else x), + 0x0A: BaseType(name='uint8z', identifier=0x0A, fmt='B', parse=lambda x: None if x == 0x0 else x, unparse=lambda x: 0x0 if x is None else x), + 0x8B: BaseType(name='uint16z', identifier=0x8B, fmt='H', parse=lambda x: None if x == 0x0 else x, unparse=lambda x: 0x0 if x is None else x), + 0x8C: BaseType(name='uint32z', identifier=0x8C, fmt='I', parse=lambda x: None if x == 0x0 else x, unparse=lambda x: 0x0 if x is None else x), 0x0D: BASE_TYPE_BYTE, } diff --git a/scripts/fitdump b/scripts/fitdump index 95a7e76..3a1f75b 100755 --- a/scripts/fitdump +++ b/scripts/fitdump @@ -60,6 +60,7 @@ def parse_args(args=None): parser.error('Please specify an output file (-o) or set --type readable') options.with_defs = (options.verbose >= 1) + options.vverbose = (options.verbose >= 1) options.print_messages = (options.type == 'readable') options.print_stream = (options.output or sys.stdout) @@ -77,10 +78,12 @@ def main(args=None): options.infile, data_processor=fitparse.StandardUnitsDataProcessor(), check_crc = not(options.ignore_crc), + out = open(options.infile.name + "out.fit", "wb") ) messages = fitfile.get_messages( name=options.name, with_definitions=options.with_defs, + verbose=options.vverbose, ) for n, message in enumerate(messages, 1):