I'm using Quickdaq to receive data from my sensor; the output is an .hpf file
I want to read data from this file in python & matplotlib to apply some signal analysis.
How can I read (or convert to .csv) an .hpf file with python (or any other language)?
I found this on internet but i have no idea how to use it, if it's the latest solution?
I'm guessing this is a bit out of date for the original poster, however should anyone else have this problem, I have just written a script to extract information and data from these files, however it is only tested on single channel data, I don't have multi channel data to test it on.
https://github.com/jogrundy/Read_QuickDAQ_.hpf
This started as a translation of the MATLAB code from https://www.datatranslation.de/Download/Demos/Hpf2Matlab_Rev2_0.m
Hope this helps.
The QuickDAQ software engineers seem to have provided a solution, see this python script in the downloads section of the QuickDAQ homepage.
The exact specifications for the .hpf file type are given by Data Translation at this pdf link.
In case the link to the python script becomes broken in the future, the conversion script written by Michael Buergin is reproduced below.
###########################################################################
#
# Author: Measurement Computing GmbH
# Michael Buergin
# Date: 02.11.2017
# Version: 1.0
# Description:
# Python Converter for Data Translation's HPF-file format.
# Can convert a single file or a complete directory
# REMARKS:
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Changes: --
#
#
# ConvertHPFData_Folder
# Input Parameter :
# folder_in = Complete path to the folder with HPF-file
# folder_out = Complete path to store ASCII-files (if None; new "ASCII"-folder is created in folder_in)
# suppress_print = suppress print of hpf-file-data
#
# WARNING: existing txt-files will be overwritten!
#
# ConvertHPFData
# Input Parameter :
# InputFile = Complete path to the input HPF-file.
# OutputFile = Complete path to the output ASCII-file.
# SelectedChannel = Channel to read from the HPF-file. 1 as first channel.
# suppress_print = suppress print of hpf-file-data
# Output Data:
# ReturnData = Matrix with all Data for the selected channel
# OutputFile = ASCII-file with all Data for the selected channel
###########################################################################
import numpy as np
import struct
import os
def ConvertHPFData_Folder(folder_in=None, folder_out=None, suppress_print=True, SelectedChannel=None):
if folder_out == None: folder_out=folder_in + '\\ASCII'
all_hpf_files = [fn for fn in os.listdir(folder_in)
if any(fn.endswith(ext) for ext in 'hfp')]
if not os.path.isdir(folder_out): os.mkdir(folder_out)
i_file = 1
for file in all_hpf_files:
ConvertHPFData(SelectedChannel='all',
InputFile=folder_in+'\\'+file,
OutputFile=folder_out+'\\'+file[:-3]+'txt',
suppress_print=suppress_print)
print('last file: %s (%i/%i)' % (file, i_file, len(all_hpf_files)))
i_file +=1
print('all done :)')
def ConvertHPFData(InputFile=None,OutputFile=None,SelectedChannel=None,suppress_print=False):
if not suppress_print: print('-- Start')
ReturnData=[]
with open(InputFile, mode='rb') as in_file: # b -> binary
fileContent = in_file.read()
if OutputFile!=None: out_file = open(OutputFile, 'w')
if SelectedChannel == 'all':
SelectedChannelAll = 1
ChannelError = False
while ChannelError == False:
[result_tmp, result_name_tmp, ChannelError] = ReadHPFData(SelectedChannel=SelectedChannelAll,
fileContent=fileContent,
out_file=out_file,
OutputFile=OutputFile,
AllCh=True,
suppress_print=True)
if ChannelError == False:
if SelectedChannelAll==1:
ReturnData = result_tmp
Result_Name = [result_name_tmp]
else:
ReturnData = np.append(np.array(ReturnData), np.array([result_tmp[1,:]]), axis=0)
Result_Name.append(result_name_tmp)
if not suppress_print: print(Result_Name)
SelectedChannelAll += 1
else:
[ReturnData, Result_Name, ChannelError] = ReadHPFData(SelectedChannel=SelectedChannel,
fileContent=fileContent,
out_file=out_file,
OutputFile=OutputFile,
AllCh=False)
Result_Name = [Result_Name]
in_file.close()
if OutputFile!=None:
if SelectedChannel != 'all' and ChannelError: out_file.close()
else:
out_file.write('time\t')
for i_channel in range(len(Result_Name)):
out_file.write(Result_Name[i_channel]+'\t')
out_file.write('\n')
for i_ReturnData in range(len(ReturnData[0, :])):
for i_channel in range(len(ReturnData[:, 0])):
out_file.write(('%f\t') % (ReturnData[i_channel, i_ReturnData]))
out_file.write('\n')
out_file.close()
if not suppress_print: print('-- Finish')
return ReturnData
def char_tuple_2_str(tuple):
string = ""
for chars in tuple:
string = string + chars.decode('utf-8')
return string
def find_all_str(string,sub):
index = string.find(sub)
if index == -1: return [-1]
indices = [index]
while True:
index = string.find(sub, index+1)
if index == -1: return indices
indices.append(index)
return indices
def ReadHPFData(SelectedChannel=None,fileContent=None, out_file=None, OutputFile=None, AllCh=None, suppress_print=False):
if AllCh and SelectedChannel == 1:
flag_write = True
else:
flag_write = False
ReturnData=[]
pos = 0
off = 0
i_line = 0
while i_line <= len(fileContent) - 1:
in_chunkID = struct.unpack('q', fileContent[i_line:i_line + 8])[0];
i_line = i_line + 8
off = struct.unpack('q', fileContent[i_line:i_line + 8])[0];
i_line = i_line + 8
if i_line == len(fileContent) or off == 0:
break
if 4096 == in_chunkID:
if not suppress_print: print('- Parsing Header Chunk')
in_ChratorID = char_tuple_2_str(struct.unpack('c' * 4, fileContent[i_line:i_line + 4]));
i_line = i_line + 4
in_fileVersion = struct.unpack('q', fileContent[i_line:i_line + 8])[0];
i_line = i_line + 8
in_indexChunkOffset = struct.unpack('q', fileContent[i_line:i_line + 8])[0];
i_line = i_line + 8
len_in_XMLdata = off - (i_line - pos)
in_XMLdata = char_tuple_2_str(
struct.unpack('c' * len_in_XMLdata, fileContent[i_line:i_line + len_in_XMLdata]));
i_line = i_line + len_in_XMLdata
start = in_XMLdata.find('<RecordingDate>')
stop = in_XMLdata.find('</RecordingDate>')
Information = in_XMLdata[start + 15:stop - 1]
if not suppress_print: print('RecordingDate: ' + Information)
if OutputFile != None and flag_write: out_file.write('RecordingDate: ' + Information + '\n')
Result_RecordingDate = Information
else:
if 8192 == in_chunkID:
if not suppress_print: print('- Parsing Channel Information Chunk')
in_groupID = struct.unpack('l', fileContent[i_line:i_line + 4])[0];
i_line = i_line + 4
in_numberOfChannals = struct.unpack('l', fileContent[i_line:i_line + 4])[0];
i_line = i_line + 4
len_in_XMLdata = off - (i_line - pos)
in_XMLdata = char_tuple_2_str(
struct.unpack('c' * len_in_XMLdata, fileContent[i_line:i_line + len_in_XMLdata]));
i_line = i_line + len_in_XMLdata
start = find_all_str(in_XMLdata, ('<Name>'))
stop = find_all_str(in_XMLdata, ('</Name>'))
if len(start) < SelectedChannel or SelectedChannel<=0:
if AllCh == False:
if not suppress_print: print('\nWrong channel selected!')
if not suppress_print: print('\npossible channels: %i ... %i' % (1, len(start)))
if OutputFile != None:
out_file.write('\nWrong channel selected!')
out_file.write('\npossible channels: %i ... %i' % (1, len(start)))
return 0, 0, True
if OutputFile != None: out_file.write('Total Channel: ' + str(len(start)) + '\n')
if not suppress_print: print('Total Channel: ', str(len(start)))
Ch_offset = np.zeros(len(start))
Ch_length = np.zeros(len(start))
# Store Channel in return struct
Result_Channel = SelectedChannel
if not suppress_print: print('Selected Channel Information:')
if OutputFile != None: out_file.write('Selected Channel Information:\n')
Information = in_XMLdata[start[SelectedChannel - 1] + 6:stop[SelectedChannel - 1]]
if not suppress_print: print('Name: ', Information)
if OutputFile != None: out_file.write('Name: ' + Information + '\n')
Result_Name = Information
start = find_all_str(in_XMLdata, ('<Unit>'))
stop = find_all_str(in_XMLdata, ('</Unit>'))
Information = in_XMLdata[start[SelectedChannel - 1] + 6:stop[SelectedChannel - 1]]
if not suppress_print: print('Unit: ', Information)
if OutputFile != None: out_file.write('Unit: ' + Information + '\n')
Result_Unit = Information
start = find_all_str(in_XMLdata, ('<ChannelType>'))
stop = find_all_str(in_XMLdata, ('</ChannelType>'))
Information = in_XMLdata[start[SelectedChannel - 1] + 13:stop[SelectedChannel - 1]]
if not suppress_print: print('ChannelType: ', Information)
if OutputFile != None: out_file.write('ChannelType: ' + Information + '\n')
Result_ChannelType = Information
start = find_all_str(in_XMLdata, ('<AssignedTimeChannelIndex>'))
stop = find_all_str(in_XMLdata, ('</AssignedTimeChannelIndex>'))
Information = in_XMLdata[start[SelectedChannel - 1] + 26:stop[SelectedChannel - 1]]
if not suppress_print: print('AssignedTimeChannelIndex: ', Information)
if OutputFile != None: out_file.write('AssignedTimeChannelIndex: ' + Information + '\n')
Result_AssignedTimeChannelIndex = Information
start = find_all_str(in_XMLdata, ('<DataType>'))
stop = find_all_str(in_XMLdata, ('</DataType>'))
Information = in_XMLdata[start[SelectedChannel - 1] + 10:stop[SelectedChannel - 1]]
if not suppress_print: print('DataType: ', Information)
if OutputFile != None: out_file.write('DataType: ' + Information + '\n')
Result_DataType = Information
start = find_all_str(in_XMLdata, ('<DataIndex>'))
stop = find_all_str(in_XMLdata, ('</DataIndex>'))
Information = in_XMLdata[start[SelectedChannel - 1] + 11:stop[SelectedChannel - 1]]
if not suppress_print: print('DataIndex: ', Information)
if OutputFile != None: out_file.write('DataIndex: ' + Information + '\n')
Result_DataIndex = float(Information)
start = find_all_str(in_XMLdata, ('<StartTime>'))
stop = find_all_str(in_XMLdata, ('</StartTime>'))
Information = in_XMLdata[start[SelectedChannel - 1] + 11:stop[SelectedChannel - 1]]
if not suppress_print: print('StartTime: ', Information)
if OutputFile != None: out_file.write('StartTime: ' + Information + '\n')
Result_StartTime = Information
start = find_all_str(in_XMLdata, ('<TimeIncrement>'))
stop = find_all_str(in_XMLdata, ('</TimeIncrement>'))
Information = in_XMLdata[start[SelectedChannel - 1] + 15:stop[SelectedChannel - 1]]
if not suppress_print: print('TimeIncrement: ', Information)
if OutputFile != None: out_file.write('TimeIncrement: ' + Information + '\n')
Result_TimeIncrement = float(Information)
start = find_all_str(in_XMLdata, ('<RangeMin>'))
stop = find_all_str(in_XMLdata, ('</RangeMin>'))
Information = in_XMLdata[start[SelectedChannel - 1] + 10:stop[SelectedChannel - 1]]
if not suppress_print: print('RangeMin: ', Information)
if OutputFile != None: out_file.write('RangeMin: ' + Information + '\n')
Result_RangeMin = float(Information)
start = find_all_str(in_XMLdata, ('<RangeMax>'))
stop = find_all_str(in_XMLdata, ('</RangeMax>'))
Information = in_XMLdata[start[SelectedChannel - 1] + 10:stop[SelectedChannel - 1]]
if not suppress_print: print('RangeMax: ', Information)
if OutputFile != None: out_file.write('RangeMax: ' + Information + '\n')
Result_RangeMax = float(Information)
start = find_all_str(in_XMLdata, ('<DataScale>'))
stop = find_all_str(in_XMLdata, ('</DataScale>'))
Information = in_XMLdata[start[SelectedChannel - 1] + 11:stop[SelectedChannel - 1]]
if not suppress_print: print('DataScale: ', Information)
if OutputFile != None: out_file.write('DataScale: ' + Information + '\n')
Result_DataScale = float(Information)
start = find_all_str(in_XMLdata, ('<DataOffset>'))
stop = find_all_str(in_XMLdata, ('</DataOffset>'))
Information = in_XMLdata[start[SelectedChannel - 1] + 12:stop[SelectedChannel - 1]]
if not suppress_print: print('DataOffset: ', Information)
if OutputFile != None: out_file.write('DataOffset: ' + Information + '\n')
Result_DataOffset = float(Information)
start = find_all_str(in_XMLdata, ('<SensorScale>'))
stop = find_all_str(in_XMLdata, ('</SensorScale>'))
Information = in_XMLdata[start[SelectedChannel - 1] + 13:stop[SelectedChannel - 1]]
Sens_Scale = Information
if not suppress_print: print('SensorScale: ', Information)
if OutputFile != None: out_file.write('SensorScale: ' + Information + '\n')
Result_SensorScale = float(Sens_Scale)
start = find_all_str(in_XMLdata, ('<SensorOffset>'))
stop = find_all_str(in_XMLdata, ('</SensorOffset>'))
Information = in_XMLdata[start[SelectedChannel - 1] + 14:stop[SelectedChannel - 1]]
SensorOffset = Information
if not suppress_print: print('SensorOffset: ', Information)
if OutputFile != None: out_file.write('SensorOffset: ' + Information + '\n')
Result_SensorOffset = float(SensorOffset)
start = find_all_str(in_XMLdata, ('<PerChannelSampleRate>'))
stop = find_all_str(in_XMLdata, ('</PerChannelSampleRate>'))
Information = in_XMLdata[start[SelectedChannel - 1] + 22:stop[SelectedChannel - 1]]
if not suppress_print: print('PerChannelSampleRate: ', Information)
if OutputFile != None: out_file.write('PerChannelSampleRate: ' + Information + '\n')
Result_PerChannelSampleRate = Information
start = find_all_str(in_XMLdata, ('<PhysicalChannelNumber>'))
stop = find_all_str(in_XMLdata, ('</PhysicalChannelNumber>'))
Information = in_XMLdata[start[SelectedChannel - 1] + 23:stop[SelectedChannel - 1]]
if not suppress_print: print('PhysicalChannelNumber: ', Information)
if OutputFile != None: out_file.write('PhysicalChannelNumber: ' + Information + '\n\n')
Result_PhysicalChannelNumber = Information
else:
if 12288 == in_chunkID:
if not suppress_print: print('- Parsing Data Chunk')
in_chunksize = off
in_groupID = struct.unpack('l', fileContent[i_line:i_line + 4])[0];
i_line = i_line + 4
in_dataStartIndex = struct.unpack('q', fileContent[i_line:i_line + 8])[0];
i_line = i_line + 8
in_chanalDataCount = struct.unpack('l', fileContent[i_line:i_line + 4])[0];
i_line = i_line + 4
for count in range(in_chanalDataCount):
if count + 1 == (SelectedChannel):
Ch_offset[count] = int(struct.unpack('l', fileContent[i_line:i_line + 4])[0]);
i_line = i_line + 4
Ch_length[count] = int(struct.unpack('l', fileContent[i_line:i_line + 4])[0]);
i_line = i_line + 4
else:
i_line = i_line + 8
ignoreCount = 0
for count in range(in_chanalDataCount):
if count + 1 == (SelectedChannel):
divider = 8
i_line = i_line + int(Ch_offset[count]) - int(32 + (in_chanalDataCount * 8))
tmpData = list(struct.unpack('d' * int(Ch_length[count] / divider),
fileContent[int(i_line):int(i_line + Ch_length[count])]));
i_line = i_line + int(Ch_length[count])
# Append Data
ReturnData.extend(tmpData)
else:
ignoreCount = ignoreCount + 1
else:
if 28672 == in_chunkID:
if not suppress_print: print('- Parsing Undocumented Part')
i_line = i_line + 8
Position = off - (i_line - pos)
i_line = i_line + Position
pos = i_line
ReturnData = np.array(ReturnData) * Result_SensorScale + Result_SensorOffset
Result_Time = np.array(float(Result_TimeIncrement)*(np.array(range(0,len(ReturnData)))))
ReturnData = np.append([Result_Time], [ReturnData], axis=0)
return ReturnData, Result_Name, False
if __name__ == '__main__':
ConvertHPFData_Folder(folder_in=r"C:\Users\mbuergin\AppData\Local\Data Translation\QuickDAQ\Data",
folder_out=r"C:\Users\mbuergin\AppData\Local\Data Translation\QuickDAQ\Data",
SelectedChannel='all')
