1
0
mirror of https://bitbucket.org/librepilot/librepilot.git synced 2025-01-17 02:52:12 +01:00

Further optimized Matlab parsing. It now does most typecast as vectorized operations.

The upshot is an approximately 50% performance boost.
This commit is contained in:
Laura Sebesta 2012-08-16 00:13:19 +02:00
parent 63e490a922
commit 69133c1427
3 changed files with 189 additions and 95 deletions

View File

@ -59,45 +59,50 @@ $(ALLOCATIONCODE)
fid = fopen(logfile);
buffer=fread(fid,Inf,'uchar=>uchar');
fseek(fid, 0, 'bof');
bufferIdx=1;
correctMsgByte=hex2dec('20');
correctSyncByte=hex2dec('3C');
unknownObjIDList=zeros(1,2);
% Parse log file, entry by entry
prebuf = fread(fid, 12, 'uint8');
log_size = dir(logfile);
log_size = log_size.bytes;
last_print = 0;
% prebuf = buffer(1:12);
last_print = -1e10;
startTime=clock;
while (1)
if (feof(fid)); break; end
%% Read message header
% get sync field (0x3C, 1 byte)
sync = fread(fid, 1, 'uint8');
if sync ~= correctSyncByte
prebuf = [prebuf(2:end); sync];
wrongSyncByte = wrongSyncByte + 1;
continue
end
%% Process header if we are aligned
timestamp = typecast(uint8(prebuf(1:4)), 'uint32');
datasize = typecast(uint8(prebuf(5:12)), 'uint64');
sync = buffer(bufferIdx+12);
if buffer(bufferIdx+12) ~= correctSyncByte
bufferIdx=bufferIdx+1;
wrongSyncByte = wrongSyncByte + 1;
continue
end
%% Process header, if we are aligned
% timestamp = typecast(buffer(bufferIdx:bufferIdx + 4-1), 'uint32'); %We do this one later
datasizeBufferIdx = bufferIdx; %Just grab the index. We'll do a typecast later, if necessary
% sync = buffer(bufferIdx+12); %This one has already been done
msgType = buffer(bufferIdx+13); % get msg type (quint8 1 byte ) should be 0x20, ignore the rest?
% msgSize = typecast(buffer(bufferIdx+14:bufferIdx+ 14+2-1), 'uint16'); % NOT USED: get msg size (quint16 2 bytes) excludes crc, include msg header and data payload
objID = typecast(buffer(bufferIdx+16:bufferIdx+ 16+4-1), 'uint32'); % get obj id (quint32 4 bytes)
% get msg type (quint8 1 byte ) should be 0x20, ignore the rest?
msgType = fread(fid, 1, 'uint8');
%Advance buffer past header
bufferIdx=bufferIdx+20;
%Check that message type is correct
if msgType ~= correctMsgByte
wrongMessageByte = wrongMessageByte + 1;
continue
end
% get msg size (quint16 2 bytes) excludes crc, include msg header and data payload
msgSize = fread(fid, 1, 'uint16');
% get obj id (quint32 4 bytes)
objID = fread(fid, 1, 'uint32');
if (isempty(objID)) %End of file
break;
@ -110,15 +115,18 @@ $(SWITCHCODE)
otherwise
unknownObjIDListIdx=find(unknownObjIDList(:,1)==objID, 1, 'first');
if isempty(unknownObjIDListIdx)
unknownObjIDList=[unknownObjIDList; objID 1];
unknownObjIDList=[unknownObjIDList; uint32(objID) 1]; %#ok<AGROW>
else
unknownObjIDList(unknownObjIDListIdx,2)=unknownObjIDList(unknownObjIDListIdx,2)+1;
unknownObjIDList(unknownObjIDListIdx,2)=unknownObjIDList(unknownObjIDListIdx,2)+1;
end
datasize = typecast(buffer(datasizeBufferIdx + 4:datasizeBufferIdx + 12-1), 'uint64');
msgBytesLeft = datasize - 1 - 1 - 2 - 4;
if msgBytesLeft > 255
msgBytesLeft = 0;
end
fread(fid, msgBytesLeft, 'uint8');
bufferIdx=bufferIdx+msgBytesLeft;
end
catch
% One of the reads failed - indicates EOF
@ -126,42 +134,41 @@ $(SWITCHCODE)
end
if (wrongSyncByte ~= lastWrongSyncByte || wrongMessageByte~=lastWrongMessageByte ) ||...
(ftell(fid) / log_size - last_print) > 0.01
bufferIdx - last_print > 5e4 %Every 50,000 bytes show the status update
lastWrongSyncByte=wrongSyncByte;
lastWrongMessageByte=wrongMessageByte;
str1=[];
for i=1:length([str2 str3 str4 str5]);
str1=[str1 sprintf('\b')];
str1=[str1 sprintf('\b')]; %#ok<AGROW>
end
str2=sprintf('wrongSyncByte instances: % 10d\n', wrongSyncByte );
str3=sprintf('wrongMessageByte instances: % 10d\n\n', wrongMessageByte );
str4=sprintf('Completed bytes: % 9d of % 9d\n', ftell(fid), log_size);
str4=sprintf('Completed bytes: % 9d of % 9d\n', bufferIdx, length(buffer));
% Arbitrary times two so that it is at least as long
estTimeRemaining=(log_size-ftell(fid))/(ftell(fid)/etime(clock,startTime)) * 2;
estTimeRemaining=(length(buffer)-bufferIdx)/(bufferIdx/etime(clock,startTime)) * 2;
h=floor(estTimeRemaining/3600);
m=floor((estTimeRemaining-h*3600)/60);
s=ceil(estTimeRemaining-h*3600-m*60);
str5=sprintf('Est. time remaining, %02dh:%02dm:%02ds \n', h,m,s);
last_print = ftell(fid) / log_size;
last_print = bufferIdx;
fprintf([str1 str2 str3 str4 str5]);
end
%Check if at end of file. If not, load next prebuffer
if bufferIdx+12-1 > length(buffer)
break;
end
% bufferIdx=bufferIdx+12;
prebuf = fread(fid, 12, 'uint8');
end
fprintf('%d records in %0.2f seconds.\n', ftell(fid), etime(clock,startTime));
for i=2:size(unknownObjIDList,1) %Don't show the first one, as it was simply a dummy placeholder
disp(['Unknown object ID: 0x' dec2hex(unknownObjIDList(i,1),8) ' appeared ' int2str(unknownObjIDList(i,2)) ' times.']);
end
for i=2:size(unknownObjIDList,1) %Don't show the first one, as it was simply a dummy placeholder
disp(['Unknown object ID: 0x' dec2hex(unknownObjIDList(i,1),8) ' appeared ' int2str(unknownObjIDList(i,2)) ' times.']);
@ -170,28 +177,23 @@ end
%% Clean Up and Save mat file
fclose(fid);
% Trim output structs
%% Prune vectors
$(CLEANUPCODE)
%% Perform typecasting on vectors
$(FUNCTIONSCODE)
if strcmpi(outputType,'mat')
matfile = strrep(logfile,'opl','mat');
save(matfile $(SAVEOBJECTSCODE));
else
$(EXPORTCSVCODE);
$(EXPORTCSVCODE)
end
fprintf('%d records in %0.2f seconds.\n', length(buffer), etime(clock,startTime));
%% Object reading functions
$(FUNCTIONSCODE)
% This function prunes the excess pre-allocated space
function [structOut]=PruneStructOfArrays(structIn, lastIndex)
fieldNames = fieldnames(structIn);
for i=1:length(fieldNames)
structOut.(fieldNames{i})=structIn.(fieldNames{i})(:,1:lastIndex);
end
function OPLog2csv(structIn, structName, logfile)
@ -229,3 +231,28 @@ function crc = compute_crc(data)
crc = crc_table(1+bitxor(data(i),crc));
end
function out=mcolon(inStart, inFinish)
%% This function was inspired by Bruno Luong's 'mcolon'. The name is kept the same as his 'mcolon'
% function, found on Matlab's file exchange. The two functions return identical
% results, although his is much faster. Unfortunately, C-compiled mex
% code would make this function non-cross-platform, so a Matlab scripted
% version is provided here.
if size(inStart,1) > 1 || size(inFinish,1) > 1
if size(inStart,2) > 1 || size(inFinish,2) > 1
error('Inputs must be vectors, i.e just one column wide.')
else
inStart=inStart';
inFinish=inFinish';
end
end
diffIn=diff([inStart; inFinish]);
numElements=sum(diffIn)+length(inStart);
out=zeros(1,numElements);
idx=1;
for i=1:length(inStart)
out(idx:idx+diffIn(i))=inStart(i):inFinish(i);
idx=idx+diffIn(i)+1;
end

View File

@ -31,7 +31,9 @@ using namespace std;
bool UAVObjectGeneratorMatlab::generate(UAVObjectParser* parser,QString templatepath,QString outputpath) {
fieldTypeStrMatlab << "int8" << "int16" << "int32"
<< "uint8" << "uint16" << "uint32" << "float32" << "uint8";
<< "uint8" << "uint16" << "uint32" << "single" << "uint8";
fieldSizeStrMatlab << "1" << "2" << "4"
<< "1" << "2" << "4" << "4" << "1";
QDir matlabTemplatePath = QDir( templatepath + QString("ground/openpilotgcs/src/plugins/uavobjects"));
QDir matlabOutputPath = QDir( outputpath + QString("matlab") );
@ -46,7 +48,8 @@ bool UAVObjectGeneratorMatlab::generate(UAVObjectParser* parser,QString template
for (int objidx = 0; objidx < parser->getNumObjects(); ++objidx) {
ObjectInfo* info=parser->getObjectByIndex(objidx);
process_object(info);
int numBytes=parser->getNumBytes(objidx);
process_object(info, numBytes);
}
matlabCodeTemplate.replace( QString("$(ALLOCATIONCODE)"), matlabAllocationCode);
@ -68,7 +71,7 @@ bool UAVObjectGeneratorMatlab::generate(UAVObjectParser* parser,QString template
/**
* Generate the matlab object files
*/
bool UAVObjectGeneratorMatlab::process_object(ObjectInfo* info)
bool UAVObjectGeneratorMatlab::process_object(ObjectInfo* info, int numBytes)
{
if (info == NULL)
return false;
@ -82,12 +85,15 @@ bool UAVObjectGeneratorMatlab::process_object(ObjectInfo* info)
QString functionCall(functionName + "(fid, timestamp, checkCRC, ");
QString objectID(QString().setNum(info->id));
QString isSingleInst = boolTo01String( info->isSingleInst );
QString numBytesString=QString("%1").arg(numBytes);
//===================================================================//
// Generate allocation code (will replace the $(ALLOCATIONCODE) tag) //
//===================================================================//
// matlabSwitchCode.append("\t\tcase " + objectID + "\n");
matlabAllocationCode.append("\n\t" + tableIdxName + " = 0;\n");
QString type;
QString allocfields;
@ -121,6 +127,9 @@ bool UAVObjectGeneratorMatlab::process_object(ObjectInfo* info)
}
matlabAllocationCode.append(allocfields);
matlabAllocationCode.append("\t" + objectTableName.toUpper() + "_OBJID=" + objectID + ";\n");
matlabAllocationCode.append("\t" + objectTableName.toUpper() + "_NUMBYTES=" + numBytesString + ";\n");
matlabAllocationCode.append("\t" + objectName + "FidIdx = [];\n");
//==============================================================//
@ -130,8 +139,8 @@ bool UAVObjectGeneratorMatlab::process_object(ObjectInfo* info)
matlabSwitchCode.append("\t\tcase " + objectTableName.toUpper() + "_OBJID\n");
// matlabSwitchCode.append("\t\t\t" + objectTableName + "(" + tableIdxName +") = " + functionCall + ";\n");
matlabSwitchCode.append("\t\t\t" + tableIdxName + " = " + tableIdxName +" + 1;\n");
matlabSwitchCode.append("\t\t\t" + tableIdxName + " = " + tableIdxName +" + 1;\n");
if(0){
matlabSwitchCode.append("\t\t\t" + objectTableName + "= " + functionCall + objectTableName + ", " + tableIdxName + ");\n");
matlabSwitchCode.append("\t\t\tif " + tableIdxName + " >= length(" + objectTableName +".timestamp) %Check to see if pre-allocated memory is exhausted\n");
matlabSwitchCode.append("\t\t\t\tFieldNames= fieldnames(" + objectTableName +");\n");
@ -139,12 +148,22 @@ bool UAVObjectGeneratorMatlab::process_object(ObjectInfo* info)
matlabSwitchCode.append("\t\t\t\t\t" + objectTableName + ".(FieldNames{i})(:," + tableIdxName + "*2+1) = 0;\n");
matlabSwitchCode.append("\t\t\t\tend\n");
matlabSwitchCode.append("\t\t\tend\n");
} else{
matlabSwitchCode.append("\t\t\t" + objectTableName + "FidIdx(" + tableIdxName + ") = bufferIdx; %#ok<*AGROW>\n");
matlabSwitchCode.append("\t\t\tbufferIdx=bufferIdx + " + objectTableName.toUpper() + "_NUMBYTES+1; %+1 is for CRC\n");
if(!info->isSingleInst){
matlabSwitchCode.append("\t\t\tbufferIdx = bufferIdx + 2; %An extra two bytes for the instance ID\n");
}
matlabSwitchCode.append("\t\t\tif " + tableIdxName + " >= length(" + objectTableName +"FidIdx) %Check to see if pre-allocated memory is exhausted\n");
matlabSwitchCode.append("\t\t\t\t" + objectTableName + "FidIdx(" + tableIdxName + "*2) = 0;\n");
matlabSwitchCode.append("\t\t\tend\n");
}
//============================================================//
// Generate 'Cleanup:' code (will replace the $(CLEANUP) tag) //
//============================================================//
matlabCleanupCode.append(objectTableName + "=PruneStructOfArrays(" + objectTableName + "," + tableIdxName +");\n" );
// //============================================================//
// // Generate 'Cleanup:' code (will replace the $(CLEANUP) tag) //
// //============================================================//
matlabCleanupCode.append(objectTableName + "FidIdx =" + objectTableName + "FidIdx(1:" + tableIdxName +");\n" );
//========================================================================//
@ -163,51 +182,98 @@ bool UAVObjectGeneratorMatlab::process_object(ObjectInfo* info)
// Generate functions code (will replace the $(FUNCTIONSCODE) tag) //
//=================================================================//
//Generate function description comment
matlabFunctionsCode.append("%%\n% " + objectName + " read function\n");
matlabFunctionsCode.append("function [" + objectName + "] = " + functionCall + objectTableName + ", " + tableIdxName + ")" + "\n");
if(0){
matlabFunctionsCode.append("%%\n% " + objectName + " read function\n");
matlabFunctionsCode.append("\t" + objectName + ".timestamp(" + tableIdxName + ")= timestamp;\n");
matlabFunctionsCode.append("\tif " + isSingleInst + "\n");
matlabFunctionsCode.append("\t\theaderSize = 8;\n");
matlabFunctionsCode.append("\telse\n");
matlabFunctionsCode.append("\t\t" + objectName + ".instanceID(" + tableIdxName + ") = (fread(fid, 1, 'uint16'));\n");
matlabFunctionsCode.append("\t\theaderSize = 10;\n");
matlabFunctionsCode.append("\tend\n\n");
matlabFunctionsCode.append("function [" + objectName + "] = " + functionCall + objectTableName + ", " + tableIdxName + ")" + "\n");
// Generate functions code, actual fields of the object
QString funcfields;
matlabFunctionsCode.append("\tstartPos = ftell(fid) - headerSize;\n");
matlabFunctionsCode.append("\t" + objectName + ".timestamp(" + tableIdxName + ")= timestamp;\n");
matlabFunctionsCode.append("\tif " + isSingleInst + "\n");
matlabFunctionsCode.append("\t\theaderSize = 8;\n");
matlabFunctionsCode.append("\telse\n");
matlabFunctionsCode.append("\t\t" + objectName + ".instanceID(" + tableIdxName + ") = (fread(fid, 1, 'uint16'));\n");
matlabFunctionsCode.append("\t\theaderSize = 10;\n");
matlabFunctionsCode.append("\tend\n\n");
for (int n = 0; n < info->fields.length(); ++n) {
// Determine type
type = fieldTypeStrMatlab[info->fields[n]->type];
// Append field
if ( info->fields[n]->numElements > 1 )
funcfields.append("\t" + objectName + "." + info->fields[n]->name + "(:," + tableIdxName + ") = double(fread(fid, " + QString::number(info->fields[n]->numElements, 10) + ", '" + type + "'));\n");
else
funcfields.append("\t" + objectName + "." + info->fields[n]->name + "(" + tableIdxName + ") = double(fread(fid, 1, '" + type + "'));\n");
// Generate functions code, actual fields of the object
QString funcfields;
matlabFunctionsCode.append("\tstartPos = ftell(fid) - headerSize;\n");
for (int n = 0; n < info->fields.length(); ++n) {
// Determine type
type = fieldTypeStrMatlab[info->fields[n]->type];
// Append field
if ( info->fields[n]->numElements > 1 )
funcfields.append("\t" + objectName + "." + info->fields[n]->name + "(:," + tableIdxName + ") = double(fread(fid, " + QString::number(info->fields[n]->numElements, 10) + ", '" + type + "'));\n");
else
funcfields.append("\t" + objectName + "." + info->fields[n]->name + "(" + tableIdxName + ") = double(fread(fid, 1, '" + type + "'));\n");
}
matlabFunctionsCode.append(funcfields);
matlabFunctionsCode.append("\tobjLen = ftell(fid) - startPos;\n");
// matlabFunctionsCode.append(QString("\tobjLen - %1 -headerSize\n").arg(numBytes));
matlabFunctionsCode.append("\t% read CRC\n");
matlabFunctionsCode.append("\tcrc_read = fread(fid, 1, '*uint8');\n");
matlabFunctionsCode.append("\tif checkCRC\n");
matlabFunctionsCode.append("\t\tfseek(fid, startPos, 'bof');\n");
matlabFunctionsCode.append("\t\tcrc_calc = compute_crc(uint8(fread(fid,objLen,'uint8')));\n");
matlabFunctionsCode.append("\t\tfread(fid,1,'uint8');\n");
matlabFunctionsCode.append("\t\tif (crc_calc ~= crc_read)\n");
matlabFunctionsCode.append("\t\t\tdisp('CRC Error')\n");
matlabFunctionsCode.append("\t\t\t" + tableIdxName + " = " + tableIdxName + " - 1;\n");
matlabFunctionsCode.append("\t\tend\n");
matlabFunctionsCode.append("\tend\n");
matlabFunctionsCode.append("\n\n");
}
matlabFunctionsCode.append(funcfields);
else{
matlabFunctionsCode.append("% " + objectName + " typecasting\n");
QString funcfields;
matlabFunctionsCode.append("\tobjLen = ftell(fid) - startPos;\n");
//Add timestamp
funcfields.append("\t" + objectName + ".timestamp = " +
"double(typecast(buffer(mcolon(" + objectName + "FidIdx "
"- 20, " + objectName + "FidIdx + 4-1 -20)), 'uint32'))';\n");
matlabFunctionsCode.append("\t% read CRC\n");
matlabFunctionsCode.append("\tcrc_read = fread(fid, 1, '*uint8');\n");
int currentIdx=0;
matlabFunctionsCode.append("\tif checkCRC\n");
matlabFunctionsCode.append("\t\tfseek(fid, startPos, 'bof');\n");
matlabFunctionsCode.append("\t\tcrc_calc = compute_crc(uint8(fread(fid,objLen,'uint8')));\n");
matlabFunctionsCode.append("\t\tfread(fid,1,'uint8');\n");
matlabFunctionsCode.append("\t\tif (crc_calc ~= crc_read)\n");
matlabFunctionsCode.append("\t\t\tdisp('CRC Error')\n");
matlabFunctionsCode.append("\t\t\t" + tableIdxName + " = " + tableIdxName + " - 1;\n");
matlabFunctionsCode.append("\t\tend\n");
matlabFunctionsCode.append("\tend\n");
//Add instance, if necessary
if(!info->isSingleInst){
funcfields.append("\t" + objectName + ".instanceID = " +
"double(typecast(buffer(mcolon(" + objectName + "FidIdx "
", " + objectName + "FidIdx + 2-1)), 'uint16'))';\n");
currentIdx+=2;
}
matlabFunctionsCode.append("\n\n");
for (int n = 0; n < info->fields.length(); ++n) {
// Determine type
type = fieldTypeStrMatlab[info->fields[n]->type];
//Determine variable length
QString size = fieldSizeStrMatlab[info->fields[n]->type];
// Append field
if ( info->fields[n]->numElements > 1 ){
funcfields.append("\t" + objectName + "." + info->fields[n]->name + " = " +
"reshape(double(typecast(buffer(mcolon(" + objectName + "FidIdx + " + QString("%1").arg(currentIdx) +
", " + objectName + "FidIdx + " + QString("%1").arg(currentIdx + size.toInt()*info->fields[n]->numElements - 1) + ")), '" + type + "')), "+ QString::number(info->fields[n]->numElements, 10) + ", [] );\n");
}
else{
funcfields.append("\t" + objectName + "." + info->fields[n]->name + " = " +
"double(typecast(buffer(mcolon(" + objectName + "FidIdx + " + QString("%1").arg(currentIdx) +
", " + objectName + "FidIdx + " + QString("%1").arg(currentIdx + size.toInt() - 1) + ")), '" + type + "'))';\n");
}
currentIdx+=size.toInt()*info->fields[n]->numElements;
}
matlabFunctionsCode.append(funcfields);
matlabFunctionsCode.append("\n");
}
// ActuatorSettings.ChannelUpdateFreq(:,actuatorsettingsIdx) = double(fread(fid, 4, 'uint16'));
return true;
}

View File

@ -35,7 +35,7 @@ public:
bool generate(UAVObjectParser* gen,QString templatepath,QString outputpath);
private:
bool process_object(ObjectInfo* info);
bool process_object(ObjectInfo* info, int numBytes);
QString matlabAllocationCode;
QString matlabSwitchCode;
QString matlabCleanupCode;
@ -43,6 +43,7 @@ private:
QString matlabExportCsvCode;
QString matlabFunctionsCode;
QStringList fieldTypeStrMatlab;
QStringList fieldSizeStrMatlab;
};