1
0
mirror of https://bitbucket.org/librepilot/librepilot.git synced 2024-12-01 09:24:10 +01:00

Merge remote-tracking branch 'origin/kenz/matlab_parser_vectorizing' into next

This commit is contained in:
James Cotton 2012-09-02 19:23:18 -05:00
commit a3b75a3cc7
3 changed files with 174 additions and 161 deletions

View File

@ -55,49 +55,54 @@ if ~strcmpi(outputType,'mat') && ~strcmpi(outputType,'csv')
error('Incorrect file format specified. Second argument must be ''mat'' or ''csv''.');
end
$(ALLOCATIONCODE)
$(INSTANTIATIONCODE)
fid = fopen(logfile);
buffer=fread(fid,Inf,'uchar=>uchar');
fseek(fid, 0, 'bof');
bufferIdx=1;
correctMsgByte=hex2dec('20');
correctSyncByte=hex2dec('3C');
unknownObjIDList=zeros(1,2);
% Parse log file, entry by entry
prebuf = fread(fid, 12, 'uint8');
log_size = dir(logfile);
log_size = log_size.bytes;
last_print = 0;
% prebuf = buffer(1:12);
last_print = -1e10;
startTime=clock;
while (1)
if (feof(fid)); break; end
%% Read message header
% get sync field (0x3C, 1 byte)
sync = fread(fid, 1, 'uint8');
if sync ~= correctSyncByte
prebuf = [prebuf(2:end); sync];
wrongSyncByte = wrongSyncByte + 1;
continue
end
%% Process header if we are aligned
timestamp = typecast(uint8(prebuf(1:4)), 'uint32');
datasize = typecast(uint8(prebuf(5:12)), 'uint64');
sync = buffer(bufferIdx+12);
if buffer(bufferIdx+12) ~= correctSyncByte
bufferIdx=bufferIdx+1;
wrongSyncByte = wrongSyncByte + 1;
continue
end
%% Process header, if we are aligned
% timestamp = typecast(buffer(bufferIdx:bufferIdx + 4-1), 'uint32'); %We do this one later
datasizeBufferIdx = bufferIdx; %Just grab the index. We'll do a typecast later, if necessary
% sync = buffer(bufferIdx+12); %This one has already been done
msgType = buffer(bufferIdx+13); % get msg type (quint8 1 byte ) should be 0x20, ignore the rest?
% msgSize = typecast(buffer(bufferIdx+14:bufferIdx+ 14+2-1), 'uint16'); % NOT USED: get msg size (quint16 2 bytes) excludes crc, include msg header and data payload
objID = typecast(buffer(bufferIdx+16:bufferIdx+ 16+4-1), 'uint32'); % get obj id (quint32 4 bytes)
% get msg type (quint8 1 byte ) should be 0x20, ignore the rest?
msgType = fread(fid, 1, 'uint8');
%Advance buffer past header
bufferIdx=bufferIdx+20;
%Check that message type is correct
if msgType ~= correctMsgByte
wrongMessageByte = wrongMessageByte + 1;
continue
end
% get msg size (quint16 2 bytes) excludes crc, include msg header and data payload
msgSize = fread(fid, 1, 'uint16');
% get obj id (quint32 4 bytes)
objID = fread(fid, 1, 'uint32');
if (isempty(objID)) %End of file
break;
@ -110,15 +115,18 @@ $(SWITCHCODE)
otherwise
unknownObjIDListIdx=find(unknownObjIDList(:,1)==objID, 1, 'first');
if isempty(unknownObjIDListIdx)
unknownObjIDList=[unknownObjIDList; objID 1];
unknownObjIDList=[unknownObjIDList; uint32(objID) 1]; %#ok<AGROW>
else
unknownObjIDList(unknownObjIDListIdx,2)=unknownObjIDList(unknownObjIDListIdx,2)+1;
unknownObjIDList(unknownObjIDListIdx,2)=unknownObjIDList(unknownObjIDListIdx,2)+1;
end
datasize = typecast(buffer(datasizeBufferIdx + 4:datasizeBufferIdx + 12-1), 'uint64');
msgBytesLeft = datasize - 1 - 1 - 2 - 4;
if msgBytesLeft > 255
msgBytesLeft = 0;
end
fread(fid, msgBytesLeft, 'uint8');
bufferIdx=bufferIdx+msgBytesLeft;
end
catch
% One of the reads failed - indicates EOF
@ -126,42 +134,41 @@ $(SWITCHCODE)
end
if (wrongSyncByte ~= lastWrongSyncByte || wrongMessageByte~=lastWrongMessageByte ) ||...
(ftell(fid) / log_size - last_print) > 0.01
bufferIdx - last_print > 5e4 %Every 50,000 bytes show the status update
lastWrongSyncByte=wrongSyncByte;
lastWrongMessageByte=wrongMessageByte;
str1=[];
for i=1:length([str2 str3 str4 str5]);
str1=[str1 sprintf('\b')];
str1=[str1 sprintf('\b')]; %#ok<AGROW>
end
str2=sprintf('wrongSyncByte instances: % 10d\n', wrongSyncByte );
str3=sprintf('wrongMessageByte instances: % 10d\n\n', wrongMessageByte );
str4=sprintf('Completed bytes: % 9d of % 9d\n', ftell(fid), log_size);
str4=sprintf('Completed bytes: % 9d of % 9d\n', bufferIdx, length(buffer));
% Arbitrary times two so that it is at least as long
estTimeRemaining=(log_size-ftell(fid))/(ftell(fid)/etime(clock,startTime)) * 2;
estTimeRemaining=(length(buffer)-bufferIdx)/(bufferIdx/etime(clock,startTime)) * 2;
h=floor(estTimeRemaining/3600);
m=floor((estTimeRemaining-h*3600)/60);
s=ceil(estTimeRemaining-h*3600-m*60);
str5=sprintf('Est. time remaining, %02dh:%02dm:%02ds \n', h,m,s);
last_print = ftell(fid) / log_size;
last_print = bufferIdx;
fprintf([str1 str2 str3 str4 str5]);
end
%Check if at end of file. If not, load next prebuffer
if bufferIdx+12-1 > length(buffer)
break;
end
% bufferIdx=bufferIdx+12;
prebuf = fread(fid, 12, 'uint8');
end
fprintf('%d records in %0.2f seconds.\n', ftell(fid), etime(clock,startTime));
for i=2:size(unknownObjIDList,1) %Don't show the first one, as it was simply a dummy placeholder
disp(['Unknown object ID: 0x' dec2hex(unknownObjIDList(i,1),8) ' appeared ' int2str(unknownObjIDList(i,2)) ' times.']);
end
for i=2:size(unknownObjIDList,1) %Don't show the first one, as it was simply a dummy placeholder
disp(['Unknown object ID: 0x' dec2hex(unknownObjIDList(i,1),8) ' appeared ' int2str(unknownObjIDList(i,2)) ' times.']);
@ -170,28 +177,25 @@ end
%% Clean Up and Save mat file
fclose(fid);
% Trim output structs
%% Prune vectors
$(CLEANUPCODE)
%% Perform typecasting on vectors
$(ALLOCATIONCODE)
%% Save data to file
if strcmpi(outputType,'mat')
matfile = strrep(logfile,'opl','mat');
[path, name]=fileparts(logfile);
matfile = fullfile(path,[name '.mat']);
save(matfile $(SAVEOBJECTSCODE));
else
$(EXPORTCSVCODE);
$(EXPORTCSVCODE)
end
fprintf('%d records in %0.2f seconds.\n', length(buffer), etime(clock,startTime));
%% Object reading functions
$(FUNCTIONSCODE)
% This function prunes the excess pre-allocated space
function [structOut]=PruneStructOfArrays(structIn, lastIndex)
fieldNames = fieldnames(structIn);
for i=1:length(fieldNames)
structOut.(fieldNames{i})=structIn.(fieldNames{i})(:,1:lastIndex);
end
function OPLog2csv(structIn, structName, logfile)
@ -229,3 +233,28 @@ function crc = compute_crc(data)
crc = crc_table(1+bitxor(data(i),crc));
end
function out=mcolon(inStart, inFinish)
%% This function was inspired by Bruno Luong's 'mcolon'. The name is kept the same as his 'mcolon'
% function, found on Matlab's file exchange. The two functions return identical
% results, although his is much faster. Unfortunately, C-compiled mex
% code would make this function non-cross-platform, so a Matlab scripted
% version is provided here.
if size(inStart,1) > 1 || size(inFinish,1) > 1
if size(inStart,2) > 1 || size(inFinish,2) > 1
error('Inputs must be vectors, i.e just one column wide.')
else
inStart=inStart';
inFinish=inFinish';
end
end
diffIn=diff([inStart; inFinish]);
numElements=sum(diffIn)+length(inStart);
out=zeros(1,numElements);
idx=1;
for i=1:length(inStart)
out(idx:idx+diffIn(i))=inStart(i):inFinish(i);
idx=idx+diffIn(i)+1;
end

View File

@ -31,7 +31,9 @@ using namespace std;
bool UAVObjectGeneratorMatlab::generate(UAVObjectParser* parser,QString templatepath,QString outputpath) {
fieldTypeStrMatlab << "int8" << "int16" << "int32"
<< "uint8" << "uint16" << "uint32" << "float32" << "uint8";
<< "uint8" << "uint16" << "uint32" << "single" << "uint8";
fieldSizeStrMatlab << "1" << "2" << "4"
<< "1" << "2" << "4" << "4" << "1";
QDir matlabTemplatePath = QDir( templatepath + QString("ground/openpilotgcs/src/plugins/uavobjects"));
QDir matlabOutputPath = QDir( outputpath + QString("matlab") );
@ -46,14 +48,15 @@ bool UAVObjectGeneratorMatlab::generate(UAVObjectParser* parser,QString template
for (int objidx = 0; objidx < parser->getNumObjects(); ++objidx) {
ObjectInfo* info=parser->getObjectByIndex(objidx);
process_object(info);
int numBytes=parser->getNumBytes(objidx);
process_object(info, numBytes);
}
matlabCodeTemplate.replace( QString("$(ALLOCATIONCODE)"), matlabAllocationCode);
matlabCodeTemplate.replace( QString("$(INSTANTIATIONCODE)"), matlabInstantiationCode);
matlabCodeTemplate.replace( QString("$(SWITCHCODE)"), matlabSwitchCode);
matlabCodeTemplate.replace( QString("$(CLEANUPCODE)"), matlabCleanupCode);
matlabCodeTemplate.replace( QString("$(SAVEOBJECTSCODE)"), matlabSaveObjectsCode);
matlabCodeTemplate.replace( QString("$(FUNCTIONSCODE)"), matlabFunctionsCode);
matlabCodeTemplate.replace( QString("$(ALLOCATIONCODE)"), matlabAllocationCode);
matlabCodeTemplate.replace( QString("$(EXPORTCSVCODE)"), matlabExportCsvCode);
bool res = writeFile( matlabOutputPath.absolutePath() + "/OPLogConvert.m", matlabCodeTemplate );
@ -68,7 +71,7 @@ bool UAVObjectGeneratorMatlab::generate(UAVObjectParser* parser,QString template
/**
* Generate the matlab object files
*/
bool UAVObjectGeneratorMatlab::process_object(ObjectInfo* info)
bool UAVObjectGeneratorMatlab::process_object(ObjectInfo* info, int numBytes)
{
if (info == NULL)
return false;
@ -78,75 +81,103 @@ bool UAVObjectGeneratorMatlab::process_object(ObjectInfo* info)
// QString objectTableName(objectName + "Objects");
QString objectTableName(objectName);
QString tableIdxName(objectName.toLower() + "Idx");
QString functionName("Read" + info->name + "Object");
QString functionCall(functionName + "(fid, timestamp, checkCRC, ");
QString objectID(QString().setNum(info->id));
QString isSingleInst = boolTo01String( info->isSingleInst );
QString numBytesString=QString("%1").arg(numBytes);
//===================================================================//
// Generate allocation code (will replace the $(ALLOCATIONCODE) tag) //
//===================================================================//
// matlabSwitchCode.append("\t\tcase " + objectID + "\n");
matlabAllocationCode.append("\n\t" + tableIdxName + " = 0;\n");
QString type;
QString allocfields;
if (0){
matlabAllocationCode.append("\t" + objectTableName + ".timestamp = 0;\n");
for (int n = 0; n < info->fields.length(); ++n) {
// Determine type
type = fieldTypeStrMatlab[info->fields[n]->type];
// Append field
if ( info->fields[n]->numElements > 1 )
allocfields.append("\t" + objectTableName + "(1)." + info->fields[n]->name + " = zeros(" + QString::number(info->fields[n]->numElements, 10) + ",1);\n");
else
allocfields.append("\t" + objectTableName + "(1)." + info->fields[n]->name + " = 0;\n");
}
}
else{
matlabAllocationCode.append("\t" + objectTableName + "=struct('timestamp', 0");
if (!info->isSingleInst) {
allocfields.append(",...\n\t\t 'instanceID', 0");
}
for (int n = 0; n < info->fields.length(); ++n) {
// Determine type
type = fieldTypeStrMatlab[info->fields[n]->type];
// Append field
if ( info->fields[n]->numElements > 1 )
allocfields.append(",...\n\t\t '" + info->fields[n]->name + "', zeros(" + QString::number(info->fields[n]->numElements, 10) + ",1)");
else
allocfields.append(",...\n\t\t '" + info->fields[n]->name + "', 0");
}
allocfields.append(");\n");
//=========================================================================//
// Generate instantiation code (will replace the $(INSTANTIATIONCODE) tag) //
//=========================================================================//
QString type;
QString instantiationFields;
matlabInstantiationCode.append("\n\t" + tableIdxName + " = 0;\n");
matlabInstantiationCode.append("\t" + objectTableName + "=struct('timestamp', 0");
if (!info->isSingleInst) {
instantiationFields.append(",...\n\t\t 'instanceID', 0");
}
matlabAllocationCode.append(allocfields);
matlabAllocationCode.append("\t" + objectTableName.toUpper() + "_OBJID=" + objectID + ";\n");
for (int n = 0; n < info->fields.length(); ++n) {
// Determine type
type = fieldTypeStrMatlab[info->fields[n]->type];
// Append field
if ( info->fields[n]->numElements > 1 )
instantiationFields.append(",...\n\t\t '" + info->fields[n]->name + "', zeros(" + QString::number(info->fields[n]->numElements, 10) + ",1)");
else
instantiationFields.append(",...\n\t\t '" + info->fields[n]->name + "', 0");
}
instantiationFields.append(");\n");
matlabInstantiationCode.append(instantiationFields);
matlabInstantiationCode.append("\t" + objectTableName.toUpper() + "_OBJID=" + objectID + ";\n");
matlabInstantiationCode.append("\t" + objectTableName.toUpper() + "_NUMBYTES=" + numBytesString + ";\n");
matlabInstantiationCode.append("\t" + objectName + "FidIdx = [];\n");
//==============================================================//
// Generate 'Switch:' code (will replace the $(SWITCHCODE) tag) //
//==============================================================//
matlabSwitchCode.append("\t\tcase " + objectTableName.toUpper() + "_OBJID\n");
// matlabSwitchCode.append("\t\t\t" + objectTableName + "(" + tableIdxName +") = " + functionCall + ";\n");
matlabSwitchCode.append("\t\t\t" + tableIdxName + " = " + tableIdxName +" + 1;\n");
matlabSwitchCode.append("\t\t\t" + objectTableName + "= " + functionCall + objectTableName + ", " + tableIdxName + ");\n");
matlabSwitchCode.append("\t\t\tif " + tableIdxName + " >= length(" + objectTableName +".timestamp) %Check to see if pre-allocated memory is exhausted\n");
matlabSwitchCode.append("\t\t\t\tFieldNames= fieldnames(" + objectTableName +");\n");
matlabSwitchCode.append("\t\t\t\tfor i=1:length(FieldNames) %Grow structure\n");
matlabSwitchCode.append("\t\t\t\t\t" + objectTableName + ".(FieldNames{i})(:," + tableIdxName + "*2+1) = 0;\n");
matlabSwitchCode.append("\t\t\t\tend\n");
matlabSwitchCode.append("\t\t\t" + tableIdxName + " = " + tableIdxName +" + 1;\n");
matlabSwitchCode.append("\t\t\t" + objectTableName + "FidIdx(" + tableIdxName + ") = bufferIdx; %#ok<*AGROW>\n");
matlabSwitchCode.append("\t\t\tbufferIdx=bufferIdx + " + objectTableName.toUpper() + "_NUMBYTES+1; %+1 is for CRC\n");
if(!info->isSingleInst){
matlabSwitchCode.append("\t\t\tbufferIdx = bufferIdx + 2; %An extra two bytes for the instance ID\n");
}
matlabSwitchCode.append("\t\t\tif " + tableIdxName + " >= length(" + objectTableName +"FidIdx) %Check to see if pre-allocated memory is exhausted\n");
matlabSwitchCode.append("\t\t\t\t" + objectTableName + "FidIdx(" + tableIdxName + "*2) = 0;\n");
matlabSwitchCode.append("\t\t\tend\n");
//============================================================//
// Generate 'Cleanup:' code (will replace the $(CLEANUP) tag) //
//============================================================//
matlabCleanupCode.append(objectTableName + "=PruneStructOfArrays(" + objectTableName + "," + tableIdxName +");\n" );
matlabCleanupCode.append(objectTableName + "FidIdx =" + objectTableName + "FidIdx(1:" + tableIdxName +");\n" );
//=================================================================//
// Generate functions code (will replace the $(ALLOCATIONCODE) tag) //
//=================================================================//
//Generate function description comment
matlabAllocationCode.append("% " + objectName + " typecasting\n");
QString allocationFields;
//Add timestamp
allocationFields.append("\t" + objectName + ".timestamp = " +
"double(typecast(buffer(mcolon(" + objectName + "FidIdx "
"- 20, " + objectName + "FidIdx + 4-1 -20)), 'uint32'))';\n");
int currentIdx=0;
//Add Instance ID, if necessary
if(!info->isSingleInst){
allocationFields.append("\t" + objectName + ".instanceID = " +
"double(typecast(buffer(mcolon(" + objectName + "FidIdx "
", " + objectName + "FidIdx + 2-1)), 'uint16'))';\n");
currentIdx+=2;
}
for (int n = 0; n < info->fields.length(); ++n) {
// Determine variabel type
type = fieldTypeStrMatlab[info->fields[n]->type];
//Determine variable type length
QString size = fieldSizeStrMatlab[info->fields[n]->type];
// Append field
if ( info->fields[n]->numElements > 1 ){
allocationFields.append("\t" + objectName + "." + info->fields[n]->name + " = " +
"reshape(double(typecast(buffer(mcolon(" + objectName + "FidIdx + " + QString("%1").arg(currentIdx) +
", " + objectName + "FidIdx + " + QString("%1").arg(currentIdx + size.toInt()*info->fields[n]->numElements - 1) + ")), '" + type + "')), "+ QString::number(info->fields[n]->numElements, 10) + ", [] );\n");
}
else{
allocationFields.append("\t" + objectName + "." + info->fields[n]->name + " = " +
"double(typecast(buffer(mcolon(" + objectName + "FidIdx + " + QString("%1").arg(currentIdx) +
", " + objectName + "FidIdx + " + QString("%1").arg(currentIdx + size.toInt() - 1) + ")), '" + type + "'))';\n");
}
currentIdx+=size.toInt()*info->fields[n]->numElements;
}
matlabAllocationCode.append(allocationFields);
matlabAllocationCode.append("\n");
//========================================================================//
// Generate objects saving code (will replace the $(SAVEOBJECTSCODE) tag) //
//========================================================================//
@ -158,56 +189,8 @@ bool UAVObjectGeneratorMatlab::process_object(ObjectInfo* info)
//==========================================================================//
matlabExportCsvCode.append("\tOPLog2csv(" + objectTableName + ", '"+objectTableName+"', logfile);\n");
// OPLog2csv(ActuatorCommand, 'ActuatorCommand', logfile)
//=================================================================//
// Generate functions code (will replace the $(FUNCTIONSCODE) tag) //
//=================================================================//
//Generate function description comment
matlabFunctionsCode.append("%%\n% " + objectName + " read function\n");
matlabFunctionsCode.append("function [" + objectName + "] = " + functionCall + objectTableName + ", " + tableIdxName + ")" + "\n");
matlabFunctionsCode.append("\t" + objectName + ".timestamp(" + tableIdxName + ")= timestamp;\n");
matlabFunctionsCode.append("\tif " + isSingleInst + "\n");
matlabFunctionsCode.append("\t\theaderSize = 8;\n");
matlabFunctionsCode.append("\telse\n");
matlabFunctionsCode.append("\t\t" + objectName + ".instanceID(" + tableIdxName + ") = (fread(fid, 1, 'uint16'));\n");
matlabFunctionsCode.append("\t\theaderSize = 10;\n");
matlabFunctionsCode.append("\tend\n\n");
// Generate functions code, actual fields of the object
QString funcfields;
matlabFunctionsCode.append("\tstartPos = ftell(fid) - headerSize;\n");
for (int n = 0; n < info->fields.length(); ++n) {
// Determine type
type = fieldTypeStrMatlab[info->fields[n]->type];
// Append field
if ( info->fields[n]->numElements > 1 )
funcfields.append("\t" + objectName + "." + info->fields[n]->name + "(:," + tableIdxName + ") = double(fread(fid, " + QString::number(info->fields[n]->numElements, 10) + ", '" + type + "'));\n");
else
funcfields.append("\t" + objectName + "." + info->fields[n]->name + "(" + tableIdxName + ") = double(fread(fid, 1, '" + type + "'));\n");
}
matlabFunctionsCode.append(funcfields);
matlabFunctionsCode.append("\tobjLen = ftell(fid) - startPos;\n");
matlabFunctionsCode.append("\t% read CRC\n");
matlabFunctionsCode.append("\tcrc_read = fread(fid, 1, '*uint8');\n");
matlabFunctionsCode.append("\tif checkCRC\n");
matlabFunctionsCode.append("\t\tfseek(fid, startPos, 'bof');\n");
matlabFunctionsCode.append("\t\tcrc_calc = compute_crc(uint8(fread(fid,objLen,'uint8')));\n");
matlabFunctionsCode.append("\t\tfread(fid,1,'uint8');\n");
matlabFunctionsCode.append("\t\tif (crc_calc ~= crc_read)\n");
matlabFunctionsCode.append("\t\t\tdisp('CRC Error')\n");
matlabFunctionsCode.append("\t\t\t" + tableIdxName + " = " + tableIdxName + " - 1;\n");
matlabFunctionsCode.append("\t\tend\n");
matlabFunctionsCode.append("\tend\n");
matlabFunctionsCode.append("\n\n");
return true;
}

View File

@ -35,14 +35,15 @@ public:
bool generate(UAVObjectParser* gen,QString templatepath,QString outputpath);
private:
bool process_object(ObjectInfo* info);
QString matlabAllocationCode;
bool process_object(ObjectInfo* info, int numBytes);
QString matlabInstantiationCode;
QString matlabSwitchCode;
QString matlabCleanupCode;
QString matlabAllocationCode;
QString matlabSaveObjectsCode;
QString matlabExportCsvCode;
QString matlabFunctionsCode;
QStringList fieldTypeStrMatlab;
QStringList fieldSizeStrMatlab;
};