Inial population of repository.
This commit is contained in:
Andras Tantos
35
simulator/dataset_extractor/Makefile
Normal file
35
simulator/dataset_extractor/Makefile
Normal file
@@ -0,0 +1,35 @@
|
||||
#################################################################################
|
||||
### MODE: determines what compiler flags to use
|
||||
### SYSTEM: determines the intermediate and final file directory names (for now)
|
||||
### DEFINES: list of predefined macros to pass to the compiler
|
||||
### INC_DIRS: list of include paths to pass to the compiler
|
||||
### DEP_INC_DIRS: list of include paths to search for dependencies in
|
||||
### LIB_DIRS: list of library paths to pass to the linker
|
||||
### TARGET_TYPE: determines whether to build executables or libraries
|
||||
### SOURCES: list of source (c++) files
|
||||
### SRC_DIRS: list of source paths. Needed to generate implicit rules
|
||||
### SOURCE_LIBS: list of locally built libs (they are included as a dependency)
|
||||
### LINK_LIBS: list of libraries to link againsg on top of SOURCE_LIBS
|
||||
### TARGETS: list of target final targets (library names or executable files)
|
||||
|
||||
ifndef MODE
|
||||
MODE=release
|
||||
#MODE=debug
|
||||
endif
|
||||
|
||||
include ../common.mak
|
||||
|
||||
TARGET_TYPE = EXECUTABLE
|
||||
|
||||
SOURCES =
|
||||
SOURCES += dataset_extractor.cpp
|
||||
|
||||
SOURCE_LIBS =
|
||||
|
||||
LINK_LIBS =
|
||||
LINK_LIBS += $(COMMON_LIBS)
|
||||
|
||||
TARGETS =
|
||||
TARGETS += dataset_extractor
|
||||
|
||||
include ../engine.mak
|
||||
307
simulator/dataset_extractor/dataset_extractor.cpp
Normal file
307
simulator/dataset_extractor/dataset_extractor.cpp
Normal file
@@ -0,0 +1,307 @@
|
||||
#define _CRT_SECURE_NO_WARNINGS
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdint.h>
|
||||
#include <string.h>
|
||||
#include "utils.h"
|
||||
#include <sstream>
|
||||
#include <boost/filesystem.hpp>
|
||||
|
||||
// Normal sequences:
|
||||
// BCW - if we're not in a dataset, this starts a dataset
|
||||
// - if we're not in a file, this starts a file
|
||||
// - if we're not in a record, this starts a record
|
||||
// - otherwise, this is just a continuation of the current record
|
||||
// EOR - if we're in a record, it ends the current record
|
||||
// - otherwise it denotes a NULL record
|
||||
// EOF - if we're in a file, it ends the current file
|
||||
// - otherwise it denotes a NULL file
|
||||
// EOD - if we're in a dataset, it ends the dataset
|
||||
// - otherwise it denotes a NULL dataset
|
||||
//
|
||||
// Ending a file without ending a record is invalid, except for a NULL file
|
||||
// Ending a dataset without ending a file is invalid
|
||||
//
|
||||
// If an EOR is followed by a BCW, that's fine, this
|
||||
// simply means that the data for the next record is after
|
||||
// the BCW. This technique can be used to skip over
|
||||
// certain sections of a dataset: since the forward index
|
||||
// of an EOR or EOF points to a BCW, the region between the
|
||||
// the two control words is assumed to not contain data
|
||||
//
|
||||
// There is some RLL compression for 'blank fields' whatever that means
|
||||
// See 2240011R part 1 2-6 - this seems like it have been removed later
|
||||
//
|
||||
// There are some new bigs that are not in the original description
|
||||
// See SR-0011P 2-15
|
||||
//
|
||||
// So, to add a file to the end:
|
||||
// 1. Remove the EOD
|
||||
// 2. Add the file, starting with a BCW
|
||||
// 3. Terminate with an EOD
|
||||
//
|
||||
// To add a file to the beginning:
|
||||
// 1. Create the file as normal, starting with a BCW
|
||||
// 2. Append the old dataset to it
|
||||
//
|
||||
// To add a file at an arbitrary position
|
||||
// A. If the EOF of the previous file points to an EOR (that is the first record of the next file
|
||||
// 1. Patch up EOF to point to immediately after itself
|
||||
// 2. Add file as normal, starting with a BCW
|
||||
// 3. Patch up our EOF to point to the first EOR of the old file
|
||||
// 4. Add the rest of the Dataset
|
||||
// B. If the EOF of the previous file points to a BCW
|
||||
// 1. Add the file as normal, starting with a BCW
|
||||
// 2. Make sure our EOF points immediately after itself
|
||||
// 2. Add the rest of the dataset
|
||||
//
|
||||
// To remove the first file
|
||||
//
|
||||
|
||||
enum CtrlWordType_e {
|
||||
CtrlWordType_BCW = 000,
|
||||
CtrlWordType_EOR = 010,
|
||||
CtrlWordType_EOF = 016,
|
||||
CtrlWordType_EOD = 017,
|
||||
CtrlWordType_UNK = 020
|
||||
};
|
||||
|
||||
// Common fields
|
||||
CtrlWordType_e CtrlWord_GetType(uint64_t aCtrlWord) {
|
||||
uint8_t CtrlType = ((uint8_t *)(&aCtrlWord))[7] >> 4;
|
||||
switch (CtrlType) {
|
||||
case CtrlWordType_BCW: return (CtrlWordType_e)CtrlType;
|
||||
case CtrlWordType_EOR: return (CtrlWordType_e)CtrlType;
|
||||
case CtrlWordType_EOF: return (CtrlWordType_e)CtrlType;
|
||||
case CtrlWordType_EOD: return (CtrlWordType_e)CtrlType;
|
||||
default: return CtrlWordType_UNK;
|
||||
}
|
||||
}
|
||||
|
||||
bool CtrlWord_GetBadData(uint64_t aCtrlWord) {
|
||||
// get bit 52 out
|
||||
return ((aCtrlWord >> 52) & 1) == 0 ? false: true;
|
||||
}
|
||||
|
||||
uint32_t CtrlWord_GetForwardWordIdx(uint64_t aCtrlWord) {
|
||||
uint32_t Lower = (uint32_t)aCtrlWord;
|
||||
return Lower & 0x1ff;
|
||||
}
|
||||
|
||||
// BCW fields
|
||||
uint32_t CtrlWord_GetBlockNumber(uint64_t aCtrlWord) {
|
||||
uint32_t Lower = (uint32_t)aCtrlWord;
|
||||
Lower >>= 9;
|
||||
return Lower;
|
||||
}
|
||||
|
||||
// EOR,EOF,EOD fields
|
||||
uint32_t CtrlWord_GetUnusedBitCnt(uint64_t aCtrlWord) {
|
||||
return ((aCtrlWord >> 54) & 63);
|
||||
}
|
||||
|
||||
bool CtrlWord_GetTransparent(uint64_t aCtrlWord) {
|
||||
// get bit 53 out
|
||||
uint8_t Byte = ((uint8_t *)(&aCtrlWord))[6];
|
||||
return (Byte & 0x20) == 0 ? false: true;
|
||||
}
|
||||
|
||||
bool CtrlWord_GetSkipRemainderSector(uint64_t aCtrlWord) {
|
||||
// get bit 51 out
|
||||
uint8_t Byte = ((uint8_t *)(&aCtrlWord))[6];
|
||||
return (Byte & 0x08) == 0 ? false: true;
|
||||
}
|
||||
|
||||
uint32_t CtrlWord_GetPreviousFileIdx(uint64_t aCtrlWord) {
|
||||
return uint32_t(GetBitsReverse(aCtrlWord, 20, 39));
|
||||
}
|
||||
|
||||
uint32_t CtrlWord_GetPreviousRecordIdx(uint64_t aCtrlWord) {
|
||||
return uint32_t(GetBitsReverse(aCtrlWord, 40, 54));
|
||||
}
|
||||
|
||||
std::string PrintCtrlWord(uint64_t aCtrlWord) {
|
||||
std::stringstream Str;
|
||||
switch (CtrlWord_GetType(aCtrlWord)) {
|
||||
case CtrlWordType_BCW: Str << "BCW: "; break;
|
||||
case CtrlWordType_EOR: Str << "EOR: "; break;
|
||||
case CtrlWordType_EOF: Str << "EOF: "; break;
|
||||
case CtrlWordType_EOD: Str << "EOD: "; break;
|
||||
default: Str << "UNK: "; break;
|
||||
}
|
||||
Str << "Forward word IDX: " << DecPrinter(CtrlWord_GetForwardWordIdx(aCtrlWord), 5) << " ";
|
||||
switch (CtrlWord_GetType(aCtrlWord)) {
|
||||
case CtrlWordType_BCW:
|
||||
Str << "Block Number: " << CtrlWord_GetBlockNumber(aCtrlWord) << " ";
|
||||
break;
|
||||
case CtrlWordType_EOR:
|
||||
case CtrlWordType_EOF:
|
||||
case CtrlWordType_EOD:
|
||||
Str << "Unused bit count: " << DecPrinter(CtrlWord_GetUnusedBitCnt(aCtrlWord),2) << " ";
|
||||
Str << "Prev file idx: " << HexPrinter(CtrlWord_GetPreviousFileIdx(aCtrlWord),7) << " ";
|
||||
Str << "Prev record idx: " << HexPrinter(CtrlWord_GetPreviousRecordIdx(aCtrlWord),6) << " ";
|
||||
Str << (CtrlWord_GetTransparent(aCtrlWord) ? "TRANS ":"");
|
||||
Str << (CtrlWord_GetSkipRemainderSector(aCtrlWord) ? "SKIP ":"");
|
||||
break;
|
||||
default: Str << "UNK: "; break;
|
||||
}
|
||||
Str << (CtrlWord_GetBadData(aCtrlWord) ? "BAD":"");
|
||||
return Str.str();
|
||||
}
|
||||
|
||||
int main(int argc, char* argv[])
|
||||
{
|
||||
if (argc != 2) {
|
||||
printf("Usage: %s <image file>\n",argv[0]);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const char *InFileName = argv[1];
|
||||
|
||||
FILE *Input = fopen(InFileName,"rb");
|
||||
if (Input == nullptr) {
|
||||
printf("Can't open input file: %s\n", InFileName);
|
||||
return 1;
|
||||
}
|
||||
uint64_t TotalFileSize = boost::filesystem::file_size(InFileName);
|
||||
|
||||
const size_t MaxFileSize = 256*1024*1024;
|
||||
if (TotalFileSize > MaxFileSize) {
|
||||
printf("Can't use files larger than %uMB\n",unsigned(MaxFileSize/1024/1024));
|
||||
fclose(Input);
|
||||
return 1;
|
||||
}
|
||||
uint8_t *File = new uint8_t[(size_t)TotalFileSize+1];
|
||||
if (File == nullptr) {
|
||||
printf("Not enough memory to allocate buffer for image\n");
|
||||
fclose(Input);
|
||||
return 1;
|
||||
}
|
||||
size_t Read = fread(File,1,(size_t)TotalFileSize,Input);
|
||||
fclose(Input);
|
||||
if (Read != TotalFileSize) {
|
||||
printf("Can't read file.\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
{
|
||||
size_t Offset = 0;
|
||||
bool InDataSet = false;
|
||||
int DataSetCnt = 0;
|
||||
int FileCnt = 0;
|
||||
int RecordCnt = 0;
|
||||
char FileName[255];
|
||||
int LastSeqCnt = -1;
|
||||
FILE *OutFile = nullptr;
|
||||
while (Offset < TotalFileSize) {
|
||||
uint64_t CtrlWord = SwapBytes(*(uint64_t*)(File+Offset));
|
||||
int SeqCount;
|
||||
|
||||
CtrlWordType_e Type = CtrlWord_GetType(CtrlWord);
|
||||
std::cout << PrintCtrlWord(CtrlWord) << std::endl;
|
||||
if (Type == CtrlWordType_UNK) {
|
||||
printf("Lost track at 0x%08x\n",unsigned(Offset));
|
||||
break;
|
||||
}
|
||||
switch (Type) {
|
||||
case CtrlWordType_BCW:
|
||||
SeqCount = (int)CtrlWord_GetBlockNumber(CtrlWord);
|
||||
if (SeqCount == 0 && InDataSet) {
|
||||
printf("BCW Start sequence inconsitency at 0x%08x\n",unsigned(Offset));
|
||||
}
|
||||
if (SeqCount != LastSeqCnt + 1) {
|
||||
printf("BCW Sequence inconsitency at 0x%08x\n",unsigned(Offset));
|
||||
}
|
||||
if (!InDataSet) {
|
||||
InDataSet = true;
|
||||
// New dataset: let's close the old file, and start a new one
|
||||
FileCnt = 0;
|
||||
RecordCnt = 0;
|
||||
}
|
||||
LastSeqCnt = SeqCount;
|
||||
break;
|
||||
case CtrlWordType_EOR:
|
||||
break;
|
||||
case CtrlWordType_EOF:
|
||||
break;
|
||||
case CtrlWordType_EOD:
|
||||
break;
|
||||
default:
|
||||
printf("Internal error at offset 0x%08x\n",unsigned(Offset));
|
||||
exit(1);
|
||||
break;
|
||||
}
|
||||
uint32_t DataSize = CtrlWord_GetForwardWordIdx(CtrlWord) * 8;
|
||||
uint32_t RecordSize = DataSize;
|
||||
// See what the next control word is and if there's anything to save
|
||||
uint32_t UnusedBitCnt = 0;
|
||||
CtrlWordType_e NextType = CtrlWordType_UNK;
|
||||
if (Type != CtrlWordType_EOD) {
|
||||
uint64_t NextCtrlWord = SwapBytes(*(uint64_t*)(File+Offset+8+RecordSize));
|
||||
NextType = CtrlWord_GetType(NextCtrlWord);
|
||||
if (NextType == CtrlWordType_EOR) UnusedBitCnt = CtrlWord_GetUnusedBitCnt(NextCtrlWord);
|
||||
}
|
||||
DataSize -= UnusedBitCnt / 8;
|
||||
if (DataSize != 0) {
|
||||
if (OutFile == nullptr) {
|
||||
sprintf(FileName,"DS%03d_F%03d.dat",DataSetCnt,FileCnt);
|
||||
OutFile = fopen(FileName,"wb");
|
||||
if (OutFile == nullptr) {
|
||||
printf("Can't create file: %s\n",FileName);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
size_t Written = fwrite(File+Offset+8,1,DataSize,OutFile);
|
||||
if (Written != DataSize) {
|
||||
printf("Can't write to file\n");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
switch (NextType) {
|
||||
case CtrlWordType_BCW:
|
||||
break;
|
||||
case CtrlWordType_EOR:
|
||||
++RecordCnt;
|
||||
//if (OutFile != nullptr) fclose(OutFile); OutFile = nullptr;
|
||||
if (OutFile == nullptr) {
|
||||
printf("EOR encountered with no file at offset 0x%08x\n",unsigned(Offset));
|
||||
exit(1);
|
||||
}
|
||||
fprintf(OutFile,"\n"); // TODO: this should only happen for text files
|
||||
break;
|
||||
case CtrlWordType_EOF:
|
||||
++FileCnt;
|
||||
RecordCnt = 0;
|
||||
if (OutFile != nullptr) fclose(OutFile); OutFile = nullptr;
|
||||
break;
|
||||
case CtrlWordType_EOD:
|
||||
++DataSetCnt;
|
||||
InDataSet = false;
|
||||
if (OutFile != nullptr) fclose(OutFile); OutFile = nullptr;
|
||||
LastSeqCnt = -1;
|
||||
break;
|
||||
case CtrlWordType_UNK:
|
||||
if (Type != CtrlWordType_EOD) {
|
||||
printf("Internal error at offset 0x%08x\n",unsigned(Offset));
|
||||
exit(1);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
printf("Internal error at offset 0x%08x\n",unsigned(Offset));
|
||||
exit(1);
|
||||
break;
|
||||
}
|
||||
|
||||
Offset += 8 + RecordSize;
|
||||
if (Type == CtrlWordType_EOD) {
|
||||
// Jump over to the next data record boundary
|
||||
Offset = (Offset & 0xfffff000) + 0x1000;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
delete[] File;
|
||||
return 0;
|
||||
}
|
||||
|
||||
20
simulator/dataset_extractor/dataset_extractor.sln
Normal file
20
simulator/dataset_extractor/dataset_extractor.sln
Normal file
@@ -0,0 +1,20 @@
|
||||
|
||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||
# Visual Studio 2012
|
||||
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "dataset_extractor", "dataset_extractor.vcxproj", "{B5F8F7C2-6676-4408-AC49-0491C5C660B0}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Win32 = Debug|Win32
|
||||
Release|Win32 = Release|Win32
|
||||
EndGlobalSection
|
||||
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||
{B5F8F7C2-6676-4408-AC49-0491C5C660B0}.Debug|Win32.ActiveCfg = Debug|Win32
|
||||
{B5F8F7C2-6676-4408-AC49-0491C5C660B0}.Debug|Win32.Build.0 = Debug|Win32
|
||||
{B5F8F7C2-6676-4408-AC49-0491C5C660B0}.Release|Win32.ActiveCfg = Release|Win32
|
||||
{B5F8F7C2-6676-4408-AC49-0491C5C660B0}.Release|Win32.Build.0 = Release|Win32
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
EndGlobalSection
|
||||
EndGlobal
|
||||
43
simulator/dataset_extractor/dataset_extractor.vcxproj
Normal file
43
simulator/dataset_extractor/dataset_extractor.vcxproj
Normal file
@@ -0,0 +1,43 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<PropertyGroup Label="Globals">
|
||||
<ProjectGuid>{B5F8F7C2-6676-4408-AC49-0491C5C660B0}</ProjectGuid>
|
||||
<RootNamespace>dataset_extractor</RootNamespace>
|
||||
<WindowsTargetPlatformVersion>8.1</WindowsTargetPlatformVersion>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||
<PlatformToolset>v140_xp</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||
<PlatformToolset>v140_xp</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<PlatformToolset>v140_xp</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<PlatformToolset>v140_xp</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
<Import Project="..\common.props" />
|
||||
<ImportGroup Label="PropertySheets">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||
<Link>
|
||||
<Profile>true</Profile>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<Link>
|
||||
<Profile>true</Profile>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="dataset_extractor.cpp" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||
</Project>
|
||||
Reference in New Issue
Block a user