From 7cf3fd93ac3d2e996c4d75ff51a6f4bacae086d0 Mon Sep 17 00:00:00 2001 From: Romein van Buren Date: Sun, 22 Jun 2025 19:14:22 +0200 Subject: [PATCH] Record parsing --- fmp/fmp_dict.go | 58 ++++++++++++++++++++++++++++++ fmp/fmp_file.go | 64 ++++++++++++++++++++++++++++++++-- fmp/fmp_sector.go | 26 +++++++------- fmp/fmp_table.go | 89 ++++++++++++++++++++--------------------------- fmp/fmp_test.go | 79 +++++++++++++++++++++++++++++------------ fmp/fmp_util.go | 84 ++++---------------------------------------- 6 files changed, 232 insertions(+), 168 deletions(-) create mode 100644 fmp/fmp_dict.go diff --git a/fmp/fmp_dict.go b/fmp/fmp_dict.go new file mode 100644 index 0000000..8613c47 --- /dev/null +++ b/fmp/fmp_dict.go @@ -0,0 +1,58 @@ +package fmp + +type FmpDict map[uint64]*FmpDictEntry + +type FmpDictEntry struct { + Value []byte + Children *FmpDict +} + +func (dict *FmpDict) GetEntry(path ...uint64) *FmpDictEntry { + for i, key := range path { + _, ok := (*dict)[key] + if !ok { + return nil + } + + if i == len(path)-1 { + return (*dict)[key] + } else { + dict = (*dict)[key].Children + if dict == nil { + return nil + } + } + } + return nil +} + +func (dict *FmpDict) GetValue(path ...uint64) []byte { + ent := dict.GetEntry(path...) + if ent != nil { + return ent.Value + } + return nil +} + +func (dict *FmpDict) GetChildren(path ...uint64) *FmpDict { + ent := dict.GetEntry(path...) + if ent != nil { + return ent.Children + } + return &FmpDict{} +} + +func (dict *FmpDict) SetValue(path []uint64, value []byte) { + for i, key := range path { + _, ok := (*dict)[key] + if !ok { + (*dict)[key] = &FmpDictEntry{Children: &FmpDict{}} + } + + if i == len(path)-1 { + (*dict)[key].Value = value + } else { + dict = (*dict)[key].Children + } + } +} diff --git a/fmp/fmp_file.go b/fmp/fmp_file.go index 3d52e8d..dd8751d 100644 --- a/fmp/fmp_file.go +++ b/fmp/fmp_file.go @@ -28,6 +28,7 @@ type FmpFile struct { Chunks []*FmpChunk Dictionary *FmpDict + tables []*FmpTable numSectors uint64 // Excludes the header sector currentSectorID uint64 @@ -85,6 +86,7 @@ func OpenFile(path string) (*FmpFile, error) { } } + ctx.readTables() return ctx, nil } @@ -128,8 +130,8 @@ func (ctx *FmpFile) readSector() (*FmpSector, error) { ID: ctx.currentSectorID, Deleted: buf[0] > 0, Level: uint8(buf[1]), - PrevID: parseVarUint64(buf[4 : 4+4]), - NextID: parseVarUint64(buf[8 : 8+4]), + PrevID: decodeVarUint64(buf[4 : 4+4]), + NextID: decodeVarUint64(buf[8 : 8+4]), Chunks: make([]*FmpChunk, 0), } @@ -148,3 +150,61 @@ func (ctx *FmpFile) readSector() (*FmpSector, error) { } return sector, nil } + +func (ctx *FmpFile) readTables() { + tables := make([]*FmpTable, 0) + ent := ctx.Dictionary.GetEntry(3, 16, 5) + + for path, tableEnt := range *ent.Children { + if path < 128 { + continue + } + + table := &FmpTable{ + ID: path, + Name: decodeString(tableEnt.Children.GetValue(16)), + Columns: map[uint64]*FmpColumn{}, + Records: map[uint64]*FmpRecord{}, + } + + tables = append(tables, table) + + for colPath, colEnt := range *ctx.Dictionary.GetChildren(table.ID, 3, 5) { + name := decodeString(colEnt.Children.GetValue(16)) + flags := colEnt.Children.GetValue(2) + + column := &FmpColumn{ + Index: colPath, + Name: name, + Type: FmpFieldType(flags[0]), + DataType: FmpDataType(flags[1]), + StorageType: FmpFieldStorageType(flags[9]), + Repetitions: flags[25], + Indexed: flags[8] == 128, + } + + if flags[11] == 1 { + column.AutoEnter = autoEnterPresetMap[flags[4]] + } else { + column.AutoEnter = autoEnterOptionMap[flags[11]] + } + + table.Columns[column.Index] = column + } + + for recPath, recEnt := range *ctx.Dictionary.GetChildren(table.ID, 5) { + record := &FmpRecord{Index: recPath, Values: make(map[uint64]string)} + table.Records[record.Index] = record + + if recPath > table.lastRecordID { + table.lastRecordID = recPath + } + + for colIndex, value := range *recEnt.Children { + record.Values[colIndex] = decodeString(value.Value) + } + } + } + + ctx.tables = tables +} diff --git a/fmp/fmp_sector.go b/fmp/fmp_sector.go index 5a18127..05a26c3 100644 --- a/fmp/fmp_sector.go +++ b/fmp/fmp_sector.go @@ -70,7 +70,7 @@ func (sect *FmpSector) processChunks(dict *FmpDict) error { for _, chunk := range sect.Chunks { switch chunk.Type { case FmpChunkPathPush, FmpChunkPathPushLong: - currentPath = append(currentPath, parseVarUint64(chunk.Value)) + currentPath = append(currentPath, decodeVarUint64(chunk.Value)) dumpPath(currentPath) case FmpChunkPathPop: @@ -134,7 +134,7 @@ func (sect *FmpSector) readChunk(payload []byte) (*FmpChunk, error) { chunk.Value = payload[3:chunk.Length] case 0x07: - valueLength := parseVarUint64(payload[2 : 2+2]) + valueLength := decodeVarUint64(payload[2 : 2+2]) chunk.Length = min(4+valueLength, uint64(len(payload))) chunk.Type = FmpChunkSegmentedData chunk.Index = uint64(payload[1]) @@ -148,13 +148,13 @@ func (sect *FmpSector) readChunk(payload []byte) (*FmpChunk, error) { case 0x09: chunk.Length = 4 chunk.Type = FmpChunkSimpleKeyValue - chunk.Key = parseVarUint64(payload[1 : 1+2]) + chunk.Key = decodeVarUint64(payload[1 : 1+2]) chunk.Value = payload[3:chunk.Length] case 0x0A, 0x0B, 0x0C, 0x0D: chunk.Length = 3 + 2*uint64(chunkCode-0x09) chunk.Type = FmpChunkSimpleKeyValue - chunk.Key = parseVarUint64(payload[1 : 1+2]) + chunk.Key = decodeVarUint64(payload[1 : 1+2]) chunk.Value = payload[3:chunk.Length] case 0x0E: @@ -167,17 +167,17 @@ func (sect *FmpSector) readChunk(payload []byte) (*FmpChunk, error) { chunk.Length = 4 + uint64(payload[3]) chunk.Type = FmpChunkSimpleKeyValue - chunk.Key = parseVarUint64(payload[1 : 1+2]) + chunk.Key = decodeVarUint64(payload[1 : 1+2]) chunk.Value = payload[4:chunk.Length] case 0x0F: - valueLength := parseVarUint64(payload[3 : 3+2]) + valueLength := decodeVarUint64(payload[3 : 3+2]) chunk.Length = uint64(len(payload)) if chunk.Length > 5+valueLength { return nil, ErrBadChunk } chunk.Type = FmpChunkSegmentedData - chunk.Index = parseVarUint64(payload[1 : 1+2]) + chunk.Index = decodeVarUint64(payload[1 : 1+2]) chunk.Value = payload[5:chunk.Length] case 0x10, 0x11: @@ -193,13 +193,13 @@ func (sect *FmpSector) readChunk(payload []byte) (*FmpChunk, error) { case 0x16: chunk.Length = 5 + uint64(payload[4]) chunk.Type = FmpChunkLongKeyValue - chunk.Key = parseVarUint64(payload[1 : 1+3]) + chunk.Key = decodeVarUint64(payload[1 : 1+3]) chunk.Value = payload[5:chunk.Length] case 0x17: - chunk.Length = 6 + parseVarUint64(payload[4:4+2]) + chunk.Length = 6 + decodeVarUint64(payload[4:4+2]) chunk.Type = FmpChunkLongKeyValue - chunk.Key = parseVarUint64(payload[1 : 1+3]) + chunk.Key = decodeVarUint64(payload[1 : 1+3]) chunk.Value = payload[6:chunk.Length] case 0x19, 0x1A, 0x1B, 0x1C, 0x1D: @@ -213,15 +213,15 @@ func (sect *FmpSector) readChunk(payload []byte) (*FmpChunk, error) { valueLength := uint64(payload[2+keyLength]) chunk.Length = 2 + keyLength + 1 + valueLength chunk.Type = FmpChunkLongKeyValue - chunk.Key = parseVarUint64(payload[2 : 2+keyLength]) + chunk.Key = decodeVarUint64(payload[2 : 2+keyLength]) chunk.Value = payload[2+keyLength+1 : chunk.Length] case 0x1F: keyLength := uint64(payload[1]) - valueLength := parseVarUint64(payload[2+keyLength : 2+keyLength+2+1]) + valueLength := decodeVarUint64(payload[2+keyLength : 2+keyLength+2+1]) chunk.Length = 2 + keyLength + 2 + valueLength chunk.Type = FmpChunkLongKeyValue - chunk.Key = parseVarUint64(payload[2 : 2+keyLength]) + chunk.Key = decodeVarUint64(payload[2 : 2+keyLength]) chunk.Value = payload[2+keyLength+2 : chunk.Length] case 0x20, 0xE0: diff --git a/fmp/fmp_table.go b/fmp/fmp_table.go index 8b36d8e..1974303 100644 --- a/fmp/fmp_table.go +++ b/fmp/fmp_table.go @@ -3,8 +3,10 @@ package fmp type FmpTable struct { ID uint64 Name string - Columns map[uint64]FmpColumn - Records map[uint64]FmpRecord + Columns map[uint64]*FmpColumn + Records map[uint64]*FmpRecord + + lastRecordID uint64 } type FmpColumn struct { @@ -19,60 +21,43 @@ type FmpColumn struct { } type FmpRecord struct { + Table *FmpTable Index uint64 Values map[uint64]string } -func (ctx *FmpFile) Tables() []*FmpTable { - tables := make([]*FmpTable, 0) - ent := ctx.Dictionary.GetEntry(3, 16, 5) - - for path, tableEnt := range *ent.Children { - if path < 128 { - continue - } - - table := &FmpTable{ - ID: path, - Name: decodeFmpString(tableEnt.Children.GetValue(16)), - Columns: map[uint64]FmpColumn{}, - Records: map[uint64]FmpRecord{}, - } - - tables = append(tables, table) - - for colPath, colEnt := range *ctx.Dictionary.GetChildren(table.ID, 3, 5) { - name := decodeFmpString(colEnt.Children.GetValue(16)) - flags := colEnt.Children.GetValue(2) - - column := FmpColumn{ - Index: colPath, - Name: name, - Type: FmpFieldType(flags[0]), - DataType: FmpDataType(flags[1]), - StorageType: FmpFieldStorageType(flags[9]), - Repetitions: flags[25], - Indexed: flags[8] == 128, - } - - if flags[11] == 1 { - column.AutoEnter = autoEnterPresetMap[flags[4]] - } else { - column.AutoEnter = autoEnterOptionMap[flags[11]] - } - - table.Columns[column.Index] = column - } - - for recPath, recEnt := range *ctx.Dictionary.GetChildren(table.ID, 5) { - record := FmpRecord{Index: recPath, Values: make(map[uint64]string)} - table.Records[record.Index] = record - - for colIndex, value := range *recEnt.Children { - record.Values[colIndex] = decodeFmpString(value.Value) - } +func (ctx *FmpFile) Table(name string) *FmpTable { + for _, table := range ctx.tables { + if table.Name == name { + return table } } - - return tables + return nil +} + +func (t *FmpTable) Column(name string) *FmpColumn { + for _, column := range t.Columns { + if column.Name == name { + return column + } + } + return nil +} + +func (t *FmpTable) NewRecord(values map[string]string) (*FmpRecord, error) { + vals := make(map[uint64]string) + for k, v := range values { + col := t.Column(k) + vals[col.Index] = v + } + + id := t.lastRecordID + 1 + t.lastRecordID = id + t.Records[id] = &FmpRecord{Table: t, Index: id, Values: vals} + + return t.Records[id], nil +} + +func (r *FmpRecord) Value(name string) string { + return r.Values[r.Table.Column(name).Index] } diff --git a/fmp/fmp_test.go b/fmp/fmp_test.go index 1f40275..57e96c9 100644 --- a/fmp/fmp_test.go +++ b/fmp/fmp_test.go @@ -1,6 +1,9 @@ package fmp -import "testing" +import ( + "slices" + "testing" +) func TestOpenFile(t *testing.T) { f, err := OpenFile("../files/Untitled.fmp12") @@ -27,50 +30,80 @@ func TestTables(t *testing.T) { if err != nil { t.Fatal(err) } - tables := f.Tables() expectedNames := []string{"Untitled"} tableNames := []string{} - for _, table := range tables { + for _, table := range f.tables { tableNames = append(tableNames, table.Name) } - if !slicesHaveSameElements(tableNames, expectedNames) { t.Errorf("tables do not match") } - var field FmpColumn - for _, table := range tables { - for _, column := range table.Columns { - if column.Name == "PrimaryKey" { - field = column - break - } - } + table := f.Table("Untitled") + if table == nil { + t.Errorf("expected table to exist, but it does not") + return + } + if table.Name != "Untitled" { + t.Errorf("expected table name to be 'Untitled', but it is '%s'", table.Name) + } + if len(table.Records) != 3 { + t.Errorf("expected table to have 3 records, but it has %d", len(table.Records)) + } + if table.Records[1].Values[1] != "629FAA83-50D8-401F-A560-C8D45217D17B" { + t.Errorf("first record has an incorrect ID '%s'", table.Records[0].Values[0]) } - if field.Type != FmpFieldSimple { + col := table.Column("PrimaryKey") + if col == nil { + t.Errorf("expected column to exist, but it does not") + return + } + if col.Name != "PrimaryKey" { + t.Errorf("expected column name to be 'PrimaryKey', but it is '%s'", col.Name) + } + if col.Type != FmpFieldSimple { t.Errorf("expected field type to be simple, but it is not") } - if field.DataType != FmpDataText { + if col.DataType != FmpDataText { t.Errorf("expected field data type to be text, but it is not") } - if field.StorageType != FmpFieldStorageRegular { + if col.StorageType != FmpFieldStorageRegular { t.Errorf("expected field storage type to be regular, but it is not") } - if field.Repetitions != 1 { - t.Errorf("expected field repetition count to be 1, but it is %d", field.Repetitions) + if col.Repetitions != 1 { + t.Errorf("expected field repetition count to be 1, but it is %d", col.Repetitions) } - if !field.Indexed { + if !col.Indexed { t.Errorf("expected field to be indexed, but it is not") } - if field.AutoEnter != FmpAutoEnterCalculationReplacingExistingValue { + if col.AutoEnter != FmpAutoEnterCalculationReplacingExistingValue { t.Errorf("expected field to have auto enter calculation replacing existing value, but it does not") } - if len(tables[0].Records) != 3 { - t.Errorf("expected table to have 3 records, but it has %d", len(tables[0].Records)) + + newRecord, err := table.NewRecord(map[string]string{"PrimaryKey": "629FAA83-50D8-401F-A560-C8D45217D17B"}) + if newRecord == nil || err != nil { + t.Errorf("expected new record to be created, but it is nil") + return } - if tables[0].Records[1].Values[1] != "629FAA83-50D8-401F-A560-C8D45217D17B" { - t.Errorf("first record has an incorrect ID '%s'", tables[0].Records[0].Values[0]) + if newRecord.Index != 4 { + t.Errorf("expected new record index to be 4, but it is %d", newRecord.Index) + } + if newRecord.Value("PrimaryKey") != "629FAA83-50D8-401F-A560-C8D45217D17B" { + t.Errorf("expected new record primary key to be '629FAA83-50D8-401F-A560-C8D45217D17B', but it is '%s'", newRecord.Value("PrimaryKey")) } } + +func slicesHaveSameElements[Type comparable](a, b []Type) bool { + if len(a) != len(b) { + return false + } + for _, av := range a { + found := slices.Contains(b, av) + if !found { + return false + } + } + return true +} diff --git a/fmp/fmp_util.go b/fmp/fmp_util.go index f523dee..cbeda38 100644 --- a/fmp/fmp_util.go +++ b/fmp/fmp_util.go @@ -1,65 +1,13 @@ package fmp -import "slices" - -type FmpDict map[uint64]*FmpDictEntry - -type FmpDictEntry struct { - Value []byte - Children *FmpDict -} - -func (dict *FmpDict) GetEntry(path ...uint64) *FmpDictEntry { - for i, key := range path { - _, ok := (*dict)[key] - if !ok { - return nil - } - - if i == len(path)-1 { - return (*dict)[key] - } else { - dict = (*dict)[key].Children - if dict == nil { - return nil - } - } +func addIf(cond bool, val uint64) uint64 { + if cond { + return val } - return nil + return 0 } -func (dict *FmpDict) GetValue(path ...uint64) []byte { - ent := dict.GetEntry(path...) - if ent != nil { - return ent.Value - } - return nil -} - -func (dict *FmpDict) GetChildren(path ...uint64) *FmpDict { - ent := dict.GetEntry(path...) - if ent != nil { - return ent.Children - } - return &FmpDict{} -} - -func (dict *FmpDict) SetValue(path []uint64, value []byte) { - for i, key := range path { - _, ok := (*dict)[key] - if !ok { - (*dict)[key] = &FmpDictEntry{Children: &FmpDict{}} - } - - if i == len(path)-1 { - (*dict)[key].Value = value - } else { - dict = (*dict)[key].Children - } - } -} - -func parseVarUint64(payload []byte) uint64 { +func decodeVarUint64(payload []byte) uint64 { var length uint64 n := min(len(payload), 8) // clamp to uint64 for i := range n { @@ -69,30 +17,10 @@ func parseVarUint64(payload []byte) uint64 { return length } -func decodeFmpString(payload []byte) string { +func decodeString(payload []byte) string { result := "" for i := range payload { result += string(payload[i] ^ 0x5A) } return result } - -func addIf(cond bool, val uint64) uint64 { - if cond { - return val - } - return 0 -} - -func slicesHaveSameElements[Type comparable](a, b []Type) bool { - if len(a) != len(b) { - return false - } - for _, av := range a { - found := slices.Contains(b, av) - if !found { - return false - } - } - return true -}