Record parsing

fmp/fmp_dict.go

@@ -0,0 +1,58 @@
+package fmp
+
+type FmpDict map[uint64]*FmpDictEntry
+
+type FmpDictEntry struct {
+	Value    []byte
+	Children *FmpDict
+}
+
+func (dict *FmpDict) GetEntry(path ...uint64) *FmpDictEntry {
+	for i, key := range path {
+		_, ok := (*dict)[key]
+		if !ok {
+			return nil
+		}
+
+		if i == len(path)-1 {
+			return (*dict)[key]
+		} else {
+			dict = (*dict)[key].Children
+			if dict == nil {
+				return nil
+			}
+		}
+	}
+	return nil
+}
+
+func (dict *FmpDict) GetValue(path ...uint64) []byte {
+	ent := dict.GetEntry(path...)
+	if ent != nil {
+		return ent.Value
+	}
+	return nil
+}
+
+func (dict *FmpDict) GetChildren(path ...uint64) *FmpDict {
+	ent := dict.GetEntry(path...)
+	if ent != nil {
+		return ent.Children
+	}
+	return &FmpDict{}
+}
+
+func (dict *FmpDict) SetValue(path []uint64, value []byte) {
+	for i, key := range path {
+		_, ok := (*dict)[key]
+		if !ok {
+			(*dict)[key] = &FmpDictEntry{Children: &FmpDict{}}
+		}
+
+		if i == len(path)-1 {
+			(*dict)[key].Value = value
+		} else {
+			dict = (*dict)[key].Children
+		}
+	}
+}

fmp/fmp_file.go

@@ -28,6 +28,7 @@ type FmpFile struct {
 	Chunks      []*FmpChunk
 	Dictionary  *FmpDict
 
+	tables          []*FmpTable
 	numSectors      uint64 // Excludes the header sector
 	currentSectorID uint64
 
@@ -85,6 +86,7 @@ func OpenFile(path string) (*FmpFile, error) {
 		}
 	}
 
+	ctx.readTables()
 	return ctx, nil
 }
 
@@ -128,8 +130,8 @@ func (ctx *FmpFile) readSector() (*FmpSector, error) {
 		ID:      ctx.currentSectorID,
 		Deleted: buf[0] > 0,
 		Level:   uint8(buf[1]),
-		PrevID:  parseVarUint64(buf[4 : 4+4]),
+		PrevID:  decodeVarUint64(buf[4 : 4+4]),
-		NextID:  parseVarUint64(buf[8 : 8+4]),
+		NextID:  decodeVarUint64(buf[8 : 8+4]),
 		Chunks:  make([]*FmpChunk, 0),
 	}
 
@@ -148,3 +150,61 @@ func (ctx *FmpFile) readSector() (*FmpSector, error) {
 	}
 	return sector, nil
 }
+
+func (ctx *FmpFile) readTables() {
+	tables := make([]*FmpTable, 0)
+	ent := ctx.Dictionary.GetEntry(3, 16, 5)
+
+	for path, tableEnt := range *ent.Children {
+		if path < 128 {
+			continue
+		}
+
+		table := &FmpTable{
+			ID:      path,
+			Name:    decodeString(tableEnt.Children.GetValue(16)),
+			Columns: map[uint64]*FmpColumn{},
+			Records: map[uint64]*FmpRecord{},
+		}
+
+		tables = append(tables, table)
+
+		for colPath, colEnt := range *ctx.Dictionary.GetChildren(table.ID, 3, 5) {
+			name := decodeString(colEnt.Children.GetValue(16))
+			flags := colEnt.Children.GetValue(2)
+
+			column := &FmpColumn{
+				Index:       colPath,
+				Name:        name,
+				Type:        FmpFieldType(flags[0]),
+				DataType:    FmpDataType(flags[1]),
+				StorageType: FmpFieldStorageType(flags[9]),
+				Repetitions: flags[25],
+				Indexed:     flags[8] == 128,
+			}
+
+			if flags[11] == 1 {
+				column.AutoEnter = autoEnterPresetMap[flags[4]]
+			} else {
+				column.AutoEnter = autoEnterOptionMap[flags[11]]
+			}
+
+			table.Columns[column.Index] = column
+		}
+
+		for recPath, recEnt := range *ctx.Dictionary.GetChildren(table.ID, 5) {
+			record := &FmpRecord{Index: recPath, Values: make(map[uint64]string)}
+			table.Records[record.Index] = record
+
+			if recPath > table.lastRecordID {
+				table.lastRecordID = recPath
+			}
+
+			for colIndex, value := range *recEnt.Children {
+				record.Values[colIndex] = decodeString(value.Value)
+			}
+		}
+	}
+
+	ctx.tables = tables
+}

fmp/fmp_sector.go

@@ -70,7 +70,7 @@ func (sect *FmpSector) processChunks(dict *FmpDict) error {
 	for _, chunk := range sect.Chunks {
 		switch chunk.Type {
 		case FmpChunkPathPush, FmpChunkPathPushLong:
-			currentPath = append(currentPath, parseVarUint64(chunk.Value))
+			currentPath = append(currentPath, decodeVarUint64(chunk.Value))
 			dumpPath(currentPath)
 
 		case FmpChunkPathPop:
@@ -134,7 +134,7 @@ func (sect *FmpSector) readChunk(payload []byte) (*FmpChunk, error) {
 		chunk.Value = payload[3:chunk.Length]
 
 	case 0x07:
-		valueLength := parseVarUint64(payload[2 : 2+2])
+		valueLength := decodeVarUint64(payload[2 : 2+2])
 		chunk.Length = min(4+valueLength, uint64(len(payload)))
 		chunk.Type = FmpChunkSegmentedData
 		chunk.Index = uint64(payload[1])
@@ -148,13 +148,13 @@ func (sect *FmpSector) readChunk(payload []byte) (*FmpChunk, error) {
 	case 0x09:
 		chunk.Length = 4
 		chunk.Type = FmpChunkSimpleKeyValue
-		chunk.Key = parseVarUint64(payload[1 : 1+2])
+		chunk.Key = decodeVarUint64(payload[1 : 1+2])
 		chunk.Value = payload[3:chunk.Length]
 
 	case 0x0A, 0x0B, 0x0C, 0x0D:
 		chunk.Length = 3 + 2*uint64(chunkCode-0x09)
 		chunk.Type = FmpChunkSimpleKeyValue
-		chunk.Key = parseVarUint64(payload[1 : 1+2])
+		chunk.Key = decodeVarUint64(payload[1 : 1+2])
 		chunk.Value = payload[3:chunk.Length]
 
 	case 0x0E:
@@ -167,17 +167,17 @@ func (sect *FmpSector) readChunk(payload []byte) (*FmpChunk, error) {
 
 		chunk.Length = 4 + uint64(payload[3])
 		chunk.Type = FmpChunkSimpleKeyValue
-		chunk.Key = parseVarUint64(payload[1 : 1+2])
+		chunk.Key = decodeVarUint64(payload[1 : 1+2])
 		chunk.Value = payload[4:chunk.Length]
 
 	case 0x0F:
-		valueLength := parseVarUint64(payload[3 : 3+2])
+		valueLength := decodeVarUint64(payload[3 : 3+2])
 		chunk.Length = uint64(len(payload))
 		if chunk.Length > 5+valueLength {
 			return nil, ErrBadChunk
 		}
 		chunk.Type = FmpChunkSegmentedData
-		chunk.Index = parseVarUint64(payload[1 : 1+2])
+		chunk.Index = decodeVarUint64(payload[1 : 1+2])
 		chunk.Value = payload[5:chunk.Length]
 
 	case 0x10, 0x11:
@@ -193,13 +193,13 @@ func (sect *FmpSector) readChunk(payload []byte) (*FmpChunk, error) {
 	case 0x16:
 		chunk.Length = 5 + uint64(payload[4])
 		chunk.Type = FmpChunkLongKeyValue
-		chunk.Key = parseVarUint64(payload[1 : 1+3])
+		chunk.Key = decodeVarUint64(payload[1 : 1+3])
 		chunk.Value = payload[5:chunk.Length]
 
 	case 0x17:
-		chunk.Length = 6 + parseVarUint64(payload[4:4+2])
+		chunk.Length = 6 + decodeVarUint64(payload[4:4+2])
 		chunk.Type = FmpChunkLongKeyValue
-		chunk.Key = parseVarUint64(payload[1 : 1+3])
+		chunk.Key = decodeVarUint64(payload[1 : 1+3])
 		chunk.Value = payload[6:chunk.Length]
 
 	case 0x19, 0x1A, 0x1B, 0x1C, 0x1D:
@@ -213,15 +213,15 @@ func (sect *FmpSector) readChunk(payload []byte) (*FmpChunk, error) {
 		valueLength := uint64(payload[2+keyLength])
 		chunk.Length = 2 + keyLength + 1 + valueLength
 		chunk.Type = FmpChunkLongKeyValue
-		chunk.Key = parseVarUint64(payload[2 : 2+keyLength])
+		chunk.Key = decodeVarUint64(payload[2 : 2+keyLength])
 		chunk.Value = payload[2+keyLength+1 : chunk.Length]
 
 	case 0x1F:
 		keyLength := uint64(payload[1])
-		valueLength := parseVarUint64(payload[2+keyLength : 2+keyLength+2+1])
+		valueLength := decodeVarUint64(payload[2+keyLength : 2+keyLength+2+1])
 		chunk.Length = 2 + keyLength + 2 + valueLength
 		chunk.Type = FmpChunkLongKeyValue
-		chunk.Key = parseVarUint64(payload[2 : 2+keyLength])
+		chunk.Key = decodeVarUint64(payload[2 : 2+keyLength])
 		chunk.Value = payload[2+keyLength+2 : chunk.Length]
 
 	case 0x20, 0xE0:

fmp/fmp_table.go

@@ -3,8 +3,10 @@ package fmp
 type FmpTable struct {
 	ID      uint64
 	Name    string
-	Columns map[uint64]FmpColumn
+	Columns map[uint64]*FmpColumn
-	Records map[uint64]FmpRecord
+	Records map[uint64]*FmpRecord
+
+	lastRecordID uint64
 }
 
 type FmpColumn struct {
@@ -19,60 +21,43 @@ type FmpColumn struct {
 }
 
 type FmpRecord struct {
+	Table  *FmpTable
 	Index  uint64
 	Values map[uint64]string
 }
 
-func (ctx *FmpFile) Tables() []*FmpTable {
+func (ctx *FmpFile) Table(name string) *FmpTable {
-	tables := make([]*FmpTable, 0)
+	for _, table := range ctx.tables {
-	ent := ctx.Dictionary.GetEntry(3, 16, 5)
+		if table.Name == name {
-
+			return table
-	for path, tableEnt := range *ent.Children {
-		if path < 128 {
-			continue
-		}
-
-		table := &FmpTable{
-			ID:      path,
-			Name:    decodeFmpString(tableEnt.Children.GetValue(16)),
-			Columns: map[uint64]FmpColumn{},
-			Records: map[uint64]FmpRecord{},
-		}
-
-		tables = append(tables, table)
-
-		for colPath, colEnt := range *ctx.Dictionary.GetChildren(table.ID, 3, 5) {
-			name := decodeFmpString(colEnt.Children.GetValue(16))
-			flags := colEnt.Children.GetValue(2)
-
-			column := FmpColumn{
-				Index:       colPath,
-				Name:        name,
-				Type:        FmpFieldType(flags[0]),
-				DataType:    FmpDataType(flags[1]),
-				StorageType: FmpFieldStorageType(flags[9]),
-				Repetitions: flags[25],
-				Indexed:     flags[8] == 128,
-			}
-
-			if flags[11] == 1 {
-				column.AutoEnter = autoEnterPresetMap[flags[4]]
-			} else {
-				column.AutoEnter = autoEnterOptionMap[flags[11]]
-			}
-
-			table.Columns[column.Index] = column
-		}
-
-		for recPath, recEnt := range *ctx.Dictionary.GetChildren(table.ID, 5) {
-			record := FmpRecord{Index: recPath, Values: make(map[uint64]string)}
-			table.Records[record.Index] = record
-
-			for colIndex, value := range *recEnt.Children {
-				record.Values[colIndex] = decodeFmpString(value.Value)
-			}
 		}
 	}
-
+	return nil
-	return tables
+}
+
+func (t *FmpTable) Column(name string) *FmpColumn {
+	for _, column := range t.Columns {
+		if column.Name == name {
+			return column
+		}
+	}
+	return nil
+}
+
+func (t *FmpTable) NewRecord(values map[string]string) (*FmpRecord, error) {
+	vals := make(map[uint64]string)
+	for k, v := range values {
+		col := t.Column(k)
+		vals[col.Index] = v
+	}
+
+	id := t.lastRecordID + 1
+	t.lastRecordID = id
+	t.Records[id] = &FmpRecord{Table: t, Index: id, Values: vals}
+
+	return t.Records[id], nil
+}
+
+func (r *FmpRecord) Value(name string) string {
+	return r.Values[r.Table.Column(name).Index]
 }

fmp/fmp_test.go

@@ -1,6 +1,9 @@
 package fmp
 
-import "testing"
+import (
+	"slices"
+	"testing"
+)
 
 func TestOpenFile(t *testing.T) {
 	f, err := OpenFile("../files/Untitled.fmp12")
@@ -27,50 +30,80 @@ func TestTables(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	tables := f.Tables()
 
 	expectedNames := []string{"Untitled"}
 	tableNames := []string{}
-	for _, table := range tables {
+	for _, table := range f.tables {
 		tableNames = append(tableNames, table.Name)
 	}
-
 	if !slicesHaveSameElements(tableNames, expectedNames) {
 		t.Errorf("tables do not match")
 	}
 
-	var field FmpColumn
+	table := f.Table("Untitled")
-	for _, table := range tables {
+	if table == nil {
-		for _, column := range table.Columns {
+		t.Errorf("expected table to exist, but it does not")
-			if column.Name == "PrimaryKey" {
+		return
-				field = column
+	}
-				break
+	if table.Name != "Untitled" {
-			}
+		t.Errorf("expected table name to be 'Untitled', but it is '%s'", table.Name)
-		}
+	}
+	if len(table.Records) != 3 {
+		t.Errorf("expected table to have 3 records, but it has %d", len(table.Records))
+	}
+	if table.Records[1].Values[1] != "629FAA83-50D8-401F-A560-C8D45217D17B" {
+		t.Errorf("first record has an incorrect ID '%s'", table.Records[0].Values[0])
 	}
 
-	if field.Type != FmpFieldSimple {
+	col := table.Column("PrimaryKey")
+	if col == nil {
+		t.Errorf("expected column to exist, but it does not")
+		return
+	}
+	if col.Name != "PrimaryKey" {
+		t.Errorf("expected column name to be 'PrimaryKey', but it is '%s'", col.Name)
+	}
+	if col.Type != FmpFieldSimple {
 		t.Errorf("expected field type to be simple, but it is not")
 	}
-	if field.DataType != FmpDataText {
+	if col.DataType != FmpDataText {
 		t.Errorf("expected field data type to be text, but it is not")
 	}
-	if field.StorageType != FmpFieldStorageRegular {
+	if col.StorageType != FmpFieldStorageRegular {
 		t.Errorf("expected field storage type to be regular, but it is not")
 	}
-	if field.Repetitions != 1 {
+	if col.Repetitions != 1 {
-		t.Errorf("expected field repetition count to be 1, but it is %d", field.Repetitions)
+		t.Errorf("expected field repetition count to be 1, but it is %d", col.Repetitions)
 	}
-	if !field.Indexed {
+	if !col.Indexed {
 		t.Errorf("expected field to be indexed, but it is not")
 	}
-	if field.AutoEnter != FmpAutoEnterCalculationReplacingExistingValue {
+	if col.AutoEnter != FmpAutoEnterCalculationReplacingExistingValue {
 		t.Errorf("expected field to have auto enter calculation replacing existing value, but it does not")
 	}
-	if len(tables[0].Records) != 3 {
+
-		t.Errorf("expected table to have 3 records, but it has %d", len(tables[0].Records))
+	newRecord, err := table.NewRecord(map[string]string{"PrimaryKey": "629FAA83-50D8-401F-A560-C8D45217D17B"})
+	if newRecord == nil || err != nil {
+		t.Errorf("expected new record to be created, but it is nil")
+		return
 	}
-	if tables[0].Records[1].Values[1] != "629FAA83-50D8-401F-A560-C8D45217D17B" {
+	if newRecord.Index != 4 {
-		t.Errorf("first record has an incorrect ID '%s'", tables[0].Records[0].Values[0])
+		t.Errorf("expected new record index to be 4, but it is %d", newRecord.Index)
+	}
+	if newRecord.Value("PrimaryKey") != "629FAA83-50D8-401F-A560-C8D45217D17B" {
+		t.Errorf("expected new record primary key to be '629FAA83-50D8-401F-A560-C8D45217D17B', but it is '%s'", newRecord.Value("PrimaryKey"))
 	}
 }
+
+func slicesHaveSameElements[Type comparable](a, b []Type) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for _, av := range a {
+		found := slices.Contains(b, av)
+		if !found {
+			return false
+		}
+	}
+	return true
+}

fmp/fmp_util.go

@@ -1,65 +1,13 @@
 package fmp
 
-import "slices"
+func addIf(cond bool, val uint64) uint64 {
-
+	if cond {
-type FmpDict map[uint64]*FmpDictEntry
+		return val
-
-type FmpDictEntry struct {
-	Value    []byte
-	Children *FmpDict
-}
-
-func (dict *FmpDict) GetEntry(path ...uint64) *FmpDictEntry {
-	for i, key := range path {
-		_, ok := (*dict)[key]
-		if !ok {
-			return nil
-		}
-
-		if i == len(path)-1 {
-			return (*dict)[key]
-		} else {
-			dict = (*dict)[key].Children
-			if dict == nil {
-				return nil
-			}
-		}
 	}
-	return nil
+	return 0
 }
 
-func (dict *FmpDict) GetValue(path ...uint64) []byte {
+func decodeVarUint64(payload []byte) uint64 {
-	ent := dict.GetEntry(path...)
-	if ent != nil {
-		return ent.Value
-	}
-	return nil
-}
-
-func (dict *FmpDict) GetChildren(path ...uint64) *FmpDict {
-	ent := dict.GetEntry(path...)
-	if ent != nil {
-		return ent.Children
-	}
-	return &FmpDict{}
-}
-
-func (dict *FmpDict) SetValue(path []uint64, value []byte) {
-	for i, key := range path {
-		_, ok := (*dict)[key]
-		if !ok {
-			(*dict)[key] = &FmpDictEntry{Children: &FmpDict{}}
-		}
-
-		if i == len(path)-1 {
-			(*dict)[key].Value = value
-		} else {
-			dict = (*dict)[key].Children
-		}
-	}
-}
-
-func parseVarUint64(payload []byte) uint64 {
 	var length uint64
 	n := min(len(payload), 8) // clamp to uint64
 	for i := range n {
@@ -69,30 +17,10 @@ func parseVarUint64(payload []byte) uint64 {
 	return length
 }
 
-func decodeFmpString(payload []byte) string {
+func decodeString(payload []byte) string {
 	result := ""
 	for i := range payload {
 		result += string(payload[i] ^ 0x5A)
 	}
 	return result
 }
-
-func addIf(cond bool, val uint64) uint64 {
-	if cond {
-		return val
-	}
-	return 0
-}
-
-func slicesHaveSameElements[Type comparable](a, b []Type) bool {
-	if len(a) != len(b) {
-		return false
-	}
-	for _, av := range a {
-		found := slices.Contains(b, av)
-		if !found {
-			return false
-		}
-	}
-	return true
-}