diff --git a/fmp/fmp_chunk.go b/fmp/fmp_chunk.go index 4d1842b..903c215 100644 --- a/fmp/fmp_chunk.go +++ b/fmp/fmp_chunk.go @@ -2,229 +2,194 @@ package fmp func (ctx *FmpFile) readChunk(payload []byte) (*FmpChunk, error) { - // Simple data + // https://github.com/evanmiller/fmptools/blob/02eb770e59e0866dab213d80e5f7d88e17648031/HACKING + // https://github.com/Rasmus20B/fmplib/blob/66245e5269275724bacfe1437fb1f73bc587a2f3/src/fmp_format/chunk.rs#L57-L60 - if payload[0] == 0x00 || payload[0] == 0x19 || payload[0] == 0x23 { - return &FmpChunk{ - Type: FMP_CHUNK_SIMPLE_DATA, - Value: payload[1 : 1+1], - Length: 2, - }, nil - } - if payload[0] == 0x08 { - return &FmpChunk{ - Type: FMP_CHUNK_SIMPLE_DATA, - Value: payload[1 : 1+2], - Length: 3, - }, nil - } - if payload[0] == 0x0E && payload[1] == 0xFF { - return &FmpChunk{ - Type: FMP_CHUNK_SIMPLE_DATA, - Value: payload[2 : 2+5], - Length: 7, - }, nil - } - if 0x10 <= payload[0] && payload[0] <= 0x11 { - valueLength := 3 + (payload[0] - 0x10) - return &FmpChunk{ - Type: FMP_CHUNK_SIMPLE_DATA, - Value: payload[1 : 1+valueLength], - Length: 1 + uint64(valueLength), - }, nil - } - if 0x12 <= payload[0] && payload[0] <= 0x15 { - valueLength := 1 + 2*(payload[0]-0x10) - return &FmpChunk{ - Type: FMP_CHUNK_SIMPLE_DATA, - Value: payload[1 : 1+valueLength], - Length: 1 + uint64(valueLength), - }, nil - } - if 0x1A <= payload[0] && payload[0] <= 0x1D { - valueLength := 2 * (payload[0] - 0x19) - return &FmpChunk{ - Type: FMP_CHUNK_SIMPLE_DATA, - Value: payload[1 : 1+valueLength], - Length: 1 + uint64(valueLength), - }, nil + chunk := &FmpChunk{} + + if (payload[0] & 0xC0) == 0xC0 { + payload[0] &= 0x3F + chunk.Delayed = true } - // Simple key-value + switch payload[0] { + case 0x00: + chunk.Type = FMP_CHUNK_SIMPLE_DATA + chunk.Value = payload[1 : 1+1] + chunk.Length = 2 - if payload[0] == 0x01 { - return &FmpChunk{ - Type: FMP_CHUNK_SIMPLE_KEY_VALUE, - Key: uint64(payload[1]), - Value: payload[2 : 2+1], - Length: 3, - }, nil - } - if 0x02 <= payload[0] && payload[0] <= 0x05 { + case 0x01: + chunk.Type = FMP_CHUNK_SIMPLE_KEY_VALUE + chunk.Key = uint64(payload[1]) + chunk.Value = payload[2 : 2+1] + chunk.Length = 3 + + case 0x02, 0x03, 0x04, 0x05: valueLength := 2 * (payload[0] - 1) - return &FmpChunk{ - Type: FMP_CHUNK_SIMPLE_KEY_VALUE, - Key: uint64(payload[1]), - Value: payload[2 : 2+valueLength], - Length: 2 + uint64(valueLength), - }, nil - } - if payload[0] == 0x06 { + chunk.Type = FMP_CHUNK_SIMPLE_KEY_VALUE + chunk.Key = uint64(payload[1]) + chunk.Value = payload[2 : 2+valueLength] + chunk.Length = 2 + uint64(valueLength) + + case 0x06: valueLength := payload[2] - return &FmpChunk{ - Type: FMP_CHUNK_SIMPLE_KEY_VALUE, - Key: uint64(payload[1]), - Value: payload[2 : 2+valueLength], // docs say offset 2? - Length: 3 + uint64(valueLength), - }, nil - } - if payload[0] == 0x09 { - return &FmpChunk{ - Type: FMP_CHUNK_SIMPLE_KEY_VALUE, - Key: parseVarUint64(payload[1 : 1+2]), - Value: payload[3 : 3+1], - Length: 4, - }, nil - } - if 0x0A <= payload[0] && payload[0] <= 0x0D { - valueLength := 2 * (payload[0] - 0x09) - return &FmpChunk{ - Type: FMP_CHUNK_SIMPLE_KEY_VALUE, - Key: parseVarUint64(payload[1 : 1+2]), - Value: payload[3 : 3+valueLength], - Length: 2 + uint64(valueLength), - }, nil - } - if payload[0] == 0x0E { - valueLength := payload[2] - return &FmpChunk{ - Type: FMP_CHUNK_SIMPLE_KEY_VALUE, - Key: parseVarUint64(payload[1 : 1+2]), - Value: payload[4 : 4+valueLength], - Length: 4 + uint64(valueLength), - }, nil - } + chunk.Type = FMP_CHUNK_SIMPLE_KEY_VALUE + chunk.Key = uint64(payload[1]) + chunk.Value = payload[3 : 3+valueLength] + chunk.Length = 3 + uint64(valueLength) - // Long key-value - - if payload[0] == 0x16 { - valueLength := payload[4] - return &FmpChunk{ - Type: FMP_CHUNK_LONG_KEY_VALUE, - Key: parseVarUint64(payload[1 : 1+3]), - Value: payload[5 : 5+valueLength], - Length: 5 + uint64(valueLength), - }, nil - } - if payload[0] == 0x17 { - valueLength := parseVarUint64(payload[4 : 4+2]) - return &FmpChunk{ - Type: FMP_CHUNK_LONG_KEY_VALUE, - Key: parseVarUint64(payload[1 : 1+3]), - Value: payload[6 : 6+valueLength], - Length: 6 + uint64(valueLength), - }, nil - } - if payload[0] == 0x1E { - keyLength := payload[1] - valueLength := payload[2+keyLength] - return &FmpChunk{ - Type: FMP_CHUNK_LONG_KEY_VALUE, - Key: parseVarUint64(payload[2 : 2+keyLength]), - Value: payload[2+keyLength+1 : 2+keyLength+1+valueLength], - Length: 2 + uint64(keyLength) + 1 + uint64(valueLength), - }, nil - } - if payload[0] == 0x1F { - keyLength := uint64(payload[1]) - valueLength := parseVarUint64(payload[2+keyLength : 2+keyLength+2+1]) - return &FmpChunk{ - Type: FMP_CHUNK_LONG_KEY_VALUE, - Key: parseVarUint64(payload[2 : 2+keyLength]), - Value: payload[2+keyLength+2 : 2+keyLength+2+valueLength], - Length: 4 + uint64(keyLength) + uint64(valueLength), - }, nil - } - - // Segmented data - - if payload[0] == 0x07 { + case 0x07: valueLength := parseVarUint64(payload[2 : 2+2]) payloadLimit := min(4+valueLength, uint64(len(payload))) - return &FmpChunk{ - Type: FMP_CHUNK_SEGMENTED_DATA, - Index: uint64(payload[1]), - Value: payload[4:payloadLimit], - Length: 4 + uint64(valueLength), - }, nil - } - if payload[0] == 0x0F { + chunk.Type = FMP_CHUNK_SEGMENTED_DATA + chunk.Index = uint64(payload[1]) + chunk.Value = payload[4:payloadLimit] + chunk.Length = 4 + uint64(valueLength) + + case 0x08: + chunk.Type = FMP_CHUNK_SIMPLE_DATA + chunk.Value = payload[1 : 1+2] + chunk.Length = 3 + + case 0x09: + chunk.Type = FMP_CHUNK_SIMPLE_KEY_VALUE + chunk.Key = parseVarUint64(payload[1 : 1+2]) + chunk.Value = payload[3 : 3+1] + chunk.Length = 4 + + case 0x0A, 0x0B, 0x0C, 0x0D: + valueLength := 2 * (payload[0] - 0x09) + chunk.Type = FMP_CHUNK_SIMPLE_KEY_VALUE + chunk.Key = parseVarUint64(payload[1 : 1+2]) + chunk.Value = payload[3 : 3+valueLength] + chunk.Length = 2 + uint64(valueLength) + + case 0x0E: + if payload[1] == 0xFE { + chunk.Type = FMP_CHUNK_PATH_PUSH + chunk.Value = payload[1 : 1+8] + chunk.Length = 10 + break + } + + if payload[1] == 0xFF { + chunk.Type = FMP_CHUNK_SIMPLE_DATA + chunk.Value = payload[2 : 2+5] + chunk.Length = 7 + break + } + + valueLength := payload[2] + chunk.Type = FMP_CHUNK_SIMPLE_KEY_VALUE + chunk.Key = parseVarUint64(payload[1 : 1+2]) + chunk.Value = payload[4 : 4+valueLength] + chunk.Length = 4 + uint64(valueLength) + + case 0x0F: valueLength := parseVarUint64(payload[3 : 3+2]) payloadLimit := min(5+valueLength, uint64(len(payload))) - return &FmpChunk{ - Type: FMP_CHUNK_SEGMENTED_DATA, - Index: parseVarUint64(payload[1 : 1+2]), - Value: payload[5:payloadLimit], - Length: 5 + valueLength, - }, nil - } + chunk.Type = FMP_CHUNK_SEGMENTED_DATA + chunk.Index = parseVarUint64(payload[1 : 1+2]) + chunk.Value = payload[5:payloadLimit] + chunk.Length = 5 + valueLength - // Path push + case 0x10, 0x11: + valueLength := 3 + (payload[0] - 0x10) + chunk.Type = FMP_CHUNK_SIMPLE_DATA + chunk.Value = payload[1 : 1+valueLength] + chunk.Length = 1 + uint64(valueLength) - if payload[0] == 0x20 || payload[0] == 0x0E { + case 0x12, 0x13, 0x14, 0x15: + valueLength := 1 + 2*(payload[0]-0x10) + chunk.Type = FMP_CHUNK_SIMPLE_DATA + chunk.Value = payload[1 : 1+valueLength] + chunk.Length = 1 + uint64(valueLength) + + case 0x16: + valueLength := payload[4] + chunk.Type = FMP_CHUNK_LONG_KEY_VALUE + chunk.Key = parseVarUint64(payload[1 : 1+3]) + chunk.Value = payload[5 : 5+valueLength] + chunk.Length = 5 + uint64(valueLength) + + case 0x17: + valueLength := parseVarUint64(payload[4 : 4+2]) + chunk.Type = FMP_CHUNK_LONG_KEY_VALUE + chunk.Key = parseVarUint64(payload[1 : 1+3]) + chunk.Value = payload[6 : 6+valueLength] + chunk.Length = 6 + uint64(valueLength) + + case 0x19: + chunk.Type = FMP_CHUNK_SIMPLE_DATA + chunk.Value = payload[1 : 1+1] + chunk.Length = 2 + + case 0x1A, 0x1B, 0x1C, 0x1D: + valueLength := 2 * (payload[0] - 0x19) + chunk.Type = FMP_CHUNK_SIMPLE_DATA + chunk.Value = payload[1 : 1+valueLength] + chunk.Length = 1 + uint64(valueLength) + + case 0x1E: + keyLength := payload[1] + valueLength := payload[2+keyLength] + chunk.Type = FMP_CHUNK_LONG_KEY_VALUE + chunk.Key = parseVarUint64(payload[2 : 2+keyLength]) + chunk.Value = payload[2+keyLength+1 : 2+keyLength+1+valueLength] + chunk.Length = 2 + uint64(keyLength) + 1 + uint64(valueLength) + + case 0x1F: + keyLength := uint64(payload[1]) + valueLength := parseVarUint64(payload[2+keyLength : 2+keyLength+2+1]) + chunk.Type = FMP_CHUNK_LONG_KEY_VALUE + chunk.Key = parseVarUint64(payload[2 : 2+keyLength]) + chunk.Value = payload[2+keyLength+2 : 2+keyLength+2+valueLength] + chunk.Length = 4 + uint64(keyLength) + uint64(valueLength) + + case 0x20: if payload[1] == 0xFE { - return &FmpChunk{ - Type: FMP_CHUNK_PATH_PUSH, - Value: payload[1 : 1+8], - Length: 10, - }, nil + chunk.Type = FMP_CHUNK_PATH_PUSH + chunk.Value = payload[1 : 1+8] + chunk.Length = 10 + break } - return &FmpChunk{ - Type: FMP_CHUNK_PATH_PUSH, - Value: payload[1 : 1+1], - Length: 2, - }, nil - } - if payload[0] == 0x28 { - return &FmpChunk{ - Type: FMP_CHUNK_PATH_PUSH, - Value: payload[1 : 1+2], - Length: 3, - }, nil - } - if payload[0] == 0x30 { - return &FmpChunk{ - Type: FMP_CHUNK_PATH_PUSH, - Value: payload[1 : 1+3], - Length: 4, - }, nil - } - if payload[0] == 0x38 { + + chunk.Type = FMP_CHUNK_PATH_PUSH + chunk.Value = payload[1 : 1+1] + chunk.Length = 2 + + case 0x23: + chunk.Type = FMP_CHUNK_SIMPLE_DATA + chunk.Value = payload[1 : 1+1] + chunk.Length = 2 + + case 0x28: + chunk.Type = FMP_CHUNK_PATH_PUSH + chunk.Value = payload[1 : 1+2] + chunk.Length = 3 + + case 0x30: + chunk.Type = FMP_CHUNK_PATH_PUSH + chunk.Value = payload[1 : 1+3] + chunk.Length = 4 + + case 0x38: valueLength := payload[1] - return &FmpChunk{ - Type: FMP_CHUNK_PATH_PUSH, - Value: payload[2 : 2+valueLength], - Length: 2 + uint64(valueLength), - }, nil + chunk.Type = FMP_CHUNK_PATH_PUSH + chunk.Value = payload[2 : 2+valueLength] + chunk.Length = 2 + uint64(valueLength) + + case 0x3D, 0x40: + chunk.Type = FMP_CHUNK_PATH_POP + chunk.Length = 1 + + case 0x80: + chunk.Type = FMP_CHUNK_NOOP + chunk.Length = 1 + + default: + return nil, ErrBadChunk } - // Path pop - - if payload[0] == 0x3D || payload[0] == 0x40 { - return &FmpChunk{ - Type: FMP_CHUNK_PATH_POP, - Length: 1, - }, nil - } - - // No-op - - if payload[0] == 0x80 { - return &FmpChunk{ - Type: FMP_CHUNK_NOOP, - Length: 1, - }, nil - } - - return nil, ErrBadChunk + return chunk, nil } diff --git a/fmp/fmp_file.go b/fmp/fmp_file.go index 966c494..7f65522 100644 --- a/fmp/fmp_file.go +++ b/fmp/fmp_file.go @@ -89,6 +89,14 @@ func OpenFile(path string) (*FmpFile, error) { case FMP_CHUNK_NOOP: // noop } + + if chunk.Delayed { + if len(currentPath) == 0 { + println("warning: delayed pop without path") + } else { + currentPath = currentPath[:len(currentPath)-1] + } + } } ctx.currentSectorID = sector.NextID @@ -129,6 +137,7 @@ func (ctx *FmpFile) readHeader() error { } func (ctx *FmpFile) readSector() (*FmpSector, error) { + println("------- Reading sector", ctx.currentSectorID) buf := make([]byte, sectorHeaderSize) n, err := ctx.stream.Read(buf) diff --git a/fmp/fmp_type.go b/fmp/fmp_type.go index 16c7d15..9bfee28 100644 --- a/fmp/fmp_type.go +++ b/fmp/fmp_type.go @@ -31,11 +31,12 @@ type FmpSector struct { } type FmpChunk struct { - Type FmpChunkType - Length uint64 - Key uint64 // If Type == FMP_CHUNK_SHORT_KEY_VALUE or FMP_CHUNK_LONG_KEY_VALUE - Index uint64 // Segment index, if Type == FMP_CHUNK_SEGMENTED_DATA - Value []byte + Type FmpChunkType + Length uint64 + Key uint64 // If Type == FMP_CHUNK_SHORT_KEY_VALUE or FMP_CHUNK_LONG_KEY_VALUE + Index uint64 // Segment index, if Type == FMP_CHUNK_SEGMENTED_DATA + Value []byte + Delayed bool } type FmpDict map[uint64]*FmpDictEntry