0
0
mirror of https://github.com/sqlite/sqlite.git synced 2024-11-22 12:17:40 +01:00

Change the checksum used in WAL files so that each frames checksum depends on the content of the WAL header and all frame headers and content up to and including the frame to which the checksum is attached.

FossilOrigin-Name: 8a53f12c83a107684b99f4a9de371b5ea3ca810a
This commit is contained in:
dan 2010-05-24 13:57:42 +00:00
parent c81791573a
commit 71d8991932
8 changed files with 146 additions and 81 deletions

View File

@ -1,8 +1,5 @@
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1
C Make\ssure\sa\sWAL\sframe\sof\sall\szeros\sis\sdetected\sas\san\sinvalid\sframe.
D 2010-05-24T13:28:36
C Change\sthe\schecksum\sused\sin\sWAL\sfiles\sso\sthat\seach\sframes\schecksum\sdepends\son\sthe\scontent\sof\sthe\sWAL\sheader\sand\sall\sframe\sheaders\sand\scontent\sup\sto\sand\sincluding\sthe\sframe\sto\swhich\sthe\schecksum\sis\sattached.
D 2010-05-24T13:57:43
F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
F Makefile.in a5cad1f8f3e021356bfcc6c77dc16f6f1952bbc3
F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@ -157,7 +154,7 @@ F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f
F src/os_os2.c 665876d5eec7585226b0a1cf5e18098de2b2da19
F src/os_unix.c 35ace483789db8ede92acc46134930c2c4267645
F src/os_win.c 1e44ee84210b59db1e098bbbc66f6dee68e20d5f
F src/pager.c d3284a6bbedeaa4ef3f5668af309d4381df97618
F src/pager.c 0fbfe2ccd98cd893f3b5254a9297e153440e5e37
F src/pager.h 76466c3a5af56943537f68b1f16567101a0cd1d0
F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e
F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf
@ -227,8 +224,8 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e
F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1
F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2
F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda
F src/wal.c e8c58e529bcc50c0fb3797bc39750e802cbace78
F src/wal.h 434f76f51225bb614e43ccb6bd2341541ba6a06e
F src/wal.c c09f4e33aad4ec3822ef3e9626f8bd7c273542af
F src/wal.h 111c6f3efd83fe2fc707b29e26431e8eff4c6f28
F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f
F src/where.c 75fee9e255b62f773fcadd1d1f25b6f63ac7a356
F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
@ -764,10 +761,10 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61
F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5
F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8
F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d
F test/wal.test 3b8ad018c1faf89d3f5bb23704775f5d20e486de
F test/wal2.test 053c9ea94194c5bce5b742429be75ff2432794ab
F test/wal.test be8ef043253ca735ffcabb92a7dac2d79ebfe8c1
F test/wal2.test d9a50d1b2e0f0735b8a21538631100eaf845364f
F test/walbak.test e7650a26eb4b8abeca9b145b1af1e63026dde432
F test/walcksum.test cc41a85d8b6f1471ebdf847f82f39dd0003a37bc
F test/walcksum.test a0712107b6a73397fc7e3f92d5b16e206caa7d3d
F test/walcrash.test f6d5fb2bb108876f04848720a488065d9deef69f
F test/walfault.test f71d4c9a13d4e27086aef55f1e0e94734ffa2f6a
F test/walhook.test 67e675127f4acb72f061a12667ce6e5460b06b78
@ -817,14 +814,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
P 51fd38152b92db637d1d346fca35ec2d3e4d4f57
R 3ae8439c7818113b3e55b5c33ec15fc6
U drh
Z 157804aa9c723c5bf0d5892d1297d942
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.6 (GNU/Linux)
iD8DBQFL+n8IoxKgR168RlERAjqiAJ93cfWX1aiOOxPNMQnGl5zdjLHO3ACfQbUq
4XUL8oGpoSF35Q/+0pQNofc=
=ktcp
-----END PGP SIGNATURE-----
P 02d99ad4b51065c67cc7689916130774be1c4c87
R 0b567cb4cfa023970e21a7a28609c3b9
U dan
Z fafe9597e653734ba29a55af1088e0f3

View File

@ -1 +1 @@
02d99ad4b51065c67cc7689916130774be1c4c87
8a53f12c83a107684b99f4a9de371b5ea3ca810a

View File

@ -221,7 +221,7 @@ struct PagerSavepoint {
Bitvec *pInSavepoint; /* Set of pages in this savepoint */
Pgno nOrig; /* Original number of pages in file */
Pgno iSubRec; /* Index of first record in sub-journal */
u32 iFrame; /* Last frame in WAL when savepoint opened */
u32 aWalData[WAL_SAVEPOINT_NDATA]; /* WAL savepoint context */
};
/*
@ -2567,7 +2567,7 @@ static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){
i64 offset = pSavepoint->iSubRec*(4+pPager->pageSize);
if( pagerUseWal(pPager) ){
rc = sqlite3WalSavepointUndo(pPager->pWal, pSavepoint->iFrame);
rc = sqlite3WalSavepointUndo(pPager->pWal, pSavepoint->aWalData);
}
for(ii=pSavepoint->iSubRec; rc==SQLITE_OK && ii<pPager->nSubRec; ii++){
assert( offset==ii*(4+pPager->pageSize) );
@ -5448,7 +5448,7 @@ int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){
return SQLITE_NOMEM;
}
if( pagerUseWal(pPager) ){
aNew[ii].iFrame = sqlite3WalSavepoint(pPager->pWal);
sqlite3WalSavepoint(pPager->pWal, aNew[ii].aWalData);
}
}

View File

@ -214,13 +214,14 @@ typedef struct WalIterator WalIterator;
** object.
*/
struct WalIndexHdr {
u32 iChange; /* Counter incremented each transaction */
u16 bigEndCksum; /* True if checksums in WAL are big-endian */
u16 szPage; /* Database page size in bytes */
u32 mxFrame; /* Index of last valid frame in the WAL */
u32 nPage; /* Size of database in pages */
u32 aSalt[2]; /* Salt-1 and salt-2 values copied from WAL header */
u32 aCksum[2]; /* Checksum over all prior fields */
u32 iChange; /* Counter incremented each transaction */
u16 bigEndCksum; /* True if checksums in WAL are big-endian */
u16 szPage; /* Database page size in bytes */
u32 mxFrame; /* Index of last valid frame in the WAL */
u32 nPage; /* Size of database in pages */
u32 aFrameCksum[2]; /* Checksum of last frame in log */
u32 aSalt[2]; /* Two salt values copied from WAL header */
u32 aCksum[2]; /* Checksum over all prior fields */
};
/* A block of WALINDEX_LOCK_RESERVED bytes beginning at
@ -424,14 +425,14 @@ static void walEncodeFrame(
u8 *aFrame /* OUT: Write encoded frame here */
){
int nativeCksum; /* True for native byte-order checksums */
u32 aCksum[2];
u32 *aCksum = pWal->hdr.aFrameCksum;
assert( WAL_FRAME_HDRSIZE==24 );
sqlite3Put4byte(&aFrame[0], iPage);
sqlite3Put4byte(&aFrame[4], nTruncate);
memcpy(&aFrame[8], pWal->hdr.aSalt, 8);
nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN);
walChecksumBytes(nativeCksum, aFrame, 16, 0, aCksum);
walChecksumBytes(nativeCksum, aFrame, 8, aCksum, aCksum);
walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum);
sqlite3Put4byte(&aFrame[16], aCksum[0]);
@ -451,8 +452,8 @@ static int walDecodeFrame(
u8 *aFrame /* Frame data */
){
int nativeCksum; /* True for native byte-order checksums */
u32 *aCksum = pWal->hdr.aFrameCksum;
u32 pgno; /* Page number of the frame */
u32 aCksum[2];
assert( WAL_FRAME_HDRSIZE==24 );
/* A frame is only valid if the salt values in the frame-header
@ -474,7 +475,7 @@ static int walDecodeFrame(
** the checksum in the last 8 bytes of the frame-header.
*/
nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN);
walChecksumBytes(nativeCksum, aFrame, 16, 0, aCksum);
walChecksumBytes(nativeCksum, aFrame, 8, aCksum, aCksum);
walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum);
if( aCksum[0]!=sqlite3Get4byte(&aFrame[16])
|| aCksum[1]!=sqlite3Get4byte(&aFrame[20])
@ -739,10 +740,10 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){
static int walIndexRecover(Wal *pWal){
int rc; /* Return Code */
i64 nSize; /* Size of log file */
WalIndexHdr hdr; /* Recovered wal-index header */
u32 aFrameCksum[2] = {0, 0};
assert( pWal->lockState>SQLITE_SHM_READ );
memset(&hdr, 0, sizeof(hdr));
memset(&pWal->hdr, 0, sizeof(WalIndexHdr));
rc = sqlite3OsFileSize(pWal->pWalFd, &nSize);
if( rc!=SQLITE_OK ){
@ -779,10 +780,13 @@ static int walIndexRecover(Wal *pWal){
){
goto finished;
}
hdr.bigEndCksum = pWal->hdr.bigEndCksum = (magic&0x00000001);
pWal->hdr.bigEndCksum = (magic&0x00000001);
pWal->szPage = szPage;
pWal->nCkpt = sqlite3Get4byte(&aBuf[12]);
memcpy(&pWal->hdr.aSalt, &aBuf[16], 8);
walChecksumBytes(pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN,
aBuf, WAL_HDRSIZE, 0, pWal->hdr.aFrameCksum
);
/* Malloc a buffer to read frames into. */
szFrame = szPage + WAL_FRAME_HDRSIZE;
@ -809,23 +813,24 @@ static int walIndexRecover(Wal *pWal){
/* If nTruncate is non-zero, this is a commit record. */
if( nTruncate ){
hdr.mxFrame = iFrame;
hdr.nPage = nTruncate;
hdr.szPage = szPage;
pWal->hdr.mxFrame = iFrame;
pWal->hdr.nPage = nTruncate;
pWal->hdr.szPage = szPage;
aFrameCksum[0] = pWal->hdr.aFrameCksum[0];
aFrameCksum[1] = pWal->hdr.aFrameCksum[1];
}
}
sqlite3_free(aFrame);
}else{
memset(&hdr, 0, sizeof(hdr));
}
finished:
if( rc==SQLITE_OK && hdr.mxFrame==0 ){
if( rc==SQLITE_OK && pWal->hdr.mxFrame==0 ){
rc = walIndexRemap(pWal, WALINDEX_MMAP_INCREMENT);
}
if( rc==SQLITE_OK ){
memcpy(&pWal->hdr, &hdr, sizeof(hdr));
pWal->hdr.aFrameCksum[0] = aFrameCksum[0];
pWal->hdr.aFrameCksum[1] = aFrameCksum[1];
walIndexWriteHdr(pWal);
}
return rc;
@ -1626,25 +1631,35 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){
return rc;
}
/* Return an integer that records the current (uncommitted) write
** position in the WAL
/*
** Argument aWalData must point to an array of WAL_SAVEPOINT_NDATA u32
** values. This function populates the array with values required to
** "rollback" the write position of the WAL handle back to the current
** point in the event of a savepoint rollback (via WalSavepointUndo()).
*/
u32 sqlite3WalSavepoint(Wal *pWal){
void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData){
assert( pWal->lockState==SQLITE_SHM_WRITE );
return pWal->hdr.mxFrame;
aWalData[0] = pWal->hdr.mxFrame;
aWalData[1] = pWal->hdr.aFrameCksum[0];
aWalData[2] = pWal->hdr.aFrameCksum[1];
}
/* Move the write position of the WAL back to iFrame. Called in
** response to a ROLLBACK TO command.
/*
** Move the write position of the WAL back to the point identified by
** the values in the aWalData[] array. aWalData must point to an array
** of WAL_SAVEPOINT_NDATA u32 values that has been previously populated
** by a call to WalSavepoint().
*/
int sqlite3WalSavepointUndo(Wal *pWal, u32 iFrame){
int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){
int rc = SQLITE_OK;
assert( pWal->lockState==SQLITE_SHM_WRITE );
assert( iFrame<=pWal->hdr.mxFrame );
if( iFrame<pWal->hdr.mxFrame ){
assert( aWalData[0]<=pWal->hdr.mxFrame );
if( aWalData[0]<pWal->hdr.mxFrame ){
rc = walIndexMap(pWal, walMappingSize(pWal->hdr.mxFrame));
pWal->hdr.mxFrame = iFrame;
pWal->hdr.mxFrame = aWalData[0];
pWal->hdr.aFrameCksum[0] = aWalData[1];
pWal->hdr.aFrameCksum[1] = aWalData[2];
if( rc==SQLITE_OK ){
walCleanupHash(pWal);
walIndexUnmap(pWal);
@ -1694,6 +1709,7 @@ int sqlite3WalFrames(
if( rc!=SQLITE_OK ){
return rc;
}
walChecksumBytes(1, aWalHdr, sizeof(aWalHdr), 0, pWal->hdr.aFrameCksum);
}
assert( pWal->szPage==szPage );

View File

@ -28,13 +28,15 @@
# define sqlite3WalDbsize(y,z)
# define sqlite3WalWriteLock(y,z) 0
# define sqlite3WalUndo(x,y,z) 0
# define sqlite3WalSavepoint(z) 0
# define sqlite3WalSavepoint(y,z)
# define sqlite3WalSavepointUndo(y,z) 0
# define sqlite3WalFrames(u,v,w,x,y,z) 0
# define sqlite3WalCheckpoint(u,v,w,x,y,z) 0
# define sqlite3WalCallback(z) 0
#else
#define WAL_SAVEPOINT_NDATA 3
/* Connection to a write-ahead log (WAL) file.
** There is one object of this type for each pager.
*/
@ -69,11 +71,11 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx);
/* Return an integer that records the current (uncommitted) write
** position in the WAL */
u32 sqlite3WalSavepoint(Wal *pWal);
void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData);
/* Move the write position of the WAL back to iFrame. Called in
** response to a ROLLBACK TO command. */
int sqlite3WalSavepointUndo(Wal *pWal, u32 iFrame);
int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData);
/* Write a frame or frames to the log. */
int sqlite3WalFrames(Wal *pWal, int, PgHdr *, Pgno, int, int);

View File

@ -1348,7 +1348,8 @@ foreach {tn pgsz works} {
set framehdr [binary format IIII $pg 5 22 23]
set c1 0
set c2 0
logcksum c1 c2 $framehdr
logcksum c1 c2 $walhdr
logcksum c1 c2 [string range $framehdr 0 7]
logcksum c1 c2 $framebody
set framehdr [binary format IIIIII $pg 5 22 23 $c1 $c2]

View File

@ -19,17 +19,24 @@ source $testdir/lock_common.tcl
ifcapable !wal {finish_test ; return }
proc set_tvfs_hdr {file args} {
# Set $nHdr to the number of bytes in the wal-index header:
set nHdr 80
set nInt [expr {$nHdr/4}]
if {[llength $args]>1} {
return -code error {wrong # args: should be "set_tvfs_hdr fileName ?val?"}
}
set blob [tvfs shm $file]
if {[llength $args]} {
set blob [binary format i16a* [lindex $args 0] [string range $blob 64 end]]
set blob [
binary format i${nInt}a* [lindex $args 0] [string range $blob $nHdr end]
]
tvfs shm $file $blob
}
binary scan $blob i16 ints
binary scan $blob i${nInt} ints
return $ints
}

View File

@ -62,18 +62,13 @@ proc readfile {filename} {
#
proc log_checksum_verify {filename iFrame endian} {
set data [readfile $filename]
set c1 0
set c2 0
binary scan [string range $data 8 11] I pgsz
set n [log_file_size [expr $iFrame-1] $pgsz]
binary scan [string range $data [expr $n+16] [expr $n+23]] II expect1 expect2
log_cksum $endian c1 c2 [string range $data $n [expr $n+15]]
log_cksum $endian c1 c2 [string range $data [expr $n+24] [expr $n+24+$pgsz-1]]
foreach {offset c1 c2} [log_checksum_calc $data $iFrame $endian] {}
binary scan [string range $data $offset [expr $offset+7]] II expect1 expect2
set expect1 [expr $expect1&0xFFFFFFFF]
set expect2 [expr $expect2&0xFFFFFFFF]
expr {$c1==$expect1 && $c2==$expect2}
}
@ -85,24 +80,37 @@ proc log_checksum_verify {filename iFrame endian} {
#
proc log_checksum_write {filename iFrame endian} {
set data [readfile $filename]
set c1 0
set c2 0
binary scan [string range $data 8 11] I pgsz
set n [log_file_size [expr $iFrame-1] $pgsz]
log_cksum $endian c1 c2 [string range $data $n [expr $n+15]]
log_cksum $endian c1 c2 [string range $data [expr $n+24] [expr $n+24+$pgsz-1]]
foreach {offset c1 c2} [log_checksum_calc $data $iFrame $endian] {}
set bin [binary format II $c1 $c2]
set fd [open $filename r+]
fconfigure $fd -encoding binary
fconfigure $fd -translation binary
seek $fd [expr $n+16]
seek $fd $offset
puts -nonewline $fd $bin
close $fd
}
proc log_checksum_calc {data iFrame endian} {
binary scan [string range $data 8 11] I pgsz
if {$iFrame > 1} {
set n [log_file_size [expr $iFrame-2] $pgsz]
binary scan [string range $data [expr $n+16] [expr $n+23]] II c1 c2
} else {
set c1 0
set c2 0
log_cksum $endian c1 c2 [string range $data 0 23]
}
set n [log_file_size [expr $iFrame-1] $pgsz]
log_cksum $endian c1 c2 [string range $data $n [expr $n+7]]
log_cksum $endian c1 c2 [string range $data [expr $n+24] [expr $n+24+$pgsz-1]]
list [expr $n+16] $c1 $c2
}
#
# File $filename must be a WAL file on disk. Set the 'magic' field of the
# WAL header to indicate that checksums are $endian-endian ($endian must be
@ -180,6 +188,7 @@ foreach endian {big little} {
# Replace all checksums in the current WAL file with $endian versions.
# Then check that it is still possible to recover and read the database.
#
log_checksum_writemagic test2.db-wal $endian
for {set f 1} {$f <= 6} {incr f} {
do_test walcksum-1.$endian.3.$f {
log_checksum_write test2.db-wal $f $endian
@ -187,7 +196,6 @@ foreach endian {big little} {
} {1}
}
do_test walcksum-1.$endian.4.1 {
log_checksum_writemagic test2.db-wal $endian
file copy -force test2.db test.db
file copy -force test2.db-wal test.db-wal
sqlite3 db test.db
@ -263,7 +271,7 @@ foreach endian {big little} {
} {1}
do_test walcksum-1.$endian.8.3 {
log_checksum_verify test.db-wal 3 $native
} [expr {$native == $endian}]
} {0}
do_test walcksum-1.$endian.9 {
execsql {
@ -276,4 +284,45 @@ foreach endian {big little} {
catch { db2 close }
}
do_test walcksum-2.1 {
file delete -force test.db test.db-wal test.db-journal
sqlite3 db test.db
execsql {
PRAGMA synchronous = NORMAL;
PRAGMA page_size = 1024;
PRAGMA journal_mode = WAL;
PRAGMA cache_size = 10;
CREATE TABLE t1(x PRIMARY KEY);
PRAGMA wal_checkpoint;
INSERT INTO t1 VALUES(randomblob(800));
BEGIN;
INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 2 */
INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 4 */
INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 8 */
INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 16 */
SAVEPOINT one;
INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 32 */
INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 64 */
INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 128 */
INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 256 */
ROLLBACK TO one;
INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 32 */
INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 64 */
INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 128 */
INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 256 */
COMMIT;
}
file copy -force test.db test2.db
file copy -force test.db-wal test2.db-wal
sqlite3 db2 test2.db
execsql {
PRAGMA integrity_check;
SELECT count(*) FROM t1;
} db2
} {ok 256}
catch { db close }
catch { db2 close }
finish_test