Add the ability to build certain tools dynamically linked to libsqlite3 instead of embedding their own copy of sqlite3.c.

FossilOrigin-Name: 84bd96a6f10823992cf865245cb2c75b69921257122fde5909b303621e288d00
2024-11-23 23:57:41 +01:00 · 2024-11-19 20:23:01 +00:00 · 2024-11-19 20:23:01 +00:00 · 9491e6d9b1
commit 9491e6d9b1
parent 5608fb36ec 457d9384a2
8 changed files with 100 additions and 44 deletions
--- a/Makefile.in
+++ b/Makefile.in
@ -287,6 +287,7 @@ AS_AUTO_DEF = $(TOP)/auto.def
 AS_AUTORECONFIG = @SQLITE_AUTORECONFIG@

 USE_AMALGAMATION ?= @USE_AMALGAMATION@
+LINK_TOOLS_DYNAMICALLY ?= @LINK_TOOLS_DYNAMICALLY@
 AMALGAMATION_GEN_FLAGS ?= --linemacros=@AMALGAMATION_LINE_MACROS@

 #
--- a/auto.def
+++ b/auto.def
@ -123,7 +123,6 @@ set DUMP_DEFINES_JSON ""; #./config.defines.json
 ########################################################################
 set flags {
  # <build-modes>
-  with-debug:=1        => {Enable debug build flags}
  shared=1             => {Disable build of shared libary}
  static=1             => {Disable build of static library (mostly)}
  amalgamation=1       => {Disable the amalgamation and instead build all files separately.}
@ -193,12 +192,19 @@ set flags {
  with-emsdk:=auto     => {Top-most dir of the Emscripten SDK installation. Default = EMSDK env var.}
  # </alternative-builds>
  # <developer>
+  with-debug:=1        => {Enable debug build flags. --with-debug does more
+    than simply builds with a -g compilation flag and will impact
+    performance by as much as 4x, as it includes large numbers of
+    assert()s in performance-critical loops.  Never use --with-debug
+    for production builds.}
+  dev                  => {Enable dev-mode build: automatically enables certain other flags}
  test-status          => {Enable status of tests}
  gcov=0               => {Enable coverage testing using gcov}
  linemacros           => {Enable #line macros in the amalgamation}
-  dev                  => {Enable dev-mode build: automatically enables certain other flags}
  dump-defines=0       => {Dump autosetup defines to $DUMP_DEFINES_TXT (for build debugging)}
+  dynlink-tools        => {Dynamically link libsqlite3 to certain tools which normally statically embed it.}
  soname:=legacy       => {SONAME for libsqlite3.so. Must be one of: none, auto, legacy}
+  # --soname has a long story behind it, as well as no small amount of uncertainty.
  # </developer>
 }
 if {"" ne $DUMP_DEFINES_JSON} {
@ -327,6 +333,8 @@ proj-if-opt-truthy dev {
  define CFLAGS [get-env CFLAGS {-O0 -g}]
 }

+define LINK_TOOLS_DYNAMICALLY [proj-opt-was-provided dynlink-tools]
+
 ########################################################################
 # Handle --with-wasi-sdk=DIR
 #
@ -468,7 +476,7 @@ apply {{} {
  if {[proj-opt-was-provided soname]} {
    set soname [opt-val soname]
  } else {
-    set soname legacy
+    set soname none; # enabling soname breaks linking for the --dynlink-tools feature
  }
  switch -exact -- $soname {
    none   { return 0 }
--- a/autosetup/proj.tcl
+++ b/autosetup/proj.tcl
@ -311,11 +311,11 @@ proc proj-first-bin-of {args} {
 ########################################################################
 # @proj-opt-was-provided key
 #
-# Returns 1 if the user specifically provided the given configure
-# flag, else 0. This can be used to distinguish between options which
-# have a default value and those which were explicitly provided by the
-# user, even if the latter is done in a way which uses the default
-# value.
+# Returns 1 if the user specifically provided the given configure flag
+# or if it was specifically set using proj-opt-set, else 0. This can
+# be used to distinguish between options which have a default value
+# and those which were explicitly provided by the user, even if the
+# latter is done in a way which uses the default value.
 #
 # For example, with a configure flag defined like:
 #
--- a/main.mk
+++ b/main.mk
@ -205,6 +205,17 @@ ENABLE_STATIC ?= 1
 #
 USE_AMALGAMATION ?= 1
 #
+# $(LINK_TOOLS_DYNAMICALLY)
+#
+# If true, certain binaries which typically statically link against
+# libsqlite3 or its component object files will instead link against
+# the DLL. The caveat is that running such builds from the source tree
+# may require that the user specifically prepend "." to their
+# $LD_LIBRARY_PATH so that the dynamic linker does not pick up a
+# libsqlite3.so from outside the source tree.
+#
+LINK_TOOLS_DYNAMICALLY ?= 0
+#
 # $(AMALGAMATION_GEN_FLAGS) =
 #
 # Optional flags for the amalgamation generator.
@ -893,6 +904,9 @@ TESTOPTS = --verbose=file --output=test-out.txt
 #
 # Extra compiler options for various shell tools
 #
+# Note that some of these will only apply when embedding sqlite3.c
+# into the shell, as these flags are not otherwise passed on to the
+# library.
 SHELL_OPT += -DSQLITE_DQS=0
 SHELL_OPT += -DSQLITE_ENABLE_FTS4
 #SHELL_OPT += -DSQLITE_ENABLE_FTS5
@ -1730,13 +1744,34 @@ smoketest:	$(TESTPROGS) fuzzcheck$(T.exe)
 shelltest:
 	$(TCLSH_CMD) $(TOP)/test/testrunner.tcl release shell

+#
+# sqlite3_analyzer.c build depends on $(LINK_TOOLS_DYNAMICALLY).
+#
+sqlite3_analyzer.c.flags.0 = -DINCLUDE_SQLITE3_C=1
+sqlite3_analyzer.c.flags.1 =
 sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl \
-                    $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in has_tclsh85
-	$(B.tclsh) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c
+                    $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in
+	$(B.tclsh) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in \
+		$(sqlite3_analyzer.c.flags.$(LINK_TOOLS_DYNAMICALLY)) \
+		$(OPT_FEATURE_FLAGS) \
+		> $@

-sqlite3_analyzer$(T.exe): $(T.tcl.env.sh) sqlite3_analyzer.c
-	$(T.link.tcl) sqlite3_analyzer.c -o $@ $$TCL_LIB_SPEC $$TCL_INCLUDE_SPEC \
-		$(LDFLAGS.libsqlite3)
+#
+# sqlite3_analyzer's build mode depends on $(LINK_TOOLS_DYNAMICALLY).
+#
+sqlite3_analyzer.flags.1 = -L. -lsqlite3 $(LDFLAGS.math)
+sqlite3_analyzer.flags.0 = $(LDFLAGS.libsqlite3)
+sqlite3_analyzer.deps.1 = $(libsqlite3.SO)
+sqlite3_analyzer.deps.0 =
+sqlite3_analyzer$(T.exe): $(T.tcl.env.sh) sqlite3_analyzer.c \
+                          $(sqlite3_analyzer.deps.$(LINK_TOOLS_DYNAMICALLY))
+	$(T.link.tcl) sqlite3_analyzer.c -o $@ \
+		$(sqlite3_analyzer.flags.$(LINK_TOOLS_DYNAMICALLY)) \
+		$$TCL_LIB_SPEC $$TCL_INCLUDE_SPEC
+# ^^^^ the order of those flags is relevant for
+# $(sqlite3_analyzer.flags.1): if the $$TCL_... flags come first they
+# can cause the $@ to link to an out-of-tree libsqlite3.so, which may
+# or may not fail or otherwise cause confusion.

 sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl \
            $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl \
@ -1918,15 +1953,27 @@ threadtest5: sqlite3.c $(TOP)/test/threadtest5.c
 	$(T.link) $(TOP)/test/threadtest5.c sqlite3.c -o $@ $(LDFLAGS.libsqlite3)
 xbin: threadtest5

-# The standard CLI is built using the amalgamation since it uses
-# special compile-time options that are interpreted by individual
-# source files within the amalgamation.
+#
+# When building sqlite3$(T.exe) we specifically embed a copy of
+# sqlite3.c, and not link to libsqlite3.so or libsqlite3.a, because
+# the shell needs to be able to enable arbitrary library features,
+# some of which have significant performance impacts. For example,,
+# SQLITE_ENABLE_EXPLAIN_COMMENTS has been measured as having a 5.2%
+# runtime performance hit, which is fine for use in the shell but is
+# not appropriate for the canonical library build.
 #
 sqlite3$(T.exe):	shell.c sqlite3.c
 	$(T.link) -o $@ \
 		shell.c sqlite3.c \
 		$(CFLAGS.readline) $(SHELL_OPT) $(CFLAGS.icu) \
 		$(LDFLAGS.libsqlite3) $(LDFLAGS.readline)
+#
+# Build sqlite3$(T.exe) by default except in wasi-sdk builds.  Yes, the
+# semantics of 0 and 1 are confusingly swapped here.
+#
+sqlite3$(T.exe)-1:
+sqlite3$(T.exe)-0: sqlite3$(T.exe)
+all: sqlite3$(T.exe)-$(HAVE_WASI_SDK)

 # The "sqlite3d" CLI is build using separate source files.  This
 # is useful during development and debugging.
@ -1937,21 +1984,19 @@ sqlite3d$(T.exe):	shell.c $(LIBOBJS0)
 		$(CFLAGS.readline) $(SHELL_OPT) \
 		$(LDFLAGS.libsqlite3) $(LDFLAGS.readline)

-#
-# Build sqlite3$(T.exe) by default except in wasi-sdk builds.  Yes, the
-# semantics of 0 and 1 are confusingly swapped here.
-#
-sqlite3$(T.exe)-1:
-sqlite3$(T.exe)-0 sqlite3$(T.exe)-: sqlite3$(T.exe)
-all: sqlite3$(T.exe)-$(HAVE_WASI_SDK)
-
 install-shell-0: sqlite3$(T.exe) $(install-dir.bin)
 	$(INSTALL) -s sqlite3$(T.exe) "$(install-dir.bin)"
-install-shell-1 install-shell-:
+install-shell-1:
 install: install-shell-$(HAVE_WASI_SDK)

-sqldiff$(T.exe):	$(TOP)/tool/sqldiff.c $(TOP)/ext/misc/sqlite3_stdio.h sqlite3.o sqlite3.h
-	$(T.link) -o $@ $(TOP)/tool/sqldiff.c sqlite3.o $(LDFLAGS.libsqlite3)
+# How to build sqldiff$(T.exe) depends on $(LINK_TOOLS_DYNAMICALLY)
+#
+sqldiff.0.deps = $(TOP)/tool/sqldiff.c $(TOP)/ext/misc/sqlite3_stdio.h sqlite3.o sqlite3.h
+sqldiff.0.rules = $(T.link) -o $@ $(TOP)/tool/sqldiff.c sqlite3.o $(LDFLAGS.libsqlite3)
+sqldiff.1.deps = $(TOP)/tool/sqldiff.c $(TOP)/ext/misc/sqlite3_stdio.h $(libsqlite3.SO)
+sqldiff.1.rules = $(T.link) -o $@ $(TOP)/tool/sqldiff.c -L. -lsqlite3 $(LDFLAGS.configure)
+sqldiff$(T.exe): $(sqldiff.$(LINK_TOOLS_DYNAMICALLY).deps)
+	$(sqldiff.$(LINK_TOOLS_DYNAMICALLY).rules)

 install-diff: sqldiff$(T.exe) $(install-dir.bin)
 	$(INSTALL) -s sqldiff$(T.exe) "$(install-dir.bin)"
--- a/25
+++ b/25
@ -1,9 +1,9 @@
-C Enhancements\sto\stool/mkccode.tcl\ssuch\sthat\sit\srecognizes\s-D\scommand\sline\narguments\sand\scan\suse\sthem\sin\sinternal\sIFDEF\sand\sIFNDEF\smacros.\s\sUpdate\nthe\stool/sqlite3_analyzer.c.in\sscript\ssuch\sthat\sit\somits\sthe\sSQLite\samalgamation\nif\s-DSQLITE_ENABLE_DBSTAT_VTAB\sis\sdefined.
-D 2024-11-19T18:26:47.422
+C Add\sthe\sability\sto\sbuild\scertain\stools\sdynamically\slinked\sto\slibsqlite3\sinstead\sof\sembedding\stheir\sown\scopy\sof\ssqlite3.c.
+D 2024-11-19T20:23:01.300
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d
-F Makefile.in c3a01e98bd2ae6a4631df02f11249ac07ffa3d540d3034e2675f66cc259f2ea7
+F Makefile.in 5461b1125a3039fef49f4896519d164ebe22ee394d20d69913ab0fdc8c464b63
 F Makefile.linux-generic bd3e3cacd369821a6241d4ea1967395c962dfe3057e38cb0a435cee0e8b789d0
 F Makefile.msc a92237976eb92c5efaa0dd2524746aec12c196e12df8d4dbff9543a4648c3312
 F README.md c3c0f19532ce28f6297a71870f3c7b424729f0e6d9ab889616d3587dd2332159
@ -13,7 +13,7 @@ F art/icon-80x90.gif 65509ce3e5f86a9cd64fe7fca2d23954199f31fe44c1e09e208c80fb83d
 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2
 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90
 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2
-F auto.def 94f0f4a697e8221d5c7ca561771a3afabb0e707922daad89b60908b87a8e399b
+F auto.def b81388775c7596c22d19ce7c5e0692d8cb6beb89ae591e2684a238084acabf61
 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903
 F autoconf/Makefile.am adedc1324b6a87fdd1265ddd336d2fb7d4f36a0e77b86ea553ae7cc4ea239347
 F autoconf/Makefile.fallback 22fe523eb36dfce31e0f6349f782eb084e86a5620b2b0b4f84a2d6133f53f5ac
@ -49,7 +49,7 @@ F autosetup/cc-shared.tcl 4f024e94a47f427ba61de1739f6381ef0080210f9fae89112d5c1d
 F autosetup/cc.tcl c0fcc50ca91deff8741e449ddad05bcd08268bc31177e613a6343bbd1fd3e45f
 F autosetup/jimsh0.c d40e381ea4526a067590e7b91bd4b2efa6d4980d286f908054c647b3df4aee14
 F autosetup/pkg-config.tcl 4e635bf39022ff65e0d5434339dd41503ea48fc53822c9c5bde88b02d3d952ba
-F autosetup/proj.tcl 96fe16b87c9feb9c1cf2682280f678c659bc52c09fca5de02afc2f7ec5bfb154
+F autosetup/proj.tcl 22556a325c964aa5377d4d881722385f41fcd7c1b60102ba8965f7814c83e9ce
 F autosetup/system.tcl 51d4be76cd9a9074704b584e5c9cbba616202c8468cf9ba8a4f8294a7ab1dba9
 F configure 9a00b21dfd13757bbfb8d89b30660a89ec1f8f3a79402b8f9f9b6fc475c3303a x
 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad
@ -696,7 +696,7 @@ F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65a
 F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2
 F ext/wasm/wasmfs.make bc8bb227f35d5bd3863a7bd2233437c37472a0d81585979f058f9b9b503bef35
 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0
-F main.mk c4e37e3015f9c2d8fd20fa3071ba86e7396959c8d5960e37e95aa85421e2b739
+F main.mk de4824958a997377f0b661329699eb0b58e81c3076d68bf45898305d074fd3ee
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421
@ -2135,7 +2135,7 @@ F tool/logest.c c34e5944318415de513d29a6098df247a9618c96d83c38d4abd88641fe46e669
 F tool/max-limits.c cbb635fbb37ae4d05f240bfb5b5270bb63c54439
 F tool/merge-test.tcl de76b62f2de2a92d4c1ca4f976bce0aea6899e0229e250479b229b2a1914b176
 F tool/mkautoconfamal.sh cbdcf993fa83dccbef7fb77b39cdeb31ef9f77d9d88c9e343b58d35ca3898a6a
-F tool/mkccode.tcl b0ddad168362ba6225c9372856ba0db90145734e882a9a5bb8951ef0d5bc6331 x
+F tool/mkccode.tcl 210159febe0ef0ecbc53c79833500663ceaba0115b2b374405818dc835b5f84b x
 F tool/mkctimec.tcl ef6a67ec82e5b6fc19152a4c79f237227b18bf67ff16d155bac7adb94355d9cf x
 F tool/mkkeywordhash.c 6b0be901c47f9ad42215fc995eb2f4384ac49213b1fba395102ec3e999acf559
 F tool/mkmsvcmin.tcl d76c45efda1cce2d4005bcea7b8a22bb752e3256009f331120fb4fecb14ebb7a
@ -2178,7 +2178,7 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd
 F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x
 F tool/split-sqlite3c.tcl 5aa60643afca558bc732b1444ae81a522326f91e1dc5665b369c54f09e20de60
 F tool/sqldiff.c 2a0987d183027c795ced13d6749061c1d2f38e24eddb428f56fa64c3a8f51e4b
-F tool/sqlite3_analyzer.c.in 39690af454d2866e87e2b475ec5eabee366f211f2307e90b918345f35bb1a643
+F tool/sqlite3_analyzer.c.in fc7735c499d226a49d843d8209b2543e4e5229eeb71a674c331323a2217b65b4
 F tool/sqlite3_rsync.c 9a1cca2ab1271c59b37a6493c15dc1bcd0ab9149197a9125926bc08dd26b83fb
 F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898
 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848
@ -2198,8 +2198,9 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350
 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7
 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 4c4d1db00bd2c522165876dcf1606116a72525d9ffc891b266213704e25cde55
-R cdb63603c9c3a16b29feb4bae93e8b25
-U drh
-Z 2a605008d2464811e071ed45a95c8ad5
+P 7fec209290aa1a6dbbca8de154edaac5d8d0ce042bc0617d27fb2095c8d580f1 50b9f6fde44f9afda27e47badde6115ab6d9ccd5e22b446a8d8127499848815f
+R 2438fd2896d39f35fc299b3fd744f13f
+T +closed 50b9f6fde44f9afda27e47badde6115ab6d9ccd5e22b446a8d8127499848815f Closed\sby\sintegrate-merge.
+U stephan
+Z 7f16e5a942c114132246c08a95b25355
 # Remove this line to create a well-formed Fossil manifest.
--- a/manifest.uuid
+++ b/manifest.uuid
@ -1 +1 @@
-7fec209290aa1a6dbbca8de154edaac5d8d0ce042bc0617d27fb2095c8d580f1
+84bd96a6f10823992cf865245cb2c75b69921257122fde5909b303621e288d00
--- a/tool/mkccode.tcl
+++ b/tool/mkccode.tcl
@ -54,7 +54,7 @@ set infile {}
 foreach ax $argv {
  if {[string match -D* $ax]} {
    if {[string match *=* $ax]} {
-      regexp {-D([^=]+)=(.*)} ax all name value
+      regexp -- {-D([^=]+)=(.*)} $ax all name value
      set DEF($name) $value
    } else {
      set DEF([string range $ax 2 end]) 1
@ -125,12 +125,12 @@ while {1} {
    puts "/* END_STRING */"
    continue
  }
-  if {[regexp {^IFNDEF +([A-Za-z_]+)} $line all name]} {
+  if {[regexp {^IFNDEF +([A-Za-z_0-9]+)} $line all name]} {
    set omit $omit[info exists DEF($name)]
    incr nomit
    continue
  }
-  if {[regexp {^IFDEF +([A-Za-z_]+)} $line all name]} {
+  if {[regexp {^IFDEF +([A-Za-z_0-9]+)} $line all name]} {
    set omit $omit[expr {![info exists DEF($name)]}]
    incr nomit
    continue
--- a/tool/sqlite3_analyzer.c.in
+++ b/tool/sqlite3_analyzer.c.in
@ -3,7 +3,8 @@
 ** text on standard output.
 */
 #define TCLSH_INIT_PROC sqlite3_analyzer_init_proc
-IFNDEF SQLITE_ENABLE_DBSTAT_VTAB
+IFDEF INCLUDE_SQLITE3_C
+#undef SQLITE_ENABLE_DBSTAT_VTAB
 #define SQLITE_ENABLE_DBSTAT_VTAB 1
 #undef SQLITE_THREADSAFE
 #define SQLITE_THREADSAFE 0