From 55b8399d7e87edf304044a198430684aa41e33ed Mon Sep 17 00:00:00 2001 From: Ryan Westlund Date: Wed, 11 Nov 2020 13:00:40 -0500 Subject: [PATCH 01/11] Enable REGEXP by connecting Crystal's stdlib Regex (#62) --- spec/db_spec.cr | 5 +++++ src/sqlite3.cr | 9 +++++++++ src/sqlite3/connection.cr | 2 ++ src/sqlite3/lib_sqlite3.cr | 7 +++++++ 4 files changed, 23 insertions(+) diff --git a/spec/db_spec.cr b/spec/db_spec.cr index 3a2fa1f..42d34b5 100644 --- a/spec/db_spec.cr +++ b/spec/db_spec.cr @@ -128,4 +128,9 @@ DB::DriverSpecs(DB::Any).run do it "handles multi-step pragma statements" do |db| db.exec %(PRAGMA journal_mode = memory) end + + it "handles REGEXP operator" do |db| + (db.scalar "select 'unmatching text' REGEXP '^m'").should eq 0 + (db.scalar "select 'matching text' REGEXP '^m'").should eq 1 + end end diff --git a/src/sqlite3.cr b/src/sqlite3.cr index 77ff98f..d1c973b 100644 --- a/src/sqlite3.cr +++ b/src/sqlite3.cr @@ -6,4 +6,13 @@ module SQLite3 # :nodoc: TIME_ZONE = Time::Location::UTC + + # :nodoc: + REGEXP_FN = ->(context : LibSQLite3::SQLite3Context, argc : Int32, argv : LibSQLite3::SQLite3Value*) do + argv = Slice.new(argv, sizeof(Void*)) + pattern = LibSQLite3.value_text(argv[0]) + text = LibSQLite3.value_text(argv[1]) + LibSQLite3.result_int(context, Regex.new(String.new(pattern)).matches?(String.new(text)).to_unsafe) + nil + end end diff --git a/src/sqlite3/connection.cr b/src/sqlite3/connection.cr index 54d1d96..ca60043 100644 --- a/src/sqlite3/connection.cr +++ b/src/sqlite3/connection.cr @@ -4,6 +4,8 @@ class SQLite3::Connection < DB::Connection filename = self.class.filename(database.uri) # TODO maybe enable Flag::URI to parse query string in the uri as additional flags check LibSQLite3.open_v2(filename, out @db, (Flag::READWRITE | Flag::CREATE), nil) + # 2 means 2 arguments; 1 is the code for UTF-8 + check LibSQLite3.create_function(@db, "regexp", 2, 1, nil, SQLite3::REGEXP_FN, nil, nil) rescue raise DB::ConnectionRefused.new end diff --git a/src/sqlite3/lib_sqlite3.cr b/src/sqlite3/lib_sqlite3.cr index d99f67c..9455f8b 100644 --- a/src/sqlite3/lib_sqlite3.cr +++ b/src/sqlite3/lib_sqlite3.cr @@ -5,6 +5,8 @@ lib LibSQLite3 type SQLite3 = Void* type Statement = Void* type SQLite3Backup = Void* + type SQLite3Context = Void* + type SQLite3Value = Void* enum Code # Successful result @@ -72,6 +74,7 @@ lib LibSQLite3 end alias Callback = (Void*, Int32, UInt8**, UInt8**) -> Int32 + alias FuncCallback = (SQLite3Context, Int32, SQLite3Value*) -> Void fun open_v2 = sqlite3_open_v2(filename : UInt8*, db : SQLite3*, flags : ::SQLite3::Flag, zVfs : UInt8*) : Int32 @@ -108,4 +111,8 @@ lib LibSQLite3 fun finalize = sqlite3_finalize(stmt : Statement) : Int32 fun close_v2 = sqlite3_close_v2(SQLite3) : Int32 fun close = sqlite3_close(SQLite3) : Int32 + + fun create_function = sqlite3_create_function(SQLite3, funcName : UInt8*, nArg : Int32, eTextRep : Int32, pApp : Void*, xFunc : FuncCallback, xStep : Void*, xFinal : Void*) : Int32 + fun value_text = sqlite3_value_text(SQLite3Value) : UInt8* + fun result_int = sqlite3_result_int(SQLite3Context, Int32) : Nil end From 367c11031d38aeae045f8eaabeae84fde33071ff Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Tue, 26 Jan 2021 16:11:20 -0300 Subject: [PATCH 02/11] Release 0.18.0 (#67) --- CHANGELOG.md | 4 ++++ shard.yml | 4 ++-- src/sqlite3/version.cr | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 662aa45..753f819 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v0.18.0 (2021-01-26) + +* Add `REGEXP` support powered by Crystal's std-lib Regex. ([#62](https://github.com/crystal-lang/crystal-sqlite3/pull/62), thanks @yujiri8) + ## v0.17.0 (2020-09-30) * Update to crystal-db ~> 0.10.0. ([#58](https://github.com/crystal-lang/crystal-sqlite3/pull/58)) diff --git a/shard.yml b/shard.yml index 4b279bd..d368f44 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: sqlite3 -version: 0.17.0 +version: 0.18.0 dependencies: db: @@ -10,6 +10,6 @@ authors: - Ary Borenszweig - Brian J. Cardiff -crystal: 0.35.0 +crystal: ">= 0.35.0, < 2.0.0" license: MIT diff --git a/src/sqlite3/version.cr b/src/sqlite3/version.cr index d5b7949..8de7608 100644 --- a/src/sqlite3/version.cr +++ b/src/sqlite3/version.cr @@ -1,3 +1,3 @@ module SQLite3 - VERSION = "0.17.0" + VERSION = "0.18.0" end From 985bfa2d7cb192671272e19c027cf9fb8a438b66 Mon Sep 17 00:00:00 2001 From: Ryan Westlund Date: Sun, 28 Feb 2021 12:18:08 -0500 Subject: [PATCH 03/11] Fix timestamp reading issue (#68) --- README.md | 2 +- spec/db_spec.cr | 2 ++ spec/result_set_spec.cr | 57 +++++++++++++++++++++++++++++++++++++++ src/sqlite3.cr | 3 ++- src/sqlite3/result_set.cr | 15 +++++++++-- src/sqlite3/statement.cr | 2 +- 6 files changed, 76 insertions(+), 5 deletions(-) create mode 100644 spec/result_set_spec.cr diff --git a/README.md b/README.md index af14447..eeef632 100644 --- a/README.md +++ b/README.md @@ -46,5 +46,5 @@ end ### DB::Any -* `Time` is implemented as `TEXT` column using `SQLite3::DATE_FORMAT` format. +* `Time` is implemented as `TEXT` column using `SQLite3::DATE_FORMAT_SUBSECOND` format (or `SQLite3::DATE_FORMAT_SECOND` if the text does not contain a dot). * `Bool` is implemented as `INT` column mapping `0`/`1` values. diff --git a/spec/db_spec.cr b/spec/db_spec.cr index 42d34b5..f3675a0 100644 --- a/spec/db_spec.cr +++ b/spec/db_spec.cr @@ -34,7 +34,9 @@ DB::DriverSpecs(DB::Any).run do sample_value 1.5_f32, "float", "1.5", type_safe_value: false sample_value 1.5, "float", "1.5" sample_value Time.utc(2016, 2, 15), "text", "'2016-02-15 00:00:00.000'", type_safe_value: false + sample_value Time.utc(2016, 2, 15, 10, 15, 30), "text", "'2016-02-15 10:15:30'", type_safe_value: false sample_value Time.utc(2016, 2, 15, 10, 15, 30), "text", "'2016-02-15 10:15:30.000'", type_safe_value: false + sample_value Time.utc(2016, 2, 15, 10, 15, 30, nanosecond: 123000000), "text", "'2016-02-15 10:15:30.123'", type_safe_value: false sample_value Time.local(2016, 2, 15, 7, 15, 30, location: Time::Location.fixed("fixed", -3*3600)), "text", "'2016-02-15 10:15:30.000'", type_safe_value: false ary = UInt8[0x53, 0x51, 0x4C, 0x69, 0x74, 0x65] diff --git a/spec/result_set_spec.cr b/spec/result_set_spec.cr new file mode 100644 index 0000000..65a02cf --- /dev/null +++ b/spec/result_set_spec.cr @@ -0,0 +1,57 @@ +require "./spec_helper" + +describe SQLite3::ResultSet do + it "reads integer data types" do + with_db do |db| + db.exec "CREATE TABLE test_table (test_int integer)" + db.exec "INSERT INTO test_table (test_int) values (?)", 42 + db.query("SELECT test_int FROM test_table") do |rs| + rs.each do + rs.read.should eq(42) + end + end + end + end + + it "reads string data types" do + with_db do |db| + db.exec "CREATE TABLE test_table (test_text text)" + db.exec "INSERT INTO test_table (test_text) values (?), (?)", "abc", "123" + db.query("SELECT test_text FROM test_table") do |rs| + rs.each do + rs.read.should match(/abc|123/) + end + end + end + end + + it "reads time data types" do + with_db do |db| + db.exec "CREATE TABLE test_table (test_date datetime)" + timestamp = Time.utc + db.exec "INSERT INTO test_table (test_date) values (current_timestamp)" + db.query("SELECT test_date FROM test_table") do |rs| + rs.each do + rs.read(Time).should be_close(timestamp, 1.second) + end + end + end + end + + it "reads time stored in text fields, too" do + with_db do |db| + db.exec "CREATE TABLE test_table (test_date text)" + timestamp = Time.utc + # Try 3 different ways: our own two formats and using SQLite's current_timestamp. + # They should all work. + db.exec "INSERT INTO test_table (test_date) values (?)", timestamp.to_s SQLite3::DATE_FORMAT_SUBSECOND + db.exec "INSERT INTO test_table (test_date) values (?)", timestamp.to_s SQLite3::DATE_FORMAT_SECOND + db.exec "INSERT INTO test_table (test_date) values (current_timestamp)" + db.query("SELECT test_date FROM test_table") do |rs| + rs.each do + rs.read(Time).should be_close(timestamp, 1.second) + end + end + end + end +end diff --git a/src/sqlite3.cr b/src/sqlite3.cr index d1c973b..4ba5365 100644 --- a/src/sqlite3.cr +++ b/src/sqlite3.cr @@ -2,7 +2,8 @@ require "db" require "./sqlite3/**" module SQLite3 - DATE_FORMAT = "%F %H:%M:%S.%L" + DATE_FORMAT_SUBSECOND = "%F %H:%M:%S.%L" + DATE_FORMAT_SECOND = "%F %H:%M:%S" # :nodoc: TIME_ZONE = Time::Location::UTC diff --git a/src/sqlite3/result_set.cr b/src/sqlite3/result_set.cr index 38e669f..48416b6 100644 --- a/src/sqlite3/result_set.cr +++ b/src/sqlite3/result_set.cr @@ -64,11 +64,22 @@ class SQLite3::ResultSet < DB::ResultSet end def read(t : Time.class) : Time - Time.parse read(String), SQLite3::DATE_FORMAT, location: SQLite3::TIME_ZONE + text = read(String) + if text.includes? "." + Time.parse text, SQLite3::DATE_FORMAT_SUBSECOND, location: SQLite3::TIME_ZONE + else + Time.parse text, SQLite3::DATE_FORMAT_SECOND, location: SQLite3::TIME_ZONE + end end def read(t : Time?.class) : Time? - read(String?).try { |v| Time.parse(v, SQLite3::DATE_FORMAT, location: SQLite3::TIME_ZONE) } + read(String?).try { |v| + if v.includes? "." + Time.parse v, SQLite3::DATE_FORMAT_SUBSECOND, location: SQLite3::TIME_ZONE + else + Time.parse v, SQLite3::DATE_FORMAT_SECOND, location: SQLite3::TIME_ZONE + end + } end def read(t : Bool.class) : Bool diff --git a/src/sqlite3/statement.cr b/src/sqlite3/statement.cr index 06289fb..368dfe3 100644 --- a/src/sqlite3/statement.cr +++ b/src/sqlite3/statement.cr @@ -70,7 +70,7 @@ class SQLite3::Statement < DB::Statement end private def bind_arg(index, value : Time) - bind_arg(index, value.in(SQLite3::TIME_ZONE).to_s(SQLite3::DATE_FORMAT)) + bind_arg(index, value.in(SQLite3::TIME_ZONE).to_s(SQLite3::DATE_FORMAT_SUBSECOND)) end private def bind_arg(index, value) From 2849fe15c885539e95b18f8fe7639063f70d21c6 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Thu, 27 Jan 2022 11:09:27 -0300 Subject: [PATCH 04/11] Update to crystal-db ~> 0.11.0 (#77) --- shard.yml | 4 ++-- src/sqlite3/result_set.cr | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/shard.yml b/shard.yml index d368f44..f6cf1de 100644 --- a/shard.yml +++ b/shard.yml @@ -4,12 +4,12 @@ version: 0.18.0 dependencies: db: github: crystal-lang/crystal-db - version: ~> 0.10.0 + version: ~> 0.11.0 authors: - Ary Borenszweig - Brian J. Cardiff -crystal: ">= 0.35.0, < 2.0.0" +crystal: ">= 1.0.0, < 2.0.0" license: MIT diff --git a/src/sqlite3/result_set.cr b/src/sqlite3/result_set.cr index 48416b6..209aab6 100644 --- a/src/sqlite3/result_set.cr +++ b/src/sqlite3/result_set.cr @@ -47,6 +47,10 @@ class SQLite3::ResultSet < DB::ResultSet value end + def next_column_index : Int32 + @column_index + end + def read(t : Int32.class) : Int32 read(Int64).to_i32 end From a9baaec310e36e37e7f6d77e941984a8709df17b Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Fri, 28 Jan 2022 15:14:33 -0300 Subject: [PATCH 05/11] Migrate CI to GitHub Actions (#78) --- .github/workflows/ci.yml | 35 +++++++++++++++++++++++++++++++++++ .travis.yml | 2 -- 2 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..63b002f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,35 @@ +name: CI + +on: + push: + pull_request: + branches: [master] + schedule: + - cron: '0 6 * * 1' # Every monday 6 AM + +jobs: + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + crystal: [1.0.0, latest, nightly] + runs-on: ${{ matrix.os }} + steps: + - name: Install Crystal + uses: crystal-lang/install-crystal@v1 + with: + crystal: ${{ matrix.crystal }} + + - name: Download source + uses: actions/checkout@v2 + + - name: Install shards + run: shards install + + - name: Run specs + run: crystal spec + + - name: Check formatting + run: crystal tool format; git diff --exit-code + if: matrix.crystal == 'latest' && matrix.os == 'ubuntu-latest' diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index dfd532a..0000000 --- a/.travis.yml +++ /dev/null @@ -1,2 +0,0 @@ -language: crystal -sudo: false From d15a8da05cad8d4da919360db0c316d4e9bbfaff Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Fri, 28 Jan 2022 15:33:18 -0300 Subject: [PATCH 06/11] Release 0.19.0 (#79) --- CHANGELOG.md | 8 ++++++++ shard.yml | 2 +- src/sqlite3/version.cr | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 753f819..c0816f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## v0.19.0 (2022-01-28) + +* Update to crystal-db ~> 0.11.0. ([#77](https://github.com/crystal-lang/crystal-sqlite3/pull/77)) +* Fix timestamps support to allow dealing with exact seconds values ([#68](https://github.com/crystal-lang/crystal-sqlite3/pull/68), thanks @yujiri8, @tenebrousedge) +* Migrate CI to GitHub Actions. ([#78](https://github.com/crystal-lang/crystal-sqlite3/pull/78)) + +This release requires Crystal 1.0.0 or later. + ## v0.18.0 (2021-01-26) * Add `REGEXP` support powered by Crystal's std-lib Regex. ([#62](https://github.com/crystal-lang/crystal-sqlite3/pull/62), thanks @yujiri8) diff --git a/shard.yml b/shard.yml index f6cf1de..82ccb5c 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: sqlite3 -version: 0.18.0 +version: 0.19.0 dependencies: db: diff --git a/src/sqlite3/version.cr b/src/sqlite3/version.cr index 8de7608..a72f1d2 100644 --- a/src/sqlite3/version.cr +++ b/src/sqlite3/version.cr @@ -1,3 +1,3 @@ module SQLite3 - VERSION = "0.18.0" + VERSION = "0.19.0" end From 84857724c104f01c01cf48f46f4868fbf62ac12e Mon Sep 17 00:00:00 2001 From: Luis Lavena Date: Sat, 18 Feb 2023 21:06:09 +0100 Subject: [PATCH 07/11] Automatically set PRAGMAs using connection query params (#85) * Automatically set PRAGMAs using connection query params Introduce the flexibility to adjust certain PRAGMAs of the SQLite3 connection without having to hardcode those in your codebase (and wait for compilation). This allows applications to use `DATABASE_URL` to dynamically fine tune their SQLite3 configuration. The change complements `#setup_connection` that offers, via code, the option to perform queries on setup of each connection. Only a few PRAGMAs necessary to allow more performant concurrent reads and reduce write locking. These pragmas are detected and combined in a single SQL string to reduce to 1 the number of calls to `sqlite3_exec` C function. There is no validation of supplied values as SQLite3 automatically ignores incorrect values for these pragmas. Closes #84 References: - https://www.sqlite.org/pragma.html * Simplify PRAGMA mapping and detection No longer prefix PRAGMAS with `_`, so the mapping between the real SQLite3 pragmas and the usage in the URI is more direct. Use macros instead of case to detect pragmas from URI params and return those as NamedTuple. * Update README.md --------- Co-authored-by: Brian J. Cardiff --- README.md | 36 +++++++++++++++++++++++++++++++ spec/connection_spec.cr | 38 +++++++++++++++++++++++++++++++++ src/sqlite3/connection.cr | 43 +++++++++++++++++++++++++++++++++++++- src/sqlite3/lib_sqlite3.cr | 1 + 4 files changed, 117 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index eeef632..639b842 100644 --- a/README.md +++ b/README.md @@ -48,3 +48,39 @@ end * `Time` is implemented as `TEXT` column using `SQLite3::DATE_FORMAT_SUBSECOND` format (or `SQLite3::DATE_FORMAT_SECOND` if the text does not contain a dot). * `Bool` is implemented as `INT` column mapping `0`/`1` values. + +### Setting PRAGMAs + +You can adjust certain [SQLite3 PRAGMAs](https://www.sqlite.org/pragma.html) +automatically when the connection is created by using the query parameters: + +```crystal +require "sqlite3" + +DB.open "sqlite3://./data.db?journal_mode=wal&synchronous=normal" do |db| + # this database now uses WAL journal and normal synchronous mode + # (defaults were `delete` and `full`, respectively) +end +``` + +The following is the list of supported options: + +| Name | Connection key | +|---------------------------|-----------------| +| [Busy Timeout][pragma-to] | `busy_timeout` | +| [Cache Size][pragma-cs] | `cache_size` | +| [Foreign Keys][pragma-fk] | `foreign_keys` | +| [Journal Mode][pragma-jm] | `journal_mode` | +| [Synchronous][pragma-sync] | `synchronous` | +| [WAL autocheckoint][pragma-walck] | `wal_autocheckpoint` | + +Please note there values passed using these connection keys are passed +directly to SQLite3 without check or evaluation. Using incorrect values result +in no error by the library. + +[pragma-to]: https://www.sqlite.org/pragma.html#pragma_busy_timeout +[pragma-cs]: https://www.sqlite.org/pragma.html#pragma_cache_size +[pragma-fk]: https://www.sqlite.org/pragma.html#pragma_foreign_keys +[pragma-jm]: https://www.sqlite.org/pragma.html#pragma_journal_mode +[pragma-sync]: https://www.sqlite.org/pragma.html#pragma_synchronous +[pragma-walck]: https://www.sqlite.org/pragma.html#pragma_wal_autocheckpoint diff --git a/spec/connection_spec.cr b/spec/connection_spec.cr index be3bf0c..84e9759 100644 --- a/spec/connection_spec.cr +++ b/spec/connection_spec.cr @@ -10,6 +10,14 @@ private def dump(source, target) end end +private def it_sets_pragma_on_connection(pragma : String, value : String, expected, file = __FILE__, line = __LINE__) + it "sets pragma '#{pragma}' to #{expected}", file, line do + with_db("#{DB_FILENAME}?#{pragma}=#{value}") do |db| + db.scalar("PRAGMA #{pragma}").should eq(expected) + end + end +end + describe Connection do it "opens a database and then backs it up to another db" do with_db do |db| @@ -68,4 +76,34 @@ describe Connection do cnn.scalar("select count(*) from person").should eq(1) end end + + # adjust busy_timeout pragma (default is 0) + it_sets_pragma_on_connection "busy_timeout", "1000", 1000 + + # adjust cache_size pragma (default is -2000, 2MB) + it_sets_pragma_on_connection "cache_size", "-4000", -4000 + + # enable foreign_keys, no need to test off (is the default) + it_sets_pragma_on_connection "foreign_keys", "1", 1 + it_sets_pragma_on_connection "foreign_keys", "yes", 1 + it_sets_pragma_on_connection "foreign_keys", "true", 1 + it_sets_pragma_on_connection "foreign_keys", "on", 1 + + # change journal_mode (default is delete) + it_sets_pragma_on_connection "journal_mode", "delete", "delete" + it_sets_pragma_on_connection "journal_mode", "truncate", "truncate" + it_sets_pragma_on_connection "journal_mode", "persist", "persist" + + # change synchronous mode (default is 2, FULL) + it_sets_pragma_on_connection "synchronous", "0", 0 + it_sets_pragma_on_connection "synchronous", "off", 0 + it_sets_pragma_on_connection "synchronous", "1", 1 + it_sets_pragma_on_connection "synchronous", "normal", 1 + it_sets_pragma_on_connection "synchronous", "2", 2 + it_sets_pragma_on_connection "synchronous", "full", 2 + it_sets_pragma_on_connection "synchronous", "3", 3 + it_sets_pragma_on_connection "synchronous", "extra", 3 + + # change wal_autocheckpoint (default is 1000) + it_sets_pragma_on_connection "wal_autocheckpoint", "0", 0 end diff --git a/src/sqlite3/connection.cr b/src/sqlite3/connection.cr index ca60043..3979120 100644 --- a/src/sqlite3/connection.cr +++ b/src/sqlite3/connection.cr @@ -2,10 +2,11 @@ class SQLite3::Connection < DB::Connection def initialize(database) super filename = self.class.filename(database.uri) - # TODO maybe enable Flag::URI to parse query string in the uri as additional flags check LibSQLite3.open_v2(filename, out @db, (Flag::READWRITE | Flag::CREATE), nil) # 2 means 2 arguments; 1 is the code for UTF-8 check LibSQLite3.create_function(@db, "regexp", 2, 1, nil, SQLite3::REGEXP_FN, nil, nil) + + process_query_params(database.uri) rescue raise DB::ConnectionRefused.new end @@ -88,4 +89,44 @@ class SQLite3::Connection < DB::Connection private def check(code) raise Exception.new(self) unless code == 0 end + + private def process_query_params(uri : URI) + return unless query = uri.query + + detected_pragmas = extract_params(query, + busy_timeout: nil, + cache_size: nil, + foreign_keys: nil, + journal_mode: nil, + synchronous: nil, + wal_autocheckpoint: nil, + ) + + # concatenate all into a single SQL string + sql = String.build do |str| + detected_pragmas.each do |key, value| + next unless value + str << "PRAGMA #{key}=#{value};" + end + end + + check LibSQLite3.exec(@db, sql, nil, nil, nil) + end + + private def extract_params(query : String, **default : **T) forall T + res = default + + URI::Params.parse(query) do |key, value| + {% begin %} + case key + {% for key in T %} + when {{ key.stringify }} + res = res.merge({{key.id}}: value) + {% end %} + end + {% end %} + end + + res + end end diff --git a/src/sqlite3/lib_sqlite3.cr b/src/sqlite3/lib_sqlite3.cr index 9455f8b..310e69c 100644 --- a/src/sqlite3/lib_sqlite3.cr +++ b/src/sqlite3/lib_sqlite3.cr @@ -86,6 +86,7 @@ lib LibSQLite3 fun backup_finish = sqlite3_backup_finish(SQLite3Backup) : Code fun prepare_v2 = sqlite3_prepare_v2(db : SQLite3, zSql : UInt8*, nByte : Int32, ppStmt : Statement*, pzTail : UInt8**) : Int32 + fun exec = sqlite3_exec(db : SQLite3, zSql : UInt8*, pCallback : Callback, pCallbackArgs : Void*, pzErrMsg : UInt8**) : Int32 fun step = sqlite3_step(stmt : Statement) : Int32 fun column_count = sqlite3_column_count(stmt : Statement) : Int32 fun column_type = sqlite3_column_type(stmt : Statement, iCol : Int32) : ::SQLite3::Type From 79615e6c85e66c4c0b3f4b27f7d4bd36e7dee99d Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Mon, 24 Apr 2023 12:53:56 -0300 Subject: [PATCH 08/11] Reset before close (#90) --- src/sqlite3/result_set.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlite3/result_set.cr b/src/sqlite3/result_set.cr index 209aab6..9955129 100644 --- a/src/sqlite3/result_set.cr +++ b/src/sqlite3/result_set.cr @@ -2,8 +2,8 @@ class SQLite3::ResultSet < DB::ResultSet @column_index = 0 protected def do_close - super LibSQLite3.reset(self) + super end # Advances to the next row. Returns `true` if there's a next row, From c08fc2befc88c1373f3f94bb26fe8f008539c1a3 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Fri, 23 Jun 2023 15:36:08 -0300 Subject: [PATCH 09/11] Update to crystal-db 0.12.0 (refactor connection factory) (#91) * Refactor connection builder * Update for ConnectionBuilder * Update to crystal-db ~> 0.12.0 * run crystal tool format --- shard.yml | 4 +- spec/db_spec.cr | 4 +- spec/driver_spec.cr | 2 +- src/sqlite3/connection.cr | 94 ++++++++++++++++++++------------------- src/sqlite3/driver.cr | 14 +++++- 5 files changed, 66 insertions(+), 52 deletions(-) diff --git a/shard.yml b/shard.yml index 82ccb5c..ec568a5 100644 --- a/shard.yml +++ b/shard.yml @@ -4,11 +4,11 @@ version: 0.19.0 dependencies: db: github: crystal-lang/crystal-db - version: ~> 0.11.0 + version: ~> 0.12.0 authors: - Ary Borenszweig - - Brian J. Cardiff + - Brian J. Cardiff crystal: ">= 1.0.0, < 2.0.0" diff --git a/spec/db_spec.cr b/spec/db_spec.cr index f3675a0..c130070 100644 --- a/spec/db_spec.cr +++ b/spec/db_spec.cr @@ -13,7 +13,7 @@ private def cast_if_blob(expr, sql_type) end end -DB::DriverSpecs(DB::Any).run do +DB::DriverSpecs(DB::Any).run do |ctx| support_unprepared false before do @@ -104,7 +104,7 @@ DB::DriverSpecs(DB::Any).run do db.exec %(insert into a (i, str) values (23, "bai bai");) 2.times do |i| - DB.open db.uri do |db| + DB.open ctx.connection_string do |db| begin db.query("SELECT i, str FROM a WHERE i = ?", 23) do |rs| rs.move_next diff --git a/spec/driver_spec.cr b/spec/driver_spec.cr index fbd64b5..6337279 100644 --- a/spec/driver_spec.cr +++ b/spec/driver_spec.cr @@ -27,7 +27,7 @@ describe Driver do it "should use database option as file to open" do with_db do |db| - db.driver.should be_a(SQLite3::Driver) + db.checkout.should be_a(SQLite3::Connection) File.exists?(DB_FILENAME).should be_true end end diff --git a/src/sqlite3/connection.cr b/src/sqlite3/connection.cr index 3979120..750776f 100644 --- a/src/sqlite3/connection.cr +++ b/src/sqlite3/connection.cr @@ -1,12 +1,56 @@ class SQLite3::Connection < DB::Connection - def initialize(database) - super - filename = self.class.filename(database.uri) - check LibSQLite3.open_v2(filename, out @db, (Flag::READWRITE | Flag::CREATE), nil) + record Options, + filename : String = ":memory:", + # pragmas + busy_timeout : String? = nil, + cache_size : String? = nil, + foreign_keys : String? = nil, + journal_mode : String? = nil, + synchronous : String? = nil, + wal_autocheckpoint : String? = nil do + def self.from_uri(uri : URI, default = Options.new) + params = HTTP::Params.parse(uri.query || "") + + Options.new( + filename: URI.decode_www_form((uri.host || "") + uri.path), + # pragmas + busy_timeout: params.fetch("busy_timeout", default.busy_timeout), + cache_size: params.fetch("cache_size", default.cache_size), + foreign_keys: params.fetch("foreign_keys", default.foreign_keys), + journal_mode: params.fetch("journal_mode", default.journal_mode), + synchronous: params.fetch("synchronous", default.synchronous), + wal_autocheckpoint: params.fetch("wal_autocheckpoint", default.wal_autocheckpoint), + ) + end + + def pragma_statement + res = String.build do |str| + pragma_append(str, "busy_timeout", busy_timeout) + pragma_append(str, "cache_size", cache_size) + pragma_append(str, "foreign_keys", foreign_keys) + pragma_append(str, "journal_mode", journal_mode) + pragma_append(str, "synchronous", synchronous) + pragma_append(str, "wal_autocheckpoint", wal_autocheckpoint) + end + + res.empty? ? nil : res + end + + private def pragma_append(io, key, value) + return unless value + io << "PRAGMA #{key}=#{value};" + end + end + + def initialize(options : ::DB::Connection::Options, sqlite3_options : Options) + super(options) + check LibSQLite3.open_v2(sqlite3_options.filename, out @db, (Flag::READWRITE | Flag::CREATE), nil) # 2 means 2 arguments; 1 is the code for UTF-8 check LibSQLite3.create_function(@db, "regexp", 2, 1, nil, SQLite3::REGEXP_FN, nil, nil) - process_query_params(database.uri) + if pragma_statement = sqlite3_options.pragma_statement + check LibSQLite3.exec(@db, pragma_statement, nil, nil, nil) + end rescue raise DB::ConnectionRefused.new end @@ -89,44 +133,4 @@ class SQLite3::Connection < DB::Connection private def check(code) raise Exception.new(self) unless code == 0 end - - private def process_query_params(uri : URI) - return unless query = uri.query - - detected_pragmas = extract_params(query, - busy_timeout: nil, - cache_size: nil, - foreign_keys: nil, - journal_mode: nil, - synchronous: nil, - wal_autocheckpoint: nil, - ) - - # concatenate all into a single SQL string - sql = String.build do |str| - detected_pragmas.each do |key, value| - next unless value - str << "PRAGMA #{key}=#{value};" - end - end - - check LibSQLite3.exec(@db, sql, nil, nil, nil) - end - - private def extract_params(query : String, **default : **T) forall T - res = default - - URI::Params.parse(query) do |key, value| - {% begin %} - case key - {% for key in T %} - when {{ key.stringify }} - res = res.merge({{key.id}}: value) - {% end %} - end - {% end %} - end - - res - end end diff --git a/src/sqlite3/driver.cr b/src/sqlite3/driver.cr index d43a512..93ae0b0 100644 --- a/src/sqlite3/driver.cr +++ b/src/sqlite3/driver.cr @@ -1,6 +1,16 @@ class SQLite3::Driver < DB::Driver - def build_connection(context : DB::ConnectionContext) : SQLite3::Connection - SQLite3::Connection.new(context) + class ConnectionBuilder < ::DB::ConnectionBuilder + def initialize(@options : ::DB::Connection::Options, @sqlite3_options : SQLite3::Connection::Options) + end + + def build : ::DB::Connection + SQLite3::Connection.new(@options, @sqlite3_options) + end + end + + def connection_builder(uri : URI) : ::DB::ConnectionBuilder + params = HTTP::Params.parse(uri.query || "") + ConnectionBuilder.new(connection_options(params), SQLite3::Connection::Options.from_uri(uri)) end end From 1f2a1cd3cb274cc0efe5a085eea2778e7f182dec Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Fri, 23 Jun 2023 16:29:58 -0300 Subject: [PATCH 10/11] Release 0.20.0 (#92) --- CHANGELOG.md | 6 ++++++ shard.yml | 2 +- src/sqlite3/version.cr | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0816f5..efb56e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## v0.20.0 (2023-06-23) + +* Update to crystal-db ~> 0.12.0. ([#91](https://github.com/crystal-lang/crystal-sqlite3/pull/91)) +* Fix result set & connection release lifecycle. ([#90](https://github.com/crystal-lang/crystal-sqlite3/pull/90)) +* Automatically set PRAGMAs using connection query params. ([#85](https://github.com/crystal-lang/crystal-sqlite3/pull/85), thanks @luislavena) + ## v0.19.0 (2022-01-28) * Update to crystal-db ~> 0.11.0. ([#77](https://github.com/crystal-lang/crystal-sqlite3/pull/77)) diff --git a/shard.yml b/shard.yml index ec568a5..1f10957 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: sqlite3 -version: 0.19.0 +version: 0.20.0 dependencies: db: diff --git a/src/sqlite3/version.cr b/src/sqlite3/version.cr index a72f1d2..bb8ef8d 100644 --- a/src/sqlite3/version.cr +++ b/src/sqlite3/version.cr @@ -1,3 +1,3 @@ module SQLite3 - VERSION = "0.19.0" + VERSION = "0.20.0" end From c58cea290c85e2a33dc8f494a5f04b519d3e0274 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Tue, 12 Dec 2023 22:15:47 -0300 Subject: [PATCH 11/11] Release 0.21.0, Update to crystal-db ~> 0.13 (#94) * Update crystal-db ~> 0.13.0 * Bump version to 0.21.0 --- CHANGELOG.md | 4 ++++ shard.yml | 4 ++-- src/sqlite3/version.cr | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index efb56e4..22359b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v0.21.0 (2023-12-12) + +* Update to crystal-db ~> 0.13.0. ([#94](https://github.com/crystal-lang/crystal-sqlite3/pull/94)) + ## v0.20.0 (2023-06-23) * Update to crystal-db ~> 0.12.0. ([#91](https://github.com/crystal-lang/crystal-sqlite3/pull/91)) diff --git a/shard.yml b/shard.yml index 1f10957..a9f613d 100644 --- a/shard.yml +++ b/shard.yml @@ -1,10 +1,10 @@ name: sqlite3 -version: 0.20.0 +version: 0.21.0 dependencies: db: github: crystal-lang/crystal-db - version: ~> 0.12.0 + version: ~> 0.13.0 authors: - Ary Borenszweig diff --git a/src/sqlite3/version.cr b/src/sqlite3/version.cr index bb8ef8d..4e4f456 100644 --- a/src/sqlite3/version.cr +++ b/src/sqlite3/version.cr @@ -1,3 +1,3 @@ module SQLite3 - VERSION = "0.20.0" + VERSION = "0.21.0" end