diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..63b002f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,35 @@ +name: CI + +on: + push: + pull_request: + branches: [master] + schedule: + - cron: '0 6 * * 1' # Every monday 6 AM + +jobs: + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + crystal: [1.0.0, latest, nightly] + runs-on: ${{ matrix.os }} + steps: + - name: Install Crystal + uses: crystal-lang/install-crystal@v1 + with: + crystal: ${{ matrix.crystal }} + + - name: Download source + uses: actions/checkout@v2 + + - name: Install shards + run: shards install + + - name: Run specs + run: crystal spec + + - name: Check formatting + run: crystal tool format; git diff --exit-code + if: matrix.crystal == 'latest' && matrix.os == 'ubuntu-latest' diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index dfd532a..0000000 --- a/.travis.yml +++ /dev/null @@ -1,2 +0,0 @@ -language: crystal -sudo: false diff --git a/CHANGELOG.md b/CHANGELOG.md index 662aa45..22359b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,25 @@ +## v0.21.0 (2023-12-12) + +* Update to crystal-db ~> 0.13.0. ([#94](https://github.com/crystal-lang/crystal-sqlite3/pull/94)) + +## v0.20.0 (2023-06-23) + +* Update to crystal-db ~> 0.12.0. ([#91](https://github.com/crystal-lang/crystal-sqlite3/pull/91)) +* Fix result set & connection release lifecycle. ([#90](https://github.com/crystal-lang/crystal-sqlite3/pull/90)) +* Automatically set PRAGMAs using connection query params. ([#85](https://github.com/crystal-lang/crystal-sqlite3/pull/85), thanks @luislavena) + +## v0.19.0 (2022-01-28) + +* Update to crystal-db ~> 0.11.0. ([#77](https://github.com/crystal-lang/crystal-sqlite3/pull/77)) +* Fix timestamps support to allow dealing with exact seconds values ([#68](https://github.com/crystal-lang/crystal-sqlite3/pull/68), thanks @yujiri8, @tenebrousedge) +* Migrate CI to GitHub Actions. ([#78](https://github.com/crystal-lang/crystal-sqlite3/pull/78)) + +This release requires Crystal 1.0.0 or later. + +## v0.18.0 (2021-01-26) + +* Add `REGEXP` support powered by Crystal's std-lib Regex. ([#62](https://github.com/crystal-lang/crystal-sqlite3/pull/62), thanks @yujiri8) + ## v0.17.0 (2020-09-30) * Update to crystal-db ~> 0.10.0. ([#58](https://github.com/crystal-lang/crystal-sqlite3/pull/58)) diff --git a/README.md b/README.md index af14447..639b842 100644 --- a/README.md +++ b/README.md @@ -46,5 +46,41 @@ end ### DB::Any -* `Time` is implemented as `TEXT` column using `SQLite3::DATE_FORMAT` format. +* `Time` is implemented as `TEXT` column using `SQLite3::DATE_FORMAT_SUBSECOND` format (or `SQLite3::DATE_FORMAT_SECOND` if the text does not contain a dot). * `Bool` is implemented as `INT` column mapping `0`/`1` values. + +### Setting PRAGMAs + +You can adjust certain [SQLite3 PRAGMAs](https://www.sqlite.org/pragma.html) +automatically when the connection is created by using the query parameters: + +```crystal +require "sqlite3" + +DB.open "sqlite3://./data.db?journal_mode=wal&synchronous=normal" do |db| + # this database now uses WAL journal and normal synchronous mode + # (defaults were `delete` and `full`, respectively) +end +``` + +The following is the list of supported options: + +| Name | Connection key | +|---------------------------|-----------------| +| [Busy Timeout][pragma-to] | `busy_timeout` | +| [Cache Size][pragma-cs] | `cache_size` | +| [Foreign Keys][pragma-fk] | `foreign_keys` | +| [Journal Mode][pragma-jm] | `journal_mode` | +| [Synchronous][pragma-sync] | `synchronous` | +| [WAL autocheckoint][pragma-walck] | `wal_autocheckpoint` | + +Please note there values passed using these connection keys are passed +directly to SQLite3 without check or evaluation. Using incorrect values result +in no error by the library. + +[pragma-to]: https://www.sqlite.org/pragma.html#pragma_busy_timeout +[pragma-cs]: https://www.sqlite.org/pragma.html#pragma_cache_size +[pragma-fk]: https://www.sqlite.org/pragma.html#pragma_foreign_keys +[pragma-jm]: https://www.sqlite.org/pragma.html#pragma_journal_mode +[pragma-sync]: https://www.sqlite.org/pragma.html#pragma_synchronous +[pragma-walck]: https://www.sqlite.org/pragma.html#pragma_wal_autocheckpoint diff --git a/shard.yml b/shard.yml index 4b279bd..a9f613d 100644 --- a/shard.yml +++ b/shard.yml @@ -1,15 +1,15 @@ name: sqlite3 -version: 0.17.0 +version: 0.21.0 dependencies: db: github: crystal-lang/crystal-db - version: ~> 0.10.0 + version: ~> 0.13.0 authors: - Ary Borenszweig - - Brian J. Cardiff + - Brian J. Cardiff -crystal: 0.35.0 +crystal: ">= 1.0.0, < 2.0.0" license: MIT diff --git a/spec/connection_spec.cr b/spec/connection_spec.cr index be3bf0c..84e9759 100644 --- a/spec/connection_spec.cr +++ b/spec/connection_spec.cr @@ -10,6 +10,14 @@ private def dump(source, target) end end +private def it_sets_pragma_on_connection(pragma : String, value : String, expected, file = __FILE__, line = __LINE__) + it "sets pragma '#{pragma}' to #{expected}", file, line do + with_db("#{DB_FILENAME}?#{pragma}=#{value}") do |db| + db.scalar("PRAGMA #{pragma}").should eq(expected) + end + end +end + describe Connection do it "opens a database and then backs it up to another db" do with_db do |db| @@ -68,4 +76,34 @@ describe Connection do cnn.scalar("select count(*) from person").should eq(1) end end + + # adjust busy_timeout pragma (default is 0) + it_sets_pragma_on_connection "busy_timeout", "1000", 1000 + + # adjust cache_size pragma (default is -2000, 2MB) + it_sets_pragma_on_connection "cache_size", "-4000", -4000 + + # enable foreign_keys, no need to test off (is the default) + it_sets_pragma_on_connection "foreign_keys", "1", 1 + it_sets_pragma_on_connection "foreign_keys", "yes", 1 + it_sets_pragma_on_connection "foreign_keys", "true", 1 + it_sets_pragma_on_connection "foreign_keys", "on", 1 + + # change journal_mode (default is delete) + it_sets_pragma_on_connection "journal_mode", "delete", "delete" + it_sets_pragma_on_connection "journal_mode", "truncate", "truncate" + it_sets_pragma_on_connection "journal_mode", "persist", "persist" + + # change synchronous mode (default is 2, FULL) + it_sets_pragma_on_connection "synchronous", "0", 0 + it_sets_pragma_on_connection "synchronous", "off", 0 + it_sets_pragma_on_connection "synchronous", "1", 1 + it_sets_pragma_on_connection "synchronous", "normal", 1 + it_sets_pragma_on_connection "synchronous", "2", 2 + it_sets_pragma_on_connection "synchronous", "full", 2 + it_sets_pragma_on_connection "synchronous", "3", 3 + it_sets_pragma_on_connection "synchronous", "extra", 3 + + # change wal_autocheckpoint (default is 1000) + it_sets_pragma_on_connection "wal_autocheckpoint", "0", 0 end diff --git a/spec/db_spec.cr b/spec/db_spec.cr index 3a2fa1f..c130070 100644 --- a/spec/db_spec.cr +++ b/spec/db_spec.cr @@ -13,7 +13,7 @@ private def cast_if_blob(expr, sql_type) end end -DB::DriverSpecs(DB::Any).run do +DB::DriverSpecs(DB::Any).run do |ctx| support_unprepared false before do @@ -34,7 +34,9 @@ DB::DriverSpecs(DB::Any).run do sample_value 1.5_f32, "float", "1.5", type_safe_value: false sample_value 1.5, "float", "1.5" sample_value Time.utc(2016, 2, 15), "text", "'2016-02-15 00:00:00.000'", type_safe_value: false + sample_value Time.utc(2016, 2, 15, 10, 15, 30), "text", "'2016-02-15 10:15:30'", type_safe_value: false sample_value Time.utc(2016, 2, 15, 10, 15, 30), "text", "'2016-02-15 10:15:30.000'", type_safe_value: false + sample_value Time.utc(2016, 2, 15, 10, 15, 30, nanosecond: 123000000), "text", "'2016-02-15 10:15:30.123'", type_safe_value: false sample_value Time.local(2016, 2, 15, 7, 15, 30, location: Time::Location.fixed("fixed", -3*3600)), "text", "'2016-02-15 10:15:30.000'", type_safe_value: false ary = UInt8[0x53, 0x51, 0x4C, 0x69, 0x74, 0x65] @@ -102,7 +104,7 @@ DB::DriverSpecs(DB::Any).run do db.exec %(insert into a (i, str) values (23, "bai bai");) 2.times do |i| - DB.open db.uri do |db| + DB.open ctx.connection_string do |db| begin db.query("SELECT i, str FROM a WHERE i = ?", 23) do |rs| rs.move_next @@ -128,4 +130,9 @@ DB::DriverSpecs(DB::Any).run do it "handles multi-step pragma statements" do |db| db.exec %(PRAGMA journal_mode = memory) end + + it "handles REGEXP operator" do |db| + (db.scalar "select 'unmatching text' REGEXP '^m'").should eq 0 + (db.scalar "select 'matching text' REGEXP '^m'").should eq 1 + end end diff --git a/spec/driver_spec.cr b/spec/driver_spec.cr index fbd64b5..6337279 100644 --- a/spec/driver_spec.cr +++ b/spec/driver_spec.cr @@ -27,7 +27,7 @@ describe Driver do it "should use database option as file to open" do with_db do |db| - db.driver.should be_a(SQLite3::Driver) + db.checkout.should be_a(SQLite3::Connection) File.exists?(DB_FILENAME).should be_true end end diff --git a/spec/result_set_spec.cr b/spec/result_set_spec.cr new file mode 100644 index 0000000..65a02cf --- /dev/null +++ b/spec/result_set_spec.cr @@ -0,0 +1,57 @@ +require "./spec_helper" + +describe SQLite3::ResultSet do + it "reads integer data types" do + with_db do |db| + db.exec "CREATE TABLE test_table (test_int integer)" + db.exec "INSERT INTO test_table (test_int) values (?)", 42 + db.query("SELECT test_int FROM test_table") do |rs| + rs.each do + rs.read.should eq(42) + end + end + end + end + + it "reads string data types" do + with_db do |db| + db.exec "CREATE TABLE test_table (test_text text)" + db.exec "INSERT INTO test_table (test_text) values (?), (?)", "abc", "123" + db.query("SELECT test_text FROM test_table") do |rs| + rs.each do + rs.read.should match(/abc|123/) + end + end + end + end + + it "reads time data types" do + with_db do |db| + db.exec "CREATE TABLE test_table (test_date datetime)" + timestamp = Time.utc + db.exec "INSERT INTO test_table (test_date) values (current_timestamp)" + db.query("SELECT test_date FROM test_table") do |rs| + rs.each do + rs.read(Time).should be_close(timestamp, 1.second) + end + end + end + end + + it "reads time stored in text fields, too" do + with_db do |db| + db.exec "CREATE TABLE test_table (test_date text)" + timestamp = Time.utc + # Try 3 different ways: our own two formats and using SQLite's current_timestamp. + # They should all work. + db.exec "INSERT INTO test_table (test_date) values (?)", timestamp.to_s SQLite3::DATE_FORMAT_SUBSECOND + db.exec "INSERT INTO test_table (test_date) values (?)", timestamp.to_s SQLite3::DATE_FORMAT_SECOND + db.exec "INSERT INTO test_table (test_date) values (current_timestamp)" + db.query("SELECT test_date FROM test_table") do |rs| + rs.each do + rs.read(Time).should be_close(timestamp, 1.second) + end + end + end + end +end diff --git a/src/sqlite3.cr b/src/sqlite3.cr index 77ff98f..4ba5365 100644 --- a/src/sqlite3.cr +++ b/src/sqlite3.cr @@ -2,8 +2,18 @@ require "db" require "./sqlite3/**" module SQLite3 - DATE_FORMAT = "%F %H:%M:%S.%L" + DATE_FORMAT_SUBSECOND = "%F %H:%M:%S.%L" + DATE_FORMAT_SECOND = "%F %H:%M:%S" # :nodoc: TIME_ZONE = Time::Location::UTC + + # :nodoc: + REGEXP_FN = ->(context : LibSQLite3::SQLite3Context, argc : Int32, argv : LibSQLite3::SQLite3Value*) do + argv = Slice.new(argv, sizeof(Void*)) + pattern = LibSQLite3.value_text(argv[0]) + text = LibSQLite3.value_text(argv[1]) + LibSQLite3.result_int(context, Regex.new(String.new(pattern)).matches?(String.new(text)).to_unsafe) + nil + end end diff --git a/src/sqlite3/connection.cr b/src/sqlite3/connection.cr index 54d1d96..750776f 100644 --- a/src/sqlite3/connection.cr +++ b/src/sqlite3/connection.cr @@ -1,9 +1,56 @@ class SQLite3::Connection < DB::Connection - def initialize(database) - super - filename = self.class.filename(database.uri) - # TODO maybe enable Flag::URI to parse query string in the uri as additional flags - check LibSQLite3.open_v2(filename, out @db, (Flag::READWRITE | Flag::CREATE), nil) + record Options, + filename : String = ":memory:", + # pragmas + busy_timeout : String? = nil, + cache_size : String? = nil, + foreign_keys : String? = nil, + journal_mode : String? = nil, + synchronous : String? = nil, + wal_autocheckpoint : String? = nil do + def self.from_uri(uri : URI, default = Options.new) + params = HTTP::Params.parse(uri.query || "") + + Options.new( + filename: URI.decode_www_form((uri.host || "") + uri.path), + # pragmas + busy_timeout: params.fetch("busy_timeout", default.busy_timeout), + cache_size: params.fetch("cache_size", default.cache_size), + foreign_keys: params.fetch("foreign_keys", default.foreign_keys), + journal_mode: params.fetch("journal_mode", default.journal_mode), + synchronous: params.fetch("synchronous", default.synchronous), + wal_autocheckpoint: params.fetch("wal_autocheckpoint", default.wal_autocheckpoint), + ) + end + + def pragma_statement + res = String.build do |str| + pragma_append(str, "busy_timeout", busy_timeout) + pragma_append(str, "cache_size", cache_size) + pragma_append(str, "foreign_keys", foreign_keys) + pragma_append(str, "journal_mode", journal_mode) + pragma_append(str, "synchronous", synchronous) + pragma_append(str, "wal_autocheckpoint", wal_autocheckpoint) + end + + res.empty? ? nil : res + end + + private def pragma_append(io, key, value) + return unless value + io << "PRAGMA #{key}=#{value};" + end + end + + def initialize(options : ::DB::Connection::Options, sqlite3_options : Options) + super(options) + check LibSQLite3.open_v2(sqlite3_options.filename, out @db, (Flag::READWRITE | Flag::CREATE), nil) + # 2 means 2 arguments; 1 is the code for UTF-8 + check LibSQLite3.create_function(@db, "regexp", 2, 1, nil, SQLite3::REGEXP_FN, nil, nil) + + if pragma_statement = sqlite3_options.pragma_statement + check LibSQLite3.exec(@db, pragma_statement, nil, nil, nil) + end rescue raise DB::ConnectionRefused.new end diff --git a/src/sqlite3/driver.cr b/src/sqlite3/driver.cr index d43a512..93ae0b0 100644 --- a/src/sqlite3/driver.cr +++ b/src/sqlite3/driver.cr @@ -1,6 +1,16 @@ class SQLite3::Driver < DB::Driver - def build_connection(context : DB::ConnectionContext) : SQLite3::Connection - SQLite3::Connection.new(context) + class ConnectionBuilder < ::DB::ConnectionBuilder + def initialize(@options : ::DB::Connection::Options, @sqlite3_options : SQLite3::Connection::Options) + end + + def build : ::DB::Connection + SQLite3::Connection.new(@options, @sqlite3_options) + end + end + + def connection_builder(uri : URI) : ::DB::ConnectionBuilder + params = HTTP::Params.parse(uri.query || "") + ConnectionBuilder.new(connection_options(params), SQLite3::Connection::Options.from_uri(uri)) end end diff --git a/src/sqlite3/lib_sqlite3.cr b/src/sqlite3/lib_sqlite3.cr index d99f67c..310e69c 100644 --- a/src/sqlite3/lib_sqlite3.cr +++ b/src/sqlite3/lib_sqlite3.cr @@ -5,6 +5,8 @@ lib LibSQLite3 type SQLite3 = Void* type Statement = Void* type SQLite3Backup = Void* + type SQLite3Context = Void* + type SQLite3Value = Void* enum Code # Successful result @@ -72,6 +74,7 @@ lib LibSQLite3 end alias Callback = (Void*, Int32, UInt8**, UInt8**) -> Int32 + alias FuncCallback = (SQLite3Context, Int32, SQLite3Value*) -> Void fun open_v2 = sqlite3_open_v2(filename : UInt8*, db : SQLite3*, flags : ::SQLite3::Flag, zVfs : UInt8*) : Int32 @@ -83,6 +86,7 @@ lib LibSQLite3 fun backup_finish = sqlite3_backup_finish(SQLite3Backup) : Code fun prepare_v2 = sqlite3_prepare_v2(db : SQLite3, zSql : UInt8*, nByte : Int32, ppStmt : Statement*, pzTail : UInt8**) : Int32 + fun exec = sqlite3_exec(db : SQLite3, zSql : UInt8*, pCallback : Callback, pCallbackArgs : Void*, pzErrMsg : UInt8**) : Int32 fun step = sqlite3_step(stmt : Statement) : Int32 fun column_count = sqlite3_column_count(stmt : Statement) : Int32 fun column_type = sqlite3_column_type(stmt : Statement, iCol : Int32) : ::SQLite3::Type @@ -108,4 +112,8 @@ lib LibSQLite3 fun finalize = sqlite3_finalize(stmt : Statement) : Int32 fun close_v2 = sqlite3_close_v2(SQLite3) : Int32 fun close = sqlite3_close(SQLite3) : Int32 + + fun create_function = sqlite3_create_function(SQLite3, funcName : UInt8*, nArg : Int32, eTextRep : Int32, pApp : Void*, xFunc : FuncCallback, xStep : Void*, xFinal : Void*) : Int32 + fun value_text = sqlite3_value_text(SQLite3Value) : UInt8* + fun result_int = sqlite3_result_int(SQLite3Context, Int32) : Nil end diff --git a/src/sqlite3/result_set.cr b/src/sqlite3/result_set.cr index 38e669f..9955129 100644 --- a/src/sqlite3/result_set.cr +++ b/src/sqlite3/result_set.cr @@ -2,8 +2,8 @@ class SQLite3::ResultSet < DB::ResultSet @column_index = 0 protected def do_close - super LibSQLite3.reset(self) + super end # Advances to the next row. Returns `true` if there's a next row, @@ -47,6 +47,10 @@ class SQLite3::ResultSet < DB::ResultSet value end + def next_column_index : Int32 + @column_index + end + def read(t : Int32.class) : Int32 read(Int64).to_i32 end @@ -64,11 +68,22 @@ class SQLite3::ResultSet < DB::ResultSet end def read(t : Time.class) : Time - Time.parse read(String), SQLite3::DATE_FORMAT, location: SQLite3::TIME_ZONE + text = read(String) + if text.includes? "." + Time.parse text, SQLite3::DATE_FORMAT_SUBSECOND, location: SQLite3::TIME_ZONE + else + Time.parse text, SQLite3::DATE_FORMAT_SECOND, location: SQLite3::TIME_ZONE + end end def read(t : Time?.class) : Time? - read(String?).try { |v| Time.parse(v, SQLite3::DATE_FORMAT, location: SQLite3::TIME_ZONE) } + read(String?).try { |v| + if v.includes? "." + Time.parse v, SQLite3::DATE_FORMAT_SUBSECOND, location: SQLite3::TIME_ZONE + else + Time.parse v, SQLite3::DATE_FORMAT_SECOND, location: SQLite3::TIME_ZONE + end + } end def read(t : Bool.class) : Bool diff --git a/src/sqlite3/statement.cr b/src/sqlite3/statement.cr index 06289fb..368dfe3 100644 --- a/src/sqlite3/statement.cr +++ b/src/sqlite3/statement.cr @@ -70,7 +70,7 @@ class SQLite3::Statement < DB::Statement end private def bind_arg(index, value : Time) - bind_arg(index, value.in(SQLite3::TIME_ZONE).to_s(SQLite3::DATE_FORMAT)) + bind_arg(index, value.in(SQLite3::TIME_ZONE).to_s(SQLite3::DATE_FORMAT_SUBSECOND)) end private def bind_arg(index, value) diff --git a/src/sqlite3/version.cr b/src/sqlite3/version.cr index d5b7949..4e4f456 100644 --- a/src/sqlite3/version.cr +++ b/src/sqlite3/version.cr @@ -1,3 +1,3 @@ module SQLite3 - VERSION = "0.17.0" + VERSION = "0.21.0" end