Release 0.21.0, Update to crystal-db ~> 0.13 (#94)

* Update crystal-db ~> 0.13.0

* Bump version to 0.21.0

.circleci/config.yml

@@ -0,0 +1,61 @@
+version: 2.1
+
+orbs:
+  crystal: manastech/crystal@1.0
+
+commands:
+  install-sqlite:
+    steps:
+      - run:
+          name: Install `sqlite`
+          command: apt-get update && apt-get install -y libsqlite3-dev
+
+jobs:
+  test:
+    parameters:
+      executor:
+        type: executor
+        default: crystal/default
+    executor: << parameters.executor >>
+    steps:
+      - install-sqlite
+      - crystal/version
+      - checkout
+      - crystal/with-shards-cache:
+          steps:
+            - crystal/shards-install
+      - crystal/spec
+      - crystal/format-check
+
+executors:
+  nightly:
+    docker:
+      - image: 'crystallang/crystal:nightly'
+    environment:
+      SHARDS_OPTS: --ignore-crystal-version
+
+workflows:
+  version: 2
+
+  build:
+    jobs:
+      - test
+      - test:
+          name: test-on-nightly
+          executor:
+            name: nightly
+
+  nightly:
+    triggers:
+      - schedule:
+          cron: "0 3 * * *"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - test:
+          name: test-on-nightly
+          executor:
+            name: nightly
+

.github/workflows/ci.yml

@@ -0,0 +1,35 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+    branches: [master]
+  schedule:
+    - cron: '0 6 * * 1'  # Every monday 6 AM
+
+jobs:
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        crystal: [1.0.0, latest, nightly]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Install Crystal
+        uses: crystal-lang/install-crystal@v1
+        with:
+          crystal: ${{ matrix.crystal }}
+
+      - name: Download source
+        uses: actions/checkout@v2
+
+      - name: Install shards
+        run: shards install
+
+      - name: Run specs
+        run: crystal spec
+
+      - name: Check formatting
+        run: crystal tool format; git diff --exit-code
+        if: matrix.crystal == 'latest' && matrix.os == 'ubuntu-latest'

.travis.yml

@@ -1,2 +0,0 @@
-language: crystal
-sudo: false

CHANGELOG.md

@@ -1,3 +1,39 @@
+## v0.21.0 (2023-12-12)
+
+* Update to crystal-db ~> 0.13.0. ([#94](https://github.com/crystal-lang/crystal-sqlite3/pull/94))
+
+## v0.20.0 (2023-06-23)
+
+* Update to crystal-db ~> 0.12.0. ([#91](https://github.com/crystal-lang/crystal-sqlite3/pull/91))
+* Fix result set & connection release lifecycle. ([#90](https://github.com/crystal-lang/crystal-sqlite3/pull/90))
+* Automatically set PRAGMAs using connection query params. ([#85](https://github.com/crystal-lang/crystal-sqlite3/pull/85), thanks @luislavena)
+
+## v0.19.0 (2022-01-28)
+
+* Update to crystal-db ~> 0.11.0. ([#77](https://github.com/crystal-lang/crystal-sqlite3/pull/77))
+* Fix timestamps support to allow dealing with exact seconds values ([#68](https://github.com/crystal-lang/crystal-sqlite3/pull/68), thanks @yujiri8, @tenebrousedge)
+* Migrate CI to GitHub Actions. ([#78](https://github.com/crystal-lang/crystal-sqlite3/pull/78))
+
+This release requires Crystal 1.0.0 or later.
+
+## v0.18.0 (2021-01-26)
+
+* Add `REGEXP` support powered by Crystal's std-lib Regex. ([#62](https://github.com/crystal-lang/crystal-sqlite3/pull/62), thanks @yujiri8)
+
+## v0.17.0 (2020-09-30)
+
+* Update to crystal-db ~> 0.10.0. ([#58](https://github.com/crystal-lang/crystal-sqlite3/pull/58))
+
+This release requires Crystal 0.35.0 or later.
+
+## v0.16.0 (2020-04-06)
+
+* Update to crystal-db ~> 0.9.0. ([#55](https://github.com/crystal-lang/crystal-sqlite3/pull/55))
+
+## v0.15.0 (2019-12-11)
+
+* Update to crystal-db ~> 0.8.0. ([#50](https://github.com/crystal-lang/crystal-sqlite3/pull/50))
+
 ## v0.14.0 (2019-09-23)
 
 * Update to crystal-db ~> 0.7.0. ([#44](https://github.com/crystal-lang/crystal-sqlite3/pull/44))

README.md

@@ -26,7 +26,7 @@ DB.open "sqlite3://./data.db" do |db|
   args = [] of DB::Any
   args << "Sarah"
   args << 33
-  db.exec "insert into contacts values (?, ?)", args
+  db.exec "insert into contacts values (?, ?)", args: args
 
   puts "max age:"
   puts db.scalar "select max(age) from contacts" # => 33
@@ -46,5 +46,41 @@ end
 
 ### DB::Any
 
-* `Time` is implemented as `TEXT` column using `SQLite3::DATE_FORMAT` format.
+* `Time` is implemented as `TEXT` column using `SQLite3::DATE_FORMAT_SUBSECOND` format (or `SQLite3::DATE_FORMAT_SECOND` if the text does not contain a dot).
 * `Bool` is implemented as `INT` column mapping `0`/`1` values.
+
+### Setting PRAGMAs
+
+You can adjust certain [SQLite3 PRAGMAs](https://www.sqlite.org/pragma.html)
+automatically when the connection is created by using the query parameters:
+
+```crystal
+require "sqlite3"
+
+DB.open "sqlite3://./data.db?journal_mode=wal&synchronous=normal" do |db|
+  # this database now uses WAL journal and normal synchronous mode
+  # (defaults were `delete` and `full`, respectively)
+end
+```
+
+The following is the list of supported options:
+
+| Name                      | Connection key  |
+|---------------------------|-----------------|
+| [Busy Timeout][pragma-to] | `busy_timeout` |
+| [Cache Size][pragma-cs] | `cache_size` |
+| [Foreign Keys][pragma-fk] | `foreign_keys` |
+| [Journal Mode][pragma-jm] | `journal_mode` |
+| [Synchronous][pragma-sync] | `synchronous` |
+| [WAL autocheckoint][pragma-walck] | `wal_autocheckpoint` |
+
+Please note there values passed using these connection keys are passed
+directly to SQLite3 without check or evaluation. Using incorrect values result
+in no error by the library.
+
+[pragma-to]: https://www.sqlite.org/pragma.html#pragma_busy_timeout
+[pragma-cs]: https://www.sqlite.org/pragma.html#pragma_cache_size
+[pragma-fk]: https://www.sqlite.org/pragma.html#pragma_foreign_keys
+[pragma-jm]: https://www.sqlite.org/pragma.html#pragma_journal_mode
+[pragma-sync]: https://www.sqlite.org/pragma.html#pragma_synchronous
+[pragma-walck]: https://www.sqlite.org/pragma.html#pragma_wal_autocheckpoint

samples/memory.cr

@@ -8,7 +8,7 @@ DB.open "sqlite3://%3Amemory%3A" do |db|
   args = [] of DB::Any
   args << "Sarah"
   args << 33
-  db.exec "insert into contacts values (?, ?)", args
+  db.exec "insert into contacts values (?, ?)", args: args
 
   puts "max age:"
   puts db.scalar "select max(age) from contacts" # => 33

shard.yml

@@ -1,15 +1,15 @@
 name: sqlite3
-version: 0.14.0
+version: 0.21.0
 
 dependencies:
   db:
     github: crystal-lang/crystal-db
-    version: ~> 0.7.0
+    version: ~> 0.13.0
 
 authors:
   - Ary Borenszweig <aborenszweig@manas.tech>
-  - Brian J. Cardiff <bcardiff@manas.tech>
+  - Brian J. Cardiff <bcardiff@gmail.com>
 
-crystal: 0.28.0
+crystal: ">= 1.0.0, < 2.0.0"
 
 license: MIT

spec/connection_spec.cr

@@ -10,6 +10,14 @@ private def dump(source, target)
   end
 end
 
+private def it_sets_pragma_on_connection(pragma : String, value : String, expected, file = __FILE__, line = __LINE__)
+  it "sets pragma '#{pragma}' to #{expected}", file, line do
+    with_db("#{DB_FILENAME}?#{pragma}=#{value}") do |db|
+      db.scalar("PRAGMA #{pragma}").should eq(expected)
+    end
+  end
+end
+
 describe Connection do
   it "opens a database and then backs it up to another db" do
     with_db do |db|
@@ -68,4 +76,34 @@ describe Connection do
       cnn.scalar("select count(*) from person").should eq(1)
     end
   end
+
+  # adjust busy_timeout pragma (default is 0)
+  it_sets_pragma_on_connection "busy_timeout", "1000", 1000
+
+  # adjust cache_size pragma (default is -2000, 2MB)
+  it_sets_pragma_on_connection "cache_size", "-4000", -4000
+
+  # enable foreign_keys, no need to test off (is the default)
+  it_sets_pragma_on_connection "foreign_keys", "1", 1
+  it_sets_pragma_on_connection "foreign_keys", "yes", 1
+  it_sets_pragma_on_connection "foreign_keys", "true", 1
+  it_sets_pragma_on_connection "foreign_keys", "on", 1
+
+  # change journal_mode (default is delete)
+  it_sets_pragma_on_connection "journal_mode", "delete", "delete"
+  it_sets_pragma_on_connection "journal_mode", "truncate", "truncate"
+  it_sets_pragma_on_connection "journal_mode", "persist", "persist"
+
+  # change synchronous mode (default is 2, FULL)
+  it_sets_pragma_on_connection "synchronous", "0", 0
+  it_sets_pragma_on_connection "synchronous", "off", 0
+  it_sets_pragma_on_connection "synchronous", "1", 1
+  it_sets_pragma_on_connection "synchronous", "normal", 1
+  it_sets_pragma_on_connection "synchronous", "2", 2
+  it_sets_pragma_on_connection "synchronous", "full", 2
+  it_sets_pragma_on_connection "synchronous", "3", 3
+  it_sets_pragma_on_connection "synchronous", "extra", 3
+
+  # change wal_autocheckpoint (default is 1000)
+  it_sets_pragma_on_connection "wal_autocheckpoint", "0", 0
 end

spec/db_spec.cr

@@ -13,7 +13,7 @@ private def cast_if_blob(expr, sql_type)
   end
 end
 
-DB::DriverSpecs(DB::Any).run do
+DB::DriverSpecs(DB::Any).run do |ctx|
   support_unprepared false
 
   before do
@@ -34,7 +34,9 @@ DB::DriverSpecs(DB::Any).run do
   sample_value 1.5_f32, "float", "1.5", type_safe_value: false
   sample_value 1.5, "float", "1.5"
   sample_value Time.utc(2016, 2, 15), "text", "'2016-02-15 00:00:00.000'", type_safe_value: false
+  sample_value Time.utc(2016, 2, 15, 10, 15, 30), "text", "'2016-02-15 10:15:30'", type_safe_value: false
   sample_value Time.utc(2016, 2, 15, 10, 15, 30), "text", "'2016-02-15 10:15:30.000'", type_safe_value: false
+  sample_value Time.utc(2016, 2, 15, 10, 15, 30, nanosecond: 123000000), "text", "'2016-02-15 10:15:30.123'", type_safe_value: false
   sample_value Time.local(2016, 2, 15, 7, 15, 30, location: Time::Location.fixed("fixed", -3*3600)), "text", "'2016-02-15 10:15:30.000'", type_safe_value: false
 
   ary = UInt8[0x53, 0x51, 0x4C, 0x69, 0x74, 0x65]
@@ -102,7 +104,7 @@ DB::DriverSpecs(DB::Any).run do
     db.exec %(insert into a (i, str) values (23, "bai bai");)
 
     2.times do |i|
-      DB.open db.uri do |db|
+      DB.open ctx.connection_string do |db|
         begin
           db.query("SELECT i, str FROM a WHERE i = ?", 23) do |rs|
             rs.move_next
@@ -128,4 +130,9 @@ DB::DriverSpecs(DB::Any).run do
   it "handles multi-step pragma statements" do |db|
     db.exec %(PRAGMA journal_mode = memory)
   end
+
+  it "handles REGEXP operator" do |db|
+    (db.scalar "select 'unmatching text' REGEXP '^m'").should eq 0
+    (db.scalar "select 'matching text' REGEXP '^m'").should eq 1
+  end
 end

spec/driver_spec.cr

@@ -27,7 +27,7 @@ describe Driver do
 
   it "should use database option as file to open" do
     with_db do |db|
-      db.driver.should be_a(SQLite3::Driver)
+      db.checkout.should be_a(SQLite3::Connection)
       File.exists?(DB_FILENAME).should be_true
     end
   end

spec/result_set_spec.cr

@@ -0,0 +1,57 @@
+require "./spec_helper"
+
+describe SQLite3::ResultSet do
+  it "reads integer data types" do
+    with_db do |db|
+      db.exec "CREATE TABLE test_table (test_int integer)"
+      db.exec "INSERT INTO test_table (test_int) values (?)", 42
+      db.query("SELECT test_int FROM test_table") do |rs|
+        rs.each do
+          rs.read.should eq(42)
+        end
+      end
+    end
+  end
+
+  it "reads string data types" do
+    with_db do |db|
+      db.exec "CREATE TABLE test_table (test_text text)"
+      db.exec "INSERT INTO test_table (test_text) values (?), (?)", "abc", "123"
+      db.query("SELECT test_text FROM test_table") do |rs|
+        rs.each do
+          rs.read.should match(/abc|123/)
+        end
+      end
+    end
+  end
+
+  it "reads time data types" do
+    with_db do |db|
+      db.exec "CREATE TABLE test_table (test_date datetime)"
+      timestamp = Time.utc
+      db.exec "INSERT INTO test_table (test_date) values (current_timestamp)"
+      db.query("SELECT test_date FROM test_table") do |rs|
+        rs.each do
+          rs.read(Time).should be_close(timestamp, 1.second)
+        end
+      end
+    end
+  end
+
+  it "reads time stored in text fields, too" do
+    with_db do |db|
+      db.exec "CREATE TABLE test_table (test_date text)"
+      timestamp = Time.utc
+      # Try 3 different ways: our own two formats and using SQLite's current_timestamp.
+      # They should all work.
+      db.exec "INSERT INTO test_table (test_date) values (?)", timestamp.to_s SQLite3::DATE_FORMAT_SUBSECOND
+      db.exec "INSERT INTO test_table (test_date) values (?)", timestamp.to_s SQLite3::DATE_FORMAT_SECOND
+      db.exec "INSERT INTO test_table (test_date) values (current_timestamp)"
+      db.query("SELECT test_date FROM test_table") do |rs|
+        rs.each do
+          rs.read(Time).should be_close(timestamp, 1.second)
+        end
+      end
+    end
+  end
+end

src/sqlite3.cr

@@ -2,8 +2,18 @@ require "db"
 require "./sqlite3/**"
 
 module SQLite3
-  DATE_FORMAT = "%F %H:%M:%S.%L"
+  DATE_FORMAT_SUBSECOND = "%F %H:%M:%S.%L"
+  DATE_FORMAT_SECOND    = "%F %H:%M:%S"
 
   # :nodoc:
   TIME_ZONE = Time::Location::UTC
+
+  # :nodoc:
+  REGEXP_FN = ->(context : LibSQLite3::SQLite3Context, argc : Int32, argv : LibSQLite3::SQLite3Value*) do
+    argv = Slice.new(argv, sizeof(Void*))
+    pattern = LibSQLite3.value_text(argv[0])
+    text = LibSQLite3.value_text(argv[1])
+    LibSQLite3.result_int(context, Regex.new(String.new(pattern)).matches?(String.new(text)).to_unsafe)
+    nil
+  end
 end

src/sqlite3/connection.cr

@@ -1,19 +1,62 @@
 class SQLite3::Connection < DB::Connection
-  def initialize(database)
+  record Options,
-    super
+    filename : String = ":memory:",
-    filename = self.class.filename(database.uri)
+    # pragmas
-    # TODO maybe enable Flag::URI to parse query string in the uri as additional flags
+    busy_timeout : String? = nil,
-    check LibSQLite3.open_v2(filename, out @db, (Flag::READWRITE | Flag::CREATE), nil)
+    cache_size : String? = nil,
+    foreign_keys : String? = nil,
+    journal_mode : String? = nil,
+    synchronous : String? = nil,
+    wal_autocheckpoint : String? = nil do
+    def self.from_uri(uri : URI, default = Options.new)
+      params = HTTP::Params.parse(uri.query || "")
+
+      Options.new(
+        filename: URI.decode_www_form((uri.host || "") + uri.path),
+        # pragmas
+        busy_timeout: params.fetch("busy_timeout", default.busy_timeout),
+        cache_size: params.fetch("cache_size", default.cache_size),
+        foreign_keys: params.fetch("foreign_keys", default.foreign_keys),
+        journal_mode: params.fetch("journal_mode", default.journal_mode),
+        synchronous: params.fetch("synchronous", default.synchronous),
+        wal_autocheckpoint: params.fetch("wal_autocheckpoint", default.wal_autocheckpoint),
+      )
+    end
+
+    def pragma_statement
+      res = String.build do |str|
+        pragma_append(str, "busy_timeout", busy_timeout)
+        pragma_append(str, "cache_size", cache_size)
+        pragma_append(str, "foreign_keys", foreign_keys)
+        pragma_append(str, "journal_mode", journal_mode)
+        pragma_append(str, "synchronous", synchronous)
+        pragma_append(str, "wal_autocheckpoint", wal_autocheckpoint)
+      end
+
+      res.empty? ? nil : res
+    end
+
+    private def pragma_append(io, key, value)
+      return unless value
+      io << "PRAGMA #{key}=#{value};"
+    end
+  end
+
+  def initialize(options : ::DB::Connection::Options, sqlite3_options : Options)
+    super(options)
+    check LibSQLite3.open_v2(sqlite3_options.filename, out @db, (Flag::READWRITE | Flag::CREATE), nil)
+    # 2 means 2 arguments; 1 is the code for UTF-8
+    check LibSQLite3.create_function(@db, "regexp", 2, 1, nil, SQLite3::REGEXP_FN, nil, nil)
+
+    if pragma_statement = sqlite3_options.pragma_statement
+      check LibSQLite3.exec(@db, pragma_statement, nil, nil, nil)
+    end
   rescue
     raise DB::ConnectionRefused.new
   end
 
   def self.filename(uri : URI)
-    {% if compare_versions(Crystal::VERSION, "0.30.0-0") >= 0 %}
     URI.decode_www_form((uri.host || "") + uri.path)
-    {% else %}
-      URI.unescape((uri.host || "") + uri.path)
-    {% end %}
   end
 
   def build_prepared_statement(query) : Statement

src/sqlite3/driver.cr

@@ -1,6 +1,16 @@
 class SQLite3::Driver < DB::Driver
-  def build_connection(context : DB::ConnectionContext) : SQLite3::Connection
+  class ConnectionBuilder < ::DB::ConnectionBuilder
-    SQLite3::Connection.new(context)
+    def initialize(@options : ::DB::Connection::Options, @sqlite3_options : SQLite3::Connection::Options)
+    end
+
+    def build : ::DB::Connection
+      SQLite3::Connection.new(@options, @sqlite3_options)
+    end
+  end
+
+  def connection_builder(uri : URI) : ::DB::ConnectionBuilder
+    params = HTTP::Params.parse(uri.query || "")
+    ConnectionBuilder.new(connection_options(params), SQLite3::Connection::Options.from_uri(uri))
   end
 end
 

src/sqlite3/lib_sqlite3.cr

@@ -5,6 +5,8 @@ lib LibSQLite3
   type SQLite3 = Void*
   type Statement = Void*
   type SQLite3Backup = Void*
+  type SQLite3Context = Void*
+  type SQLite3Value = Void*
 
   enum Code
     # Successful result
@@ -72,6 +74,7 @@ lib LibSQLite3
   end
 
   alias Callback = (Void*, Int32, UInt8**, UInt8**) -> Int32
+  alias FuncCallback = (SQLite3Context, Int32, SQLite3Value*) -> Void
 
   fun open_v2 = sqlite3_open_v2(filename : UInt8*, db : SQLite3*, flags : ::SQLite3::Flag, zVfs : UInt8*) : Int32
 
@@ -83,6 +86,7 @@ lib LibSQLite3
   fun backup_finish = sqlite3_backup_finish(SQLite3Backup) : Code
 
   fun prepare_v2 = sqlite3_prepare_v2(db : SQLite3, zSql : UInt8*, nByte : Int32, ppStmt : Statement*, pzTail : UInt8**) : Int32
+  fun exec = sqlite3_exec(db : SQLite3, zSql : UInt8*, pCallback : Callback, pCallbackArgs : Void*, pzErrMsg : UInt8**) : Int32
   fun step = sqlite3_step(stmt : Statement) : Int32
   fun column_count = sqlite3_column_count(stmt : Statement) : Int32
   fun column_type = sqlite3_column_type(stmt : Statement, iCol : Int32) : ::SQLite3::Type
@@ -108,4 +112,8 @@ lib LibSQLite3
   fun finalize = sqlite3_finalize(stmt : Statement) : Int32
   fun close_v2 = sqlite3_close_v2(SQLite3) : Int32
   fun close = sqlite3_close(SQLite3) : Int32
+
+  fun create_function = sqlite3_create_function(SQLite3, funcName : UInt8*, nArg : Int32, eTextRep : Int32, pApp : Void*, xFunc : FuncCallback, xStep : Void*, xFinal : Void*) : Int32
+  fun value_text = sqlite3_value_text(SQLite3Value) : UInt8*
+  fun result_int = sqlite3_result_int(SQLite3Context, Int32) : Nil
 end

src/sqlite3/result_set.cr

@@ -2,8 +2,8 @@ class SQLite3::ResultSet < DB::ResultSet
   @column_index = 0
 
   protected def do_close
-    super
     LibSQLite3.reset(self)
+    super
   end
 
   # Advances to the next row. Returns `true` if there's a next row,
@@ -47,6 +47,10 @@ class SQLite3::ResultSet < DB::ResultSet
     value
   end
 
+  def next_column_index : Int32
+    @column_index
+  end
+
   def read(t : Int32.class) : Int32
     read(Int64).to_i32
   end
@@ -64,11 +68,22 @@ class SQLite3::ResultSet < DB::ResultSet
   end
 
   def read(t : Time.class) : Time
-    Time.parse read(String), SQLite3::DATE_FORMAT, location: SQLite3::TIME_ZONE
+    text = read(String)
+    if text.includes? "."
+      Time.parse text, SQLite3::DATE_FORMAT_SUBSECOND, location: SQLite3::TIME_ZONE
+    else
+      Time.parse text, SQLite3::DATE_FORMAT_SECOND, location: SQLite3::TIME_ZONE
+    end
   end
 
   def read(t : Time?.class) : Time?
-    read(String?).try { |v| Time.parse(v, SQLite3::DATE_FORMAT, location: SQLite3::TIME_ZONE) }
+    read(String?).try { |v|
+      if v.includes? "."
+        Time.parse v, SQLite3::DATE_FORMAT_SUBSECOND, location: SQLite3::TIME_ZONE
+      else
+        Time.parse v, SQLite3::DATE_FORMAT_SECOND, location: SQLite3::TIME_ZONE
+      end
+    }
   end
 
   def read(t : Bool.class) : Bool

src/sqlite3/statement.cr

@@ -1,7 +1,7 @@
 class SQLite3::Statement < DB::Statement
-  def initialize(connection, sql)
+  def initialize(connection, command)
-    super(connection)
+    super(connection, command)
-    check LibSQLite3.prepare_v2(sqlite3_connection, sql, sql.bytesize + 1, out @stmt, nil)
+    check LibSQLite3.prepare_v2(sqlite3_connection, command, command.bytesize + 1, out @stmt, nil)
   end
 
   protected def perform_query(args : Enumerable) : DB::ResultSet
@@ -70,7 +70,7 @@ class SQLite3::Statement < DB::Statement
   end
 
   private def bind_arg(index, value : Time)
-    bind_arg(index, value.in(SQLite3::TIME_ZONE).to_s(SQLite3::DATE_FORMAT))
+    bind_arg(index, value.in(SQLite3::TIME_ZONE).to_s(SQLite3::DATE_FORMAT_SUBSECOND))
   end
 
   private def bind_arg(index, value)

src/sqlite3/version.cr

@@ -1,3 +1,3 @@
 module SQLite3
-  VERSION = "0.14.0"
+  VERSION = "0.21.0"
 end