From fd5905910176d11307e43570ba81ac317b08863d Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 7 Dec 2022 14:57:07 -0300 Subject: [PATCH 1/7] add draft for config schema validation --- config.lua | 94 ++++++++++++++++++++++++++++++++- main.lua | 4 +- scripts/webfinger_allowlist.lua | 4 +- test.lua | 1 + tests/schema_validation.lua | 43 +++++++++++++++ 5 files changed, 140 insertions(+), 6 deletions(-) create mode 100644 tests/schema_validation.lua diff --git a/config.lua b/config.lua index f83449b..cc148bd 100644 --- a/config.lua +++ b/config.lua @@ -29,10 +29,100 @@ local function findConfigFile() return nil end +local function insertError(errors, key, message) + errors[key] = errors[key] or {} + table.insert(errors[key], message) + return errors +end + +local function fakeTableSchema(value_schema) + return setmetatable({ __list = true }, { + __index = function (self, key) + return value_schema + end + }) +end + +local SPECIAL_KEYS = {__list = true} + +local function validateSchema(schema, input, errors) + local errors = errors or {} + if schema.__list then + -- generate full schema for lists that are same size as input + for k, _ in pairs(input) do schema[k] = schema.__schema_value end + end + + for key, field_schema in pairs(schema) do + if not SPECIAL_KEYS[key] then + assert(field_schema, 'schema not provided') + local input_value = input[key] + local input_type = type(input_value) + local wanted_type = field_schema.type + + local actual_wanted_type = wanted_type + if wanted_type == 'list' then + actual_wanted_type = 'table' + end + + if input_type == actual_wanted_type then + if wanted_type == 'table' then + -- recursive schema validation for generic tables + errors[key] = errors[key] or {} + validateSchema(field_schema.schema, input_value, errors[key]) + if not errors[key] then + errors[key] = nil + end + elseif wanted_type == 'list' then + -- for lists (which only have schemas for values), interpret + -- it differently + errors[key] = errors[key] or {} + validateSchema(fakeTableSchema(field_schema.schema), input_value, errors[key]) + if next(errors[key]) == nil then + errors[key] = nil + end + end + else + insertError( + errors, key, + string.format('wanted %s but got %s', tostring(wanted_type), tostring(input_type)) + ) + end + end + end + return errors +end + +local function validateConfigFile(config_object) + local all_schema_errors = {} + for module_name, module_config in pairs(config_object.wantedScripts) do + local module_manifest = require('scripts.' .. module_name) + local config_schema = module_manifest.config + local schema_errors = validateSchema(config_schema, module_config) + if schema_errors then + all_schema_errors[module_name] = schema_errors + end + end + return all_schema_errors +end + +local function writeSchemaErrors(errors, out) + out('sex') +end + local function loadConfigFile() local config_file_data = assert(findConfigFile(), 'no config file found, config path: ' .. config_path) local config_file_function = assert(loadstring(config_file_data)) - return config_file_function() + local config_object = config_file_function() + local schema_errors = validateConfigFile(config_object) + if schema_errors then + for name, errors in pairs(schema_errors) do + log('CONFIG ERROR' .. writeSchemaErrors(errors, log)) + end + end + return config_object end -return loadConfigFile() +return { + loadConfigFile=loadConfigFile, + validateSchema=validateSchema, +} diff --git a/main.lua b/main.lua index c2e8e40..2be3b01 100644 --- a/main.lua +++ b/main.lua @@ -7,9 +7,9 @@ -- local config = loadConfig() local ctx = require('ctx') -local conf = require('config') +local config = require('config') -ctx:loadFromConfig(conf) +ctx:loadFromConfig(config.loadConfigFile()) return function() ctx:onRequest() diff --git a/scripts/webfinger_allowlist.lua b/scripts/webfinger_allowlist.lua index b5f5c7e..41d5c80 100644 --- a/scripts/webfinger_allowlist.lua +++ b/scripts/webfinger_allowlist.lua @@ -37,8 +37,8 @@ return { }, config={ ['accounts'] = { - type='table', - value={ + type='list', + schema={ type='string', description='ap id' }, diff --git a/test.lua b/test.lua index 63d3a08..76e17fa 100644 --- a/test.lua +++ b/test.lua @@ -74,4 +74,5 @@ function onRequest() end require('tests.webfinger_allowlist') +require('tests.schema_validation') os.exit(lu.LuaUnit.run()) diff --git a/tests/schema_validation.lua b/tests/schema_validation.lua new file mode 100644 index 0000000..7b4d8f8 --- /dev/null +++ b/tests/schema_validation.lua @@ -0,0 +1,43 @@ +TestConfigSchemaValidator = {} + +local config = require('config') + +local function len(t) + local count = 0 + for _ in pairs(t) do count = count + 1 end + return count +end + +function pprint(t, ident) + local ident = ident or 0 + if type(t) == 'table' then + local count = 0 + for k, v in pairs(t) do + print(string.rep('\t', ident) .. k) + pprint(v, ident + 1) + count = count + 1 + end + if count == 0 then + print('') + end + else + print(string.rep('\t', ident) .. tostring(t)) + end +end + +function TestConfigSchemaValidator:testBasicFields() + local errors = config.validateSchema({a={type='string'}}, {a='test'}) + lu.assertIs(len(errors), 0) + local errors = config.validateSchema({a={type='number'}}, {a=123}) + lu.assertIs(len(errors), 0) + local errors = config.validateSchema({a={type='string'}}, {a=123}) + lu.assertIs(len(errors), 1) +end + +function TestConfigSchemaValidator:testList() + local errors = config.validateSchema({a={type='list', schema={type='number'}}}, {a={1,2,3}}) + lu.assertIs(len(errors), 0) + + local errors = config.validateSchema({a={type='list', schema={type='number'}}}, {a={1,2,3,'asd'}}) + lu.assertIs(len(errors), 1) +end From 48917659ca8cedb9477e87a402c8afc8590e62c8 Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 7 Dec 2022 15:12:43 -0300 Subject: [PATCH 2/7] fix validation for full tables --- config.lua | 2 +- tests/schema_validation.lua | 25 ++++++++++++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/config.lua b/config.lua index cc148bd..58b1750 100644 --- a/config.lua +++ b/config.lua @@ -69,7 +69,7 @@ local function validateSchema(schema, input, errors) -- recursive schema validation for generic tables errors[key] = errors[key] or {} validateSchema(field_schema.schema, input_value, errors[key]) - if not errors[key] then + if next(errors[key]) == nil then errors[key] = nil end elseif wanted_type == 'list' then diff --git a/tests/schema_validation.lua b/tests/schema_validation.lua index 7b4d8f8..74ead89 100644 --- a/tests/schema_validation.lua +++ b/tests/schema_validation.lua @@ -1,4 +1,4 @@ -TestConfigSchemaValidator = {} +TestSchemaValidator = {} local config = require('config') @@ -25,7 +25,7 @@ function pprint(t, ident) end end -function TestConfigSchemaValidator:testBasicFields() +function TestSchemaValidator:testBasicFields() local errors = config.validateSchema({a={type='string'}}, {a='test'}) lu.assertIs(len(errors), 0) local errors = config.validateSchema({a={type='number'}}, {a=123}) @@ -34,10 +34,29 @@ function TestConfigSchemaValidator:testBasicFields() lu.assertIs(len(errors), 1) end -function TestConfigSchemaValidator:testList() +function TestSchemaValidator:testList() local errors = config.validateSchema({a={type='list', schema={type='number'}}}, {a={1,2,3}}) lu.assertIs(len(errors), 0) local errors = config.validateSchema({a={type='list', schema={type='number'}}}, {a={1,2,3,'asd'}}) lu.assertIs(len(errors), 1) end + +function TestSchemaValidator:testTable() + local errors = config.validateSchema( + { + a={ + type='table', + schema={ + b={ + type='number' + } + } + } + }, + {a= + {b=2} + } + ) + lu.assertIs(len(errors), 0) +end From d0fba27097962585c151a4c6f5799a20bb2d4ed9 Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 7 Dec 2022 15:13:33 -0300 Subject: [PATCH 3/7] add test for incorrect table schema --- tests/schema_validation.lua | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/schema_validation.lua b/tests/schema_validation.lua index 74ead89..32e7573 100644 --- a/tests/schema_validation.lua +++ b/tests/schema_validation.lua @@ -43,8 +43,7 @@ function TestSchemaValidator:testList() end function TestSchemaValidator:testTable() - local errors = config.validateSchema( - { + local TEST_SCHEMA = { a={ type='table', schema={ @@ -53,10 +52,21 @@ function TestSchemaValidator:testTable() } } } - }, + } + + local errors = config.validateSchema( + TEST_SCHEMA, {a= {b=2} } ) lu.assertIs(len(errors), 0) + + local errors = config.validateSchema( + TEST_SCHEMA, + {a= + {b='sex'} + } + ) + lu.assertIs(len(errors), 1) end From 2d2a68b1c3e5d79b86c5d7139f064bfb4911a02f Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 7 Dec 2022 15:26:11 -0300 Subject: [PATCH 4/7] validate schema on module tests --- test.lua | 13 ++++++++++--- tests/schema_validation.lua | 37 +++++++------------------------------ util.lua | 25 +++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 33 deletions(-) create mode 100644 util.lua diff --git a/test.lua b/test.lua index 76e17fa..968117b 100644 --- a/test.lua +++ b/test.lua @@ -1,5 +1,6 @@ lu = require('luaunit') local rex = require('rex_pcre2') +require('util') function createNgx() local ngx = { @@ -54,12 +55,18 @@ function setupFakeRequest(path, options) end local ctx = require('ctx') -function setupTest(module_require_path, config) +local config = require('config') +function setupTest(module_require_path, input_config) resetNgx() local module = require(module_require_path) - state = module.init(config) + + local schema_errors = config.validateSchema(module.config, input_config) + local count = table.pprint(schema_errors) + lu.assertIs(count, 0) + + state = module.init(input_config) ctx.compiled_chain = { - {module, config, state} + {module, input_config, state} } return module end diff --git a/tests/schema_validation.lua b/tests/schema_validation.lua index 32e7573..7834930 100644 --- a/tests/schema_validation.lua +++ b/tests/schema_validation.lua @@ -2,44 +2,21 @@ TestSchemaValidator = {} local config = require('config') -local function len(t) - local count = 0 - for _ in pairs(t) do count = count + 1 end - return count -end - -function pprint(t, ident) - local ident = ident or 0 - if type(t) == 'table' then - local count = 0 - for k, v in pairs(t) do - print(string.rep('\t', ident) .. k) - pprint(v, ident + 1) - count = count + 1 - end - if count == 0 then - print('') - end - else - print(string.rep('\t', ident) .. tostring(t)) - end -end - function TestSchemaValidator:testBasicFields() local errors = config.validateSchema({a={type='string'}}, {a='test'}) - lu.assertIs(len(errors), 0) + lu.assertIs(table.len(errors), 0) local errors = config.validateSchema({a={type='number'}}, {a=123}) - lu.assertIs(len(errors), 0) + lu.assertIs(table.len(errors), 0) local errors = config.validateSchema({a={type='string'}}, {a=123}) - lu.assertIs(len(errors), 1) + lu.assertIs(table.len(errors), 1) end function TestSchemaValidator:testList() local errors = config.validateSchema({a={type='list', schema={type='number'}}}, {a={1,2,3}}) - lu.assertIs(len(errors), 0) + lu.assertIs(table.len(errors), 0) local errors = config.validateSchema({a={type='list', schema={type='number'}}}, {a={1,2,3,'asd'}}) - lu.assertIs(len(errors), 1) + lu.assertIs(table.len(errors), 1) end function TestSchemaValidator:testTable() @@ -60,7 +37,7 @@ function TestSchemaValidator:testTable() {b=2} } ) - lu.assertIs(len(errors), 0) + lu.assertIs(table.len(errors), 0) local errors = config.validateSchema( TEST_SCHEMA, @@ -68,5 +45,5 @@ function TestSchemaValidator:testTable() {b='sex'} } ) - lu.assertIs(len(errors), 1) + lu.assertIs(table.len(errors), 1) end diff --git a/util.lua b/util.lua new file mode 100644 index 0000000..a510a1c --- /dev/null +++ b/util.lua @@ -0,0 +1,25 @@ +function table.len(t) + local count = 0 + for _ in pairs(t) do count = count + 1 end + return count +end + +function table.pprint(t, ident, total_count) + local ident = ident or 0 + local total_count = total_count or 0 + if type(t) == 'table' then + local count = 0 + for k, v in pairs(t) do + print(string.rep('\t', ident) .. k) + count = count + 1 + total_count = pprint(v, ident + 1, total_count) + end + if count == 0 then + --print('') + end + else + print(string.rep('\t', ident) .. tostring(t)) + total_count = total_count + 1 + end + return total_count +end From 8354478e724fff3691621d4dc2ded153acf05713 Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 7 Dec 2022 15:30:22 -0300 Subject: [PATCH 5/7] properly print config errors when loading config file --- config.lua | 9 +++++---- main.lua | 1 + util.lua | 11 +++++++---- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/config.lua b/config.lua index 58b1750..846d6b4 100644 --- a/config.lua +++ b/config.lua @@ -114,10 +114,11 @@ local function loadConfigFile() local config_file_function = assert(loadstring(config_file_data)) local config_object = config_file_function() local schema_errors = validateConfigFile(config_object) - if schema_errors then - for name, errors in pairs(schema_errors) do - log('CONFIG ERROR' .. writeSchemaErrors(errors, log)) - end + + local total_count = table.pprint(schema_errors, {call=function() end}) + if total_count > 0 then + log('CONFIG ERROR') + table.pprint(schema_errors, {call=log}) end return config_object end diff --git a/main.lua b/main.lua index 2be3b01..60fbb8e 100644 --- a/main.lua +++ b/main.lua @@ -8,6 +8,7 @@ local ctx = require('ctx') local config = require('config') +require('util') ctx:loadFromConfig(config.loadConfigFile()) diff --git a/util.lua b/util.lua index a510a1c..4db2229 100644 --- a/util.lua +++ b/util.lua @@ -4,21 +4,24 @@ function table.len(t) return count end -function table.pprint(t, ident, total_count) +function table.pprint(t, options, ident, total_count) local ident = ident or 0 local total_count = total_count or 0 + + local options = options or {} + local print_function = options.call or print if type(t) == 'table' then local count = 0 for k, v in pairs(t) do - print(string.rep('\t', ident) .. k) + print_function(string.rep('\t', ident) .. k) count = count + 1 - total_count = pprint(v, ident + 1, total_count) + total_count = table.pprint(v, options, ident + 1, total_count) end if count == 0 then --print('') end else - print(string.rep('\t', ident) .. tostring(t)) + print_function(string.rep('\t', ident) .. tostring(t)) total_count = total_count + 1 end return total_count From 4d92a58349dc9ae58b58b272ee5ce2b177543750 Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 7 Dec 2022 15:40:48 -0300 Subject: [PATCH 6/7] add hook for init_by_lua_block --- main.lua | 12 +++++++++--- nginx.conf | 10 +++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/main.lua b/main.lua index 60fbb8e..9f2049d 100644 --- a/main.lua +++ b/main.lua @@ -12,6 +12,12 @@ require('util') ctx:loadFromConfig(config.loadConfigFile()) -return function() - ctx:onRequest() -end +return { + init=function () + -- validate config and print out errors + config.loadConfigFile() + end, + access=function() + ctx:onRequest() + end +} diff --git a/nginx.conf b/nginx.conf index 81904be..4b6e8bf 100644 --- a/nginx.conf +++ b/nginx.conf @@ -1,4 +1,8 @@ - server { +init_by_lua_block { + require("aproxy.main").init() +} + +server { listen 80; lua_code_cache off; @@ -7,9 +11,9 @@ # must happen before proxy_pass access_by_lua_block { - require("aproxy.main")() + require("aproxy.main").access() } proxy_pass http://localhost:9999; } - } +} From 06b8173ecbc52357e78efe500a965cf0107d3560 Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 7 Dec 2022 15:49:45 -0300 Subject: [PATCH 7/7] only run config validations on init callback --- config.lua | 18 ++++++++++++------ main.lua | 2 +- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/config.lua b/config.lua index 846d6b4..278ee9b 100644 --- a/config.lua +++ b/config.lua @@ -109,17 +109,23 @@ local function writeSchemaErrors(errors, out) out('sex') end -local function loadConfigFile() +local function loadConfigFile(options) + local options = options or {} + local config_file_data = assert(findConfigFile(), 'no config file found, config path: ' .. config_path) local config_file_function = assert(loadstring(config_file_data)) local config_object = config_file_function() - local schema_errors = validateConfigFile(config_object) - local total_count = table.pprint(schema_errors, {call=function() end}) - if total_count > 0 then - log('CONFIG ERROR') - table.pprint(schema_errors, {call=log}) + if options.validate then + local schema_errors = validateConfigFile(config_object) + + local total_count = table.pprint(schema_errors, {call=function() end}) + if total_count > 0 then + log('CONFIG ERROR') + table.pprint(schema_errors, {call=log}) + end end + return config_object end diff --git a/main.lua b/main.lua index 9f2049d..b0cf8ff 100644 --- a/main.lua +++ b/main.lua @@ -15,7 +15,7 @@ ctx:loadFromConfig(config.loadConfigFile()) return { init=function () -- validate config and print out errors - config.loadConfigFile() + config.loadConfigFile({validate = true}) end, access=function() ctx:onRequest()