Use deserialization utils

This commit is contained in:
jaina heartles 2022-12-01 01:56:17 -08:00
parent 83af6a40e4
commit 8400cd74fd
5 changed files with 181 additions and 517 deletions

View file

@ -607,7 +607,7 @@ fn parseBodyFromRequest(comptime T: type, content_type: ?[]const u8, reader: any
return try util.deepClone(alloc, body);
},
.url_encoded => return query_utils.parseQuery(alloc, T, buf) catch |err| switch (err) {
error.NoQuery => error.NoBody,
//error.NoQuery => error.NoBody,
else => err,
},
.multipart_formdata => {
@ -615,12 +615,13 @@ fn parseBodyFromRequest(comptime T: type, content_type: ?[]const u8, reader: any
const params = query_utils.parseQuery(alloc, struct {
boundary: []const u8,
}, param_string) catch |err| return switch (err) {
error.NoQuery => error.MissingBoundary,
//error.NoQuery => error.MissingBoundary,
else => err,
};
defer util.deepFree(alloc, params);
try @import("./multipart.zig").parseFormData(params.boundary, reader, alloc);
unreachable;
//try @import("./multipart.zig").parseFormData(params.boundary, reader, alloc);
},
else => return error.UnsupportedMediaType,
}
@ -688,9 +689,9 @@ test "parseBodyFromRequest" {
try testCase("application/json", "{\"id\": 3}", Struct{ .id = 3 });
try testCase("application/x-www-form-urlencoded", "id=3", Struct{ .id = 3 });
try testCase("multipart/form-data; ",
\\
, Struct{ .id = 3 });
//try testCase("multipart/form-data; ",
//\\
//, Struct{ .id = 3 });
}
test "parseBody" {

View file

@ -1,17 +1,105 @@
const std = @import("std");
const util = @import("util");
const http = @import("./lib.zig");
const max_boundary = 70;
const read_ahead = max_boundary + 4;
const FormFieldResult = struct {
field: FormField,
more: bool,
};
pub fn MultipartStream(comptime ReaderType: type) type {
return struct {
const Multipart = @This();
const FormField = struct {
value: []const u8,
params: FormDataParams,
};
pub const PartReader = std.io.Reader(*Part, ReaderType.Error, Part.read);
stream: std.io.PeekStream(.{ .Static = read_ahead }, ReaderType),
boundary: []const u8,
pub fn next(self: *Multipart, alloc: std.mem.Allocator) !?Part {
const reader = self.stream.reader();
while (true) {
try reader.skipUntilDelimiterOrEof('\r');
var line_buf: [read_ahead]u8 = undefined;
const len = try reader.readAll(line_buf[0 .. self.boundary.len + 3]);
const line = line_buf[0..len];
if (line.len == 0) return null;
if (std.mem.startsWith(u8, line, "\n--") and std.mem.endsWith(u8, line, self.boundary)) {
// match, check for end thing
var more_buf: [2]u8 = undefined;
if (try reader.readAll(&more_buf) != 2) return error.EndOfStream;
const more = !(more_buf[0] == '-' and more_buf[1] == '-');
try self.stream.putBack(&more_buf);
try reader.skipUntilDelimiterOrEof('\n');
if (more) return try Part.open(self, alloc) else return null;
}
}
}
pub const Part = struct {
base: ?*Multipart,
fields: http.Fields,
pub fn open(base: *Multipart, alloc: std.mem.Allocator) !Part {
var fields = try @import("./request/parser.zig").parseHeaders(alloc, base.stream.reader());
return .{ .base = base, .fields = fields };
}
pub fn reader(self: *Part) PartReader {
return .{ .context = self };
}
pub fn close(self: *Part) void {
self.fields.deinit();
}
pub fn read(self: *Part, buf: []u8) ReaderType.Error!usize {
const base = self.base orelse return 0;
const r = base.stream.reader();
var count: usize = 0;
while (count < buf.len) {
const byte = r.readByte() catch |err| switch (err) {
error.EndOfStream => {
self.base = null;
return count;
},
else => |e| return e,
};
buf[count] = byte;
count += 1;
if (byte != '\r') continue;
var line_buf: [read_ahead]u8 = undefined;
const line = line_buf[0..try r.readAll(line_buf[0 .. base.boundary.len + 3])];
if (!std.mem.startsWith(u8, line, "\n--") or !std.mem.endsWith(u8, line, base.boundary)) {
base.stream.putBack(line) catch unreachable;
continue;
} else {
base.stream.putBack(line) catch unreachable;
base.stream.putBackByte('\r') catch unreachable;
self.base = null;
return count - 1;
}
}
return count;
}
};
};
}
pub fn openMultipart(boundary: []const u8, reader: anytype) !MultipartStream(@TypeOf(reader)) {
if (boundary.len > max_boundary) return error.BoundaryTooLarge;
var stream = .{
.stream = std.io.peekStream(read_ahead, reader),
.boundary = boundary,
};
stream.stream.putBack("\r\n") catch unreachable;
return stream;
}
const ParamIter = struct {
str: []const u8,
@ -26,6 +114,10 @@ const ParamIter = struct {
return .{ .str = str, .index = std.mem.indexOfScalar(u8, str, ';') orelse str.len };
}
pub fn fieldValue(self: *ParamIter) []const u8 {
return std.mem.sliceTo(self.str, ';');
}
pub fn next(self: *ParamIter) ?Param {
if (self.index >= self.str.len) return null;
@ -47,313 +139,27 @@ const ParamIter = struct {
}
};
const FormDataParams = struct {
name: ?[]const u8 = null,
filename: ?[]const u8 = null,
charset: ?[]const u8 = null,
};
fn parseParams(alloc: std.mem.Allocator, comptime T: type, str: []const u8) !T {
var result = T{};
errdefer util.deepFree(alloc, result);
var iter = ParamIter.from(str);
while (iter.next()) |param| {
inline for (comptime std.meta.fieldNames(T)) |f| {
if (std.mem.eql(u8, param.name, f)) {
@field(result, f) = try util.deepClone(alloc, param.value);
}
}
}
return result;
}
fn isFinalPart(peek_stream: anytype) !bool {
const reader = peek_stream.reader();
var buf: [2]u8 = undefined;
const end = try reader.readAll(&buf);
const end_line = buf[0..end];
const terminal = std.mem.eql(u8, end_line, "--");
if (!terminal) try peek_stream.putBack(end_line);
// Skip whitespace
while (true) {
const b = reader.readByte() catch |err| switch (err) {
error.EndOfStream => {
if (terminal) break else return error.InvalidMultipartBoundary;
},
else => return err,
};
if (std.mem.indexOfScalar(u8, " \t\r\n", b) == null) {
try peek_stream.putBackByte(b);
break;
}
}
return terminal;
}
fn parseFormField(boundary: []const u8, peek_stream: anytype, alloc: std.mem.Allocator) !FormFieldResult {
const reader = peek_stream.reader();
// TODO: refactor
var headers = try @import("./request/parser.zig").parseHeaders(alloc, reader);
defer headers.deinit();
var value = std.ArrayList(u8).init(alloc);
errdefer value.deinit();
line_loop: while (true) {
// parse crlf--
var buf: [4]u8 = undefined;
try reader.readNoEof(&buf);
if (!std.mem.eql(u8, &buf, "\r\n--")) {
try value.append(buf[0]);
try peek_stream.putBack(buf[1..]);
var ch = try reader.readByte();
while (ch != '\r') : (ch = try reader.readByte()) try value.append(ch);
try peek_stream.putBackByte(ch);
continue;
}
for (boundary) |ch, i| {
const b = try reader.readByte();
if (b != ch) {
try value.appendSlice("\r\n--");
try value.appendSlice(boundary[0 .. i + 1]);
continue :line_loop;
}
}
// Boundary parsed. See if its a terminal or not
break;
}
const terminal = try isFinalPart(peek_stream);
const disposition = headers.get("Content-Disposition") orelse return error.NoDisposition;
return FormFieldResult{
.field = .{
.value = value.toOwnedSlice(),
.params = try parseParams(alloc, FormDataParams, disposition),
},
.more = !terminal,
};
}
pub fn parseFormData(comptime T: type, boundary: []const u8, reader: anytype, alloc: std.mem.Allocator) !T {
if (boundary.len > max_boundary) return error.BoundaryTooLarge;
var multipart = try openMultipart(boundary, reader);
var stream = std.io.peekStream(72, reader);
{
var buf: [72]u8 = undefined;
const count = try stream.reader().readAll(buf[0 .. boundary.len + 2]);
var line = buf[0..count];
if (line.len != boundary.len + 2) return error.InvalidMultipartBoundary;
if (!std.mem.startsWith(u8, line, "--")) return error.InvalidMultipartBoundary;
if (!std.mem.endsWith(u8, line, boundary)) return error.InvalidMultipartBoundary;
if (try isFinalPart(&stream)) return error.NoForm;
}
var fields = Intermediary(T){};
var ds = util.Deserializer(T){};
while (true) {
const form_field = try parseFormField(boundary, &stream, alloc);
var part = (try multipart.next(alloc)) orelse break;
defer part.close();
inline for (std.meta.fields(Intermediary(T))) |field| {
if (std.ascii.eqlIgnoreCase(field.name[2..], form_field.field.params.name.?)) {
@field(fields, field.name) = form_field.field;
break;
}
} else {
std.log.debug("unknown form field {?s}", .{form_field.field.params.name});
util.deepFree(alloc, form_field);
}
const disposition = part.fields.get("Content-Disposition") orelse return error.InvalidForm;
var iter = ParamIter.from(disposition);
if (!std.ascii.eqlIgnoreCase("form-data", iter.fieldValue())) return error.InvalidForm;
const name = while (iter.next()) |param| {
if (!std.ascii.eqlIgnoreCase("name", param.name)) @panic("Not implemented");
break param.value;
} else return error.InvalidForm;
if (!form_field.more) break;
const value = try part.reader().readAllAlloc(alloc, 1 << 32);
try ds.setSerializedField(name, value);
}
return (try parse(alloc, T, "", "", fields)).?;
}
fn parse(
alloc: std.mem.Allocator,
comptime T: type,
comptime prefix: []const u8,
comptime name: []const u8,
fields: anytype,
) !?T {
if (comptime isScalar(T)) return try parseFormValue(alloc, T, @field(fields, prefix ++ "." ++ name));
switch (@typeInfo(T)) {
.Union => |info| {
var result: ?T = null;
inline for (info.fields) |field| {
const F = field.field_type;
const maybe_value = try parse(alloc, F, prefix, field.name, fields);
if (maybe_value) |value| {
if (result != null) return error.DuplicateUnionField;
result = @unionInit(T, field.name, value);
}
}
std.log.debug("{any}", .{result});
return result;
},
.Struct => |info| {
var result: T = undefined;
var fields_specified: usize = 0;
errdefer inline for (info.fields) |field, i| {
if (fields_specified < i) util.deepFree(alloc, @field(result, field.name));
};
inline for (info.fields) |field| {
const F = field.field_type;
var maybe_value: ?F = null;
if (try parse(alloc, F, prefix ++ "." ++ name, field.name, fields)) |v| {
maybe_value = v;
} else if (field.default_value) |default| {
if (comptime @sizeOf(F) != 0) {
maybe_value = try util.deepClone(alloc, @ptrCast(*const F, @alignCast(@alignOf(F), default)).*);
} else {
maybe_value = std.mem.zeroes(F);
}
}
if (maybe_value) |v| {
fields_specified += 1;
@field(result, field.name) = v;
}
}
if (fields_specified == 0) {
return null;
} else if (fields_specified != info.fields.len) {
std.log.debug("{} {s} {s}", .{ T, prefix, name });
return error.PartiallySpecifiedStruct;
} else {
return result;
}
},
// Only applies to non-scalar optionals
.Optional => |info| return try parse(alloc, info.child, prefix, name, fields),
else => @compileError("tmp"),
}
}
fn recursiveFieldPaths(comptime T: type, comptime prefix: []const u8) []const []const u8 {
comptime {
if (std.meta.trait.is(.Optional)(T)) return recursiveFieldPaths(std.meta.Child(T), prefix);
var fields: []const []const u8 = &.{};
for (std.meta.fields(T)) |f| {
const full_name = prefix ++ f.name;
if (isScalar(f.field_type)) {
fields = fields ++ @as([]const []const u8, &.{full_name});
} else {
const field_prefix = if (@typeInfo(f.field_type) == .Union) prefix else full_name ++ ".";
fields = fields ++ recursiveFieldPaths(f.field_type, field_prefix);
}
}
return fields;
}
}
fn Intermediary(comptime T: type) type {
const field_names = recursiveFieldPaths(T, "..");
var fields: [field_names.len]std.builtin.Type.StructField = undefined;
for (field_names) |name, i| fields[i] = .{
.name = name,
.field_type = ?FormField,
.default_value = &@as(?FormField, null),
.is_comptime = false,
.alignment = @alignOf(?FormField),
};
return @Type(.{ .Struct = .{
.layout = .Auto,
.fields = &fields,
.decls = &.{},
.is_tuple = false,
} });
}
const FormFile = struct {
filename: ?[]const u8,
data: []const u8,
};
fn parseFormValue(alloc: std.mem.Allocator, comptime T: type, f: ?FormField) !T {
const field = f orelse unreachable;
if (comptime std.meta.trait.isZigString(T)) return field.value;
if (T == FormFile) {
return FormFile{
.filename = field.filename,
.data = field.value,
};
}
const result = if (comptime std.meta.trait.isIntegral(T))
try std.fmt.parseInt(T, field.value, 0)
else if (comptime std.meta.trait.isFloat(T))
try std.fmt.parseFloat(T, field.value)
else if (comptime std.meta.trait.is(.Enum)(T)) blk: {
const val = std.ascii.lowerStringAlloc(alloc, field.value);
defer alloc.free(val);
break :blk std.meta.stringToEnum(T, val) orelse return error.InvalidEnumValue;
} else if (T == bool) blk: {
const val = std.ascii.lowerStringAlloc(alloc, field.value);
defer alloc.free(val);
break :blk bool_map.get(val) orelse return error.InvalidBool;
} else if (comptime std.meta.trait.hasFn("parse")(T))
try T.parse(field.value)
else
@compileError("Invalid type " ++ @typeName(T));
return result;
}
const bool_map = std.ComptimeStringMap(bool, .{
.{ "true", true },
.{ "t", true },
.{ "yes", true },
.{ "y", true },
.{ "1", true },
.{ "false", false },
.{ "f", false },
.{ "no", false },
.{ "n", false },
.{ "0", false },
});
fn isScalar(comptime T: type) bool {
if (comptime std.meta.trait.isZigString(T)) return true;
if (comptime std.meta.trait.isIntegral(T)) return true;
if (comptime std.meta.trait.isFloat(T)) return true;
if (comptime std.meta.trait.is(.Enum)(T)) return true;
if (comptime std.meta.trait.is(.EnumLiteral)(T)) return true;
if (T == bool) return true;
if (T == FormFile) return true;
if (comptime std.meta.trait.hasFn("parse")(T)) return true;
if (comptime std.meta.trait.is(.Optional)(T) and isScalar(std.meta.Child(T))) return true;
return false;
return try ds.finish(alloc);
}
fn toCrlf(comptime str: []const u8) []const u8 {
@ -377,7 +183,8 @@ fn toCrlf(comptime str: []const u8) []const u8 {
}
}
test "parseFormData" {
// TODO: Fix these tests
test "MultipartStream" {
const body = toCrlf(
\\--abcd
\\Content-Disposition: form-data; name=first; charset=utf8
@ -395,15 +202,31 @@ test "parseFormData" {
\\
);
const T = struct {
first: []const u8,
second: []const u8,
third: []const u8,
};
var stream = std.io.StreamSource{ .const_buffer = std.io.fixedBufferStream(body) };
const result = try parseFormData(T, "abcd", stream.reader(), std.testing.allocator);
std.debug.print("\nfirst: {s}\n\n", .{result.first});
std.debug.print("\nsecond: {s}\n\n", .{result.second});
std.debug.print("\nthird: {s}\n\n", .{result.third});
std.debug.print("\n{any}\n\n", .{result});
var src = std.io.StreamSource{ .const_buffer = std.io.fixedBufferStream(body) };
var stream = try openMultipart("abcd", src.reader());
while (try stream.next(std.testing.allocator)) |p| {
var part = p;
defer part.close();
std.debug.print("\n{?s}\n", .{part.fields.get("content-disposition")});
var buf: [64]u8 = undefined;
std.debug.print("\"{s}\"\n", .{buf[0..try part.reader().readAll(&buf)]});
}
}
test "parseFormData" {
const body = toCrlf(
\\--abcd
\\Content-Disposition: form-data; name=foo
\\
\\content
\\--abcd--
\\
);
var src = std.io.StreamSource{ .const_buffer = std.io.fixedBufferStream(body) };
const val = try parseFormData(struct {
foo: []const u8,
}, "abcd", src.reader(), std.testing.allocator);
std.debug.print("\n\n\n\"{any}\"\n\n\n", .{val});
}

View file

@ -68,21 +68,40 @@ const QueryIter = util.QueryIter;
/// `?foo.baz=12345`
///
pub fn parseQuery(alloc: std.mem.Allocator, comptime T: type, query: []const u8) !T {
if (comptime !std.meta.trait.isContainer(T)) @compileError("T must be a struct");
var iter = QueryIter.from(query);
var fields = Intermediary(T){};
var deserializer = Deserializer(T){};
while (iter.next()) |pair| {
// TODO: Hash map
inline for (std.meta.fields(Intermediary(T))) |field| {
if (std.ascii.eqlIgnoreCase(field.name[2..], pair.key)) {
@field(fields, field.name) = if (pair.value) |v| .{ .value = v } else .{ .no_value = {} };
break;
}
} else std.log.debug("unknown param {s}", .{pair.key});
try deserializer.setSerializedField(pair.key, pair.value);
}
return (try parse(alloc, T, "", "", fields)) orelse error.NoQuery;
return try deserializer.finish(alloc);
}
fn Deserializer(comptime Result: type) type {
return util.DeserializerContext(Result, ?[]const u8, struct {
pub const options = util.serialize.default_options;
pub fn deserializeScalar(_: @This(), alloc: std.mem.Allocator, comptime T: type, maybe_val: ?[]const u8) !T {
const is_optional = comptime std.meta.trait.is(.Optional)(T);
if (maybe_val) |val| {
if (val.len == 0 and is_optional) return null;
const decoded = try decodeString(alloc, val);
defer alloc.free(decoded);
return try util.serialize.deserializeString(alloc, T, decoded);
} else {
// If param is present, but without an associated value
return if (is_optional)
null
else if (T == bool)
true
else
error.InvalidValue;
}
}
});
}
pub fn parseQueryFree(alloc: std.mem.Allocator, val: anytype) void {
@ -110,186 +129,6 @@ fn decodeString(alloc: std.mem.Allocator, val: []const u8) ![]u8 {
return list.toOwnedSlice();
}
fn parseScalar(alloc: std.mem.Allocator, comptime T: type, comptime name: []const u8, fields: anytype) !?T {
const param = @field(fields, name);
return switch (param) {
.not_specified => null,
.no_value => try parseQueryValue(alloc, T, null),
.value => |v| try parseQueryValue(alloc, T, v),
};
}
fn parse(
alloc: std.mem.Allocator,
comptime T: type,
comptime prefix: []const u8,
comptime name: []const u8,
fields: anytype,
) !?T {
if (comptime isScalar(T)) return parseScalar(alloc, T, prefix ++ "." ++ name, fields);
switch (@typeInfo(T)) {
.Union => |info| {
var result: ?T = null;
inline for (info.fields) |field| {
const F = field.field_type;
const maybe_value = try parse(alloc, F, prefix, field.name, fields);
if (maybe_value) |value| {
if (result != null) return error.DuplicateUnionField;
result = @unionInit(T, field.name, value);
}
}
std.log.debug("{any}", .{result});
return result;
},
.Struct => |info| {
var result: T = undefined;
var fields_specified: usize = 0;
errdefer inline for (info.fields) |field, i| {
if (fields_specified < i) util.deepFree(alloc, @field(result, field.name));
};
inline for (info.fields) |field| {
const F = field.field_type;
var maybe_value: ?F = null;
if (try parse(alloc, F, prefix ++ "." ++ name, field.name, fields)) |v| {
maybe_value = v;
} else if (field.default_value) |default| {
if (comptime @sizeOf(F) != 0) {
maybe_value = try util.deepClone(alloc, @ptrCast(*const F, @alignCast(@alignOf(F), default)).*);
} else {
maybe_value = std.mem.zeroes(F);
}
}
if (maybe_value) |v| {
fields_specified += 1;
@field(result, field.name) = v;
}
}
if (fields_specified == 0) {
return null;
} else if (fields_specified != info.fields.len) {
std.log.debug("{} {s} {s}", .{ T, prefix, name });
return error.PartiallySpecifiedStruct;
} else {
return result;
}
},
// Only applies to non-scalar optionals
.Optional => |info| return try parse(alloc, info.child, prefix, name, fields),
else => @compileError("tmp"),
}
}
fn recursiveFieldPaths(comptime T: type, comptime prefix: []const u8) []const []const u8 {
comptime {
if (std.meta.trait.is(.Optional)(T)) return recursiveFieldPaths(std.meta.Child(T), prefix);
var fields: []const []const u8 = &.{};
for (std.meta.fields(T)) |f| {
const full_name = prefix ++ f.name;
if (isScalar(f.field_type)) {
fields = fields ++ @as([]const []const u8, &.{full_name});
} else {
const field_prefix = if (@typeInfo(f.field_type) == .Union) prefix else full_name ++ ".";
fields = fields ++ recursiveFieldPaths(f.field_type, field_prefix);
}
}
return fields;
}
}
const QueryParam = union(enum) {
not_specified: void,
no_value: void,
value: []const u8,
};
fn Intermediary(comptime T: type) type {
const field_names = recursiveFieldPaths(T, "..");
var fields: [field_names.len]std.builtin.Type.StructField = undefined;
for (field_names) |name, i| fields[i] = .{
.name = name,
.field_type = QueryParam,
.default_value = &QueryParam{ .not_specified = {} },
.is_comptime = false,
.alignment = @alignOf(QueryParam),
};
return @Type(.{ .Struct = .{
.layout = .Auto,
.fields = &fields,
.decls = &.{},
.is_tuple = false,
} });
}
fn parseQueryValue(alloc: std.mem.Allocator, comptime T: type, maybe_value: ?[]const u8) !T {
const is_optional = comptime std.meta.trait.is(.Optional)(T);
if (maybe_value) |value| {
const Eff = if (is_optional) std.meta.Child(T) else T;
if (value.len == 0 and is_optional) return null;
const decoded = try decodeString(alloc, value);
errdefer alloc.free(decoded);
if (comptime std.meta.trait.isZigString(Eff)) return decoded;
defer alloc.free(decoded);
const result = if (comptime std.meta.trait.isIntegral(Eff))
try std.fmt.parseInt(Eff, decoded, 0)
else if (comptime std.meta.trait.isFloat(Eff))
try std.fmt.parseFloat(Eff, decoded)
else if (comptime std.meta.trait.is(.Enum)(Eff)) blk: {
_ = std.ascii.lowerString(decoded, decoded);
break :blk std.meta.stringToEnum(Eff, decoded) orelse return error.InvalidEnumValue;
} else if (Eff == bool) blk: {
_ = std.ascii.lowerString(decoded, decoded);
break :blk bool_map.get(decoded) orelse return error.InvalidBool;
} else if (comptime std.meta.trait.hasFn("parse")(Eff))
try Eff.parse(value)
else
@compileError("Invalid type " ++ @typeName(T));
return result;
} else {
// If param is present, but without an associated value
return if (is_optional)
null
else if (T == bool)
true
else
error.InvalidValue;
}
}
const bool_map = std.ComptimeStringMap(bool, .{
.{ "true", true },
.{ "t", true },
.{ "yes", true },
.{ "y", true },
.{ "1", true },
.{ "false", false },
.{ "f", false },
.{ "no", false },
.{ "n", false },
.{ "0", false },
});
fn isScalar(comptime T: type) bool {
if (comptime std.meta.trait.isZigString(T)) return true;
if (comptime std.meta.trait.isIntegral(T)) return true;

View file

@ -1,5 +1,6 @@
test {
_ = @import("./request/test_parser.zig");
_ = @import("./middleware.zig");
_ = @import("./multipart.zig");
_ = @import("./query.zig");
}

View file

@ -3,7 +3,7 @@ const util = @import("./lib.zig");
const FieldRef = []const []const u8;
fn defaultIsScalar(comptime T: type) bool {
pub fn defaultIsScalar(comptime T: type) bool {
if (comptime std.meta.trait.is(.Optional)(T) and defaultIsScalar(std.meta.Child(T))) return true;
if (comptime std.meta.trait.isZigString(T)) return true;
if (comptime std.meta.trait.isIntegral(T)) return true;