From 7f96c7c7f32a3bf2f2c1958fc80836c03b95d90d Mon Sep 17 00:00:00 2001 From: Dmitri Tikhonov Date: Fri, 4 Dec 2020 11:29:14 -0500 Subject: [PATCH] Release 2.25.0 - [API, FEATURE] Add es_delay_onclose option to delay on_close until all data is ACKed. Use new function lsquic_stream_has_unacked_data() to learn whether peer acknowledged all data written to stream. - [API] Add optional on_reset() stream callback to get notifications when RESET or STOP_SENDING frames are received. - [BUGFIX] On STOP_SENDING, make conn tickable is _writeable_, not readable. --- CHANGELOG | 10 ++ bin/http_server.c | 3 +- bin/test_common.c | 5 + docs/apiref.rst | 33 ++++++ docs/conf.py | 4 +- include/lsquic.h | 35 +++++- src/liblsquic/lsquic_engine.c | 1 + src/liblsquic/lsquic_full_conn.c | 3 + src/liblsquic/lsquic_full_conn_ietf.c | 6 + src/liblsquic/lsquic_stream.c | 66 ++++++++--- src/liblsquic/lsquic_stream.h | 14 ++- tests/test_send_headers.c | 17 +++ tests/test_stream.c | 161 ++++++++++++++++++++------ 13 files changed, 300 insertions(+), 58 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index b09af53..f583389 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,13 @@ +2020-12-04 + - 2.25.0 + - [API, FEATURE] Add es_delay_onclose option to delay on_close until all + data is ACKed. Use new function lsquic_stream_has_unacked_data() to + learn whether peer acknowledged all data written to stream. + - [API] Add optional on_reset() stream callback to get notifications + when RESET or STOP_SENDING frames are received. + - [BUGFIX] On STOP_SENDING, make conn tickable is _writeable_, not + readable. + 2020-11-24 - 2.24.5 - [FEATURE] Improve Delayed ACKs extension and turn it on by default. diff --git a/bin/http_server.c b/bin/http_server.c index ee2fa31..a5bb8d5 100644 --- a/bin/http_server.c +++ b/bin/http_server.c @@ -1045,7 +1045,8 @@ http_server_on_close (lsquic_stream_t *stream, lsquic_stream_ctx_t *st_h) if (st_h->req) interop_server_hset_destroy(st_h->req); free(st_h); - LSQ_INFO("%s called", __func__); + LSQ_INFO("%s called, has unacked data: %d", __func__, + lsquic_stream_has_unacked_data(stream)); } diff --git a/bin/test_common.c b/bin/test_common.c index e5aad38..6a933cb 100644 --- a/bin/test_common.c +++ b/bin/test_common.c @@ -1931,6 +1931,11 @@ set_engine_option (struct lsquic_engine_settings *settings, settings->es_ptpc_int_gain = atof(val); return 0; } + if (0 == strncmp(name, "delay_onclose", 13)) + { + settings->es_delay_onclose = atoi(val); + return 0; + } break; case 14: if (0 == strncmp(name, "max_streams_in", 14)) diff --git a/docs/apiref.rst b/docs/apiref.rst index cade5f7..1c55db6 100644 --- a/docs/apiref.rst +++ b/docs/apiref.rst @@ -856,6 +856,16 @@ settings structure: Default value is :macro:`LSQUIC_DF_QPACK_EXPERIMENT` + .. member:: int es_delay_onclose + + When set to true, :member:`lsquic_stream_if.on_close` will be delayed until the + peer acknowledges all data sent on the stream. (Or until the connection + is destroyed in some manner -- either explicitly closed by the user or + as a result of an engine shutdown.) To find out whether all data written + to peer has been acknowledged, use `lsquic_stream_has_unacked_data()`. + + Default value is :macro:`LSQUIC_DF_DELAY_ONCLOSE` + To initialize the settings structure to library defaults, use the following convenience function: @@ -1088,6 +1098,10 @@ out of date. Please check your :file:`lsquic.h` for actual values.* By default, QPACK experiments are turned off. +.. macro:: LSQUIC_DF_DELAY_ONCLOSE + + By default, calling :member:`lsquic_stream_if.on_close()` is not delayed. + Receiving Packets ----------------- @@ -1272,6 +1286,20 @@ the engine to communicate with the user code: This callback is mandatory. + .. member:: void (*on_reset) (lsquic_stream_t *s, lsquic_stream_ctx_t *h, int how) + + This callback is called as soon as the peer resets a stream. + The argument `how` is either 0, 1, or 2, meaning "read", "write", and + "read and write", respectively (just like in ``shutdown(2)``). This + signals the user to stop reading, writing, or both. + + Note that resets differ in gQUIC and IETF QUIC. In gQUIC, `how` is + always 2; in IETF QUIC, `how` is either 0 or 1 because on can reset + just one direction in IETF QUIC. + + This callback is optional. The reset error can still be collected + during next "on read" or "on write" event. + .. member:: void (*on_hsk_done)(lsquic_conn_t *c, enum lsquic_hsk_status s) When handshake is completed, this callback is called. @@ -1945,6 +1973,11 @@ Miscellaneous Stream Functions Returns true if this stream was rejected, false otherwise. Use this as an aid to distinguish between errors. +.. function:: int lsquic_stream_has_unacked_data (const lsquic_stream_t *stream) + + Return true if peer has not ACKed all data written to the stream. This + includes both packetized and buffered data. + Other Functions --------------- diff --git a/docs/conf.py b/docs/conf.py index 381560d..f142c15 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -24,9 +24,9 @@ copyright = u'2020, LiteSpeed Technologies' author = u'LiteSpeed Technologies' # The short X.Y version -version = u'2.24' +version = u'2.25' # The full version, including alpha/beta/rc tags -release = u'2.24.5' +release = u'2.25.0' # -- General configuration --------------------------------------------------- diff --git a/include/lsquic.h b/include/lsquic.h index 4b39dc7..75bcdbd 100644 --- a/include/lsquic.h +++ b/include/lsquic.h @@ -24,8 +24,8 @@ extern "C" { #endif #define LSQUIC_MAJOR_VERSION 2 -#define LSQUIC_MINOR_VERSION 24 -#define LSQUIC_PATCH_VERSION 5 +#define LSQUIC_MINOR_VERSION 25 +#define LSQUIC_PATCH_VERSION 0 /** * Engine flags: @@ -210,6 +210,17 @@ struct lsquic_stream_if { * perform a session resumption next time around. */ void (*on_sess_resume_info)(lsquic_conn_t *c, const unsigned char *, size_t); + /** + * Optional callback is called as soon as the peer resets a stream. + * The argument `how' is either 0, 1, or 2, meaning "read", "write", and + * "read and write", respectively (just like in shutdown(2)). This + * signals the user to stop reading, writing, or both. + * + * Note that resets differ in gQUIC and IETF QUIC. In gQUIC, `how' is + * always 2; in IETF QUIC, `how' is either 0 or 1 because on can reset + * just one direction in IETF QUIC. + */ + void (*on_reset) (lsquic_stream_t *s, lsquic_stream_ctx_t *h, int how); }; struct ssl_ctx_st; @@ -411,6 +422,9 @@ typedef struct ssl_ctx_st * (*lsquic_lookup_cert_f)( /** By default, we use the minimum timer of 1000 milliseconds */ #define LSQUIC_DF_MTU_PROBE_TIMER 1000 +/** By default, calling on_close() is not delayed */ +#define LSQUIC_DF_DELAY_ONCLOSE 0 + struct lsquic_engine_settings { /** * This is a bit mask wherein each bit corresponds to a value in @@ -1005,6 +1019,16 @@ struct lsquic_engine_settings { es_ptpc_int_gain, /* LSQUIC_DF_PTPC_INT_GAIN */ es_ptpc_err_thresh, /* LSQUIC_DF_PTPC_ERR_THRESH */ es_ptpc_err_divisor; /* LSQUIC_DF_PTPC_ERR_DIVISOR */ + + /** + * When set to true, the on_close() callback will be delayed until the + * peer acknowledges all data sent on the stream. (Or until the connection + * is destroyed in some manner -- either explicitly closed by the user or + * as a result of an engine shutdown.) + * + * Default value is @ref LSQUIC_DF_DELAY_ONCLOSE + */ + int es_delay_onclose; }; /* Initialize `settings' to default values */ @@ -1636,6 +1660,13 @@ int lsquic_stream_shutdown(lsquic_stream_t *s, int how); int lsquic_stream_close(lsquic_stream_t *s); +/** + * Return true if peer has not ACKed all data written to the stream. This + * includes both packetized and buffered data. + */ +int +lsquic_stream_has_unacked_data (lsquic_stream_t *s); + /** * Get certificate chain returned by the server. This can be used for * server certificate verification. diff --git a/src/liblsquic/lsquic_engine.c b/src/liblsquic/lsquic_engine.c index 2b11dec..35162a0 100644 --- a/src/liblsquic/lsquic_engine.c +++ b/src/liblsquic/lsquic_engine.c @@ -390,6 +390,7 @@ lsquic_engine_init_settings (struct lsquic_engine_settings *settings, settings->es_ptpc_int_gain = LSQUIC_DF_PTPC_INT_GAIN; settings->es_ptpc_err_thresh = LSQUIC_DF_PTPC_ERR_THRESH; settings->es_ptpc_err_divisor= LSQUIC_DF_PTPC_ERR_DIVISOR; + settings->es_delay_onclose = LSQUIC_DF_DELAY_ONCLOSE; } diff --git a/src/liblsquic/lsquic_full_conn.c b/src/liblsquic/lsquic_full_conn.c index cd25982..dae1b67 100644 --- a/src/liblsquic/lsquic_full_conn.c +++ b/src/liblsquic/lsquic_full_conn.c @@ -1330,6 +1330,8 @@ new_stream (struct full_conn *conn, lsquic_stream_id_t stream_id, flags |= SCF_HTTP; if (conn->fc_enpub->enp_settings.es_rw_once) flags |= SCF_DISP_RW_ONCE; + if (conn->fc_enpub->enp_settings.es_delay_onclose) + flags |= SCF_DELAY_ONCLOSE; return new_stream_ext(conn, stream_id, STREAM_IF_STD, flags); } @@ -4025,6 +4027,7 @@ headers_stream_on_push_promise (void *ctx, struct uncompressed_headers *uh) } stream = new_stream_ext(conn, uh->uh_oth_stream_id, STREAM_IF_STD, + (conn->fc_enpub->enp_settings.es_delay_onclose?SCF_DELAY_ONCLOSE:0)| SCF_DI_AUTOSWITCH|(conn->fc_enpub->enp_settings.es_rw_once ? SCF_DISP_RW_ONCE : 0)); if (!stream) diff --git a/src/liblsquic/lsquic_full_conn_ietf.c b/src/liblsquic/lsquic_full_conn_ietf.c index 15edb40..fe12561 100644 --- a/src/liblsquic/lsquic_full_conn_ietf.c +++ b/src/liblsquic/lsquic_full_conn_ietf.c @@ -1079,6 +1079,8 @@ create_bidi_stream_out (struct ietf_full_conn *conn) flags = SCF_IETF|SCF_DI_AUTOSWITCH; if (conn->ifc_enpub->enp_settings.es_rw_once) flags |= SCF_DISP_RW_ONCE; + if (conn->ifc_enpub->enp_settings.es_delay_onclose) + flags |= SCF_DELAY_ONCLOSE; if (conn->ifc_flags & IFC_HTTP) { flags |= SCF_HTTP; @@ -1117,6 +1119,8 @@ create_push_stream (struct ietf_full_conn *conn) flags = SCF_IETF|SCF_HTTP; if (conn->ifc_enpub->enp_settings.es_rw_once) flags |= SCF_DISP_RW_ONCE; + if (conn->ifc_enpub->enp_settings.es_delay_onclose) + flags |= SCF_DELAY_ONCLOSE; stream_id = generate_stream_id(conn, SD_UNI); stream = lsquic_stream_new(stream_id, &conn->ifc_pub, @@ -5267,6 +5271,8 @@ new_stream (struct ietf_full_conn *conn, lsquic_stream_id_t stream_id, stream_ctx = conn->ifc_enpub->enp_stream_if_ctx; if (conn->ifc_enpub->enp_settings.es_rw_once) flags |= SCF_DISP_RW_ONCE; + if (conn->ifc_enpub->enp_settings.es_delay_onclose) + flags |= SCF_DELAY_ONCLOSE; if (conn->ifc_flags & IFC_HTTP) { flags |= SCF_HTTP; diff --git a/src/liblsquic/lsquic_stream.c b/src/liblsquic/lsquic_stream.c index b5638f5..b250415 100644 --- a/src/liblsquic/lsquic_stream.c +++ b/src/liblsquic/lsquic_stream.c @@ -1,20 +1,6 @@ /* Copyright (c) 2017 - 2020 LiteSpeed Technologies Inc. See LICENSE. */ /* * lsquic_stream.c -- stream processing - * - * To clear up terminology, here are some of our stream states (in order). - * They are not codified, but they are referred to in both code and comments. - * - * CLOSED STREAM_U_READ_DONE and STREAM_U_WRITE_DONE are set. At this - * point, on_close() gets called. - * FINISHED FIN or RST has been sent to peer. Stream is scheduled to be - * finished (freed): it gets put onto the `service_streams' - * list for connection to clean it up. - * DESTROYED All remaining memory associated with the stream is released. - * If on_close() has not been called yet, it is called now. - * The stream pointer is now invalid. - * - * When connection is aborted, a stream may go directly to DESTROYED state. */ #include @@ -476,7 +462,7 @@ lsquic_stream_new (lsquic_stream_id_t id, stream->sm_readable = stream_readable_non_http; if ((ctor_flags & (SCF_HTTP|SCF_HTTP_PRIO)) == (SCF_HTTP|SCF_HTTP_PRIO)) - lsquic_stream_set_priority_internal(stream, LSQUIC_DEF_HTTP_URGENCY); + lsquic_stream_set_priority_internal(stream, LSQUIC_DEF_HTTP_URGENCY); else lsquic_stream_set_priority_internal(stream, LSQUIC_STREAM_DEFAULT_PRIO); @@ -708,13 +694,16 @@ lsquic_stream_destroy (lsquic_stream_t *stream) static int -stream_is_finished (const lsquic_stream_t *stream) +stream_is_finished (struct lsquic_stream *stream) { return lsquic_stream_is_closed(stream) + && (stream->sm_bflags & SMBF_DELAY_ONCLOSE ? + /* Need a stricter check when on_close() is delayed: */ + !lsquic_stream_has_unacked_data(stream) : /* n_unacked checks that no outgoing packets that reference this * stream are outstanding: */ - && 0 == stream->n_unacked + 0 == stream->n_unacked) && 0 == (stream->sm_qflags & ( /* This checks that no packets that reference this stream will * become outstanding: @@ -756,6 +745,8 @@ maybe_schedule_call_on_close (lsquic_stream_t *stream) if ((stream->stream_flags & (STREAM_U_READ_DONE|STREAM_U_WRITE_DONE| STREAM_ONNEW_DONE|STREAM_ONCLOSE_DONE)) == (STREAM_U_READ_DONE|STREAM_U_WRITE_DONE|STREAM_ONNEW_DONE) + && (!(stream->sm_bflags & SMBF_DELAY_ONCLOSE) + || !lsquic_stream_has_unacked_data(stream)) && !(stream->sm_qflags & SMQF_CALL_ONCLOSE)) { if (0 == (stream->sm_qflags & SMQF_SERVICE_FLAGS)) @@ -1224,6 +1215,28 @@ lsquic_stream_rst_in (lsquic_stream_t *stream, uint64_t offset, return -1; } + if (stream->stream_if->on_reset + && !(stream->stream_flags & STREAM_ONCLOSE_DONE)) + { + if (stream->sm_bflags & SMBF_IETF) + { + if (!(stream->sm_dflags & SMDF_ONRESET0)) + { + stream->stream_if->on_reset(stream, stream->st_ctx, 0); + stream->sm_dflags |= SMDF_ONRESET0; + } + } + else + { + if ((stream->sm_dflags & (SMDF_ONRESET0|SMDF_ONRESET1)) + != (SMDF_ONRESET0|SMDF_ONRESET1)) + { + stream->stream_if->on_reset(stream, stream->st_ctx, 2); + stream->sm_dflags |= SMDF_ONRESET0|SMDF_ONRESET1; + } + } + } + /* Let user collect error: */ maybe_conn_to_tickable_if_readable(stream); @@ -1270,8 +1283,15 @@ lsquic_stream_stop_sending_in (struct lsquic_stream *stream, SM_HISTORY_APPEND(stream, SHE_STOP_SENDIG_IN); stream->stream_flags |= STREAM_SS_RECVD; + if (stream->stream_if->on_reset && !(stream->sm_dflags & SMDF_ONRESET1) + && !(stream->stream_flags & STREAM_ONCLOSE_DONE)) + { + stream->stream_if->on_reset(stream, stream->st_ctx, 1); + stream->sm_dflags |= SMDF_ONRESET1; + } + /* Let user collect error: */ - maybe_conn_to_tickable_if_readable(stream); + maybe_conn_to_tickable_if_writeable(stream, 0); lsquic_sfcw_consume_rem(&stream->fc); drop_frames_in(stream); @@ -4283,7 +4303,10 @@ lsquic_stream_acked (struct lsquic_stream *stream, stream->stream_flags |= STREAM_RST_ACKED; } if (0 == stream->n_unacked) + { + maybe_schedule_call_on_close(stream); maybe_finish_stream(stream); + } } @@ -5411,3 +5434,10 @@ lsquic_stream_set_http_prio (struct lsquic_stream *stream, else return -1; } + + +int +lsquic_stream_has_unacked_data (struct lsquic_stream *stream) +{ + return stream->n_unacked > 0 || stream->sm_n_buffered > 0; +} diff --git a/src/liblsquic/lsquic_stream.h b/src/liblsquic/lsquic_stream.h index 8c23453..864f214 100644 --- a/src/liblsquic/lsquic_stream.h +++ b/src/liblsquic/lsquic_stream.h @@ -191,7 +191,17 @@ enum stream_b_flags SMBF_HTTP_PRIO = 1 <<10, /* Extensible HTTP Priorities are used */ SMBF_INCREMENTAL = 1 <<11, /* Value of the "incremental" HTTP Priority parameter */ SMBF_HPRIO_SET = 1 <<12, /* Extensible HTTP Priorities have been set once */ -#define N_SMBF_FLAGS 13 + SMBF_DELAY_ONCLOSE= 1 <<13, /* Delay calling on_close() until peer ACKs everything */ +#define N_SMBF_FLAGS 14 +}; + + +/* Stream "callback done" flags */ +/* TODO: move STREAM.*DONE flags from stream_flags here */ +enum stream_d_flags +{ + SMDF_ONRESET0 = 1 << 0, /* Called on_reset(0) */ + SMDF_ONRESET1 = 1 << 1, /* Called on_reset(1) */ }; @@ -364,6 +374,7 @@ struct lsquic_stream SSHS_ENC_SENDING, /* Sending encoder stream data */ SSHS_HBLOCK_SENDING,/* Sending header block data */ } sm_send_headers_state:8; + enum stream_d_flags sm_dflags:8; signed char sm_saved_want_write; signed char sm_has_frame; @@ -396,6 +407,7 @@ enum stream_ctor_flags SCF_CRYPTO = SMBF_CRYPTO, SCF_HEADERS = SMBF_HEADERS, SCF_HTTP_PRIO = SMBF_HTTP_PRIO, + SCF_DELAY_ONCLOSE = SMBF_DELAY_ONCLOSE, }; diff --git a/tests/test_send_headers.c b/tests/test_send_headers.c index db99704..3931ece 100644 --- a/tests/test_send_headers.c +++ b/tests/test_send_headers.c @@ -89,10 +89,24 @@ on_write (lsquic_stream_t *stream, lsquic_stream_ctx_t *h) } +static struct reset_call_ctx { + struct lsquic_stream *stream; + int how; +} s_onreset_called = { NULL, -1, }; + + +static void +on_reset (lsquic_stream_t *stream, lsquic_stream_ctx_t *h, int how) +{ + s_onreset_called = (struct reset_call_ctx) { stream, how, }; +} + + const struct lsquic_stream_if stream_if = { .on_new_stream = on_new_stream, .on_write = on_write, .on_close = on_close, + .on_reset = on_reset, }; @@ -355,7 +369,10 @@ test_flushes_and_closes (void) lsquic_stream_acked(stream, QUIC_FRAME_STREAM); lsquic_stream_call_on_close(stream); assert(!(stream->sm_qflags & SMQF_FREE_STREAM)); /* Not yet */ + s_onreset_called = (struct reset_call_ctx) { NULL, -1, }; lsquic_stream_rst_in(stream, 0, 0); + assert(s_onreset_called.stream == NULL); + assert(s_onreset_called.how == -1); assert(!(stream->sm_qflags & (SMQF_SEND_STOP_SENDING|SMQF_WAIT_FIN_OFF))); assert(stream->sm_qflags & SMQF_FREE_STREAM); lsquic_stream_destroy(stream); diff --git a/tests/test_stream.c b/tests/test_stream.c index b36b1a8..7874830 100644 --- a/tests/test_stream.c +++ b/tests/test_stream.c @@ -177,9 +177,23 @@ on_close (lsquic_stream_t *stream, lsquic_stream_ctx_t *st_h) } +static struct reset_call_ctx { + struct lsquic_stream *stream; + int how; +} s_onreset_called = { NULL, -1, }; + + +static void +on_reset (lsquic_stream_t *stream, lsquic_stream_ctx_t *h, int how) +{ + s_onreset_called = (struct reset_call_ctx) { stream, how, }; +} + + const struct lsquic_stream_if stream_if = { .on_new_stream = on_new_stream, .on_close = on_close, + .on_reset = on_reset, }; @@ -462,6 +476,8 @@ new_stream_ext (struct test_objs *tobjs, unsigned stream_id, uint64_t send_off) ctor_flags = SCF_CRITICAL; else ctor_flags = 0; + if ((1 << tobjs->lconn.cn_version) & LSQUIC_IETF_VERSIONS) + ctor_flags |= SCF_IETF; return lsquic_stream_new(stream_id, &tobjs->conn_pub, tobjs->stream_if, tobjs->stream_if_ctx, tobjs->initial_stream_window, send_off, tobjs->ctor_flags | ctor_flags); @@ -791,8 +807,14 @@ test_rem_data_loc_close_and_rst_in (struct test_objs *tobjs) assert(!TAILQ_EMPTY(&tobjs->conn_pub.service_streams)); assert((stream->sm_qflags & (SMQF_SERVICE_FLAGS)) == SMQF_CALL_ONCLOSE); + s_onreset_called = (struct reset_call_ctx) { NULL, -1, }; s = lsquic_stream_rst_in(stream, 100, 1); assert(0 == s); + assert(s_onreset_called.stream == stream); + if (stream->sm_bflags & SMBF_IETF) + assert(s_onreset_called.how == 0); + else + assert(s_onreset_called.how == 2); assert(!(stream->sm_qflags & SMQF_FREE_STREAM)); /* Not yet */ assert(stream->sm_qflags & SMQF_CALL_ONCLOSE); @@ -926,8 +948,14 @@ test_loc_FIN_rem_RST (struct test_objs *tobjs) s = lsquic_stream_frame_in(stream, new_frame_in(tobjs, 0, 100, 0)); assert(0 == s); + s_onreset_called = (struct reset_call_ctx) { NULL, -1, }; s = lsquic_stream_rst_in(stream, 100, 0); assert(0 == s); + assert(s_onreset_called.stream == stream); + if (stream->sm_bflags & SMBF_IETF) + assert(s_onreset_called.how == 0); + else + assert(s_onreset_called.how == 2); /* No RST to send, we already sent FIN */ assert(0 == lsquic_send_ctl_n_scheduled(&tobjs->send_ctl)); @@ -1001,13 +1029,27 @@ test_loc_data_rem_RST (struct test_objs *tobjs) s = lsquic_stream_frame_in(stream, new_frame_in(tobjs, 0, 100, 0)); assert(0 == s); - s = lsquic_stream_rst_in(stream, 200, 0); - assert(0 == s); + s_onreset_called = (struct reset_call_ctx) { NULL, -1, }; + if (stream->sm_bflags & SMBF_IETF) + lsquic_stream_stop_sending_in(stream, 12345); + else + { + s = lsquic_stream_rst_in(stream, 200, 0); + assert(0 == s); + } + assert(s_onreset_called.stream == stream); + if (stream->sm_bflags & SMBF_IETF) + assert(s_onreset_called.how == 1); + else + assert(s_onreset_called.how == 2); ack_packet(&tobjs->send_ctl, 1); - assert(!TAILQ_EMPTY(&tobjs->conn_pub.sending_streams)); - assert((stream->sm_qflags & SMQF_SENDING_FLAGS) == SMQF_SEND_RST); + if (!(stream->sm_bflags & SMBF_IETF)) + { + assert(!TAILQ_EMPTY(&tobjs->conn_pub.sending_streams)); + assert((stream->sm_qflags & SMQF_SENDING_FLAGS) == SMQF_SEND_RST); + } /* Not yet closed: error needs to be collected */ assert(TAILQ_EMPTY(&tobjs->conn_pub.service_streams)); @@ -1021,18 +1063,26 @@ test_loc_data_rem_RST (struct test_objs *tobjs) assert(!TAILQ_EMPTY(&tobjs->conn_pub.service_streams)); assert((stream->sm_qflags & SMQF_SERVICE_FLAGS) == SMQF_CALL_ONCLOSE); + if (stream->sm_bflags & SMBF_IETF) + lsquic_stream_ss_frame_sent(stream); lsquic_stream_rst_frame_sent(stream); lsquic_stream_call_on_close(stream); assert(TAILQ_EMPTY(&tobjs->conn_pub.sending_streams)); - assert(!TAILQ_EMPTY(&tobjs->conn_pub.service_streams)); - assert((stream->sm_qflags & SMQF_SERVICE_FLAGS) == SMQF_FREE_STREAM); + if (stream->sm_bflags & SMBF_IETF) + assert(stream->sm_qflags & SMQF_WAIT_FIN_OFF); + else + { + assert(!TAILQ_EMPTY(&tobjs->conn_pub.service_streams)); + assert((stream->sm_qflags & SMQF_SERVICE_FLAGS) == SMQF_FREE_STREAM); + } lsquic_stream_destroy(stream); assert(TAILQ_EMPTY(&tobjs->conn_pub.service_streams)); - assert(200 == tobjs->conn_pub.cfcw.cf_max_recv_off); - assert(200 == tobjs->conn_pub.cfcw.cf_read_off); + const unsigned expected_nread = stream->sm_bflags & SMBF_IETF ? 100 : 200; + assert(expected_nread == tobjs->conn_pub.cfcw.cf_max_recv_off); + assert(expected_nread == tobjs->conn_pub.cfcw.cf_read_off); } @@ -1158,9 +1208,20 @@ test_gapless_elision_middle (struct test_objs *tobjs) assert(n == written_to_A); assert(0 == memcmp(buf, buf_out, written_to_A)); - /* Now reset stream A: */ - s = lsquic_stream_rst_in(streamB, 0, 0); - assert(s == 0); + /* Now reset stream B: */ + s_onreset_called = (struct reset_call_ctx) { NULL, -1, }; + if (streamB->sm_bflags & SMBF_IETF) + lsquic_stream_stop_sending_in(streamB, 12345); + else + { + s = lsquic_stream_rst_in(streamB, 0, 0); + assert(s == 0); + } + assert(s_onreset_called.stream == streamB); + if (streamB->sm_bflags & SMBF_IETF) + assert(s_onreset_called.how == 1); + else + assert(s_onreset_called.how == 2); assert(2 == lsquic_send_ctl_n_scheduled(&tobjs->send_ctl)); /* Verify A again: */ n = read_from_scheduled_packets(&tobjs->send_ctl, streamA->id, buf, @@ -1229,9 +1290,16 @@ test_gapless_elision_beginning (struct test_objs *tobjs) assert(n == written_to_A); assert(0 == memcmp(buf, buf_out, written_to_A)); - /* Now reset stream A: */ - s = lsquic_stream_rst_in(streamB, 0, 0); - assert(s == 0); + /* Now reset stream B: */ + assert(!(streamB->stream_flags & STREAM_FRAMES_ELIDED)); + if (streamB->sm_bflags & SMBF_IETF) + lsquic_stream_stop_sending_in(streamB, 12345); + else + { + s = lsquic_stream_rst_in(streamB, 0, 0); + assert(s == 0); + } + assert(streamB->stream_flags & STREAM_FRAMES_ELIDED); assert(2 == lsquic_send_ctl_n_scheduled(&tobjs->send_ctl)); /* Verify A again: */ n = read_from_scheduled_packets(&tobjs->send_ctl, streamA->id, buf, @@ -1250,6 +1318,17 @@ test_gapless_elision_beginning (struct test_objs *tobjs) packet_out = lsquic_send_ctl_next_packet_to_send(&tobjs->send_ctl, 0); assert(!packet_out); + /* Test on_reset() behavior. This is unrelated to the gapless elision + * test, but convenient to do here. + */ + if (streamA->sm_bflags & SMBF_IETF) + { + s_onreset_called = (struct reset_call_ctx) { NULL, -1, }; + lsquic_stream_stop_sending_in(streamA, 12345); + assert(s_onreset_called.stream == streamA); + assert(s_onreset_called.how == 1); + } + /* Now we can call on_close: */ lsquic_stream_destroy(streamA); lsquic_stream_destroy(streamB); @@ -1384,26 +1463,40 @@ static void test_termination (void) { struct test_objs tobjs; - unsigned i; - void (*const test_funcs[])(struct test_objs *) = { - test_loc_FIN_rem_FIN, - test_rem_FIN_loc_FIN, - test_rem_data_loc_close_and_rst_in, - test_rem_data_loc_close, - test_loc_FIN_rem_RST, - test_loc_data_rem_RST, - test_loc_RST_rem_FIN, - test_gapless_elision_beginning, - test_gapless_elision_middle, - }; + const struct { + int gquic; + int ietf; + void (*func)(struct test_objs *); + } test_funcs[] = { + { 1, 1, test_loc_FIN_rem_FIN, }, + { 1, 1, test_rem_FIN_loc_FIN, }, + { 1, 0, test_rem_data_loc_close_and_rst_in, }, + { 1, 0, test_rem_data_loc_close, }, + { 1, 1, test_loc_FIN_rem_RST, }, + { 1, 1, test_loc_data_rem_RST, }, + { 1, 0, test_loc_RST_rem_FIN, }, + { 1, 1, test_gapless_elision_beginning, }, + { 1, 1, test_gapless_elision_middle, }, + }, *tf; - for (i = 0; i < sizeof(test_funcs) / sizeof(test_funcs[0]); ++i) + for (tf = test_funcs; tf < test_funcs + sizeof(test_funcs) / sizeof(test_funcs[0]); ++tf) { - init_test_ctl_settings(&g_ctl_settings); - g_ctl_settings.tcs_schedule_stream_packets_immediately = 1; - init_test_objs(&tobjs, 0x4000, 0x4000, NULL); - test_funcs[i](&tobjs); - deinit_test_objs(&tobjs); + if (tf->gquic) + { + init_test_ctl_settings(&g_ctl_settings); + g_ctl_settings.tcs_schedule_stream_packets_immediately = 1; + init_test_objs(&tobjs, 0x4000, 0x4000, select_pf_by_ver(LSQVER_043)); + tf->func(&tobjs); + deinit_test_objs(&tobjs); + } + if (tf->ietf) + { + init_test_ctl_settings(&g_ctl_settings); + g_ctl_settings.tcs_schedule_stream_packets_immediately = 1; + init_test_objs(&tobjs, 0x4000, 0x4000, select_pf_by_ver(LSQVER_ID27)); + tf->func(&tobjs); + deinit_test_objs(&tobjs); + } } } @@ -2799,7 +2892,7 @@ test_resize_buffered (void) init_test_objs(&tobjs, 0x100000, 0x100000, pf); tobjs.send_ctl.sc_flags |= SC_IETF; /* work around asserts lsquic_send_ctl_resize() */ network_path.np_pack_size = 4096; - streams[0] = new_stream_ext(&tobjs, 7, 0x100000); + streams[0] = new_stream_ext(&tobjs, 8, 0x100000); nw = lsquic_stream_write(streams[0], buf, sizeof(buf)); assert(nw == sizeof(buf)); @@ -2860,7 +2953,7 @@ test_resize_scheduled (void) init_test_objs(&tobjs, 0x100000, 0x100000, pf); tobjs.send_ctl.sc_flags |= SC_IETF; /* work around asserts lsquic_send_ctl_resize() */ network_path.np_pack_size = 4096; - streams[0] = new_stream_ext(&tobjs, 7, 0x100000); + streams[0] = new_stream_ext(&tobjs, 8, 0x100000); nw = lsquic_stream_write(streams[0], buf, sizeof(buf)); assert(nw == sizeof(buf));