Update README.md

.cirrus.yml

@@ -1,5 +1,5 @@
 freebsd_instance:
-    image: freebsd-12-1-release-amd64
+    image: freebsd-13-2-release-amd64
 
 task:
     install_script:

.readthedocs.yaml

@@ -0,0 +1,35 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+    # You can also specify other tool versions:
+    # nodejs: "20"
+    # rust: "1.70"
+    # golang: "1.20"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+  # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
+  # builder: "dirhtml"
+  # Fail on all warnings to avoid broken references
+  # fail_on_warning: true
+
+# Optionally build your docs in additional formats such as PDF and ePub
+# formats:
+#   - pdf
+#   - epub
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+    - requirements: docs/requirements.txt

CHANGELOG

@@ -1,3 +1,27 @@
+2024-03-12
+    - 4.0.8
+    - Fix RETIRE_CONNECTION_ID frame abuse.
+    - Fix some assert failures.
+
+2024-02-28
+    - 4.0.7
+    - Fix overly strict 0-RTT packet DCID validation.
+    - Update docker build.
+
+2024-02-23
+    - 4.0.6
+    - Fix ACK handling.
+    - Do not apply congestion control to CONNECTION_CLOSE frame.
+
+2024-02-07
+    - 4.0.5
+    - Fix CPU spinning due to STREAM_BLOCKED frame.
+
+2024-01-08
+    - 4.0.4
+    - Fix DCID validation.
+    - Fix CPU spinning due to pending STREAM_BLOCKED frame.
+
 2023-12-25
     - 4.0.3
     - Fix session resumption bug introduced in 4.0.2.

Dockerfile

@@ -1,13 +1,17 @@
-FROM ubuntu:16.04
+FROM ubuntu:20.04 as build-lsquic
+
+ENV DEBIAN_FRONTEND noninteractive
 
 RUN apt-get update && \
-    apt-get install -y build-essential git cmake software-properties-common \
+    apt-get install -y apt-utils build-essential git cmake software-properties-common \
                        zlib1g-dev libevent-dev
 
-RUN add-apt-repository ppa:gophers/archive && \
+RUN add-apt-repository ppa:longsleep/golang-backports && \
     apt-get update && \
-    apt-get install -y golang-1.9-go && \
-    cp /usr/lib/go-1.9/bin/go* /usr/bin/.
+    apt-get install -y golang-1.21-go && \
+    cp /usr/lib/go-1.21/bin/go* /usr/bin/.
+
+ENV GOROOT /usr/lib/go-1.21
 
 RUN mkdir /src
 WORKDIR /src
@@ -15,14 +19,21 @@ WORKDIR /src
 RUN mkdir /src/lsquic
 COPY ./ /src/lsquic/
 
-RUN git clone https://boringssl.googlesource.com/boringssl && \
+RUN git clone https://github.com/google/boringssl.git && \
     cd boringssl && \
-    git checkout a2278d4d2cabe73f6663e3299ea7808edfa306b9 && \
+    git checkout 9fc1c33e9c21439ce5f87855a6591a9324e569fd && \
     cmake . && \
     make
 
+ENV EXTRA_CFLAGS -DLSQUIC_QIR=1
 RUN cd /src/lsquic && \
     cmake -DBORINGSSL_DIR=/src/boringssl . && \
     make
 
-RUN cd lsquic && make test && cp bin/http_client /usr/bin/ && cp bin/http_server /usr/bin
+RUN cd lsquic && cp bin/http_client /usr/bin/ && cp bin/http_server /usr/bin
+
+FROM martenseemann/quic-network-simulator-endpoint:latest as lsquic-qir
+COPY --from=build-lsquic /usr/bin/http_client /usr/bin/http_server /usr/bin/
+COPY qir/run_endpoint.sh .
+RUN chmod +x run_endpoint.sh
+ENTRYPOINT [ "./run_endpoint.sh" ]

README.md

@@ -22,22 +22,23 @@ Standard Compliance
 
 LiteSpeed QUIC is mostly compliant to the follow RFCs:
 
-- [RFC 9000](https://www.rfc-editor.org/rfc/rfc9000)
-- [RFC 9001](https://www.rfc-editor.org/rfc/rfc9001)
-- [RFC 9002](https://www.rfc-editor.org/rfc/rfc9002)
-- [RFC 9114](https://www.rfc-editor.org/rfc/rfc9114)
-- [RFC 9204](https://www.rfc-editor.org/rfc/rfc9204)
+- [RFC 9000](https://www.rfc-editor.org/rfc/rfc9000) QUIC: A UDP-Based Multiplexed and Secure Transport
+- [RFC 9001](https://www.rfc-editor.org/rfc/rfc9001) Using TLS to Secure QUIC
+- [RFC 9002](https://www.rfc-editor.org/rfc/rfc9002) QUIC Loss Detection and Congestion Control
+- [RFC 9114](https://www.rfc-editor.org/rfc/rfc9114) HTTP/3
+- [RFC 9204](https://www.rfc-editor.org/rfc/rfc9204) QPACK: Field Compression for HTTP/3
 
 QUIC protocol extensions
 ------------------------
 
 The following QUIC protocol extensions are implemented:
 
-- [QUIC Version 2](https://www.rfc-editor.org/authors/rfc9369.html)
-- [Compatible Version Negotiation](https://datatracker.ietf.org/doc/draft-ietf-quic-version-negotiation/)
-- [Datagrams](https://datatracker.ietf.org/doc/html/rfc9221)
+- [RFC 9368](https://www.rfc-editor.org/rfc/rfc9368) Compatible Version Negotiation for QUIC
+- [RFC 9369](https://www.rfc-editor.org/rfc/rfc9369) QUIC Version 2
+- [RFC 9218](https://www.rfc-editor.org/rfc/rfc9218) Extensible Prioritization Scheme for HTTP
+- [RFC 9221](https://www.rfc-editor.org/rfc/rfc9221) An Unreliable Datagram Extension to QUIC
+- [RFC 9287](https://www.rfc-editor.org/rfc/rfc9287) Greasing the QUIC Bit
 - [ACK Frequency](https://datatracker.ietf.org/doc/draft-ietf-quic-ack-frequency/)
-- [Greasing the QUIC Bit](https://datatracker.ietf.org/doc/html/rfc9287)
 
 Documentation
 -------------
@@ -71,7 +72,7 @@ You may need to install pre-requisites like zlib and libevent.
 2. Use specific BoringSSL version
 
 ```
-git checkout 31bad2514d21f6207f3925ba56754611c462a873
+git checkout 9fc1c33e9c21439ce5f87855a6591a9324e569fd
 ```
 Or, just try the latest master branch.
 
@@ -111,8 +112,7 @@ as follows:
 ```
 git clone https://github.com/litespeedtech/lsquic.git
 cd lsquic
-git submodule init
-git submodule update
+git submodule update --init
 ```
 
 2. Compile the library
@@ -147,8 +147,7 @@ The library and the example client and server can be built with Docker.
 Initialize Git submodules:
 ```
 cd lsquic
-git submodule init
-git submodule update
+git submodule update --init
 ```
 
 Build the Docker image:

bin/prog.c

@@ -171,6 +171,11 @@ prog_print_common_options (const struct prog *prog, FILE *out)
 "                   sndbuf=12345    # Sets SO_SNDBUF\n"
 "                   rcvbuf=12345    # Sets SO_RCVBUF\n"
 "   -W          Use stock PMI (malloc & free)\n"
+"   -A CC_ALGO  Congestion control algorithm.  The following algorithms are\n"
+"                 supported.\n"
+"                   1: Cubic\n"
+"                   2: BBRv1\n"
+"                   3: Adaptive congestion control (this is the default).\n"
     );
 
 #if HAVE_SENDMMSG
@@ -266,6 +271,9 @@ prog_set_opt (struct prog *prog, int opt, const char *arg)
     case 'W':
         prog->prog_use_stock_pmi = 1;
         return 0;
+    case 'A':
+        prog->prog_settings.es_cc_algo = atoi(optarg);
+        return 0;
     case 'c':
         if (prog->prog_engine_flags & LSENG_SERVER)
         {

bin/prog.h

@@ -74,7 +74,7 @@ prog_init (struct prog *, unsigned lsquic_engine_flags, struct sport_head *,
 #   define IP_DONTFRAG_FLAG ""
 #endif
 
-#define PROG_OPTS "i:km:c:y:L:l:o:H:s:S:Y:z:G:W" RECVMMSG_FLAG SENDMMSG_FLAG \
+#define PROG_OPTS "i:km:c:y:L:l:o:H:s:S:Y:z:G:WA:" RECVMMSG_FLAG SENDMMSG_FLAG \
                                                             IP_DONTFRAG_FLAG
 
 /* Returns:

docs/conf.py

@@ -26,7 +26,7 @@ author = u'LiteSpeed Technologies'
 # The short X.Y version
 version = u'4.0'
 # The full version, including alpha/beta/rc tags
-release = u'4.0.3'
+release = u'4.0.8'
 
 
 # -- General configuration ---------------------------------------------------
@@ -41,6 +41,7 @@ release = u'4.0.3'
 extensions = [
 # To make ours look like readthedocs.io, change theme to "sphinx_rtd_theme",
 # pip install sphinx_rtd_theme, and uncomment extensions:
+    "sphinx_rtd_theme",
 #    "sphinx.ext.intersphinx",
 #    "sphinx.ext.autodoc",
 #    "sphinx.ext.mathjax",
@@ -83,8 +84,8 @@ primary_domain = 'c'
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-#html_theme = 'alabaster'
-html_style = '/default.css'
+html_theme = "sphinx_rtd_theme"
+#html_style = '/default.css'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the

docs/requirements.txt

@@ -0,0 +1 @@
+sphinx-rtd-theme

include/lsquic.h

@@ -27,7 +27,7 @@ extern "C" {
 
 #define LSQUIC_MAJOR_VERSION 4
 #define LSQUIC_MINOR_VERSION 0
-#define LSQUIC_PATCH_VERSION 3
+#define LSQUIC_PATCH_VERSION 8
 
 /**
  * Engine flags:
@@ -318,7 +318,7 @@ typedef struct ssl_ctx_st * (*lsquic_lookup_cert_f)(
 #define LSQUIC_DF_STTL               86400
 #define LSQUIC_DF_MAX_INCHOATE     (1 * 1000 * 1000)
 
-#define LSQUIC_DF_SUPPORT_SREJ_SERVER  0
+#define LSQUIC_DF_SUPPORT_SREJ_SERVER  1
 #define LSQUIC_DF_SUPPORT_SREJ_CLIENT  0
 
 /** Do not use NSTP by default */

qir/run_endpoint.sh

@@ -0,0 +1,139 @@
+#!/bin/bash
+#
+# run_endpoint.sh -- QUIC Interop Runner script for lsquic
+#
+
+/setup.sh
+
+if [ "$ROLE" == "client" ]; then
+    # Wait for the simulator to start up.
+    /wait-for-it.sh sim:57832 -s -t 30
+fi
+
+echo TEST_PARAMS: $TEST_PARAMS
+echo REQUESTS: "'$REQUESTS'"
+eval $(perl <<'PERL'
+    @paths = split /\s+/, $ENV{REQUESTS};
+    s~^https?://[^/]+~-p ~ for @paths;
+    print "PATHS='@paths'\n";
+    $server = $ENV{REQUESTS};
+    $server =~ s~^https?://~~;
+    $server =~ s~/.*~~;
+    ($server, $port) = split /:/, $server;
+    print "SERVER=$server\n";
+    print "PORT=$port\n";
+    print "N_REQS=", scalar(@paths), "\n";
+    print "N_reqs=", scalar(@paths), "\n";
+    if (@paths > 100) {
+        print "W=100\n";
+    } else {
+        print "W=1\n";
+    }
+PERL
+)
+echo paths: $PATHS
+echo server: $SERVER
+echo port: $PORT
+
+# lsquic command-line tools create one file per connection when -G option
+# is used.  Here we make a copy and give it required name.
+#
+function maybe_create_keylog() {
+    local NAME=/logs/keys.log
+    if ls /logs/*.keys; then
+        # There may be more than one of these, as one file is created per
+        # connection.
+        cat /logs/*.keys > $NAME
+    fi
+    if [ -f $NAME ]; then
+        echo $NAME exists
+    else
+        echo $NAME does not exit
+    fi
+}
+
+if [ "$ROLE" = server ]; then
+    if [ ! -z "$TESTCASE" ]; then
+        case "$TESTCASE" in
+            http3)
+                VERSIONS='-o version=h3-29 -o version=h3'
+                ;;
+            v2)
+                VERSIONS='-o version=h3-v2 -o version=h3 -Q hq-interop'
+                ;;
+            handshake|transfer|longrtt|resumption|blackhole|multiconnect|chacha20|zerortt)
+                VERSIONS='-o version=h3-29 -o version=h3 -o scid_iss_rate=0 -Q hq-interop'
+                ;;
+            retry)
+                VERSIONS='-o version=h3-29 -o version=h3 -o srej=1 -Q hq-interop'
+                FORCE_RETRY=1
+                ;;
+            ecn)
+                VERSIONS='-o version=h3-29 -o version=h3 -Q hq-interop'
+                ECN='-o ecn=1'
+                ;;
+            *) exit 127 ;;
+        esac
+    fi
+    echo SERVER_PARAMS: $SERVER_PARAMS
+    exec env LSQUIC_FORCE_RETRY=$FORCE_RETRY /usr/bin/http_server $VERSIONS $ECN \
+        -c server,/certs/cert.pem,/certs/priv.key \
+        -c server4,/certs/cert.pem,/certs/priv.key \
+        -c server6,/certs/cert.pem,/certs/priv.key \
+        -c server46,/certs/cert.pem,/certs/priv.key \
+        -s ::0:443 -s 0.0.0.0:443 -s 193.167.100.100:12345 \
+        -r /www -L debug 2>/logs/$TESTCASE.out
+elif [ "$ROLE" = debug-server ]; then
+    exec /usr/bin/http_server $SERVER_PARAMS
+elif [ "$ROLE" = client ]; then
+    if [ ! -z "$TESTCASE" ]; then
+        case "$TESTCASE" in
+            http3)
+                VERSIONS='-o version=h3'
+                ;;
+            v2)
+                VERSIONS='-o version=h3-v2 -o version=h3 -Q hq-interop'
+                ;;
+            handshake|transfer|longrtt|retry|multiplexing|blackhole)
+                VERSIONS='-o version=h3 -Q hq-interop'
+                ;;
+            multiconnect)
+                VERSIONS='-o version=h3 -Q hq-interop'
+                N_REQS=1
+                ;;
+            ecn)
+                VERSIONS='-o version=h3 -Q hq-interop'
+                ECN='-o ecn=1'
+                ;;
+            resumption)
+                VERSIONS='-o version=h3 -Q hq-interop'
+                RESUME='-0 /logs/resume.file'
+                ;;
+            *) exit 127 ;;
+        esac
+    fi
+    echo CLIENT_PARAMS: $CLIENT_PARAMS
+    if [ "$TESTCASE" = resumption ]; then
+        # Fetch first file:
+        /usr/bin/http_client $VERSIONS -s $SERVER:$PORT $PATHS \
+            -r 1 -R 1 $RESUME \
+            -B -7 /downloads -G /logs \
+            -L debug 2>/logs/$TESTCASE-req1.out || exit $?
+        PATHS=`echo "$PATHS" | sed 's~-p /[^ ]* ~~'`
+        N_REQS=1
+        N_reqs=1
+        W=1
+        echo "first request successful, new args: $N_REQS; $N_reqs; $PATHS"
+    fi
+    /usr/bin/http_client $VERSIONS -s $SERVER:$PORT $PATHS \
+        -r $N_reqs -R $N_REQS -w $W $ECN $RESUME \
+        -B -7 /downloads -G /logs \
+        -L debug 2>/logs/$TESTCASE.out
+    EXIT_CODE=$?
+    maybe_create_keylog
+    sync
+    exit $EXIT_CODE
+else
+    echo hi
+    exit 127
+fi

src/liblsquic/lsquic_full_conn.c

@@ -904,12 +904,12 @@ lsquic_gquic_full_conn_server_new (struct lsquic_engine_public *enpub,
 
     for (n = 0; received; ++n)
     {
-        if (received & (1U << n))
+        if (received & (1ULL << n))
             /* Setting `now' to zero is OK here, as we should have had at
              * least one other packet above.
              */
             lsquic_rechist_received(&conn->fc_rechist, n + 1, 0);
-        received &= ~(1U << n);
+        received &= ~(1ULL << n);
     }
 
     /* Mini connection sends out packets 1, 2, 3... and so on.  It deletes

src/liblsquic/lsquic_full_conn_ietf.c

@@ -453,6 +453,7 @@ struct ietf_full_conn
     unsigned                    ifc_n_slack_akbl[N_PNS];
     unsigned                    ifc_n_slack_all;    /* App PNS only */
     unsigned                    ifc_max_retx_since_last_ack;
+    unsigned short              ifc_max_udp_payload;    /* Cached TP */
     lsquic_time_t               ifc_max_ack_delay;
     uint64_t                    ifc_ecn_counts_in[N_PNS][4];
     lsquic_stream_id_t          ifc_max_req_id;
@@ -466,6 +467,7 @@ struct ietf_full_conn
     }                           ifc_peer_hq_settings;
     struct dcid_elem           *ifc_dces[MAX_IETF_CONN_DCIDS];
     TAILQ_HEAD(, dcid_elem)     ifc_to_retire;
+    unsigned                    ifc_n_to_retire;
     unsigned                    ifc_scid_seqno;
     lsquic_time_t               ifc_scid_timestamp[MAX_SCID];
     /* Last 8 packets had ECN markings? */
@@ -491,7 +493,8 @@ struct ietf_full_conn
     unsigned                    ifc_max_pack_tol_sent;
 #endif
     unsigned                    ifc_max_ack_freq_seqno; /* Incoming */
-    unsigned short              ifc_max_udp_payload;    /* Cached TP */
+    unsigned short              ifc_min_dg_sz,
+                                ifc_max_dg_sz;
     lsquic_time_t               ifc_last_live_update;
     struct conn_path            ifc_paths[N_PATHS];
     union {
@@ -520,8 +523,6 @@ struct ietf_full_conn
     lsquic_time_t               ifc_ping_period;
     struct lsquic_hash         *ifc_bpus;
     uint64_t                    ifc_last_max_data_off_sent;
-    unsigned short              ifc_min_dg_sz,
-                                ifc_max_dg_sz;
     struct packet_tolerance_stats
                                 ifc_pts;
 #if LSQUIC_CONN_STATS
@@ -755,6 +756,7 @@ blocked_ka_alarm_expired (enum alarm_id al_id, void *ctx,
     struct ietf_full_conn *const conn = (struct ietf_full_conn *) ctx;
     struct lsquic_stream *stream;
     struct lsquic_hash_elem *el;
+    int has_send_flag;
 
     if (lsquic_conn_cap_avail(&conn->ifc_pub.conn_cap) == 0)
     {
@@ -767,13 +769,21 @@ blocked_ka_alarm_expired (enum alarm_id al_id, void *ctx,
                          el = lsquic_hash_next(conn->ifc_pub.all_streams))
     {
         stream = lsquic_hashelem_getdata(el);
-        if (lsquic_stream_is_blocked(stream))
+        if (lsquic_stream_is_blocked(stream)
+            && !lsquic_stream_is_write_reset(stream))
         {
-            if (!(stream->sm_qflags & SMQF_SENDING_FLAGS))
-                TAILQ_INSERT_TAIL(&conn->ifc_pub.sending_streams, stream,
-                                                            next_send_stream);
+            has_send_flag = (stream->sm_qflags & SMQF_SENDING_FLAGS);
             stream->sm_qflags |= SMQF_SEND_BLOCKED;
             LSQ_DEBUG("set SEND_BLOCKED flag on stream %"PRIu64, stream->id);
+            if (!lsquic_sendctl_gen_stream_blocked_frame(
+                        stream->conn_pub->send_ctl, stream))
+            {
+                LSQ_DEBUG("failed to send STREAM_BLOCKED frame for"
+                        " stream %"PRIu64 " immedately, postpone.", stream->id);
+                if (!has_send_flag)
+                    TAILQ_INSERT_TAIL(&conn->ifc_pub.sending_streams, stream,
+                                                            next_send_stream);
+            }
             return;
         }
     }
@@ -1268,6 +1278,7 @@ ietf_full_conn_init (struct ietf_full_conn *conn,
     TAILQ_INIT(&conn->ifc_pub.service_streams);
     STAILQ_INIT(&conn->ifc_stream_ids_to_ss);
     TAILQ_INIT(&conn->ifc_to_retire);
+    conn->ifc_n_to_retire = 0;
 
     lsquic_alarmset_init(&conn->ifc_alset, &conn->ifc_conn);
     lsquic_alarmset_init_alarm(&conn->ifc_alset, AL_IDLE, idle_alarm_expired, conn);
@@ -1924,7 +1935,7 @@ generate_ack_frame_for_pns (struct ietf_full_conn *conn,
         conn->ifc_flags |= IFC_ACK_HAD_MISS;
     else
         conn->ifc_flags &= ~IFC_ACK_HAD_MISS;
-    LSQ_DEBUG("Put %d bytes of ACK frame into packet %" PRIu64
+    LSQ_DEBUG("Put %d bytes of ACK frame into packet #%" PRIu64
               " on outgoing queue", w, packet_out->po_packno);
     if (conn->ifc_n_cons_unretx >= conn->ifc_ping_unretx_thresh &&
                 !lsquic_send_ctl_have_outgoing_retx_frames(&conn->ifc_send_ctl))
@@ -1935,6 +1946,7 @@ generate_ack_frame_for_pns (struct ietf_full_conn *conn,
         /* This gives a range [12, 27]: */
         conn->ifc_ping_unretx_thresh = 12
                     + lsquic_crand_get_nybble(conn->ifc_enpub->enp_crand);
+        conn->ifc_n_cons_unretx = 0;
     }
 
     conn->ifc_n_slack_akbl[pns] = 0;
@@ -2283,6 +2295,7 @@ generate_retire_cid_frame (struct ietf_full_conn *conn)
     lsquic_send_ctl_incr_pack_sz(&conn->ifc_send_ctl, packet_out, w);
 
     TAILQ_REMOVE(&conn->ifc_to_retire, dce, de_next_to_ret);
+    --conn->ifc_n_to_retire;
     lsquic_malo_put(dce);
 
     if (TAILQ_EMPTY(&conn->ifc_to_retire))
@@ -2297,6 +2310,13 @@ generate_retire_cid_frames (struct ietf_full_conn *conn, lsquic_time_t now)
 {
     int s;
 
+    if (conn->ifc_n_to_retire >= MAX_IETF_CONN_DCIDS * 3)
+    {
+        ABORT_QUIETLY(0, TEC_CONNECTION_ID_LIMIT_ERROR,
+            "too many (%d) CIDs to retire", conn->ifc_n_to_retire);
+        return;
+    }
+
     do
         s = generate_retire_cid_frame(conn);
     while (0 == s && (conn->ifc_send_flags & SF_SEND_RETIRE_CID));
@@ -2875,10 +2895,26 @@ process_stream_ready_to_send (struct ietf_full_conn *conn,
                                             struct lsquic_stream *stream)
 {
     int r = 1;
+
+    LSQ_DEBUG("process_stream_ready_to_send: stream: %"PRIu64", "
+              "sm_qflags: %d. stream_flags: %d, sm_bflags: %d, ", stream->id,
+              stream->sm_qflags, stream->stream_flags, stream->sm_bflags);
+
     if (stream->sm_qflags & SMQF_SEND_MAX_STREAM_DATA)
         r &= generate_max_stream_data_frame(conn, stream);
     if (stream->sm_qflags & SMQF_SEND_BLOCKED)
-        r &= lsquic_sendctl_gen_stream_blocked_frame(&conn->ifc_send_ctl, stream);
+    {
+        if (lsquic_stream_is_write_reset(stream))
+        {
+            stream->sm_qflags &= ~SMQF_SEND_BLOCKED;
+            if (!(stream->sm_qflags & SMQF_SENDING_FLAGS))
+                TAILQ_REMOVE(&stream->conn_pub->sending_streams, stream,
+                             next_send_stream);
+        }
+        else
+            r &= lsquic_sendctl_gen_stream_blocked_frame(&conn->ifc_send_ctl,
+                                                         stream);
+    }
     if (stream->sm_qflags & SMQF_SEND_RST)
         r &= generate_rst_stream_frame(conn, stream);
     if (stream->sm_qflags & SMQF_SEND_STOP_SENDING)
@@ -3001,6 +3037,7 @@ retire_dcid (struct ietf_full_conn *conn, struct dcid_elem **dce)
     if ((*dce)->de_hash_el.qhe_flags & QHE_HASHED)
         lsquic_hash_erase(conn->ifc_enpub->enp_srst_hash, &(*dce)->de_hash_el);
     TAILQ_INSERT_TAIL(&conn->ifc_to_retire, *dce, de_next_to_ret);
+    ++conn->ifc_n_to_retire;
     LSQ_DEBUG("prepare to retire DCID seqno %"PRIu32"", (*dce)->de_seqno);
     *dce = NULL;
     conn->ifc_send_flags |= SF_SEND_RETIRE_CID;
@@ -3018,6 +3055,7 @@ retire_seqno (struct ietf_full_conn *conn, unsigned seqno)
         memset(dce, 0, sizeof(*dce));
         dce->de_seqno = seqno;
         TAILQ_INSERT_TAIL(&conn->ifc_to_retire, dce, de_next_to_ret);
+        ++conn->ifc_n_to_retire;
         LSQ_DEBUG("prepare to retire DCID seqno %"PRIu32, seqno);
         conn->ifc_send_flags |= SF_SEND_RETIRE_CID;
     }
@@ -3149,6 +3187,7 @@ ietf_full_conn_ci_destroy (struct lsquic_conn *lconn)
     while ((dce = TAILQ_FIRST(&conn->ifc_to_retire)))
     {
         TAILQ_REMOVE(&conn->ifc_to_retire, dce, de_next_to_ret);
+        --conn->ifc_n_to_retire;
         lsquic_malo_put(dce);
     }
     lsquic_send_ctl_cleanup(&conn->ifc_send_ctl);
@@ -3350,6 +3389,7 @@ retire_cid_from_tp (struct ietf_full_conn *conn,
                                                     sizeof(dce->de_srst));
     dce->de_flags = DE_SRST;
     TAILQ_INSERT_TAIL(&conn->ifc_to_retire, dce, de_next_to_ret);
+    ++conn->ifc_n_to_retire;
     LSQ_DEBUG("prepare to retire DCID seqno %"PRIu32, dce->de_seqno);
     conn->ifc_send_flags |= SF_SEND_RETIRE_CID;
 }
@@ -7276,7 +7316,8 @@ on_dcid_change (struct ietf_full_conn *conn, const lsquic_cid_t *dcid_in)
     struct lsquic_conn *const lconn = &conn->ifc_conn;  /* Shorthand */
     struct conn_cid_elem *cce;
 
-    LSQ_DEBUG("peer switched its DCID, attempt to switch own SCID");
+    LSQ_DEBUGC("peer switched its DCID to %"CID_FMT
+              ", attempt to switch own SCID", CID_BITS(dcid_in));
 
     for (cce = lconn->cn_cces; cce < END_OF_CCES(lconn); ++cce)
         if (cce - lconn->cn_cces != lconn->cn_cur_cce_idx
@@ -7387,6 +7428,7 @@ process_regular_packet (struct ietf_full_conn *conn,
     enum was_missing was_missing;
     int is_rechist_empty;
     unsigned char saved_path_id;
+    int is_dcid_changed;
 
     if (HETY_RETRY == packet_in->pi_header_type)
         return process_retry_packet(conn, packet_in);
@@ -7489,16 +7531,31 @@ process_regular_packet (struct ietf_full_conn *conn,
         }
     }
 
+    is_dcid_changed = !LSQUIC_CIDS_EQ(CN_SCID(&conn->ifc_conn),
+                                        &packet_in->pi_dcid);
     if (pns == PNS_INIT)
         conn->ifc_conn.cn_esf.i->esfi_set_iscid(conn->ifc_conn.cn_enc_session,
                                                                     packet_in);
-    else if (pns == PNS_HSK)
+    else
     {
-        if ((conn->ifc_flags & (IFC_SERVER | IFC_IGNORE_INIT)) == IFC_SERVER)
-            ignore_init(conn);
-        lsquic_send_ctl_maybe_calc_rough_rtt(&conn->ifc_send_ctl, pns - 1);
+        if (is_dcid_changed && HETY_0RTT != packet_in->pi_header_type)
+        {
+            if (LSQUIC_CIDS_EQ(&conn->ifc_conn.cn_cces[0].cce_cid,
+                            &packet_in->pi_dcid)
+                && !(conn->ifc_conn.cn_cces[0].cce_flags & CCE_SEQNO))
+            {
+                ABORT_QUIETLY(0, TEC_PROTOCOL_VIOLATION,
+                            "protocol violation detected bad dcid");
+                return -1;
+            }
+        }
+        if (pns == PNS_HSK)
+        {
+            if ((conn->ifc_flags & (IFC_SERVER | IFC_IGNORE_INIT)) == IFC_SERVER)
+                ignore_init(conn);
+            lsquic_send_ctl_maybe_calc_rough_rtt(&conn->ifc_send_ctl, pns - 1);
+        }
     }
-
     EV_LOG_PACKET_IN(LSQUIC_LOG_CONN_ID, packet_in);
 
     is_rechist_empty = lsquic_rechist_is_empty(&conn->ifc_rechist[pns]);
@@ -7522,8 +7579,9 @@ process_regular_packet (struct ietf_full_conn *conn,
                                                     << packet_in->pi_path_id);
                 }
             }
-            else if (!LSQUIC_CIDS_EQ(CN_SCID(&conn->ifc_conn),
-                                                    &packet_in->pi_dcid))
+            else if (is_dcid_changed
+                && !LSQUIC_CIDS_EQ(CN_SCID(&conn->ifc_conn),
+                                   &packet_in->pi_dcid))
             {
                 if (0 != on_dcid_change(conn, &packet_in->pi_dcid))
                     return -1;
@@ -7898,7 +7956,7 @@ ietf_full_conn_ci_packet_sent (struct lsquic_conn *lconn,
     struct ietf_full_conn *const conn = (struct ietf_full_conn *) lconn;
     int s;
 
-    if (packet_out->po_frame_types & (IQUIC_FRAME_RETX_MASK | QUIC_FTBIT_ACK))
+    if (packet_out->po_frame_types & (IQUIC_FRAME_RETX_MASK))
         conn->ifc_n_cons_unretx = 0;
     else
         ++conn->ifc_n_cons_unretx;
@@ -8531,7 +8589,8 @@ ietf_full_conn_ci_tick (struct lsquic_conn *lconn, lsquic_time_t now)
                     || 0 != lsquic_send_ctl_n_scheduled(&conn->ifc_send_ctl))
                 )
         {
-            RETURN_IF_OUT_OF_PACKETS();
+            /* CONNECTION_CLOSE frame should not be congestion controlled.
+            RETURN_IF_OUT_OF_PACKETS(); */
             generate_connection_close_packet(conn);
             tick |= TICK_SEND|TICK_CLOSE;
         }

src/liblsquic/lsquic_parse_common.c

@@ -232,10 +232,12 @@ lsquic_dcid_from_packet (const unsigned char *buf, size_t bufsz,
     {
     /* Xs vary, Gs are iGnored: */
     /* 1X11 XGGG: */
+    /*
     case (0x80|0x40|0x20|0x10|0x08) >> 3:
     case (0x80|0x00|0x20|0x10|0x08) >> 3:
     case (0x80|0x40|0x20|0x10|0x00) >> 3:
     case (0x80|0x00|0x20|0x10|0x00) >> 3:
+    */
   Q046_long:
         /* lsquic_Q046_parse_packet_in_long_begin */
         if (bufsz < 14)

src/liblsquic/lsquic_send_ctl.c

@@ -1606,7 +1606,7 @@ lsquic_send_ctl_cleanup (lsquic_send_ctl_t *ctl)
         send_ctl_destroy_packet(ctl, packet_out);
     }
     assert(0 == ctl->sc_n_scheduled);
-    assert(0 == ctl->sc_bytes_scheduled);
+    //assert(0 == ctl->sc_bytes_scheduled);
     for (pns = PNS_INIT; pns < N_PNS; ++pns)
         while ((packet_out = TAILQ_FIRST(&ctl->sc_unacked_packets[pns])))
         {
@@ -1704,16 +1704,23 @@ lsquic_send_ctl_pacer_blocked (struct lsquic_send_ctl *ctl)
 static int
 send_ctl_can_send (struct lsquic_send_ctl *ctl)
 {
+    uint64_t cwnd = ctl->sc_ci->cci_get_cwnd(CGP(ctl));
     const unsigned n_out = send_ctl_all_bytes_out(ctl);
-    LSQ_DEBUG("%s: n_out: %u (unacked_all: %u); cwnd: %"PRIu64
-        "; ccfc: %"PRIu64"/%"PRIu64, __func__,
-        n_out, ctl->sc_bytes_unacked_all,
-        ctl->sc_ci->cci_get_cwnd(CGP(ctl)),
+    LSQ_DEBUG("%s: sc_flags: 0x%X, b_out: %u = (%u + %u); b_retx: %u; cwnd: %"PRIu64
+        "; ccfc: %"PRIu64"/%"PRIu64"; n_scheduled: %d, n_in_flight_all: %d"
+        "; pa_burst: %d; pa_next: %lu; pa_now: %lu"
+        , __func__, ctl->sc_flags,
+        n_out, ctl->sc_bytes_unacked_all, ctl->sc_bytes_scheduled,
+        ctl->sc_bytes_unacked_retx, cwnd,
         ctl->sc_conn_pub->conn_cap.cc_sent,
-        ctl->sc_conn_pub->conn_cap.cc_max);
+        ctl->sc_conn_pub->conn_cap.cc_max,
+        ctl->sc_n_scheduled, ctl->sc_n_in_flight_all,
+        ctl->sc_pacer.pa_burst_tokens,
+        ctl->sc_pacer.pa_next_sched, ctl->sc_pacer.pa_now);
+
     if (ctl->sc_flags & SC_PACE)
     {
-        if (n_out >= ctl->sc_ci->cci_get_cwnd(CGP(ctl)))
+        if (n_out >= cwnd)
             return 0;
         if (lsquic_pacer_can_schedule(&ctl->sc_pacer,
                                ctl->sc_n_scheduled + ctl->sc_n_in_flight_all))
@@ -1728,7 +1735,7 @@ send_ctl_can_send (struct lsquic_send_ctl *ctl)
         return 0;
     }
     else
-        return n_out < ctl->sc_ci->cci_get_cwnd(CGP(ctl));
+        return n_out < cwnd;
 }
 
 
@@ -2976,13 +2983,23 @@ lsquic_sendctl_gen_stream_blocked_frame (struct lsquic_send_ctl *ctl,
     unsigned need;
     uint64_t off;
     int sz;
+    int is_err;
 
     off = lsquic_stream_combined_send_off(stream);
     need = pf->pf_stream_blocked_frame_size(stream->id, off);
     packet_out = lsquic_send_ctl_get_packet_for_stream(ctl, need,
                        stream->conn_pub->path, stream);
     if (!packet_out)
-        return 0;
+    {
+        LSQ_DEBUG("failed to get packet_out with lsquic_send_ctl_get_packet_for_stream");
+        packet_out = lsquic_send_ctl_get_writeable_packet(ctl,
+                            PNS_APP, need, stream->conn_pub->path, 0, &is_err);
+        if (!packet_out)
+        {
+            LSQ_DEBUG("cannot get writeable packet for STREAM_BLOCKED frame");
+            return 0;
+        }
+    }
     sz = pf->pf_gen_stream_blocked_frame(
                          packet_out->po_data + packet_out->po_data_sz,
                          lsquic_packet_out_avail(packet_out), stream->id, off);

src/liblsquic/lsquic_send_ctl.h

@@ -285,32 +285,41 @@ lsquic_send_ctl_ack_to_front (struct lsquic_send_ctl *, unsigned n_acks);
 #define lsquic_send_ctl_n_stop_waiting(ctl) \
                                     (+(ctl)->sc_n_stop_waiting)
 
-#define lsquic_send_ctl_n_stop_waiting_reset(ctl) do {      \
-    (ctl)->sc_n_stop_waiting = 0;                           \
-} while (0)
+static inline void
+lsquic_send_ctl_n_stop_waiting_reset(lsquic_send_ctl_t *ctl)
+{
+    ctl->sc_n_stop_waiting = 0;
+}
 
 void
 lsquic_send_ctl_drop_scheduled (lsquic_send_ctl_t *);
 
-#define lsquic_send_ctl_tick_in(ctl, now) do {              \
-    if ((ctl)->sc_flags & SC_PACE)                          \
-    {                                                       \
-        (ctl)->sc_flags |= SC_SCHED_TICK;                   \
-        lsquic_pacer_tick_in(&(ctl)->sc_pacer, now);        \
-    }                                                       \
-    (ctl)->sc_flags &= ~SC_APP_LIMITED;                     \
-} while (0)
+static inline void
+lsquic_send_ctl_tick_in(lsquic_send_ctl_t *ctl, lsquic_time_t now)
+{
+    if ((ctl)->sc_flags & SC_PACE)
+    {
+        (ctl)->sc_flags |= SC_SCHED_TICK;
+        lsquic_pacer_tick_in(&(ctl)->sc_pacer, now);
+    }
+    (ctl)->sc_flags &= ~SC_APP_LIMITED;
+}
 
-#define lsquic_send_ctl_tick_out(ctl) do {                  \
-    if ((ctl)->sc_flags & SC_PACE)                          \
-        lsquic_pacer_tick_out(&(ctl)->sc_pacer);            \
-} while (0)
+static inline void
+lsquic_send_ctl_tick_out(lsquic_send_ctl_t *ctl)
+{
+    if ((ctl)->sc_flags & SC_PACE)
+        lsquic_pacer_tick_out(&(ctl)->sc_pacer);
+}
 
-#define lsquic_send_ctl_next_pacer_time(ctl) (              \
-    ((ctl)->sc_flags & SC_PACE)                             \
-        && lsquic_pacer_delayed(&(ctl)->sc_pacer)           \
-        ? lsquic_pacer_next_sched(&(ctl)->sc_pacer)         \
-        : 0 )
+static inline lsquic_time_t
+lsquic_send_ctl_next_pacer_time(lsquic_send_ctl_t *ctl)
+{
+    return ((ctl)->sc_flags & SC_PACE)
+            && lsquic_pacer_delayed(&(ctl)->sc_pacer)
+            ? lsquic_pacer_next_sched(&(ctl)->sc_pacer)
+            : 0;
+}
 
 enum packno_bits
 lsquic_send_ctl_packno_bits (struct lsquic_send_ctl *, enum packnum_space);

src/liblsquic/lsquic_stream.c

@@ -1334,6 +1334,15 @@ lsquic_stream_stop_sending_in (struct lsquic_stream *stream,
                                     && !(stream->sm_qflags & SMQF_SEND_RST))
         stream_reset(stream, 0, 0);
 
+    if (stream->sm_qflags & (SMQF_SEND_WUF | SMQF_SEND_BLOCKED \
+                             | SMQF_SEND_STOP_SENDING))
+    {
+        stream->sm_qflags &= ~(SMQF_SEND_WUF | SMQF_SEND_BLOCKED \
+                               | SMQF_SEND_STOP_SENDING);
+        if (!(stream->sm_qflags & SMQF_SENDING_FLAGS))
+            TAILQ_REMOVE(&stream->conn_pub->sending_streams, stream, next_send_stream);
+    }
+
     maybe_finish_stream(stream);
     maybe_schedule_call_on_close(stream);
 }
@@ -1879,6 +1888,11 @@ stream_shutdown_write (lsquic_stream_t *stream)
             {
                 LSQ_DEBUG("turned on FIN flag in the yet-unsent STREAM frame");
                 stream->stream_flags |= STREAM_FIN_SENT;
+                if (stream->sm_qflags & SMQF_WANT_FLUSH)
+                {
+                    LSQ_DEBUG("turned off SMQF_WANT_FLUSH flag as FIN flag is turned on.");
+                    maybe_remove_from_write_q(stream, SMQF_WANT_FLUSH);
+                }
             }
             else
             {
@@ -2437,6 +2451,12 @@ lsquic_stream_dispatch_write_events (lsquic_stream_t *stream)
     else
         stream_dispatch_write_events_loop(stream);
 
+    if ((stream->sm_qflags & SMQF_SEND_BLOCKED) &&
+        (stream->sm_bflags & SMBF_IETF))
+    {
+        lsquic_sendctl_gen_stream_blocked_frame(stream->conn_pub->send_ctl, stream);
+    }
+
     /* Progress means either flags or offsets changed: */
     progress = !((stream->sm_qflags & SMQF_WRITE_Q_FLAGS) == q_flags &&
                         stream->tosend_off == tosend_off &&
@@ -3444,6 +3464,11 @@ stream_write_to_packets (lsquic_stream_t *stream, struct lsquic_reader *reader,
                 if (use_framing && seen_ok)
                     maybe_close_varsize_hq_frame(stream);
                 stream->stream_flags |= STREAM_FIN_SENT;
+                if (stream->sm_qflags & SMQF_WANT_FLUSH)
+                {
+                    LSQ_DEBUG("turned off SMQF_WANT_FLUSH flag as FIN has been sent.");
+                    maybe_remove_from_write_q(stream, SMQF_WANT_FLUSH);
+                }
                 goto end;
             }
             else