Update README.md

After some time without activity (only keepalive), the ping frame was send in loop.

.cirrus.yml

@@ -1,5 +1,5 @@
 freebsd_instance:
-    image: freebsd-12-1-release-amd64
+    image: freebsd-13-2-release-amd64
 
 task:
     install_script:

.readthedocs.yaml

@@ -0,0 +1,35 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+    # You can also specify other tool versions:
+    # nodejs: "20"
+    # rust: "1.70"
+    # golang: "1.20"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+  # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
+  # builder: "dirhtml"
+  # Fail on all warnings to avoid broken references
+  # fail_on_warning: true
+
+# Optionally build your docs in additional formats such as PDF and ePub
+# formats:
+#   - pdf
+#   - epub
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+    - requirements: docs/requirements.txt

CHANGELOG

@@ -1,3 +1,52 @@
+2024-03-12
+    - 4.0.8
+    - Fix RETIRE_CONNECTION_ID frame abuse.
+    - Fix some assert failures.
+
+2024-02-28
+    - 4.0.7
+    - Fix overly strict 0-RTT packet DCID validation.
+    - Update docker build.
+
+2024-02-23
+    - 4.0.6
+    - Fix ACK handling.
+    - Do not apply congestion control to CONNECTION_CLOSE frame.
+
+2024-02-07
+    - 4.0.5
+    - Fix CPU spinning due to STREAM_BLOCKED frame.
+
+2024-01-08
+    - 4.0.4
+    - Fix DCID validation.
+    - Fix CPU spinning due to pending STREAM_BLOCKED frame.
+
+2023-12-25
+    - 4.0.3
+    - Fix session resumption bug introduced in 4.0.2.
+
+2023-12-19
+    - 4.0.2
+    - More strict RFC compliance.
+    - Minimize packet generated when send STREAM_BLOCKED frame
+    - Fix memory management bugs.
+    - Fix memory leak in unit tests.
+
+2023-05-14
+    - 4.0.1
+    - Fix send_ctl bug.
+    - Make logid consistent
+    - Reduce memory usage for storing CIDs.
+
+2023-03-14
+    - 4.0.0
+    - Add support for QUICv2 (RFC9369).
+    - Add support for version negotiation (RFC9368).
+    - Retry packet for address validation.
+    - Improve handshake under high packet loss/corruption.
+    - Improve QUIC interop results
+
 2023-01-26
     - 3.3.1
     - Fix blocked header encoding stream due to connection flow control congestion.

CMakeLists.txt

@@ -106,6 +106,7 @@ SET(MY_CMAKE_FLAGS "${MY_CMAKE_FLAGS} /wd4245")	# conversion from 'int' to 'unsi
 SET(MY_CMAKE_FLAGS "${MY_CMAKE_FLAGS} /wd4267")	# integer conversion
 SET(MY_CMAKE_FLAGS "${MY_CMAKE_FLAGS} /wd4214")	# nonstandard extension used: bit field types other than int
 SET(MY_CMAKE_FLAGS "${MY_CMAKE_FLAGS} /wd4295")	# array is too small to include a terminating null character
+SET(MY_CMAKE_FLAGS "${MY_CMAKE_FLAGS} /wd4324")	# structure was padded due to alignment specifier
 SET(MY_CMAKE_FLAGS "${MY_CMAKE_FLAGS} /wd4334")	# result of 32-bit shift implicitly converted to 64 bits (was 64-bit shift intended?)
 SET(MY_CMAKE_FLAGS "${MY_CMAKE_FLAGS} /wd4456")	# hide previous local declaration
 SET(MY_CMAKE_FLAGS "${MY_CMAKE_FLAGS} /wd4459")	# hide global declaration
@@ -271,7 +272,8 @@ ENDIF()
 SET(LIBS lsquic ${BORINGSSL_LIB_ssl} ${BORINGSSL_LIB_crypto} ${ZLIB_LIB} ${LIBS})
 
 IF (LSQUIC_BIN)
-    FIND_PATH(EVENT_INCLUDE_DIR NAMES event2/event.h)
+    FIND_PATH(EVENT_INCLUDE_DIR NAMES event2/event.h
+              PATHS ${PROJECT_SOURCE_DIR}/../third-party/include)
     IF (EVENT_INCLUDE_DIR)
         INCLUDE_DIRECTORIES(${EVENT_INCLUDE_DIR})
     ELSE()
@@ -284,7 +286,8 @@ IF (LSQUIC_BIN)
     IF (CMAKE_SYSTEM_NAME STREQUAL Windows)
         FIND_LIBRARY(EVENT_LIB event)
     ELSE()
-        FIND_LIBRARY(EVENT_LIB libevent${LIB_SUFFIX})
+        FIND_LIBRARY(EVENT_LIB libevent${LIB_SUFFIX}
+                    PATHS ${PROJECT_SOURCE_DIR}/../third-party/lib)
         IF(NOT EVENT_LIB)
             FIND_LIBRARY(EVENT_LIB libevent.so)
         ENDIF()

Dockerfile

@@ -1,13 +1,17 @@
-FROM ubuntu:16.04
+FROM ubuntu:20.04 as build-lsquic
+
+ENV DEBIAN_FRONTEND noninteractive
 
 RUN apt-get update && \
-    apt-get install -y build-essential git cmake software-properties-common \
+    apt-get install -y apt-utils build-essential git cmake software-properties-common \
                        zlib1g-dev libevent-dev
 
-RUN add-apt-repository ppa:gophers/archive && \
+RUN add-apt-repository ppa:longsleep/golang-backports && \
     apt-get update && \
-    apt-get install -y golang-1.9-go && \
-    cp /usr/lib/go-1.9/bin/go* /usr/bin/.
+    apt-get install -y golang-1.21-go && \
+    cp /usr/lib/go-1.21/bin/go* /usr/bin/.
+
+ENV GOROOT /usr/lib/go-1.21
 
 RUN mkdir /src
 WORKDIR /src
@@ -15,14 +19,21 @@ WORKDIR /src
 RUN mkdir /src/lsquic
 COPY ./ /src/lsquic/
 
-RUN git clone https://boringssl.googlesource.com/boringssl && \
+RUN git clone https://github.com/google/boringssl.git && \
     cd boringssl && \
-    git checkout a2278d4d2cabe73f6663e3299ea7808edfa306b9 && \
+    git checkout 9fc1c33e9c21439ce5f87855a6591a9324e569fd && \
     cmake . && \
     make
 
+ENV EXTRA_CFLAGS -DLSQUIC_QIR=1
 RUN cd /src/lsquic && \
     cmake -DBORINGSSL_DIR=/src/boringssl . && \
     make
 
-RUN cd lsquic && make test && cp bin/http_client /usr/bin/ && cp bin/http_server /usr/bin
+RUN cd lsquic && cp bin/http_client /usr/bin/ && cp bin/http_server /usr/bin
+
+FROM martenseemann/quic-network-simulator-endpoint:latest as lsquic-qir
+COPY --from=build-lsquic /usr/bin/http_client /usr/bin/http_server /usr/bin/
+COPY qir/run_endpoint.sh .
+RUN chmod +x run_endpoint.sh
+ENTRYPOINT [ "./run_endpoint.sh" ]

README.md

@@ -14,9 +14,32 @@ and HTTP/3 functionality for servers and clients.  Most of the code in this
 distribution is used in our own products: LiteSpeed Web Server, LiteSpeed ADC,
 and OpenLiteSpeed.
 
-Currently supported QUIC versions are v1, Internet-Draft versions 29, and 27;
+Currently supported QUIC versions are v1, v2, Internet-Draft versions 29, and 27;
 and the older "Google" QUIC versions Q043, Q046, an Q050.
 
+Standard Compliance
+-------------------
+
+LiteSpeed QUIC is mostly compliant to the follow RFCs:
+
+- [RFC 9000](https://www.rfc-editor.org/rfc/rfc9000) QUIC: A UDP-Based Multiplexed and Secure Transport
+- [RFC 9001](https://www.rfc-editor.org/rfc/rfc9001) Using TLS to Secure QUIC
+- [RFC 9002](https://www.rfc-editor.org/rfc/rfc9002) QUIC Loss Detection and Congestion Control
+- [RFC 9114](https://www.rfc-editor.org/rfc/rfc9114) HTTP/3
+- [RFC 9204](https://www.rfc-editor.org/rfc/rfc9204) QPACK: Field Compression for HTTP/3
+
+QUIC protocol extensions
+------------------------
+
+The following QUIC protocol extensions are implemented:
+
+- [RFC 9368](https://www.rfc-editor.org/rfc/rfc9368) Compatible Version Negotiation for QUIC
+- [RFC 9369](https://www.rfc-editor.org/rfc/rfc9369) QUIC Version 2
+- [RFC 9218](https://www.rfc-editor.org/rfc/rfc9218) Extensible Prioritization Scheme for HTTP
+- [RFC 9221](https://www.rfc-editor.org/rfc/rfc9221) An Unreliable Datagram Extension to QUIC
+- [RFC 9287](https://www.rfc-editor.org/rfc/rfc9287) Greasing the QUIC Bit
+- [ACK Frequency](https://datatracker.ietf.org/doc/draft-ietf-quic-ack-frequency/)
+
 Documentation
 -------------
 
@@ -49,7 +72,7 @@ You may need to install pre-requisites like zlib and libevent.
 2. Use specific BoringSSL version
 
 ```
-git checkout 31bad2514d21f6207f3925ba56754611c462a873
+git checkout 9fc1c33e9c21439ce5f87855a6591a9324e569fd
 ```
 Or, just try the latest master branch.
 
@@ -89,8 +112,7 @@ as follows:
 ```
 git clone https://github.com/litespeedtech/lsquic.git
 cd lsquic
-git submodule init
-git submodule update
+git submodule update --init
 ```
 
 2. Compile the library
@@ -125,8 +147,7 @@ The library and the example client and server can be built with Docker.
 Initialize Git submodules:
 ```
 cd lsquic
-git submodule init
-git submodule update
+git submodule update --init
 ```
 
 Build the Docker image:
@@ -136,7 +157,7 @@ docker build -t lsquic .
 
 Then you can use the examples from the command line.  For example:
 ```
-sudo docker run -it --rm lsquic http_client -s www.google.com  -p / -o version=h3-29
+sudo docker run -it --rm lsquic http_client -s www.google.com  -p / -o version=h3
 sudo docker run -p 12345:12345/udp -v /path/to/certs:/mnt/certs -it --rm lsquic http_server -c www.example.com,/mnt/certs/chain,/mnt/certs/key
 ```
 

bin/http_client.c

@@ -348,7 +348,10 @@ http_client_on_conn_closed (lsquic_conn_t *conn)
     if (conn_h->client_ctx->hcc_flags & HCC_ABORT_ON_INCOMPLETE)
     {
         if (!(conn_h->client_ctx->hcc_flags & HCC_SEEN_FIN))
-            abort();
+        {
+            LSQ_INFO("abort incomplete connection");
+            exit(1);
+        }
     }
     --conn_h->client_ctx->hcc_n_open_conns;
 
@@ -1011,6 +1014,7 @@ usage (const char *prog)
 "                 content-type: application/octet-stream and\n"
 "                 content-length\n"
 "   -K          Discard server response\n"
+"   -B          Header bypass\n"
 "   -I          Abort on incomplete reponse from server\n"
 "   -4          Prefer IPv4 when resolving hostname\n"
 "   -6          Prefer IPv6 when resolving hostname\n"

bin/prog.c

@@ -171,6 +171,11 @@ prog_print_common_options (const struct prog *prog, FILE *out)
 "                   sndbuf=12345    # Sets SO_SNDBUF\n"
 "                   rcvbuf=12345    # Sets SO_RCVBUF\n"
 "   -W          Use stock PMI (malloc & free)\n"
+"   -A CC_ALGO  Congestion control algorithm.  The following algorithms are\n"
+"                 supported.\n"
+"                   1: Cubic\n"
+"                   2: BBRv1\n"
+"                   3: Adaptive congestion control (this is the default).\n"
     );
 
 #if HAVE_SENDMMSG
@@ -266,6 +271,9 @@ prog_set_opt (struct prog *prog, int opt, const char *arg)
     case 'W':
         prog->prog_use_stock_pmi = 1;
         return 0;
+    case 'A':
+        prog->prog_settings.es_cc_algo = atoi(optarg);
+        return 0;
     case 'c':
         if (prog->prog_engine_flags & LSENG_SERVER)
         {

bin/prog.h

@@ -74,7 +74,7 @@ prog_init (struct prog *, unsigned lsquic_engine_flags, struct sport_head *,
 #   define IP_DONTFRAG_FLAG ""
 #endif
 
-#define PROG_OPTS "i:km:c:y:L:l:o:H:s:S:Y:z:G:W" RECVMMSG_FLAG SENDMMSG_FLAG \
+#define PROG_OPTS "i:km:c:y:L:l:o:H:s:S:Y:z:G:WA:" RECVMMSG_FLAG SENDMMSG_FLAG \
                                                             IP_DONTFRAG_FLAG
 
 /* Returns:

bin/test_common.c

@@ -804,7 +804,7 @@ sport_init_server (struct service_port *sport, struct lsquic_engine *engine,
 #ifndef WIN32
     int flags;
 #endif
-    SOCKOPT_VAL on;
+    SOCKOPT_VAL on = 1;
     socklen_t socklen;
     char addr_str[0x20];
 
@@ -828,6 +828,14 @@ sport_init_server (struct service_port *sport, struct lsquic_engine *engine,
     if (-1 == sockfd)
         return -1;
 
+    if (AF_INET6 == sa_local->sa_family
+        && setsockopt(sockfd, IPPROTO_IPV6, IPV6_V6ONLY,
+                      CHAR_CAST &on, sizeof(on)) == -1)
+    {
+        close(sockfd);
+        return -1;
+    }
+
     if (0 != bind(sockfd, sa_local, socklen)) {
         saved_errno = errno;
         LSQ_WARN("bind failed: %s", strerror(errno));
@@ -977,9 +985,21 @@ sport_init_server (struct service_port *sport, struct lsquic_engine *engine,
 #if ECN_SUPPORTED
     on = 1;
     if (AF_INET == sa_local->sa_family)
-        s = setsockopt(sockfd, IPPROTO_IP, IP_RECVTOS, CHAR_CAST &on, sizeof(on));
+    {
+        s = setsockopt(sockfd, IPPROTO_IP, IP_RECVTOS,
+                       CHAR_CAST &on, sizeof(on));
+        if (!s)
+            s = setsockopt(sockfd, IPPROTO_IP, IP_TOS,
+                           CHAR_CAST &on, sizeof(on));
+    }
     else
-        s = setsockopt(sockfd, IPPROTO_IPV6, IPV6_RECVTCLASS, CHAR_CAST &on, sizeof(on));
+    {
+        s = setsockopt(sockfd, IPPROTO_IPV6, IPV6_RECVTCLASS,
+                       CHAR_CAST &on, sizeof(on));
+        if (!s)
+            s = setsockopt(sockfd, IPPROTO_IPV6, IPV6_TCLASS,
+                           CHAR_CAST &on, sizeof(on));
+    }
     if (0 != s)
     {
         saved_errno = errno;
@@ -987,6 +1007,8 @@ sport_init_server (struct service_port *sport, struct lsquic_engine *engine,
         errno = saved_errno;
         return -1;
     }
+    LSQ_DEBUG("server ECN support is enabled.");
+
 #endif
 
     if (sport->sp_flags & SPORT_SET_SNDBUF)
@@ -1177,11 +1199,21 @@ sport_init_client (struct service_port *sport, struct lsquic_engine *engine,
     {
         int on = 1;
         if (AF_INET == sa_local->sa_family)
+        {
             s = setsockopt(sockfd, IPPROTO_IP, IP_RECVTOS,
-                                            CHAR_CAST &on, sizeof(on));
+                        CHAR_CAST &on, sizeof(on));
+            if (!s)
+                s = setsockopt(sockfd, IPPROTO_IP, IP_TOS,
+                            CHAR_CAST &on, sizeof(on));
+        }
         else
+        {
             s = setsockopt(sockfd, IPPROTO_IPV6, IPV6_RECVTCLASS,
-                                            CHAR_CAST &on, sizeof(on));
+                        CHAR_CAST &on, sizeof(on));
+            if (!s)
+                s = setsockopt(sockfd, IPPROTO_IPV6, IPV6_TCLASS,
+                            CHAR_CAST &on, sizeof(on));
+        }
         if (0 != s)
         {
             saved_errno = errno;
@@ -1189,6 +1221,7 @@ sport_init_client (struct service_port *sport, struct lsquic_engine *engine,
             errno = saved_errno;
             return -1;
         }
+        LSQ_DEBUG("client ECN support is enabled.");
     }
 #endif
 
@@ -1796,6 +1829,11 @@ set_engine_option (struct lsquic_engine_settings *settings,
             settings->es_spin = atoi(val);
             return 0;
         }
+        if (0 == strncmp(name, "srej", 4))
+        {
+            settings->es_support_srej = atoi(val);
+            return 0;
+        }
         break;
     case 7:
         if (0 == strncmp(name, "version", 7))
@@ -1933,6 +1971,11 @@ set_engine_option (struct lsquic_engine_settings *settings,
             settings->es_handshake_to = atoi(val);
             return 0;
         }
+        if (0 == strncmp(name, "support_srej", 12))
+        {
+            settings->es_support_srej = atoi(val);
+            return 0;
+        }
         if (0 == strncmp(name, "delayed_acks", 12))
         {
             settings->es_delayed_acks = atoi(val);

docs/conf.py

@@ -24,9 +24,9 @@ copyright = u'2023, LiteSpeed Technologies'
 author = u'LiteSpeed Technologies'
 
 # The short X.Y version
-version = u'3.3'
+version = u'4.0'
 # The full version, including alpha/beta/rc tags
-release = u'3.3.1'
+release = u'4.0.8'
 
 
 # -- General configuration ---------------------------------------------------
@@ -41,6 +41,7 @@ release = u'3.3.1'
 extensions = [
 # To make ours look like readthedocs.io, change theme to "sphinx_rtd_theme",
 # pip install sphinx_rtd_theme, and uncomment extensions:
+    "sphinx_rtd_theme",
 #    "sphinx.ext.intersphinx",
 #    "sphinx.ext.autodoc",
 #    "sphinx.ext.mathjax",
@@ -83,8 +84,8 @@ primary_domain = 'c'
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-#html_theme = 'alabaster'
-html_style = '/default.css'
+html_theme = "sphinx_rtd_theme"
+#html_style = '/default.css'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the

docs/requirements.txt

@@ -0,0 +1 @@
+sphinx-rtd-theme

include/lsquic.h

@@ -25,9 +25,9 @@ struct sockaddr;
 extern "C" {
 #endif
 
-#define LSQUIC_MAJOR_VERSION 3
-#define LSQUIC_MINOR_VERSION 3
-#define LSQUIC_PATCH_VERSION 1
+#define LSQUIC_MAJOR_VERSION 4
+#define LSQUIC_MINOR_VERSION 0
+#define LSQUIC_PATCH_VERSION 8
 
 /**
  * Engine flags:
@@ -82,12 +82,23 @@ enum lsquic_version
     LSQVER_I001,
 
     /**
-     * Special version to trigger version negotiation.
-     * [draft-ietf-quic-transport-11], Section 3.
+     * IETF QUIC v2.
      */
-    LSQVER_VERNEG,
+    LSQVER_I002,
 
-    N_LSQVER
+    /**
+     * Reserved version to trigger version negotiation.
+     * [rfc9000], Section 15.
+     */
+    LSQVER_RESVED,
+
+    N_LSQVER,
+
+    /**
+     *  The version 0x00000000 is reserved to represent version negotiation.
+     * [rfc9000], Section 15.
+     */
+    LSQVER_VERNEG
 };
 
 /**
@@ -103,19 +114,19 @@ enum lsquic_version
 #define LSQUIC_FORCED_TCID0_VERSIONS ((1 << LSQVER_046)|(1 << LSQVER_050))
 
 #define LSQUIC_EXPERIMENTAL_VERSIONS ( \
-                            (1 << LSQVER_VERNEG))
+                            (1 << LSQVER_RESVED))
 
 #define LSQUIC_DEPRECATED_VERSIONS ((1 << LSQVER_ID27))
 
 #define LSQUIC_GQUIC_HEADER_VERSIONS (1 << LSQVER_043)
 
 #define LSQUIC_IETF_VERSIONS ((1 << LSQVER_ID27) \
-                          | (1 << LSQVER_ID29) \
-                          | (1 << LSQVER_I001) | (1 << LSQVER_VERNEG))
+                          | (1 << LSQVER_ID29) | (1 << LSQVER_I001) \
+                          | (1 << LSQVER_I002) | (1 << LSQVER_RESVED))
 
 #define LSQUIC_IETF_DRAFT_VERSIONS ((1 << LSQVER_ID27) \
                                   | (1 << LSQVER_ID29) \
-                                  | (1 << LSQVER_VERNEG))
+                                  | (1 << LSQVER_RESVED))
 
 enum lsquic_hsk_status
 {
@@ -306,6 +317,10 @@ typedef struct ssl_ctx_st * (*lsquic_lookup_cert_f)(
 
 #define LSQUIC_DF_STTL               86400
 #define LSQUIC_DF_MAX_INCHOATE     (1 * 1000 * 1000)
+
+#define LSQUIC_DF_SUPPORT_SREJ_SERVER  1
+#define LSQUIC_DF_SUPPORT_SREJ_CLIENT  0
+
 /** Do not use NSTP by default */
 #define LSQUIC_DF_SUPPORT_NSTP     0
 /** TODO: IETF QUIC clients do not support push */
@@ -355,6 +370,9 @@ typedef struct ssl_ctx_st * (*lsquic_lookup_cert_f)(
 /** Allow migration by default */
 #define LSQUIC_DF_ALLOW_MIGRATION 1
 
+/** Default retry token duration. */    /* Do not set this value to zero. */
+#define LSQUIC_DF_RETRY_TOKEN_DURATION 10
+
 /** Use QL loss bits by default */
 #define LSQUIC_DF_QL_BITS 2
 
@@ -544,6 +562,17 @@ struct lsquic_engine_settings {
      */
     unsigned        es_max_inchoate;
 
+    /**
+     * Support SREJ: for client side, this means supporting server's SREJ
+     * responses (this does not work yet) and for server side, this means
+     * generating SREJ instead of REJ when appropriate.
+     *
+     * For IETF QUIC, this sending stateless retries when appropriate.
+     * The IETF client always supports stateless retries and knows how to
+     * handle them.
+     */
+    int             es_support_srej;
+
     /**
      * Setting this value to 0 means that
      *
@@ -863,6 +892,13 @@ struct lsquic_engine_settings {
      */
     int             es_allow_migration;
 
+    /**
+     * Amount of time, in seconds, after which the server token included in
+     * a stateless retry expires.  If set to zero, the default value is
+     * used, which is @ref LSQUIC_DF_RETRY_TOKEN_DURATION
+     */
+     unsigned       es_retry_token_duration;
+
     /**
      * Use QL loss bits.  Allowed values are:
      *  0:  Do not use loss bits

include/lsquic_types.h

@@ -13,23 +13,28 @@
 #define MAX_CID_LEN 20
 #define GQUIC_CID_LEN 8
 
+#if defined(_MSC_VER)
+#define ALIGNED_(x) __declspec(align(x))
+#else
+#if defined(__GNUC__)
+#define ALIGNED_(x) __attribute__ ((aligned(x)))
+#endif
+#endif
+
 /**
  * Connection ID
  */
 typedef struct lsquic_cid
 {
-    uint_fast8_t    len;
-    union {
-        uint8_t     buf[MAX_CID_LEN];
-        uint64_t    id;
-    }               u_cid;
-#define idbuf u_cid.buf
-}
+    uint8_t     buf[MAX_CID_LEN];
+#define idbuf buf
+    uint_fast8_t len;
+} ALIGNED_(8)
 lsquic_cid_t;
 
 
 #define LSQUIC_CIDS_EQ(a, b) ((a)->len == 8 ? \
-    (b)->len == 8 && (a)->u_cid.id == (b)->u_cid.id : \
+    (b)->len == 8 && *(uint64_t *)((a)->buf) == *(uint64_t *)((b)->buf) : \
     (a)->len == (b)->len && 0 == memcmp((a)->idbuf, (b)->idbuf, (a)->len))
 
 /** Stream ID */

include/lsxpack_header.h

@@ -1,6 +1,6 @@
 /* Copyright (c) 2017 - 2022 LiteSpeed Technologies Inc.  See LICENSE. */
-#ifndef LSXPACK_HEADER_H_v207
-#define LSXPACK_HEADER_H_v207
+#ifndef LSXPACK_HEADER_H_v208
+#define LSXPACK_HEADER_H_v208
 
 #ifdef __cplusplus
 extern "C" {
@@ -14,6 +14,7 @@ extern "C" {
 #define LSXPACK_MAX_STRLEN UINT16_MAX
 #endif
 
+typedef int32_t lsxpack_offset_t;
 #if LSXPACK_MAX_STRLEN == UINT16_MAX
 typedef uint16_t lsxpack_strlen_t;
 #elif LSXPACK_MAX_STRLEN == UINT32_MAX
@@ -49,9 +50,9 @@ struct lsxpack_header
     char             *buf;          /* the buffer for headers */
     uint32_t          name_hash;    /* hash value for name */
     uint32_t          nameval_hash; /* hash value for name + value */
-    lsxpack_strlen_t  name_offset;  /* the offset for name in the buffer */
+    lsxpack_offset_t  name_offset;  /* the offset for name in the buffer */
+    lsxpack_offset_t  val_offset;   /* the offset for value in the buffer */
     lsxpack_strlen_t  name_len;     /* the length of name */
-    lsxpack_strlen_t  val_offset;   /* the offset for value in the buffer */
     lsxpack_strlen_t  val_len;      /* the length of value */
     uint16_t          chain_next_idx; /* mainly for cookie value chain */
     uint8_t           hpack_index;  /* HPACK static table index */
@@ -99,11 +100,11 @@ lsxpack_header_set_offset(lsxpack_header_t *hdr, const char *buf,
 {
     memset(hdr, 0, sizeof(*hdr));
     hdr->buf = (char *)buf;
-    hdr->name_offset = (lsxpack_strlen_t)name_offset;
+    hdr->name_offset = (lsxpack_offset_t)name_offset;
     assert(name_len <= LSXPACK_MAX_STRLEN);
     hdr->name_len = (lsxpack_strlen_t)name_len;
     assert(name_offset + name_len + 2 <= LSXPACK_MAX_STRLEN);
-    hdr->val_offset = (lsxpack_strlen_t)(name_offset + name_len + 2);
+    hdr->val_offset = (lsxpack_offset_t)(name_offset + name_len + 2);
     assert(val_len <= LSXPACK_MAX_STRLEN);
     hdr->val_len = (lsxpack_strlen_t)val_len;
 }
@@ -116,11 +117,11 @@ lsxpack_header_set_offset2(lsxpack_header_t *hdr, const char *buf,
 {
     memset(hdr, 0, sizeof(*hdr));
     hdr->buf = (char *)buf;
-    hdr->name_offset = (lsxpack_strlen_t)name_offset;
+    hdr->name_offset = (lsxpack_offset_t)name_offset;
     assert(name_len <= LSXPACK_MAX_STRLEN);
     hdr->name_len = (lsxpack_strlen_t)name_len;
     assert(val_offset <= LSXPACK_MAX_STRLEN);
-    hdr->val_offset = (lsxpack_strlen_t)val_offset;
+    hdr->val_offset = (lsxpack_offset_t)val_offset;
     assert(val_len <= LSXPACK_MAX_STRLEN);
     hdr->val_len = (lsxpack_strlen_t)val_len;
 }
@@ -133,7 +134,7 @@ lsxpack_header_prepare_decode(lsxpack_header_t *hdr,
     memset(hdr, 0, sizeof(*hdr));
     hdr->buf = out;
     assert(offset <= LSXPACK_MAX_STRLEN);
-    hdr->name_offset = (lsxpack_strlen_t)offset;
+    hdr->name_offset = (lsxpack_offset_t)offset;
     if (len > LSXPACK_MAX_STRLEN)
         hdr->val_len = LSXPACK_MAX_STRLEN;
     else
@@ -166,4 +167,4 @@ lsxpack_header_mark_val_changed(lsxpack_header_t *hdr)
 }
 #endif
 
-#endif //LSXPACK_HEADER_H_v207
+#endif //LSXPACK_HEADER_H_v208

qir/run_endpoint.sh

@@ -0,0 +1,139 @@
+#!/bin/bash
+#
+# run_endpoint.sh -- QUIC Interop Runner script for lsquic
+#
+
+/setup.sh
+
+if [ "$ROLE" == "client" ]; then
+    # Wait for the simulator to start up.
+    /wait-for-it.sh sim:57832 -s -t 30
+fi
+
+echo TEST_PARAMS: $TEST_PARAMS
+echo REQUESTS: "'$REQUESTS'"
+eval $(perl <<'PERL'
+    @paths = split /\s+/, $ENV{REQUESTS};
+    s~^https?://[^/]+~-p ~ for @paths;
+    print "PATHS='@paths'\n";
+    $server = $ENV{REQUESTS};
+    $server =~ s~^https?://~~;
+    $server =~ s~/.*~~;
+    ($server, $port) = split /:/, $server;
+    print "SERVER=$server\n";
+    print "PORT=$port\n";
+    print "N_REQS=", scalar(@paths), "\n";
+    print "N_reqs=", scalar(@paths), "\n";
+    if (@paths > 100) {
+        print "W=100\n";
+    } else {
+        print "W=1\n";
+    }
+PERL
+)
+echo paths: $PATHS
+echo server: $SERVER
+echo port: $PORT
+
+# lsquic command-line tools create one file per connection when -G option
+# is used.  Here we make a copy and give it required name.
+#
+function maybe_create_keylog() {
+    local NAME=/logs/keys.log
+    if ls /logs/*.keys; then
+        # There may be more than one of these, as one file is created per
+        # connection.
+        cat /logs/*.keys > $NAME
+    fi
+    if [ -f $NAME ]; then
+        echo $NAME exists
+    else
+        echo $NAME does not exit
+    fi
+}
+
+if [ "$ROLE" = server ]; then
+    if [ ! -z "$TESTCASE" ]; then
+        case "$TESTCASE" in
+            http3)
+                VERSIONS='-o version=h3-29 -o version=h3'
+                ;;
+            v2)
+                VERSIONS='-o version=h3-v2 -o version=h3 -Q hq-interop'
+                ;;
+            handshake|transfer|longrtt|resumption|blackhole|multiconnect|chacha20|zerortt)
+                VERSIONS='-o version=h3-29 -o version=h3 -o scid_iss_rate=0 -Q hq-interop'
+                ;;
+            retry)
+                VERSIONS='-o version=h3-29 -o version=h3 -o srej=1 -Q hq-interop'
+                FORCE_RETRY=1
+                ;;
+            ecn)
+                VERSIONS='-o version=h3-29 -o version=h3 -Q hq-interop'
+                ECN='-o ecn=1'
+                ;;
+            *) exit 127 ;;
+        esac
+    fi
+    echo SERVER_PARAMS: $SERVER_PARAMS
+    exec env LSQUIC_FORCE_RETRY=$FORCE_RETRY /usr/bin/http_server $VERSIONS $ECN \
+        -c server,/certs/cert.pem,/certs/priv.key \
+        -c server4,/certs/cert.pem,/certs/priv.key \
+        -c server6,/certs/cert.pem,/certs/priv.key \
+        -c server46,/certs/cert.pem,/certs/priv.key \
+        -s ::0:443 -s 0.0.0.0:443 -s 193.167.100.100:12345 \
+        -r /www -L debug 2>/logs/$TESTCASE.out
+elif [ "$ROLE" = debug-server ]; then
+    exec /usr/bin/http_server $SERVER_PARAMS
+elif [ "$ROLE" = client ]; then
+    if [ ! -z "$TESTCASE" ]; then
+        case "$TESTCASE" in
+            http3)
+                VERSIONS='-o version=h3'
+                ;;
+            v2)
+                VERSIONS='-o version=h3-v2 -o version=h3 -Q hq-interop'
+                ;;
+            handshake|transfer|longrtt|retry|multiplexing|blackhole)
+                VERSIONS='-o version=h3 -Q hq-interop'
+                ;;
+            multiconnect)
+                VERSIONS='-o version=h3 -Q hq-interop'
+                N_REQS=1
+                ;;
+            ecn)
+                VERSIONS='-o version=h3 -Q hq-interop'
+                ECN='-o ecn=1'
+                ;;
+            resumption)
+                VERSIONS='-o version=h3 -Q hq-interop'
+                RESUME='-0 /logs/resume.file'
+                ;;
+            *) exit 127 ;;
+        esac
+    fi
+    echo CLIENT_PARAMS: $CLIENT_PARAMS
+    if [ "$TESTCASE" = resumption ]; then
+        # Fetch first file:
+        /usr/bin/http_client $VERSIONS -s $SERVER:$PORT $PATHS \
+            -r 1 -R 1 $RESUME \
+            -B -7 /downloads -G /logs \
+            -L debug 2>/logs/$TESTCASE-req1.out || exit $?
+        PATHS=`echo "$PATHS" | sed 's~-p /[^ ]* ~~'`
+        N_REQS=1
+        N_reqs=1
+        W=1
+        echo "first request successful, new args: $N_REQS; $N_reqs; $PATHS"
+    fi
+    /usr/bin/http_client $VERSIONS -s $SERVER:$PORT $PATHS \
+        -r $N_reqs -R $N_REQS -w $W $ECN $RESUME \
+        -B -7 /downloads -G /logs \
+        -L debug 2>/logs/$TESTCASE.out
+    EXIT_CODE=$?
+    maybe_create_keylog
+    sync
+    exit $EXIT_CODE
+else
+    echo hi
+    exit 127
+fi

src/liblsquic/ls-qpack

@@ -1 +1 @@
-Subproject commit bbd707065f4547a1b5a25bbc1cf6c1963e86b8cf
+Subproject commit 521792711f2839f05565fd3dd617708cfb5b80ee

src/liblsquic/lsquic_conn.c

@@ -274,9 +274,7 @@ lsquic_conn_status (struct lsquic_conn *lconn, char *errbuf, size_t bufsz)
 const lsquic_cid_t *
 lsquic_conn_log_cid (const struct lsquic_conn *lconn)
 {
-    if (lconn->cn_if && lconn->cn_if->ci_get_log_cid)
-        return lconn->cn_if->ci_get_log_cid(lconn);
-    return CN_SCID(lconn);
+    return &lconn->cn_logid;
 }
 
 

src/liblsquic/lsquic_conn.h

@@ -40,7 +40,7 @@ enum lsquic_conn_flags {
     LSCONN_HASHED         = (1 << 2),
     LSCONN_MINI           = (1 << 3),   /* This is a mini connection */
     LSCONN_IMMED_CLOSE    = (1 << 4),
-    LSCONN_UNUSED_5       = (1 << 5),
+    LSCONN_PROMOTE_FAIL   = (1 << 5),
     LSCONN_HANDSHAKE_DONE = (1 << 6),
     LSCONN_CLOSING        = (1 << 7),
     LSCONN_PEER_GOING_AWAY= (1 << 8),
@@ -60,6 +60,8 @@ enum lsquic_conn_flags {
     LSCONN_SERVER         = (1 <<22),
     LSCONN_IETF           = (1 <<23),
     LSCONN_RETRY_CONN     = (1 <<24),   /* This is a retry connection */
+    LSCONN_VER_UPDATED    = (1 <<25),
+    LSCONN_NO_BL          = (1 <<26),
 };
 
 /* A connection may have things to send and be closed at the same time.
@@ -68,6 +70,7 @@ enum tick_st {
     TICK_SEND    = (1 << 0),
     TICK_CLOSE   = (1 << 1),
     TICK_PROMOTE = (1 << 2), /* Promote mini connection to full connection */
+    TICK_RETRY   = (1 << 3), /* Send retry packet -- used by mini conns */
 };
 
 #define TICK_QUIET 0
@@ -252,9 +255,6 @@ struct conn_iface
     (*ci_record_addrs) (struct lsquic_conn *, void *peer_ctx,
         const struct sockaddr *local_sa, const struct sockaddr *peer_sa);
 
-    const lsquic_cid_t *
-    (*ci_get_log_cid) (const struct lsquic_conn *);
-
     /* Optional method.  Only used by the IETF client code. */
     void
     (*ci_drop_crypto_streams) (struct lsquic_conn *);
@@ -341,6 +341,7 @@ struct lsquic_conn
     const struct conn_iface     *cn_if;
     const struct parse_funcs    *cn_pf;
     struct attq_elem            *cn_attq_elem;
+    lsquic_cid_t                 cn_logid;
     lsquic_time_t                cn_last_sent;
     lsquic_time_t                cn_last_ticked;
     struct conn_cid_elem        *cn_cces;   /* At least one is available */

src/liblsquic/lsquic_di_hash.c

@@ -443,7 +443,6 @@ hash_di_insert_frame (struct data_in *data_in,
     if (ins == INS_FRAME_OK)
     {
         lsquic_packet_in_put(hdi->hdi_conn_pub->mm, new_frame->packet_in);
-        lsquic_malo_put(new_frame);
     }
     return ins;
 }

src/liblsquic/lsquic_enc_sess.h

@@ -39,10 +39,10 @@ enum lsquic_version;
 /* This enum maps to the list above */
 enum enc_level
 {
-    ENC_LEV_CLEAR,
-    ENC_LEV_EARLY,
     ENC_LEV_INIT,
-    ENC_LEV_FORW,
+    ENC_LEV_0RTT,
+    ENC_LEV_HSK,
+    ENC_LEV_APP,
     N_ENC_LEVS
 };
 
@@ -300,7 +300,9 @@ struct enc_session_funcs_iquic
                            void *(crypto_streams)[4],
                            const struct crypto_stream_if *,
                            const struct lsquic_cid *odcid,
-                           const struct lsquic_cid *iscid);
+                           const struct lsquic_cid *iscid,
+                           const struct lsquic_cid *rscid
+                            );
 
     void
     (*esfi_shake_stream)(enc_session_t *, struct lsquic_stream *,
@@ -340,10 +342,7 @@ struct enc_session_funcs_gquic lsquic_enc_session_gquic_gquic_1;
 LSQUIC_EXTERN const struct enc_session_funcs_iquic lsquic_enc_session_iquic_ietf_v1;
 
 #define select_esf_common_by_ver(ver) ( \
-    ver == LSQVER_ID27 ? &lsquic_enc_session_common_ietf_v1 : \
-    ver == LSQVER_ID29 ? &lsquic_enc_session_common_ietf_v1 : \
-    ver == LSQVER_I001 ? &lsquic_enc_session_common_ietf_v1 : \
-    ver == LSQVER_VERNEG ? &lsquic_enc_session_common_ietf_v1 : \
+    ver > LSQVER_050 ? &lsquic_enc_session_common_ietf_v1 : \
     ver == LSQVER_050 ? &lsquic_enc_session_common_gquic_2 : \
     &lsquic_enc_session_common_gquic_1 )
 
@@ -376,4 +375,16 @@ lsquic_enc_sess_ietf_gen_quic_ctx (
                 const struct lsquic_engine_settings *settings,
                 enum lsquic_version version, unsigned char *buf, size_t bufsz);
 
+struct enc_sess_iquic;
+int
+iquic_esfi_init_server_tp (struct enc_sess_iquic *const enc_sess);
+
+int
+iquic_esfi_switch_version (enc_session_t *enc_session_p, lsquic_cid_t *dcid,
+                           int backup_keys);
+
+int
+iquic_esf_is_enc_level_ready (enc_session_t *enc_session_p,
+                              enum enc_level level);
+
 #endif

src/liblsquic/lsquic_enc_sess_common.c

@@ -12,10 +12,10 @@
 
 const char *const lsquic_enclev2str[] =
 {
-    [ENC_LEV_EARLY] = "early",
-    [ENC_LEV_CLEAR] = "clear",
-    [ENC_LEV_INIT]  = "initial",
-    [ENC_LEV_FORW]  = "forw-secure",
+    [ENC_LEV_0RTT] = "0RTT",
+    [ENC_LEV_INIT] = "INIT",
+    [ENC_LEV_HSK]  = "HSK",
+    [ENC_LEV_APP]  = "APP",
 };
 
 

src/liblsquic/lsquic_enc_sess_ietf.c

@@ -58,13 +58,6 @@
 #define LSQUIC_LOG_CONN_ID lsquic_conn_log_cid(enc_sess->esi_conn)
 #include "lsquic_logger.h"
 
-#define KEY_LABEL "quic key"
-#define KEY_LABEL_SZ (sizeof(KEY_LABEL) - 1)
-#define IV_LABEL "quic iv"
-#define IV_LABEL_SZ (sizeof(IV_LABEL) - 1)
-#define PN_LABEL "quic hp"
-#define PN_LABEL_SZ (sizeof(PN_LABEL) - 1)
-
 #define N_HSK_PAIRS (N_ENC_LEVS - 1)
 
 static const struct alpn_map {
@@ -74,7 +67,8 @@ static const struct alpn_map {
     {   LSQVER_ID27, (unsigned char *) "\x05h3-27",     },
     {   LSQVER_ID29, (unsigned char *) "\x05h3-29",     },
     {   LSQVER_I001, (unsigned char *) "\x02h3",        },
-    {   LSQVER_VERNEG, (unsigned char *) "\x02h3",      },
+    {   LSQVER_I002, (unsigned char *) "\x02h3",        },
+    {   LSQVER_RESVED, (unsigned char *) "\x02h3",      },
 };
 
 struct enc_sess_iquic;
@@ -159,11 +153,36 @@ struct crypto_ctx_pair
 };
 
 
+struct hsk_crypto
+{
+    struct crypto_ctx_pair pair;
+    struct header_prot     hp;
+};
+
+
+struct label_set
+{
+    const char *key;
+    const char *iv;
+    const char *hp;
+    int key_len;
+    int iv_len;
+    int hp_len;
+};
+
+static struct label_set hkdf_labels[2] =
+{
+    {   "quic key", "quic iv", "quic hp", 8, 7, 7 },
+    {   "quicv2 key", "quicv2 iv", "quicv2 hp", 10, 9, 9 }
+};
+
+
 /* [draft-ietf-quic-tls-12] Section 5.3.6 */
 static int
 init_crypto_ctx (struct crypto_ctx *crypto_ctx, const EVP_MD *md,
                  const EVP_AEAD *aead, const unsigned char *secret,
-                 size_t secret_sz, enum evp_aead_direction_t dir)
+                 size_t secret_sz, enum evp_aead_direction_t dir,
+                 struct label_set *key_iv)
 {
     crypto_ctx->yk_key_sz = EVP_AEAD_key_length(aead);
     crypto_ctx->yk_iv_sz = EVP_AEAD_nonce_length(aead);
@@ -174,9 +193,9 @@ init_crypto_ctx (struct crypto_ctx *crypto_ctx, const EVP_MD *md,
         return -1;
     }
 
-    lsquic_qhkdf_expand(md, secret, secret_sz, KEY_LABEL, KEY_LABEL_SZ,
+    lsquic_qhkdf_expand(md, secret, secret_sz, key_iv->key, key_iv->key_len,
         crypto_ctx->yk_key_buf, crypto_ctx->yk_key_sz);
-    lsquic_qhkdf_expand(md, secret, secret_sz, IV_LABEL, IV_LABEL_SZ,
+    lsquic_qhkdf_expand(md, secret, secret_sz, key_iv->iv, key_iv->iv_len,
         crypto_ctx->yk_iv_buf, crypto_ctx->yk_iv_sz);
     if (!EVP_AEAD_CTX_init_with_direction(&crypto_ctx->yk_aead_ctx, aead,
             crypto_ctx->yk_key_buf, crypto_ctx->yk_key_sz, IQUIC_TAG_LEN, dir))
@@ -216,13 +235,9 @@ struct enc_sess_iquic
     struct header_prot   esi_hp;
     struct crypto_ctx_pair
                          esi_pairs[2];
-    /* These are used during handshake.  There are three of them.
-     * esi_hsk_pairs and esi_hsk_hps are allocated and freed
-     * together.
-     */
-    struct crypto_ctx_pair *
-                         esi_hsk_pairs;
-    struct header_prot  *esi_hsk_hps;
+    /* These are used during handshake.  There are three of them. */
+    struct hsk_crypto   *esi_hsk_crypto;
+    struct hsk_crypto   *esi_vn_save;
     lsquic_packno_t      esi_max_packno[N_PNS];
     lsquic_cid_t         esi_odcid;
     lsquic_cid_t         esi_rscid; /* Retry SCID */
@@ -250,6 +265,7 @@ struct enc_sess_iquic
         ESI_MAX_PACKNO_HSK  = ESI_MAX_PACKNO_INIT << PNS_HSK,
         ESI_MAX_PACKNO_APP  = ESI_MAX_PACKNO_INIT << PNS_APP,
         ESI_HAVE_0RTT_TP = 1 << 20,
+        ESI_SWITCH_VER   = 1 << 21,
     }                    esi_flags;
     enum enc_level       esi_last_w;
     unsigned             esi_trasec_sz;
@@ -496,6 +512,20 @@ strip_hp (struct enc_sess_iquic *enc_sess,
 }
 
 
+static void
+set_tp_version_info (struct transport_params *params,
+                     unsigned versions, enum lsquic_version ver)
+{
+    assert(params->tp_version_cnt == 0);
+    params->tp_version_info[params->tp_version_cnt++] = ver;
+    if (versions & (1 << LSQVER_I002))
+        params->tp_version_info[params->tp_version_cnt++] = LSQVER_I002;
+    if (versions & (1 << LSQVER_I001))
+        params->tp_version_info[params->tp_version_cnt++] = LSQVER_I001;
+    params->tp_set |= 1 << TPI_VERSION_INFORMATION;
+}
+
+
 static int
 gen_trans_params (struct enc_sess_iquic *enc_sess, unsigned char *buf,
                                                                 size_t bufsz)
@@ -525,6 +555,11 @@ gen_trans_params (struct enc_sess_iquic *enc_sess, unsigned char *buf,
             params.tp_original_dest_cid = enc_sess->esi_odcid;
             params.tp_set |= 1 << TPI_ORIGINAL_DEST_CID;
         }
+        if (enc_sess->esi_flags & ESI_RSCID)
+        {
+            params.tp_retry_source_cid = enc_sess->esi_rscid;
+            params.tp_set |= 1 << TPI_RETRY_SOURCE_CID;
+        }
 #if LSQUIC_PREFERRED_ADDR
         char addr_buf[INET6_ADDRSTRLEN + 6 /* port */ + 1];
         const char *s, *colon;
@@ -661,6 +696,14 @@ gen_trans_params (struct enc_sess_iquic *enc_sess, unsigned char *buf,
         params.tp_set |= 1 << TPI_MAX_DATAGRAM_FRAME_SIZE;
     }
 
+    if (enc_sess->esi_ver_neg && enc_sess->esi_ver_neg->vn_supp
+                                != (1u << enc_sess->esi_ver_neg->vn_ver))
+        set_tp_version_info(&params, enc_sess->esi_ver_neg->vn_supp,
+                            enc_sess->esi_ver_neg->vn_ver);
+    else if (enc_sess->esi_conn->cn_flags & LSCONN_VER_UPDATED)
+        set_tp_version_info(&params, settings->es_versions,
+                            enc_sess->esi_conn->cn_version);
+
     len = (version == LSQVER_ID27 ? lsquic_tp_encode_27 : lsquic_tp_encode)(
                         &params, enc_sess->esi_flags & ESI_SERVER, buf, bufsz);
     if (len >= 0)
@@ -1018,7 +1061,8 @@ iquic_esfi_create_server (struct lsquic_engine_public *enpub,
                     void *(crypto_streams)[4],
                     const struct crypto_stream_if *cryst_if,
                     const struct lsquic_cid *odcid,
-                    const struct lsquic_cid *iscid)
+                    const struct lsquic_cid *iscid,
+                    const struct lsquic_cid *rscid)
 {
     struct enc_sess_iquic *enc_sess;
 
@@ -1042,6 +1086,11 @@ iquic_esfi_create_server (struct lsquic_engine_public *enpub,
         enc_sess->esi_odcid = *odcid;
         enc_sess->esi_flags |= ESI_ODCID;
     }
+    if (rscid)
+    {
+        enc_sess->esi_rscid = *rscid;
+        enc_sess->esi_flags |= ESI_RSCID;
+    }
     enc_sess->esi_iscid = *iscid;
     enc_sess->esi_flags |= ESI_ISCID;
 
@@ -1099,6 +1148,18 @@ log_crypto_pair (const struct enc_sess_iquic *enc_sess,
 }
 
 
+static const unsigned char *const lsquic_ver2salt[N_LSQVER] = {
+    [LSQVER_043] = HSK_SALT_PRE29,
+    [LSQVER_046] = HSK_SALT_PRE29,
+    [LSQVER_050] = HSK_SALT_PRE29,
+    [LSQVER_ID27] = HSK_SALT_PRE29,
+    [LSQVER_ID29] = HSK_SALT_PRE33,
+    [LSQVER_I001] = HSK_SALT,
+    [LSQVER_I002] = HSK_SALT_V2,
+    [LSQVER_RESVED] = HSK_SALT,
+};
+
+
 /* [draft-ietf-quic-tls-12] Section 5.3.2 */
 static int
 setup_handshake_keys (struct enc_sess_iquic *enc_sess, const lsquic_cid_t *cid)
@@ -1118,30 +1179,20 @@ setup_handshake_keys (struct enc_sess_iquic *enc_sess, const lsquic_cid_t *cid)
     unsigned char secret[2][SHA256_DIGEST_LENGTH];  /* client, server */
     unsigned char key[2][EVP_MAX_KEY_LENGTH];
     char hexbuf[EVP_MAX_MD_SIZE * 2 + 1];
+    struct label_set *labels;
 
-    if (!enc_sess->esi_hsk_pairs)
+    if (!enc_sess->esi_hsk_crypto)
     {
-        enc_sess->esi_hsk_pairs = calloc(N_HSK_PAIRS,
-                                            sizeof(enc_sess->esi_hsk_pairs[0]));
-        enc_sess->esi_hsk_hps = calloc(N_HSK_PAIRS,
-                                            sizeof(enc_sess->esi_hsk_hps[0]));
-        if (!(enc_sess->esi_hsk_pairs && enc_sess->esi_hsk_hps))
-        {
-            free(enc_sess->esi_hsk_pairs);
-            free(enc_sess->esi_hsk_hps);
+        enc_sess->esi_hsk_crypto = calloc(N_HSK_PAIRS,
+                                          sizeof(enc_sess->esi_hsk_crypto[0]));
+        if (!enc_sess->esi_hsk_crypto)
             return -1;
-        }
     }
-    pair = &enc_sess->esi_hsk_pairs[ENC_LEV_CLEAR];
+    pair = &enc_sess->esi_hsk_crypto[ENC_LEV_INIT].pair;
     pair->ykp_thresh = IQUIC_INVALID_PACKNO;
-    hp = &enc_sess->esi_hsk_hps[ENC_LEV_CLEAR];
+    hp = &enc_sess->esi_hsk_crypto[ENC_LEV_INIT].hp;
 
-    if (enc_sess->esi_conn->cn_version < LSQVER_ID29)
-        salt = HSK_SALT_PRE29;
-    else if (enc_sess->esi_conn->cn_version < LSQVER_I001)
-        salt = HSK_SALT_PRE33;
-    else
-        salt = HSK_SALT;
+    salt = lsquic_ver2salt[enc_sess->esi_conn->cn_version];
     HKDF_extract(hsk_secret, &hsk_secret_sz, md, cid->idbuf, cid->len,
                     salt, HSK_SALT_SZ);
     if (enc_sess->esi_flags & ESI_LOG_SECRETS)
@@ -1163,21 +1214,22 @@ setup_handshake_keys (struct enc_sess_iquic *enc_sess, const lsquic_cid_t *cid)
             HEXSTR(secret[1], sizeof(secret[1]), hexbuf));
     }
 
+    labels = &hkdf_labels[enc_sess->esi_conn->cn_version == LSQVER_I002];
     cliser = !!(enc_sess->esi_flags & ESI_SERVER);
     if (0 != init_crypto_ctx(&pair->ykp_ctx[!cliser], md, aead, secret[0],
-                sizeof(secret[0]), rw2dir(!cliser)))
+                sizeof(secret[0]), rw2dir(!cliser), labels))
         goto err;
     if (0 != init_crypto_ctx(&pair->ykp_ctx[cliser], md, aead, secret[1],
-                sizeof(secret[1]), rw2dir(cliser)))
+                sizeof(secret[1]), rw2dir(cliser), labels))
         goto err;
 
     hp->hp_gen_mask = gen_hp_mask_aes;
-    hp->hp_enc_level = ENC_LEV_CLEAR;
+    hp->hp_enc_level = ENC_LEV_INIT;
     key_len = EVP_AEAD_key_length(aead);
-    lsquic_qhkdf_expand(md, secret[!cliser], sizeof(secret[0]), PN_LABEL,
-        PN_LABEL_SZ, key[0], key_len);
-    lsquic_qhkdf_expand(md, secret[cliser], sizeof(secret[0]), PN_LABEL,
-        PN_LABEL_SZ, key[1], key_len);
+    lsquic_qhkdf_expand(md, secret[!cliser], sizeof(secret[0]), labels->hp,
+        labels->hp_len, key[0], key_len);
+    lsquic_qhkdf_expand(md, secret[cliser], sizeof(secret[0]), labels->hp,
+        labels->hp_len, key[1], key_len);
     if (enc_sess->esi_flags & ESI_LOG_SECRETS)
     {
         log_crypto_pair(enc_sess, pair, "handshake");
@@ -1217,30 +1269,43 @@ cleanup_hp (struct header_prot *hp)
 }
 
 
+static void
+cleanup_hsk_crypto (struct hsk_crypto *c)
+{
+    cleanup_crypto_ctx(&c->pair.ykp_ctx[0]);
+    cleanup_crypto_ctx(&c->pair.ykp_ctx[1]);
+    cleanup_hp(&c->hp);
+}
+
+
+static void
+free_vn_save (struct enc_sess_iquic *enc_sess)
+{
+    if (enc_sess->esi_vn_save)
+    {
+        cleanup_hsk_crypto(enc_sess->esi_vn_save);
+        free(enc_sess->esi_vn_save);
+        enc_sess->esi_vn_save = NULL;
+    }
+}
+
+
 static void
 free_handshake_keys (struct enc_sess_iquic *enc_sess)
 {
-    struct crypto_ctx_pair *pair;
-    unsigned i;
+    struct hsk_crypto *c;
 
-    if (enc_sess->esi_hsk_pairs)
+    if (enc_sess->esi_hsk_crypto)
     {
-        assert(enc_sess->esi_hsk_hps);
-        for (pair = enc_sess->esi_hsk_pairs; pair <
-                enc_sess->esi_hsk_pairs + N_HSK_PAIRS; ++pair)
+        for (c = enc_sess->esi_hsk_crypto; c <
+                enc_sess->esi_hsk_crypto + N_HSK_PAIRS; ++c)
         {
-            cleanup_crypto_ctx(&pair->ykp_ctx[0]);
-            cleanup_crypto_ctx(&pair->ykp_ctx[1]);
+            cleanup_hsk_crypto(c);
         }
-        free(enc_sess->esi_hsk_pairs);
-        enc_sess->esi_hsk_pairs = NULL;
-        for (i = 0; i < N_HSK_PAIRS; ++i)
-            cleanup_hp(&enc_sess->esi_hsk_hps[i]);
-        free(enc_sess->esi_hsk_hps);
-        enc_sess->esi_hsk_hps = NULL;
+        free(enc_sess->esi_hsk_crypto);
+        enc_sess->esi_hsk_crypto = NULL;
     }
-    else
-        assert(!enc_sess->esi_hsk_hps);
+    free_vn_save(enc_sess);
 }
 
 
@@ -1342,6 +1407,33 @@ iquic_esf_set_conn (enc_session_t *enc_session_p, struct lsquic_conn *lconn)
 }
 
 
+int
+iquic_esfi_init_server_tp (struct enc_sess_iquic *const enc_sess)
+{
+    unsigned char trans_params[sizeof(struct transport_params)
+#if LSQUIC_TEST_QUANTUM_READINESS
+            + 4 + lsquic_tp_get_quantum_sz()
+#endif
+        ];
+    int transpa_len;
+    char errbuf[ERR_ERROR_STRING_BUF_LEN];
+    transpa_len = gen_trans_params(enc_sess, trans_params,
+                                   sizeof(trans_params));
+    if (transpa_len < 0)
+        return -1;
+
+    if (1 != SSL_set_quic_transport_params(enc_sess->esi_ssl, trans_params,
+                                                            transpa_len))
+    {
+        LSQ_ERROR("cannot set QUIC transport params: %s",
+            ERR_error_string(ERR_get_error(), errbuf));
+        return -1;
+    }
+
+    return 0;
+}
+
+
 static int
 iquic_esfi_init_server (enc_session_t *enc_session_p)
 {
@@ -1349,16 +1441,7 @@ iquic_esfi_init_server (enc_session_t *enc_session_p)
     struct network_path *path;
     const struct alpn_map *am;
     unsigned quic_ctx_idx;
-    int transpa_len;
     SSL_CTX *ssl_ctx = NULL;
-    union {
-        char errbuf[ERR_ERROR_STRING_BUF_LEN];
-        unsigned char trans_params[sizeof(struct transport_params)
-#if LSQUIC_TEST_QUANTUM_READINESS
-            + 4 + lsquic_tp_get_quantum_sz()
-#endif
-        ];
-    } u;
 
     if (enc_sess->esi_enpub->enp_alpn)
         enc_sess->esi_alpn = enc_sess->esi_enpub->enp_alpn;
@@ -1388,8 +1471,9 @@ iquic_esfi_init_server (enc_session_t *enc_session_p)
     enc_sess->esi_ssl = SSL_new(ssl_ctx);
     if (!enc_sess->esi_ssl)
     {
+        char errbuf[ERR_ERROR_STRING_BUF_LEN];
         LSQ_ERROR("cannot create SSL object: %s",
-            ERR_error_string(ERR_get_error(), u.errbuf));
+            ERR_error_string(ERR_get_error(), errbuf));
         return -1;
     }
 #if BORINGSSL_API_VERSION >= 13
@@ -1410,18 +1494,8 @@ iquic_esfi_init_server (enc_session_t *enc_session_p)
         return -1;
     }
 
-    transpa_len = gen_trans_params(enc_sess, u.trans_params,
-                                                    sizeof(u.trans_params));
-    if (transpa_len < 0)
-        return -1;
-
-    if (1 != SSL_set_quic_transport_params(enc_sess->esi_ssl, u.trans_params,
-                                                            transpa_len))
-    {
-        LSQ_ERROR("cannot set QUIC transport params: %s",
-            ERR_error_string(ERR_get_error(), u.errbuf));
-        return -1;
-    }
+//     if (iquic_esfi_init_server_tp(enc_sess) == -1)
+//         return -1;
 
     SSL_clear_options(enc_sess->esi_ssl, SSL_OP_NO_TLSv1_3);
     if (enc_sess->esi_enpub->enp_lookup_cert)
@@ -1786,6 +1860,7 @@ get_peer_transport_params (struct enc_sess_iquic *enc_sess)
                                                             cidbuf[0]),
                     CID_BITS_B(cids[TP_CID_IDX(tpi)], cidbuf[1]));
             }
+            enc_sess->esi_conn->cn_flags |= LSCONN_NO_BL;
             return -1;
         }
     }
@@ -2026,30 +2101,46 @@ iquic_esfi_destroy (enc_session_t *enc_session_p)
 /* See [draft-ietf-quic-tls-14], Section 4 */
 static const enum enc_level hety2el[] =
 {
-    [HETY_NOT_SET]   = ENC_LEV_FORW,
+    [HETY_SHORT]     = ENC_LEV_APP,
     [HETY_VERNEG]    = 0,
-    [HETY_INITIAL]   = ENC_LEV_CLEAR,
+    [HETY_INITIAL]   = ENC_LEV_INIT,
     [HETY_RETRY]     = 0,
-    [HETY_HANDSHAKE] = ENC_LEV_INIT,
-    [HETY_0RTT]      = ENC_LEV_EARLY,
+    [HETY_HANDSHAKE] = ENC_LEV_HSK,
+    [HETY_0RTT]      = ENC_LEV_0RTT,
 };
 
 
 static const enum enc_level pns2enc_level[2][N_PNS] =
 {
     [0] = {
-        [PNS_INIT]  = ENC_LEV_CLEAR,
-        [PNS_HSK]   = ENC_LEV_INIT,
-        [PNS_APP]   = ENC_LEV_EARLY,
+        [PNS_INIT]  = ENC_LEV_INIT,
+        [PNS_HSK]   = ENC_LEV_HSK,
+        [PNS_APP]   = ENC_LEV_0RTT,
     },
     [1] = {
-        [PNS_INIT]  = ENC_LEV_CLEAR,
-        [PNS_HSK]   = ENC_LEV_INIT,
-        [PNS_APP]   = ENC_LEV_FORW,
+        [PNS_INIT]  = ENC_LEV_INIT,
+        [PNS_HSK]   = ENC_LEV_HSK,
+        [PNS_APP]   = ENC_LEV_APP,
     },
 };
 
 
+int
+iquic_esf_is_enc_level_ready (enc_session_t *enc_session_p,
+                              enum enc_level level)
+{
+    const struct enc_sess_iquic *enc_sess = enc_session_p;
+    const struct header_prot *hp;
+    if (level == ENC_LEV_APP)
+        hp = &enc_sess->esi_hp;
+    else if (enc_sess->esi_hsk_crypto)
+        hp = &enc_sess->esi_hsk_crypto[level].hp;
+    else
+        return 0;
+    return header_prot_inited(hp, 0);
+}
+
+
 static enum enc_packout
 iquic_esf_encrypt_packet (enc_session_t *enc_session_p,
     const struct lsquic_engine_public *enpub, struct lsquic_conn *lconn_UNUSED,
@@ -2075,17 +2166,17 @@ iquic_esf_encrypt_packet (enc_session_t *enc_session_p,
     pns = lsquic_packet_out_pns(packet_out);
     enc_level = pns2enc_level[ enc_sess->esi_have_forw ][ pns ];
 
-    if (enc_level == ENC_LEV_FORW)
+    if (enc_level == ENC_LEV_APP)
     {
         pair = &enc_sess->esi_pairs[ enc_sess->esi_key_phase ];
         crypto_ctx = &pair->ykp_ctx[ 1 ];
         hp = &enc_sess->esi_hp;
     }
-    else if (enc_sess->esi_hsk_pairs)
+    else if (enc_sess->esi_hsk_crypto)
     {
-        pair = &enc_sess->esi_hsk_pairs[ enc_level ];
+        pair = &enc_sess->esi_hsk_crypto[ enc_level ].pair;
         crypto_ctx = &pair->ykp_ctx[ 1 ];
-        hp = &enc_sess->esi_hsk_hps[ enc_level ];
+        hp = &enc_sess->esi_hsk_crypto[ enc_level ].hp;
     }
     else
     {
@@ -2146,7 +2237,7 @@ iquic_esf_encrypt_packet (enc_session_t *enc_session_p,
                                             dst_sz, &packno_off, &packno_len);
     if (header_sz < 0)
         goto err;
-    if (enc_level == ENC_LEV_FORW)
+    if (enc_level == ENC_LEV_APP)
         dst[0] |= enc_sess->esi_key_phase << 2;
     dst[0] &= enc_sess->esi_grease | packet_out->po_path->np_dcid.idbuf[0];
 
@@ -2183,7 +2274,7 @@ iquic_esf_encrypt_packet (enc_session_t *enc_session_p,
     lsquic_packet_out_set_enc_level(packet_out, enc_level);
     lsquic_packet_out_set_kp(packet_out, enc_sess->esi_key_phase);
 
-    if (enc_level == ENC_LEV_FORW && hp->hp_gen_mask != gen_hp_mask_chacha20)
+    if (enc_level == ENC_LEV_APP && hp->hp_gen_mask != gen_hp_mask_chacha20)
         apply_hp_batch(enc_sess, hp, packet_out, packno_off, packno_len);
     else
         apply_hp_immediately(enc_sess, hp, packet_out, packno_off, packno_len);
@@ -2218,9 +2309,14 @@ static struct ku_label
 }
 
 
-select_ku_label (const struct enc_sess_iquic *enc_sess)
+select_ku_label (enum lsquic_version version)
 {
-    return (struct ku_label) { "quic ku", 7, };
+    static struct ku_label labels[2] =
+    {
+        { "quic ku", 7, },
+        { "quicv2 ku", 9, }
+    };
+    return labels[version == LSQVER_I002];
 }
 
 
@@ -2267,10 +2363,10 @@ iquic_esf_decrypt_packet (enc_session_t *enc_session_p,
     }
 
     enc_level = hety2el[packet_in->pi_header_type];
-    if (enc_level == ENC_LEV_FORW)
+    if (enc_level == ENC_LEV_APP)
         hp = &enc_sess->esi_hp;
-    else if (enc_sess->esi_hsk_pairs)
-        hp = &enc_sess->esi_hsk_hps[ enc_level ];
+    else if (enc_sess->esi_hsk_crypto)
+        hp = &enc_sess->esi_hsk_crypto[ enc_level ].hp;
     else
         hp = NULL;
 
@@ -2299,7 +2395,7 @@ iquic_esf_decrypt_packet (enc_session_t *enc_session_p,
         packet_in->pi_data + sample_off,
         dst, packet_in->pi_header_sz, &packno_len);
 
-    if (enc_level == ENC_LEV_FORW)
+    if (enc_level == ENC_LEV_APP)
     {
         key_phase = (dst[0] & 0x04) > 0;
         pair = &enc_sess->esi_pairs[ key_phase ];
@@ -2314,7 +2410,7 @@ iquic_esf_decrypt_packet (enc_session_t *enc_session_p,
                 || packet_in->pi_packno
                     > enc_sess->esi_pairs[enc_sess->esi_key_phase].ykp_thresh)
         {
-            const struct ku_label kl = select_ku_label(enc_sess);
+            const struct ku_label kl = select_ku_label(enc_sess->esi_conn->cn_version);
             lsquic_qhkdf_expand(enc_sess->esi_md,
                 enc_sess->esi_traffic_secrets[0], enc_sess->esi_trasec_sz,
                 kl.str, kl.len, new_secret, enc_sess->esi_trasec_sz);
@@ -2329,7 +2425,9 @@ iquic_esf_decrypt_packet (enc_session_t *enc_session_p,
             crypto_ctx->yk_flags = 0;
             s = init_crypto_ctx(crypto_ctx, enc_sess->esi_md,
                         enc_sess->esi_aead, new_secret, enc_sess->esi_trasec_sz,
-                        evp_aead_open);
+                        evp_aead_open,
+                        &hkdf_labels[enc_sess->esi_conn->cn_version == LSQVER_I002]
+                               );
             if (s != 0)
             {
                 LSQ_ERROR("could not init open crypto ctx (key phase)");
@@ -2352,8 +2450,8 @@ iquic_esf_decrypt_packet (enc_session_t *enc_session_p,
     else
     {
         key_phase = 0;
-        assert(enc_sess->esi_hsk_pairs);
-        pair = &enc_sess->esi_hsk_pairs[ enc_level ];
+        assert(enc_sess->esi_hsk_crypto);
+        pair = &enc_sess->esi_hsk_crypto[ enc_level ].pair;
         crypto_ctx = &pair->ykp_ctx[ 0 ];
         if (UNLIKELY(0 == (crypto_ctx->yk_flags & YK_INITED)))
         {
@@ -2411,7 +2509,7 @@ iquic_esf_decrypt_packet (enc_session_t *enc_session_p,
         if (dst[0] & 0x08)
             packet_in->pi_flags |= PI_LOSS_BIT;
     }
-    else if (dst[0] & (0x0C << (packet_in->pi_header_type == HETY_NOT_SET)))
+    else if (dst[0] & (0x0C << (packet_in->pi_header_type == HETY_SHORT)))
     {
         LSQ_DEBUG("reserved bits are not set to zero");
         dec_packin = DECPI_VIOLATION;
@@ -2422,7 +2520,7 @@ iquic_esf_decrypt_packet (enc_session_t *enc_session_p,
     {
         LSQ_DEBUG("decryption in the new key phase %u successful, rotate "
             "keys", key_phase);
-        const struct ku_label kl = select_ku_label(enc_sess);
+        const struct ku_label kl = select_ku_label(enc_sess->esi_conn->cn_version);
         pair->ykp_thresh = packet_in->pi_packno;
         pair->ykp_ctx[ 0 ] = crypto_ctx_buf;
         memcpy(enc_sess->esi_traffic_secrets[ 0 ], new_secret,
@@ -2434,7 +2532,8 @@ iquic_esf_decrypt_packet (enc_session_t *enc_session_p,
                                                 enc_sess->esi_trasec_sz);
         s = init_crypto_ctx(&pair->ykp_ctx[1], enc_sess->esi_md,
                     enc_sess->esi_aead, new_secret, enc_sess->esi_trasec_sz,
-                    evp_aead_seal);
+                    evp_aead_seal,
+                    &hkdf_labels[enc_sess->esi_conn->cn_version == LSQVER_I002]);
         if (s != 0)
         {
             LSQ_ERROR("could not init seal crypto ctx (key phase)");
@@ -2457,8 +2556,6 @@ iquic_esf_decrypt_packet (enc_session_t *enc_session_p,
     packet_in->pi_data = dst;
     packet_in->pi_flags |= PI_OWN_DATA | PI_DECRYPTED
                         | (enc_level << PIBIT_ENC_LEV_SHIFT);
-    EV_LOG_CONN_EVENT(LSQUIC_LOG_CONN_ID, "decrypted packet %"PRIu64,
-                                                    packet_in->pi_packno);
     pns = lsquic_enclev2pns[enc_level];
     if (packet_in->pi_packno > enc_sess->esi_max_packno[pns]
             || !(enc_sess->esi_flags & (ESI_MAX_PACKNO_INIT << pns)))
@@ -2622,23 +2719,58 @@ iquic_esfi_set_iscid (enc_session_t *enc_session_p,
 }
 
 
+int
+iquic_esfi_switch_version (enc_session_t *enc_session_p, lsquic_cid_t *dcid,
+                           int backup_keys)
+{
+    struct enc_sess_iquic *const enc_sess = enc_session_p;
+
+    enc_sess->esi_flags |= ESI_SWITCH_VER;
+
+    /* Free previous handshake keys */
+    assert(enc_sess->esi_hsk_crypto);
+    if (backup_keys)
+    {
+        if (!enc_sess->esi_vn_save)
+        {
+            enc_sess->esi_vn_save = calloc(1, sizeof(*enc_sess->esi_vn_save));
+            //skip error, non fatal
+        }
+        else
+            cleanup_hsk_crypto(enc_sess->esi_vn_save);
+        if (enc_sess->esi_vn_save)
+            *enc_sess->esi_vn_save = enc_sess->esi_hsk_crypto[ENC_LEV_INIT];
+    }
+    if (!enc_sess->esi_vn_save)
+        cleanup_hsk_crypto(&enc_sess->esi_hsk_crypto[ENC_LEV_INIT]);
+    else
+        memset(&enc_sess->esi_hsk_crypto[ENC_LEV_INIT], 0,
+               sizeof(enc_sess->esi_hsk_crypto[ENC_LEV_INIT]));
+
+    if (0 == setup_handshake_keys(enc_sess, dcid ? dcid : &enc_sess->esi_odcid))
+    {
+        LSQ_INFO("update handshake keys to version %s",
+                 lsquic_ver2str[enc_sess->esi_conn->cn_version]);
+        return 0;
+    }
+    else
+        return -1;
+}
+
+
 static int
 iquic_esfi_reset_dcid (enc_session_t *enc_session_p,
         const lsquic_cid_t *old_dcid, const lsquic_cid_t *new_dcid)
 {
     struct enc_sess_iquic *const enc_sess = enc_session_p;
-    struct crypto_ctx_pair *pair;
 
     enc_sess->esi_odcid = *old_dcid;
     enc_sess->esi_rscid = *new_dcid;
     enc_sess->esi_flags |= ESI_ODCID|ESI_RSCID|ESI_RETRY;
 
     /* Free previous handshake keys */
-    assert(enc_sess->esi_hsk_pairs);
-    pair = &enc_sess->esi_hsk_pairs[ENC_LEV_CLEAR];
-    cleanup_crypto_ctx(&pair->ykp_ctx[0]);
-    cleanup_crypto_ctx(&pair->ykp_ctx[1]);
-    cleanup_hp(&enc_sess->esi_hsk_hps[ENC_LEV_CLEAR]);
+    assert(enc_sess->esi_hsk_crypto);
+    cleanup_hsk_crypto(&enc_sess->esi_hsk_crypto[ENC_LEV_INIT]);
 
     if (0 == setup_handshake_keys(enc_sess, new_dcid))
     {
@@ -2812,7 +2944,7 @@ maybe_drop_SSL (struct enc_sess_iquic *enc_sess)
     if ((enc_sess->esi_flags & (ESI_HSK_CONFIRMED|ESI_HANDSHAKE_OK))
                             == (ESI_HSK_CONFIRMED|ESI_HANDSHAKE_OK)
         && enc_sess->esi_ssl
-        && lsquic_frab_list_empty(&enc_sess->esi_frals[ENC_LEV_FORW]))
+        && lsquic_frab_list_empty(&enc_sess->esi_frals[ENC_LEV_APP]))
     {
         if ((enc_sess->esi_flags & (ESI_SERVER|ESI_WANT_TICKET))
                                                             != ESI_WANT_TICKET)
@@ -2841,10 +2973,10 @@ no_sess_ticket (enum alarm_id alarm_id, void *ctx,
 
 
 typedef char enums_have_the_same_value[
-    (int) ssl_encryption_initial     == (int) ENC_LEV_CLEAR &&
-    (int) ssl_encryption_early_data  == (int) ENC_LEV_EARLY &&
-    (int) ssl_encryption_handshake   == (int) ENC_LEV_INIT  &&
-    (int) ssl_encryption_application == (int) ENC_LEV_FORW      ? 1 : -1];
+    (int) ssl_encryption_initial     == (int) ENC_LEV_INIT &&
+    (int) ssl_encryption_early_data  == (int) ENC_LEV_0RTT &&
+    (int) ssl_encryption_handshake   == (int) ENC_LEV_HSK  &&
+    (int) ssl_encryption_application == (int) ENC_LEV_APP      ? 1 : -1];
 
 static int
 set_secret (SSL *ssl, enum ssl_encryption_level_t level,
@@ -2862,6 +2994,7 @@ set_secret (SSL *ssl, enum ssl_encryption_level_t level,
     unsigned char key[EVP_MAX_KEY_LENGTH];
     char errbuf[ERR_ERROR_STRING_BUF_LEN];
 #define hexbuf errbuf
+    struct label_set *labels;
 
     enc_sess = SSL_get_ex_data(ssl, s_idx);
     if (!enc_sess)
@@ -2894,11 +3027,11 @@ set_secret (SSL *ssl, enum ssl_encryption_level_t level,
         secrets[0] = write_secret, secrets[1] = read_secret;
         */
 
-    if (enc_level < ENC_LEV_FORW)
+    if (enc_level < ENC_LEV_APP)
     {
-        assert(enc_sess->esi_hsk_pairs);
-        pair = &enc_sess->esi_hsk_pairs[enc_level];
-        hp = &enc_sess->esi_hsk_hps[enc_level];
+        assert(enc_sess->esi_hsk_crypto);
+        pair = &enc_sess->esi_hsk_crypto[enc_level].pair;
+        hp = &enc_sess->esi_hsk_crypto[enc_level].hp;
     }
     else
     {
@@ -2924,8 +3057,9 @@ set_secret (SSL *ssl, enum ssl_encryption_level_t level,
     else
         LSQ_DEBUG("set %s for level %u", rw2str[rw], enc_level);
 
+    labels = &hkdf_labels[enc_sess->esi_conn->cn_version == LSQVER_I002];
     if (0 != init_crypto_ctx(&pair->ykp_ctx[rw], crypa.md,
-                crypa.aead, secret, secret_len, rw2dir(rw)))
+                crypa.aead, secret, secret_len, rw2dir(rw), labels))
         goto err;
 
     if (pair->ykp_ctx[!rw].yk_flags & YK_INITED)
@@ -2943,8 +3077,8 @@ set_secret (SSL *ssl, enum ssl_encryption_level_t level,
     key_len = EVP_AEAD_key_length(crypa.aead);
     if (hp->hp_gen_mask == gen_hp_mask_aes)
     {
-        lsquic_qhkdf_expand(crypa.md, secret, secret_len, PN_LABEL, PN_LABEL_SZ,
-            key, key_len);
+        lsquic_qhkdf_expand(crypa.md, secret, secret_len, labels->hp,
+                            labels->hp_len, key, key_len);
         EVP_CIPHER_CTX_init(&hp->hp_u.cipher_ctx[rw]);
         if (!EVP_EncryptInit_ex(&hp->hp_u.cipher_ctx[rw], crypa.hp, NULL, key, 0))
         {
@@ -2953,8 +3087,8 @@ set_secret (SSL *ssl, enum ssl_encryption_level_t level,
         }
     }
     else
-        lsquic_qhkdf_expand(crypa.md, secret, secret_len, PN_LABEL, PN_LABEL_SZ,
-            hp->hp_u.buf[rw], key_len);
+        lsquic_qhkdf_expand(crypa.md, secret, secret_len, labels->hp,
+                            labels->hp_len, hp->hp_u.buf[rw], key_len);
     hp->hp_flags |= 1 << rw;
 
     if (enc_sess->esi_flags & ESI_LOG_SECRETS)
@@ -2965,7 +3099,7 @@ set_secret (SSL *ssl, enum ssl_encryption_level_t level,
             key_len, hexbuf));
     }
 
-    if (rw && enc_level == ENC_LEV_FORW)
+    if (rw && enc_level == ENC_LEV_APP)
         enc_sess->esi_have_forw = 1;
 
     return 1;
@@ -3106,7 +3240,7 @@ chsk_ietf_on_new_stream (void *stream_if_ctx, struct lsquic_stream *stream)
     enum enc_level enc_level;
 
     enc_level = enc_sess->esi_cryst_if->csi_enc_level(stream);
-    if (enc_level == ENC_LEV_CLEAR)
+    if (enc_level == ENC_LEV_INIT)
         enc_sess->esi_cryst_if->csi_wantwrite(stream, 1);
 
     LSQ_DEBUG("handshake stream created successfully");
@@ -3344,6 +3478,9 @@ const unsigned char *const lsquic_retry_key_buf[N_IETF_RETRY_VERSIONS] =
     /* [draft-ietf-quic-tls-33] Section 5.8 */
     (unsigned char *)
         "\xbe\x0c\x69\x0b\x9f\x66\x57\x5a\x1d\x76\x6b\x54\xe3\x68\xc8\x4e",
+    /* [draft-draft-ietf-quic-v2] Section 3.3.3 */
+    (unsigned char *)
+        "\x8f\xb4\xb0\x1b\x56\xac\x48\xe2\x60\xfb\xcb\xce\xad\x7c\xcc\x92",
 };
 
 
@@ -3355,6 +3492,8 @@ const unsigned char *const lsquic_retry_nonce_buf[N_IETF_RETRY_VERSIONS] =
     (unsigned char *) "\xe5\x49\x30\xf9\x7f\x21\x36\xf0\x53\x0a\x8c\x1c",
     /* [draft-ietf-quic-tls-33] Section 5.8 */
     (unsigned char *) "\x46\x15\x99\xd3\x5d\x63\x2b\xf2\x23\x98\x25\xbb",
+    /* [draft-draft-ietf-quic-v2] Section 3.3.3 */
+    (unsigned char *) "\xd8\x69\x69\xbc\x2d\x7c\x6d\x99\x90\xef\xb0\x4a",
 };
 
 

src/liblsquic/lsquic_engine.c

@@ -226,6 +226,7 @@ struct lsquic_engine
                                          */
         ENG_CONNS_BY_ADDR
                         = (1 <<  9),    /* Connections are hashed by address */
+        ENG_FORCE_RETRY = (1 << 10),    /* Will force retry packets to be sent */
 #ifndef NDEBUG
         ENG_COALESCE    = (1 << 24),    /* Packet coalescing is enabled */
 #endif
@@ -307,6 +308,7 @@ lsquic_engine_init_settings (struct lsquic_engine_settings *settings,
     {
         settings->es_cfcw        = LSQUIC_DF_CFCW_SERVER;
         settings->es_sfcw        = LSQUIC_DF_SFCW_SERVER;
+        settings->es_support_srej= LSQUIC_DF_SUPPORT_SREJ_SERVER;
         settings->es_init_max_data
                                  = LSQUIC_DF_INIT_MAX_DATA_SERVER;
         settings->es_init_max_stream_data_bidi_remote
@@ -325,6 +327,7 @@ lsquic_engine_init_settings (struct lsquic_engine_settings *settings,
     {
         settings->es_cfcw        = LSQUIC_DF_CFCW_CLIENT;
         settings->es_sfcw        = LSQUIC_DF_SFCW_CLIENT;
+        settings->es_support_srej= LSQUIC_DF_SUPPORT_SREJ_CLIENT;
         settings->es_init_max_data
                                  = LSQUIC_DF_INIT_MAX_DATA_CLIENT;
         settings->es_init_max_stream_data_bidi_remote
@@ -373,6 +376,7 @@ lsquic_engine_init_settings (struct lsquic_engine_settings *settings,
     settings->es_qpack_enc_max_size = LSQUIC_DF_QPACK_ENC_MAX_SIZE;
     settings->es_qpack_enc_max_blocked = LSQUIC_DF_QPACK_ENC_MAX_BLOCKED;
     settings->es_allow_migration = LSQUIC_DF_ALLOW_MIGRATION;
+    settings->es_retry_token_duration = LSQUIC_DF_RETRY_TOKEN_DURATION;
     settings->es_ql_bits         = LSQUIC_DF_QL_BITS;
     settings->es_spin            = LSQUIC_DF_SPIN;
     settings->es_delayed_acks    = LSQUIC_DF_DELAYED_ACKS;
@@ -756,7 +760,7 @@ lsquic_engine_new (unsigned flags,
     if (flags & LSENG_HTTP)
         engine->pub.enp_flags |= ENPUB_HTTP;
 
-#ifndef NDEBUG
+#if !defined(NDEBUG) || LSQUIC_QIR
     {
         const char *env;
         env = getenv("LSQUIC_LOSE_PACKETS_RE");
@@ -775,6 +779,15 @@ lsquic_engine_new (unsigned flags,
                                                                         env);
         }
 #endif
+        env = getenv("LSQUIC_FORCE_RETRY");
+        if (env)
+        {
+            if (atoi(env))
+            {
+                engine->flags |= ENG_FORCE_RETRY;
+                LSQ_WARN("will force retry");
+            }
+        }
         env = getenv("LSQUIC_COALESCE");
         if (env)
         {
@@ -908,7 +921,7 @@ destroy_conn (struct lsquic_engine *engine, struct lsquic_conn *conn,
     struct purga_el *puel;
 
     engine->mini_conns_count -= !!(conn->cn_flags & LSCONN_MINI);
-    if (engine->purga
+    if (engine->purga && !(conn->cn_flags & LSCONN_NO_BL)
         /* Blacklist all CIDs except for promoted mini connections */
             && (conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED))
                                         != (LSCONN_MINI|LSCONN_PROMOTED))
@@ -1139,12 +1152,56 @@ new_full_conn_server (lsquic_engine_t *engine, lsquic_conn_t *mini_conn,
         destroy_conn(engine, conn, now);
         return NULL;
     }
-    assert(!(conn->cn_flags & CONN_REF_FLAGS));
+    assert(!(conn->cn_flags & (CONN_REF_FLAGS & ~LSCONN_TICKABLE)));
     conn->cn_flags |= LSCONN_HASHED;
     return conn;
 }
 
 
+static void
+remove_conn_from_hash (lsquic_engine_t *engine, lsquic_conn_t *conn);
+
+
+static int
+promote_mini_conn (lsquic_engine_t *engine, lsquic_conn_t *mini_conn,
+                                                        lsquic_time_t now)
+{
+    lsquic_conn_t *new_conn;
+    EV_LOG_CONN_EVENT(lsquic_conn_log_cid( mini_conn ),
+                                        "promote to full conn");
+    assert( mini_conn->cn_flags & LSCONN_MINI);
+
+    lsquic_mini_conn_ietf_pre_promote((struct ietf_mini_conn *)mini_conn, now);
+
+    new_conn = new_full_conn_server(engine, mini_conn, now);
+    if (new_conn)
+    {
+        new_conn->cn_last_sent = engine->last_sent;
+        eng_hist_inc(&engine->history, now, sl_new_full_conns);
+        mini_conn->cn_flags |= LSCONN_PROMOTED;
+        assert(engine->curr_conn == mini_conn);
+        engine->curr_conn = new_conn;
+
+        if (mini_conn->cn_flags & LSCONN_ATTQ)
+        {
+            lsquic_attq_remove(engine->attq, mini_conn);
+            (void) engine_decref_conn(engine, mini_conn, LSCONN_ATTQ);
+        }
+        if (mini_conn->cn_flags & LSCONN_HASHED)
+            remove_conn_from_hash(engine, mini_conn);
+
+        if (!(new_conn->cn_flags & LSCONN_TICKABLE))
+        {
+            lsquic_mh_insert(&engine->conns_tickable, new_conn,
+                            new_conn->cn_last_ticked);
+            engine_incref_conn(new_conn, LSCONN_TICKABLE);
+        }
+        return 0;
+    }
+    return -1;
+}
+
+
 static enum
 {
     VER_NOT_SPECIFIED,
@@ -1203,6 +1260,38 @@ schedule_req_packet (struct lsquic_engine *engine, enum packet_req_type type,
 }
 
 
+static void
+schedule_mini_retry (struct lsquic_engine *engine, struct lsquic_conn *conn,
+                                                            lsquic_time_t now)
+{
+    const struct network_path *path;
+
+    assert(engine->pr_queue);
+    path = conn->cn_if->ci_get_path(conn, NULL);
+    if (!path)
+    {
+        LSQ_WARN("cannot fetch default path");
+        return;
+    }
+
+    assert(conn->cn_flags & LSCONN_IETF);
+    if (0 == lsquic_prq_new_req_ext(engine->pr_queue, PACKET_REQ_RETRY,
+                    0 /* Only supporting retry on IETF mini conns for now */,
+                    conn->cn_version, path->np_pack_size, &conn->cn_cid,
+                    &path->np_dcid, path->np_peer_ctx, NP_LOCAL_SA(path),
+                    NP_PEER_SA(path)
+                    ))
+        LSQ_DEBUGC("scheduled %s packet for mini conn %"CID_FMT,
+                        lsquic_preqt2str[PACKET_REQ_RETRY],
+                        CID_BITS(lsquic_conn_log_cid(conn)));
+    else
+        LSQ_DEBUGC("could not schedule %s packet for mini conn %"CID_FMT,
+                        lsquic_preqt2str[PACKET_REQ_RETRY],
+                        CID_BITS(lsquic_conn_log_cid(conn)));
+
+}
+
+
 static unsigned short
 sa2port (const struct sockaddr *sa)
 {
@@ -1423,8 +1512,60 @@ find_or_create_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in,
 
     if ((1 << version) & LSQUIC_IETF_VERSIONS)
     {
+        lsquic_cid_t odcid;
+        if (engine->pub.enp_settings.es_support_srej
+                                && HETY_INITIAL == packet_in->pi_header_type)
+        {
+            /* XXX Need to handle condition when packets are reordered? */
+            const int has_token
+                = packet_in->pi_token && packet_in->pi_token_size;
+            const int need_retry =
+                !!(engine->flags & ENG_FORCE_RETRY);
+            switch ((need_retry << 1) | has_token)
+            {
+            case (0 << 1) | 0:
+                odcid.len = 0;
+                goto create_ietf_mini_conn;
+            case (1 << 1) | 1:
+            case (0 << 1) | 1:
+                odcid.len = 0;
+                if (0 == lsquic_tg_validate_token(engine->pub.enp_tokgen,
+                                                packet_in, sa_peer, &odcid))
+                    goto create_ietf_mini_conn;
+        /* From [draft-ietf-quic-transport-30] Section 8.1.2:
+         " In response to processing an Initial containing a token that was
+         " provided in a Retry packet, a server cannot send another Retry
+         " packet; it can only refuse the connection or permit it to proceed.
+         */
+                if (TOKEN_RETRY == packet_in->pi_data[packet_in->pi_token])
+                {
+                    LSQ_DEBUGC("CID %"CID_FMT" has invalid Retry token",
+                                            CID_BITS(&packet_in->pi_conn_id));
+                    return NULL;
+                }
+                /* According to the spec, we SHOULD send CONNECTION_CLOSE
+                 * when receiving an invalid Retry token.  We don't do it
+                 * because it's a lot of code change for an event that is
+                 * not likely to happen: a major browser copying the token
+                 * incorrectly.
+                 */
+                break;
+            default:
+                assert(0);
+                /* fall-through */
+            case (1 << 1) | 0:
+                break;
+            }
+            schedule_req_packet(engine, PACKET_REQ_RETRY, packet_in, sa_local,
+                                                            sa_peer, peer_ctx);
+            return NULL;
+        }
+        else
+            odcid.len = 0;
+  create_ietf_mini_conn:
         conn = lsquic_mini_conn_ietf_new(&engine->pub, packet_in, version,
-                    sa_peer->sa_family == AF_INET, NULL, packet_in_size);
+                    sa_peer->sa_family == AF_INET, odcid.len ? &odcid : NULL,
+                    packet_in_size);
     }
     else
     {
@@ -1621,6 +1762,12 @@ process_packet_in (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in,
 #endif
     QLOG_PACKET_RX(lsquic_conn_log_cid(conn), packet_in, packet_in_data, packet_in_size);
     lsquic_packet_in_put(&engine->pub.enp_mm, packet_in);
+    if ((conn->cn_flags & (LSCONN_MINI | LSCONN_HANDSHAKE_DONE | LSCONN_IETF))
+                    == (LSCONN_MINI | LSCONN_HANDSHAKE_DONE | LSCONN_IETF))
+    {
+        if (promote_mini_conn(engine, conn, lsquic_time_now()) == -1)
+            conn->cn_flags |= LSCONN_PROMOTE_FAIL;
+    }
     return 0;
 }
 
@@ -1900,7 +2047,8 @@ engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag)
     assert(flag & CONN_REF_FLAGS);
     assert(!(conn->cn_flags & flag));
     conn->cn_flags |= flag;
-    LSQ_DEBUGC("incref conn %"CID_FMT", '%s' -> '%s'",
+    LSQ_DEBUGC("incref %sconn %"CID_FMT", '%s' -> '%s'",
+                    (conn->cn_flags &LSCONN_MINI) ? "mini-" : "",
                     CID_BITS(lsquic_conn_log_cid(conn)),
                     (refflags2str(conn->cn_flags & ~flag, str[0]), str[0]),
                     (refflags2str(conn->cn_flags, str[1]), str[1]));
@@ -1920,7 +2068,8 @@ engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn,
         assert(0 == (conn->cn_flags & LSCONN_HASHED));
 #endif
     conn->cn_flags &= ~flags;
-    LSQ_DEBUGC("decref conn %"CID_FMT", '%s' -> '%s'",
+    LSQ_DEBUGC("decref %sconn %"CID_FMT", '%s' -> '%s'",
+                    (conn->cn_flags &LSCONN_MINI) ? "mini-" : "",
                     CID_BITS(lsquic_conn_log_cid(conn)),
                     (refflags2str(conn->cn_flags | flags, str[0]), str[0]),
                     (refflags2str(conn->cn_flags, str[1]), str[1]));
@@ -2296,7 +2445,7 @@ lose_matching_packets (const lsquic_engine_t *engine, struct out_batch *batch,
 
     for (i = 0; i < n; ++i)
     {
-        snprintf(packno_str, sizeof(packno_str), "%"PRIu64,
+        snprintf(packno_str, sizeof(packno_str), "#%"PRIu64,
                                                 batch->packets[i]->po_packno);
         if (0 == regexec(&engine->lose_packets_re, packno_str, 0, NULL, 0))
         {
@@ -2662,7 +2811,7 @@ send_packets_out (struct lsquic_engine *engine,
                 goto end_for;
             }
         }
-        LSQ_DEBUGC("batched packet %"PRIu64" for connection %"CID_FMT,
+        LSQ_DEBUGC("batched packet #%"PRIu64" for connection %"CID_FMT,
                     packet_out->po_packno, CID_BITS(lsquic_conn_log_cid(conn)));
         if (packet_out->po_flags & PO_ENCRYPTED)
         {
@@ -2928,6 +3077,11 @@ process_connections (lsquic_engine_t *engine, conn_iter_f next_conn,
             }
             tick_st |= TICK_CLOSE;  /* Destroy mini connection */
         }
+        if (tick_st & TICK_RETRY)
+        {
+            assert(conn->cn_flags & LSCONN_MINI);
+            schedule_mini_retry(engine, conn, now);
+        }
         if (tick_st & TICK_SEND)
         {
             if (!(conn->cn_flags & LSCONN_HAS_OUTGOING))
@@ -3078,6 +3232,7 @@ lsquic_engine_packet_in (lsquic_engine_t *engine,
          * by PI_OWN_DATA flag.
          */
         packet_in->pi_data = (unsigned char *) packet_in_data;
+        packet_in->pi_pkt_size = packet_in_size;
         if (0 != parse_packet_in_begin(packet_in, packet_end - packet_in_data,
                                 engine->flags & ENG_SERVER,
                                 engine->pub.enp_settings.es_scid_len, &ppstate))
@@ -3214,6 +3369,8 @@ lsquic_engine_earliest_adv_tick (lsquic_engine_t *engine, int *diff)
                                     && lsquic_mh_count(&engine->conns_out))
     {
 #if LSQUIC_DEBUG_NEXT_ADV_TICK
+    if (LSQ_LOG_ENABLED(L))
+    {
         conn = lsquic_mh_peek(&engine->conns_out);
         engine->last_logged_conn = 0;
         LSQ_LOGC(L, "next advisory tick is now: went past deadline last time "
@@ -3221,6 +3378,7 @@ lsquic_engine_earliest_adv_tick (lsquic_engine_t *engine, int *diff)
             lsquic_mh_count(&engine->conns_out),
             lsquic_mh_count(&engine->conns_out) != 1, "s",
             CID_BITS(lsquic_conn_log_cid(conn)));
+    }
 #endif
 #if LSQUIC_CONN_STATS
         conn = lsquic_mh_peek(&engine->conns_out);
@@ -3234,8 +3392,11 @@ lsquic_engine_earliest_adv_tick (lsquic_engine_t *engine, int *diff)
         && engine->pr_queue && lsquic_prq_have_pending(engine->pr_queue))
     {
 #if LSQUIC_DEBUG_NEXT_ADV_TICK
+    if (LSQ_LOG_ENABLED(L))
+    {
         engine->last_logged_conn = 0;
         LSQ_LOG(L, "next advisory tick is now: have pending PRQ elements");
+    }
 #endif
         *diff = 0;
         return 1;
@@ -3244,6 +3405,8 @@ lsquic_engine_earliest_adv_tick (lsquic_engine_t *engine, int *diff)
     if (lsquic_mh_count(&engine->conns_tickable))
     {
 #if LSQUIC_DEBUG_NEXT_ADV_TICK
+    if (LSQ_LOG_ENABLED(L))
+    {
         conn = lsquic_mh_peek(&engine->conns_tickable);
         engine->last_logged_conn = 0;
         LSQ_LOGC(L, "next advisory tick is now: have %u tickable "
@@ -3251,6 +3414,7 @@ lsquic_engine_earliest_adv_tick (lsquic_engine_t *engine, int *diff)
             lsquic_mh_count(&engine->conns_tickable),
             lsquic_mh_count(&engine->conns_tickable) != 1, "s",
             CID_BITS(lsquic_conn_log_cid(conn)));
+    }
 #endif
 #if LSQUIC_CONN_STATS
         conn = lsquic_mh_peek(&engine->conns_tickable);
@@ -3286,28 +3450,31 @@ lsquic_engine_earliest_adv_tick (lsquic_engine_t *engine, int *diff)
     now = lsquic_time_now();
     *diff = (int) ((int64_t) next_time - (int64_t) now);
 #if LSQUIC_DEBUG_NEXT_ADV_TICK
-    if (next_attq)
+    if (LSQ_LOG_ENABLED(L))
     {
-        /* Deduplicate consecutive log messages about the same reason for the
-         * same connection.
-         * If diff is always zero or diff reset to a higher value, event is
-         * still logged.
-         */
-        if (!((unsigned) next_attq->ae_why == engine->last_logged_ae_why
-                    && (uintptr_t) next_attq->ae_conn
-                                            == engine->last_logged_conn
-                    && *diff < engine->last_tick_diff))
+        if (next_attq)
         {
-            engine->last_logged_conn = (uintptr_t) next_attq->ae_conn;
-            engine->last_logged_ae_why = (unsigned) next_attq->ae_why;
-            engine->last_tick_diff = *diff;
-            LSQ_LOGC(L, "next advisory tick is %d usec away: conn %"CID_FMT
-                ": %s", *diff, CID_BITS(lsquic_conn_log_cid(next_attq->ae_conn)),
-                lsquic_attq_why2str(next_attq->ae_why));
+            /* Deduplicate consecutive log messages about the same reason for the
+             * same connection.
+             * If diff is always zero or diff reset to a higher value, event is
+             * still logged.
+             */
+            if (!((unsigned) next_attq->ae_why == engine->last_logged_ae_why
+                        && (uintptr_t) next_attq->ae_conn
+                                                == engine->last_logged_conn
+                        && *diff < engine->last_tick_diff))
+            {
+                engine->last_logged_conn = (uintptr_t) next_attq->ae_conn;
+                engine->last_logged_ae_why = (unsigned) next_attq->ae_why;
+                engine->last_tick_diff = *diff;
+                LSQ_LOGC(L, "next advisory tick is %d usec away: conn %"CID_FMT
+                    ": %s", *diff, CID_BITS(lsquic_conn_log_cid(next_attq->ae_conn)),
+                    lsquic_attq_why2str(next_attq->ae_why));
+            }
         }
+        else
+            LSQ_LOG(L, "next advisory tick is %d usec away: resume sending", *diff);
     }
-    else
-        LSQ_LOG(L, "next advisory tick is %d usec away: resume sending", *diff);
 #endif
 
 #if LSQUIC_CONN_STATS

src/liblsquic/lsquic_ev_log.c

@@ -57,23 +57,23 @@ lsquic_ev_log_packet_in (const lsquic_cid_t *cid,
     switch (packet_in->pi_flags & (PI_FROM_MINI|PI_GQUIC))
     {
     case PI_FROM_MINI|PI_GQUIC:
-        LCID("packet in: %"PRIu64" (from mini)", packet_in->pi_packno);
+        LCID("RX packet #%"PRIu64" (mini)", packet_in->pi_packno);
         break;
     case PI_FROM_MINI:
-        LCID("packet in: %"PRIu64" (from mini), type: %s, ecn: %u",
+        LCID("RX packet #%"PRIu64" %s (mini), ecn: %u",
             packet_in->pi_packno, lsquic_hety2str[packet_in->pi_header_type],
             lsquic_packet_in_ecn(packet_in));
         break;
     case PI_GQUIC:
         packet_sz = packet_in->pi_data_sz
             + (packet_in->pi_flags & PI_DECRYPTED ? GQUIC_PACKET_HASH_SZ : 0);
-        LCID("packet in: %"PRIu64", size: %u", packet_in->pi_packno, packet_sz);
+        LCID("RX packet #%"PRIu64", size: %u", packet_in->pi_packno, packet_sz);
         break;
     default:
         packet_sz = packet_in->pi_data_sz
             + (packet_in->pi_flags & PI_DECRYPTED ? IQUIC_TAG_LEN : 0);
         if (packet_in->pi_flags & PI_LOG_QL_BITS)
-            LCID("packet in: %"PRIu64", type: %s, size: %u; ecn: %u, spin: %d; "
+            LCID("RX packet #%"PRIu64" %s, size: %u; ecn: %u, spin: %d; "
                 "path: %hhu; Q: %d; L: %d",
                 packet_in->pi_packno, lsquic_hety2str[packet_in->pi_header_type],
                 packet_sz,
@@ -83,7 +83,7 @@ lsquic_ev_log_packet_in (const lsquic_cid_t *cid,
                 ((packet_in->pi_flags & PI_SQUARE_BIT) > 0),
                 ((packet_in->pi_flags & PI_LOSS_BIT) > 0));
         else
-            LCID("packet in: %"PRIu64", type: %s, size: %u; ecn: %u, spin: %d; "
+            LCID("RX packet #%"PRIu64" %s, size: %u; ecn: %u, spin: %d; "
                 "path: %hhu",
                 packet_in->pi_packno, lsquic_hety2str[packet_in->pi_header_type],
                 packet_sz,
@@ -102,7 +102,7 @@ lsquic_ev_log_ack_frame_in (const lsquic_cid_t *cid,
     char buf[MAX_ACKI_STR_SZ];
 
     lsquic_acki2str(acki, buf, sizeof(buf));
-    LCID("ACK frame in: %s", buf);
+    LCID("RX ACK frame: %s", buf);
 }
 
 
@@ -110,7 +110,7 @@ void
 lsquic_ev_log_stream_frame_in (const lsquic_cid_t *cid,
                                         const struct stream_frame *frame)
 {
-    LCID("STREAM frame in: stream %"PRIu64"; offset %"PRIu64"; size %"PRIu16
+    LCID("RX STREAM frame: stream %"PRIu64"; offset %"PRIu64"; size %"PRIu16
         "; fin: %d", frame->stream_id, frame->data_frame.df_offset,
         frame->data_frame.df_size, (int) frame->data_frame.df_fin);
 }
@@ -120,7 +120,7 @@ void
 lsquic_ev_log_crypto_frame_in (const lsquic_cid_t *cid,
                         const struct stream_frame *frame, unsigned enc_level)
 {
-    LCID("CRYPTO frame in: level %u; offset %"PRIu64"; size %"PRIu16,
+    LCID("RX CRYPTO frame: level %u; offset %"PRIu64"; size %"PRIu16,
         enc_level, frame->data_frame.df_offset, frame->data_frame.df_size);
 }
 
@@ -129,7 +129,7 @@ void
 lsquic_ev_log_stop_waiting_frame_in (const lsquic_cid_t *cid,
                                                         lsquic_packno_t least)
 {
-    LCID("STOP_WAITING frame in: least unacked packno %"PRIu64, least);
+    LCID("RX STOP_WAITING frame: least unacked packno %"PRIu64, least);
 }
 
 
@@ -137,7 +137,7 @@ void
 lsquic_ev_log_window_update_frame_in (const lsquic_cid_t *cid,
                                 lsquic_stream_id_t stream_id, uint64_t offset)
 {
-    LCID("WINDOW_UPDATE frame in: stream %"PRIu64"; offset %"PRIu64,
+    LCID("RX WINDOW_UPDATE frame: stream %"PRIu64"; offset %"PRIu64,
         stream_id, offset);
 }
 
@@ -146,7 +146,7 @@ void
 lsquic_ev_log_blocked_frame_in (const lsquic_cid_t *cid,
                                             lsquic_stream_id_t stream_id)
 {
-    LCID("BLOCKED frame in: stream %"PRIu64, stream_id);
+    LCID("RX BLOCKED frame: stream %"PRIu64, stream_id);
 }
 
 
@@ -154,7 +154,7 @@ void
 lsquic_ev_log_connection_close_frame_in (const lsquic_cid_t *cid,
                     uint64_t error_code, int reason_len, const char *reason)
 {
-    LCID("CONNECTION_CLOSE frame in: error code %"PRIu64", reason: %.*s",
+    LCID("RX CONNECTION_CLOSE frame: error code %"PRIu64", reason: %.*s",
         error_code, reason_len, reason);
 }
 
@@ -163,7 +163,7 @@ void
 lsquic_ev_log_goaway_frame_in (const lsquic_cid_t *cid, uint32_t error_code,
             lsquic_stream_id_t stream_id, int reason_len, const char *reason)
 {
-    LCID("GOAWAY frame in: error code %"PRIu32", stream %"PRIu64
+    LCID("RX GOAWAY frame: error code %"PRIu32", stream %"PRIu64
         ", reason: %.*s", error_code, stream_id, reason_len, reason);
 }
 
@@ -172,7 +172,7 @@ void
 lsquic_ev_log_rst_stream_frame_in (const lsquic_cid_t *cid,
         lsquic_stream_id_t stream_id, uint64_t offset, uint64_t error_code)
 {
-    LCID("RST_STREAM frame in: error code %"PRIu64", stream %"PRIu64
+    LCID("RX RST_STREAM frame: error code %"PRIu64", stream %"PRIu64
         ", offset: %"PRIu64, error_code, stream_id, offset);
 }
 
@@ -181,7 +181,7 @@ void
 lsquic_ev_log_stop_sending_frame_in (const lsquic_cid_t *cid,
                         lsquic_stream_id_t stream_id, uint64_t error_code)
 {
-    LCID("STOP_SENDING frame in: error code %"PRIu64", stream %"PRIu64,
+    LCID("RX STOP_SENDING frame: error code %"PRIu64", stream %"PRIu64,
                                                      error_code, stream_id);
 }
 
@@ -189,14 +189,14 @@ lsquic_ev_log_stop_sending_frame_in (const lsquic_cid_t *cid,
 void
 lsquic_ev_log_padding_frame_in (const lsquic_cid_t *cid, size_t len)
 {
-    LCID("PADDING frame in of %zd bytes", len);
+    LCID("RX PADDING frame of %zd bytes", len);
 }
 
 
 void
 lsquic_ev_log_ping_frame_in (const lsquic_cid_t *cid)
 {
-    LCID("PING frame in");
+    LCID("RX PING frame");
 }
 
 
@@ -204,8 +204,9 @@ void
 lsquic_ev_log_packet_created (const lsquic_cid_t *cid,
                                 const struct lsquic_packet_out *packet_out)
 {
-    LCID("created packet %"PRIu64"; flags: version=%d, nonce=%d, conn_id=%d",
+    LCID("created packet #%"PRIu64" %s; flags: version=%d, nonce=%d, conn_id=%d",
         packet_out->po_packno,
+        lsquic_hety2str[packet_out->po_header_type],
         !!(packet_out->po_flags & PO_VERSION),
         !!(packet_out->po_flags & PO_NONCE),
         !!(packet_out->po_flags & PO_CONN_ID));
@@ -218,34 +219,34 @@ lsquic_ev_log_packet_sent (const lsquic_cid_t *cid,
 {
     char frames[lsquic_frame_types_str_sz];
     if (lsquic_packet_out_verneg(packet_out))
-        LCID("sent version negotiation packet, size %hu",
+        LCID("TX version negotiation packet, size %hu",
                                                     packet_out->po_data_sz);
     else if (lsquic_packet_out_retry(packet_out))
-        LCID("sent stateless retry packet, size %hu", packet_out->po_data_sz);
+        LCID("TX stateless retry packet, size %hu", packet_out->po_data_sz);
     else if (lsquic_packet_out_pubres(packet_out))
-        LCID("sent public reset packet, size %hu", packet_out->po_data_sz);
+        LCID("TX public reset packet, size %hu", packet_out->po_data_sz);
     else if (packet_out->po_lflags & POL_GQUIC)
-        LCID("sent packet %"PRIu64", size %hu, frame types: %s",
-            packet_out->po_packno, packet_out->po_enc_data_sz,
+        LCID("TX packet #%"PRIu64" (%s), size %hu",
+            packet_out->po_packno,
                 /* Frame types is a list of different frames types contained
                  * in the packet, no more.  Count and order of frames is not
                  * printed.
                  */
                 lsquic_frame_types_to_str(frames, sizeof(frames),
-                                                packet_out->po_frame_types));
+                                          packet_out->po_frame_types),
+             packet_out->po_enc_data_sz);
     else if (packet_out->po_lflags & POL_LOG_QL_BITS)
-        LCID("sent packet %"PRIu64", type %s, crypto: %s, size %hu, frame "
-            "types: %s, ecn: %u, spin: %d; kp: %u, path: %hhu, flags: %u; "
+        LCID("TX packet #%"PRIu64" %s (%s), size %hu, "
+            "ecn: %u, spin: %d; kp: %u, path: %hhu, flags: %u; "
             "Q: %u; L: %u",
             packet_out->po_packno, lsquic_hety2str[packet_out->po_header_type],
-            lsquic_enclev2str[ lsquic_packet_out_enc_level(packet_out) ],
-            packet_out->po_enc_data_sz,
                 /* Frame types is a list of different frames types contained
                  * in the packet, no more.  Count and order of frames is not
                  * printed.
                  */
                 lsquic_frame_types_to_str(frames, sizeof(frames),
                                                 packet_out->po_frame_types),
+                packet_out->po_enc_data_sz,
                 lsquic_packet_out_ecn(packet_out),
                 /* spin bit value is only valid for short packet headers */
                 lsquic_packet_out_spin_bit(packet_out),
@@ -255,17 +256,16 @@ lsquic_ev_log_packet_sent (const lsquic_cid_t *cid,
                 lsquic_packet_out_square_bit(packet_out),
                 lsquic_packet_out_loss_bit(packet_out));
     else
-        LCID("sent packet %"PRIu64", type %s, crypto: %s, size %hu, frame "
-            "types: %s, ecn: %u, spin: %d; kp: %u, path: %hhu, flags: %u",
+        LCID("TX packet #%"PRIu64" %s (%s), size %hu, "
+            "ecn: %u, spin: %d; kp: %u, path: %hhu, flags: %u",
             packet_out->po_packno, lsquic_hety2str[packet_out->po_header_type],
-            lsquic_enclev2str[ lsquic_packet_out_enc_level(packet_out) ],
-            packet_out->po_enc_data_sz,
                 /* Frame types is a list of different frames types contained
                  * in the packet, no more.  Count and order of frames is not
                  * printed.
                  */
                 lsquic_frame_types_to_str(frames, sizeof(frames),
                                                 packet_out->po_frame_types),
+                packet_out->po_enc_data_sz,
                 lsquic_packet_out_ecn(packet_out),
                 /* spin bit value is only valid for short packet headers */
                 lsquic_packet_out_spin_bit(packet_out),
@@ -280,13 +280,14 @@ lsquic_ev_log_packet_not_sent (const lsquic_cid_t *cid,
                                 const struct lsquic_packet_out *packet_out)
 {
     char frames[lsquic_frame_types_str_sz];
-    LCID("unsent packet %"PRIu64", size %hu, frame types: %s",
-        packet_out->po_packno, packet_out->po_enc_data_sz,
+    LCID("unsent packet #%"PRIu64" %s, size %hu",
+        packet_out->po_packno,
             /* Frame types is a list of different frames types contained in
              * the packet, no more.  Count and order of frames is not printed.
              */
             lsquic_frame_types_to_str(frames, sizeof(frames),
-                                                packet_out->po_frame_types));
+                                      packet_out->po_frame_types),
+            packet_out->po_enc_data_sz);
 }
 
 

src/liblsquic/lsquic_full_conn.c

@@ -617,6 +617,7 @@ new_conn_common (lsquic_cid_t cid, struct lsquic_engine_public *enpub,
     conn->fc_conn.cn_cces = conn->fc_cces;
     conn->fc_conn.cn_cces_mask = 1;
     conn->fc_conn.cn_cid = cid;
+    conn->fc_conn.cn_logid = cid;
     conn->fc_flags = flags;
     conn->fc_enpub = enpub;
     conn->fc_pub.enpub = enpub;
@@ -903,12 +904,12 @@ lsquic_gquic_full_conn_server_new (struct lsquic_engine_public *enpub,
 
     for (n = 0; received; ++n)
     {
-        if (received & (1U << n))
+        if (received & (1ULL << n))
             /* Setting `now' to zero is OK here, as we should have had at
              * least one other packet above.
              */
             lsquic_rechist_received(&conn->fc_rechist, n + 1, 0);
-        received &= ~(1U << n);
+        received &= ~(1ULL << n);
     }
 
     /* Mini connection sends out packets 1, 2, 3... and so on.  It deletes
@@ -1608,7 +1609,7 @@ process_stream_frame (struct full_conn *conn, lsquic_packet_in_t *packet_in,
 
     enc_level = lsquic_packet_in_enc_level(packet_in);
     if (!is_handshake_stream_id(conn, stream_frame->stream_id)
-        && enc_level == ENC_LEV_CLEAR)
+        && enc_level == ENC_LEV_INIT)
     {
         lsquic_malo_put(stream_frame);
         ABORT_ERROR("received unencrypted data for stream %"PRIu64,

src/liblsquic/lsquic_full_conn_ietf.c

@@ -81,13 +81,14 @@
 #include "lsquic_qpack_exp.h"
 
 #define LSQUIC_LOGGER_MODULE LSQLM_CONN
-#define LSQUIC_LOG_CONN_ID ietf_full_conn_ci_get_log_cid(&conn->ifc_conn)
+#define LSQUIC_LOG_CONN_ID lsquic_conn_log_cid(&conn->ifc_conn)
 #include "lsquic_logger.h"
 
 #define MAX_RETR_PACKETS_SINCE_LAST_ACK 2
 #define MAX_ANY_PACKETS_SINCE_LAST_ACK 20
 #define ACK_TIMEOUT                    (TP_DEF_MAX_ACK_DELAY * 1000)
 #define INITIAL_CHAL_TIMEOUT            250000
+#define HSK_PING_TIMEOUT                200000
 
 /* Retire original CID after this much time has elapsed: */
 #define RET_CID_TIMEOUT                 2000000
@@ -191,6 +192,7 @@ enum send
     SEND_MAX_STREAMS_BIDI = SEND_MAX_STREAMS + SD_BIDI,
     SEND_MAX_STREAMS_UNI = SEND_MAX_STREAMS + SD_UNI,
     SEND_STOP_SENDING,
+    SEND_NEW_TOKEN,
     SEND_HANDSHAKE_DONE,
     SEND_ACK_FREQUENCY,
     N_SEND
@@ -220,6 +222,7 @@ enum send_flags
     SF_SEND_MAX_STREAMS_BIDI        = 1 << SEND_MAX_STREAMS_BIDI,
     SF_SEND_MAX_STREAMS_UNI         = 1 << SEND_MAX_STREAMS_UNI,
     SF_SEND_STOP_SENDING            = 1 << SEND_STOP_SENDING,
+    SF_SEND_NEW_TOKEN               = 1 << SEND_NEW_TOKEN,
     SF_SEND_HANDSHAKE_DONE          = 1 << SEND_HANDSHAKE_DONE,
     SF_SEND_ACK_FREQUENCY           = 1 << SEND_ACK_FREQUENCY,
 };
@@ -450,6 +453,7 @@ struct ietf_full_conn
     unsigned                    ifc_n_slack_akbl[N_PNS];
     unsigned                    ifc_n_slack_all;    /* App PNS only */
     unsigned                    ifc_max_retx_since_last_ack;
+    unsigned short              ifc_max_udp_payload;    /* Cached TP */
     lsquic_time_t               ifc_max_ack_delay;
     uint64_t                    ifc_ecn_counts_in[N_PNS][4];
     lsquic_stream_id_t          ifc_max_req_id;
@@ -463,6 +467,7 @@ struct ietf_full_conn
     }                           ifc_peer_hq_settings;
     struct dcid_elem           *ifc_dces[MAX_IETF_CONN_DCIDS];
     TAILQ_HEAD(, dcid_elem)     ifc_to_retire;
+    unsigned                    ifc_n_to_retire;
     unsigned                    ifc_scid_seqno;
     lsquic_time_t               ifc_scid_timestamp[MAX_SCID];
     /* Last 8 packets had ECN markings? */
@@ -488,7 +493,8 @@ struct ietf_full_conn
     unsigned                    ifc_max_pack_tol_sent;
 #endif
     unsigned                    ifc_max_ack_freq_seqno; /* Incoming */
-    unsigned short              ifc_max_udp_payload;    /* Cached TP */
+    unsigned short              ifc_min_dg_sz,
+                                ifc_max_dg_sz;
     lsquic_time_t               ifc_last_live_update;
     struct conn_path            ifc_paths[N_PATHS];
     union {
@@ -517,8 +523,6 @@ struct ietf_full_conn
     lsquic_time_t               ifc_ping_period;
     struct lsquic_hash         *ifc_bpus;
     uint64_t                    ifc_last_max_data_off_sent;
-    unsigned short              ifc_min_dg_sz,
-                                ifc_max_dg_sz;
     struct packet_tolerance_stats
                                 ifc_pts;
 #if LSQUIC_CONN_STATS
@@ -574,9 +578,6 @@ ignore_hsk (struct ietf_full_conn *);
 static unsigned
 ietf_full_conn_ci_n_avail_streams (const struct lsquic_conn *);
 
-static const lsquic_cid_t *
-ietf_full_conn_ci_get_log_cid (const struct lsquic_conn *);
-
 static void
 ietf_full_conn_ci_destroy (struct lsquic_conn *);
 
@@ -755,6 +756,7 @@ blocked_ka_alarm_expired (enum alarm_id al_id, void *ctx,
     struct ietf_full_conn *const conn = (struct ietf_full_conn *) ctx;
     struct lsquic_stream *stream;
     struct lsquic_hash_elem *el;
+    int has_send_flag;
 
     if (lsquic_conn_cap_avail(&conn->ifc_pub.conn_cap) == 0)
     {
@@ -767,13 +769,21 @@ blocked_ka_alarm_expired (enum alarm_id al_id, void *ctx,
                          el = lsquic_hash_next(conn->ifc_pub.all_streams))
     {
         stream = lsquic_hashelem_getdata(el);
-        if (lsquic_stream_is_blocked(stream))
+        if (lsquic_stream_is_blocked(stream)
+            && !lsquic_stream_is_write_reset(stream))
         {
-            if (!(stream->sm_qflags & SMQF_SENDING_FLAGS))
-                TAILQ_INSERT_TAIL(&conn->ifc_pub.sending_streams, stream,
-                                                            next_send_stream);
+            has_send_flag = (stream->sm_qflags & SMQF_SENDING_FLAGS);
             stream->sm_qflags |= SMQF_SEND_BLOCKED;
             LSQ_DEBUG("set SEND_BLOCKED flag on stream %"PRIu64, stream->id);
+            if (!lsquic_sendctl_gen_stream_blocked_frame(
+                        stream->conn_pub->send_ctl, stream))
+            {
+                LSQ_DEBUG("failed to send STREAM_BLOCKED frame for"
+                        " stream %"PRIu64 " immedately, postpone.", stream->id);
+                if (!has_send_flag)
+                    TAILQ_INSERT_TAIL(&conn->ifc_pub.sending_streams, stream,
+                                                            next_send_stream);
+            }
             return;
         }
     }
@@ -1268,6 +1278,7 @@ ietf_full_conn_init (struct ietf_full_conn *conn,
     TAILQ_INIT(&conn->ifc_pub.service_streams);
     STAILQ_INIT(&conn->ifc_stream_ids_to_ss);
     TAILQ_INIT(&conn->ifc_to_retire);
+    conn->ifc_n_to_retire = 0;
 
     lsquic_alarmset_init(&conn->ifc_alset, &conn->ifc_conn);
     lsquic_alarmset_init_alarm(&conn->ifc_alset, AL_IDLE, idle_alarm_expired, conn);
@@ -1353,10 +1364,13 @@ lsquic_ietf_full_conn_client_new (struct lsquic_engine_public *enpub,
                                                 / sizeof(conn->ifc_cces[0]);
     if (!ietf_full_conn_add_scid(conn, enpub, CCE_USED, now))
         goto err1;
-
+    conn->ifc_conn.cn_logid = *CN_SCID(&conn->ifc_conn);
     assert(versions);
     versions &= LSQUIC_IETF_VERSIONS;
-    ver = highest_bit_set(versions);
+    if (versions & (1 << LSQVER_I001))
+        ver = LSQVER_I001;
+    else
+        ver = highest_bit_set(versions);
     if (sess_resume)
     {
         sess_resume_version = lsquic_sess_resume_version(sess_resume, sess_resume_sz);
@@ -1432,13 +1446,13 @@ lsquic_ietf_full_conn_client_new (struct lsquic_engine_public *enpub,
     if (!conn->ifc_conn.cn_enc_session)
         goto err2;
 
-    conn->ifc_u.cli.crypto_streams[ENC_LEV_CLEAR] = lsquic_stream_new_crypto(
-        ENC_LEV_CLEAR, &conn->ifc_pub, &lsquic_cry_sm_if,
+    conn->ifc_u.cli.crypto_streams[ENC_LEV_INIT] = lsquic_stream_new_crypto(
+                ENC_LEV_INIT, &conn->ifc_pub, &lsquic_cry_sm_if,
         conn->ifc_conn.cn_enc_session,
         SCF_IETF|SCF_DI_AUTOSWITCH|SCF_CALL_ON_NEW|SCF_CRITICAL);
-    if (!conn->ifc_u.cli.crypto_streams[ENC_LEV_CLEAR])
+    if (!conn->ifc_u.cli.crypto_streams[ENC_LEV_INIT])
         goto err3;
-    if (!lsquic_stream_get_ctx(conn->ifc_u.cli.crypto_streams[ENC_LEV_CLEAR]))
+    if (!lsquic_stream_get_ctx(conn->ifc_u.cli.crypto_streams[ENC_LEV_INIT]))
         goto err4;
     conn->ifc_pub.packet_out_malo =
                         lsquic_malo_create(sizeof(struct lsquic_packet_out));
@@ -1450,8 +1464,7 @@ lsquic_ietf_full_conn_client_new (struct lsquic_engine_public *enpub,
                         lsquic_ver2str[conn->ifc_u.cli.ifcli_ver_neg.vn_ver]);
     conn->ifc_process_incoming_packet = process_incoming_packet_verneg;
     conn->ifc_created = now;
-    LSQ_DEBUG("logging using %s SCID",
-        LSQUIC_LOG_CONN_ID == CN_SCID(&conn->ifc_conn) ? "client" : "server");
+    LSQ_DEBUG("logging using client SCID");
     if (sess_resume && (params
             = conn->ifc_conn.cn_esf.i->esfi_get_peer_transport_params(
                             conn->ifc_conn.cn_enc_session), params != NULL))
@@ -1467,7 +1480,7 @@ lsquic_ietf_full_conn_client_new (struct lsquic_engine_public *enpub,
     return &conn->ifc_conn;
 
   err4:
-    lsquic_stream_destroy(conn->ifc_u.cli.crypto_streams[ENC_LEV_CLEAR]);
+    lsquic_stream_destroy(conn->ifc_u.cli.crypto_streams[ENC_LEV_INIT]);
   err3:
     conn->ifc_conn.cn_esf.i->esfi_destroy(conn->ifc_conn.cn_enc_session);
   err2:
@@ -1531,7 +1544,7 @@ lsquic_ietf_full_conn_server_new (struct lsquic_engine_public *enpub,
             conn->ifc_scid_timestamp[i] = now;
         }
     ++conn->ifc_scid_seqno;
-
+    conn->ifc_conn.cn_logid = mini_conn->cn_logid;
     /* Set the flags early so that correct CID is used for logging */
     conn->ifc_conn.cn_flags |= LSCONN_IETF | LSCONN_SERVER;
 
@@ -1544,6 +1557,8 @@ lsquic_ietf_full_conn_server_new (struct lsquic_engine_public *enpub,
         goto err1;
     if (imc->imc_flags & IMC_IGNORE_INIT)
         conn->ifc_flags |= IFC_IGNORE_INIT;
+    if (enpub->enp_settings.es_support_srej)
+        conn->ifc_send_flags |= SF_SEND_NEW_TOKEN;
 
     conn->ifc_paths[0].cop_path = imc->imc_path;
     conn->ifc_paths[0].cop_flags = COP_VALIDATED|COP_INITIALIZED|COP_ALLOW_MTU_PADDING;
@@ -1603,6 +1618,11 @@ lsquic_ietf_full_conn_server_new (struct lsquic_engine_public *enpub,
     conn->ifc_process_incoming_packet = process_incoming_packet_fast;
 
     conn->ifc_send_ctl.sc_cur_packno = imc->imc_next_packno - 1;
+    conn->ifc_incoming_ecn = imc->imc_incoming_ecn;
+    conn->ifc_pub.rtt_stats = imc->imc_rtt_stats;
+
+    conn->ifc_last_live_update = now;
+
     lsquic_send_ctl_begin_optack_detection(&conn->ifc_send_ctl);
 
     for (pns = 0; pns < IMICO_N_PNS; ++pns)
@@ -1675,15 +1695,6 @@ lsquic_ietf_full_conn_server_new (struct lsquic_engine_public *enpub,
         {
             conn->ifc_ecn_counts_in[pns][i]  = imc->imc_ecn_counts_in[pns][i];
         }
-    conn->ifc_incoming_ecn = imc->imc_incoming_ecn;
-    conn->ifc_pub.rtt_stats = imc->imc_rtt_stats;
-
-    lsquic_alarmset_init_alarm(&conn->ifc_alset, AL_RET_CIDS,
-                                            ret_cids_alarm_expired, conn);
-    lsquic_alarmset_set(&conn->ifc_alset, AL_RET_CIDS,
-                                            now + RET_CID_TIMEOUT);
-
-    conn->ifc_last_live_update = now;
 
     if (0 != handshake_ok(&conn->ifc_conn))
         goto err3;
@@ -1693,10 +1704,10 @@ lsquic_ietf_full_conn_server_new (struct lsquic_engine_public *enpub,
                         conn->ifc_enpub->enp_stream_if_ctx, &conn->ifc_conn);
     conn->ifc_idle_to = conn->ifc_settings->es_idle_timeout * 1000000;
 
-    conn->ifc_created = imc->imc_created;
+    conn->ifc_created = now;
     if (conn->ifc_idle_to)
         lsquic_alarmset_set(&conn->ifc_alset, AL_IDLE,
-                                        imc->imc_created + conn->ifc_idle_to);
+                                        now + conn->ifc_idle_to);
     while ((packet_in = TAILQ_FIRST(&imc->imc_app_packets)))
     {
         TAILQ_REMOVE(&imc->imc_app_packets, packet_in, pi_next);
@@ -1901,6 +1912,16 @@ generate_ack_frame_for_pns (struct ietf_full_conn *conn,
                         packet_out->po_data + packet_out->po_data_sz, w);
     lsquic_send_ctl_scheduled_ack(&conn->ifc_send_ctl, pns,
                                                     packet_out->po_ack2ed);
+
+    // NOTE: Add a PING frame after ACK frame before HANDSHAKE_DONE, in a hacky way
+    if (!(conn->ifc_conn.cn_flags & LSCONN_HANDSHAKE_DONE)
+        && packet_out->po_data_sz + w < packet_out->po_n_alloc)
+    {
+        LSQ_DEBUG("add a PING frame before HANDSHAKE_DONE");
+        *(packet_out->po_data + packet_out->po_data_sz + w) = '\x01';
+        ++w;
+    }
+
     packet_out->po_frame_types |= 1 << QUIC_FRAME_ACK;
     if (0 != lsquic_packet_out_add_frame(packet_out, conn->ifc_pub.mm, 0,
                             QUIC_FRAME_ACK, packet_out->po_data_sz, w))
@@ -1914,7 +1935,8 @@ generate_ack_frame_for_pns (struct ietf_full_conn *conn,
         conn->ifc_flags |= IFC_ACK_HAD_MISS;
     else
         conn->ifc_flags &= ~IFC_ACK_HAD_MISS;
-    LSQ_DEBUG("Put %d bytes of ACK frame into packet on outgoing queue", w);
+    LSQ_DEBUG("Put %d bytes of ACK frame into packet #%" PRIu64
+              " on outgoing queue", w, packet_out->po_packno);
     if (conn->ifc_n_cons_unretx >= conn->ifc_ping_unretx_thresh &&
                 !lsquic_send_ctl_have_outgoing_retx_frames(&conn->ifc_send_ctl))
     {
@@ -1924,6 +1946,7 @@ generate_ack_frame_for_pns (struct ietf_full_conn *conn,
         /* This gives a range [12, 27]: */
         conn->ifc_ping_unretx_thresh = 12
                     + lsquic_crand_get_nybble(conn->ifc_enpub->enp_crand);
+        conn->ifc_n_cons_unretx = 0;
     }
 
     conn->ifc_n_slack_akbl[pns] = 0;
@@ -2034,6 +2057,58 @@ generate_max_data_frame (struct ietf_full_conn *conn)
 }
 
 
+static void
+generate_new_token_frame (struct ietf_full_conn *conn, lsquic_time_t now)
+{
+    struct lsquic_packet_out *packet_out;
+    const struct network_path *path;
+    ssize_t token_sz;
+    size_t need;
+    int w;
+    unsigned char token_buf[MAX_RETRY_TOKEN_LEN];
+
+    path = &conn->ifc_paths[conn->ifc_cur_path_id].cop_path;
+    token_sz = lsquic_tg_token_size(conn->ifc_enpub->enp_tokgen, TOKEN_RESUME,
+                                                            NP_PEER_SA(path));
+    need = conn->ifc_conn.cn_pf->pf_new_token_frame_size(token_sz);
+    packet_out = get_writeable_packet(conn, need);
+    if (!packet_out)
+        return;
+
+    token_sz = lsquic_tg_generate_resume(conn->ifc_enpub->enp_tokgen, token_buf,
+                                        sizeof(token_buf), NP_PEER_SA(path));
+    if (token_sz < 0)
+    {
+        LSQ_WARN("could not generate resume token");
+        conn->ifc_send_flags &= ~SF_SEND_NEW_TOKEN; /* Let's not try again */
+        return;
+    }
+
+    w = conn->ifc_conn.cn_pf->pf_gen_new_token_frame(
+            packet_out->po_data + packet_out->po_data_sz,
+            lsquic_packet_out_avail(packet_out), token_buf, token_sz);
+    if (w < 0)
+    {
+        ABORT_ERROR("generating NEW_TOKEN frame failed: %d", errno);
+        return;
+    }
+    LSQ_DEBUG("generated %d-byte NEW_TOKEN frame", w);
+    EV_LOG_GENERATED_NEW_TOKEN_FRAME(LSQUIC_LOG_CONN_ID, conn->ifc_conn.cn_pf,
+                        packet_out->po_data + packet_out->po_data_sz, w);
+    if (0 != lsquic_packet_out_add_frame(packet_out, conn->ifc_pub.mm, 0,
+                            QUIC_FRAME_NEW_TOKEN, packet_out->po_data_sz, w))
+    {
+        ABORT_ERROR("adding frame to packet failed: %d", errno);
+        return;
+    }
+    packet_out->po_frame_types |= QUIC_FTBIT_NEW_TOKEN;
+    lsquic_send_ctl_incr_pack_sz(&conn->ifc_send_ctl, packet_out, w);
+
+    conn->ifc_send_flags &= ~SF_SEND_NEW_TOKEN;
+    (void) token_sz;
+}
+
+
 static int
 can_issue_cids (const struct ietf_full_conn *conn)
 {
@@ -2220,6 +2295,7 @@ generate_retire_cid_frame (struct ietf_full_conn *conn)
     lsquic_send_ctl_incr_pack_sz(&conn->ifc_send_ctl, packet_out, w);
 
     TAILQ_REMOVE(&conn->ifc_to_retire, dce, de_next_to_ret);
+    --conn->ifc_n_to_retire;
     lsquic_malo_put(dce);
 
     if (TAILQ_EMPTY(&conn->ifc_to_retire))
@@ -2234,6 +2310,13 @@ generate_retire_cid_frames (struct ietf_full_conn *conn, lsquic_time_t now)
 {
     int s;
 
+    if (conn->ifc_n_to_retire >= MAX_IETF_CONN_DCIDS * 3)
+    {
+        ABORT_QUIETLY(0, TEC_CONNECTION_ID_LIMIT_ERROR,
+            "too many (%d) CIDs to retire", conn->ifc_n_to_retire);
+        return;
+    }
+
     do
         s = generate_retire_cid_frame(conn);
     while (0 == s && (conn->ifc_send_flags & SF_SEND_RETIRE_CID));
@@ -2428,46 +2511,6 @@ generate_max_stream_data_frame (struct ietf_full_conn *conn,
 }
 
 
-/* Return true if generated, false otherwise */
-static int
-generate_stream_blocked_frame (struct ietf_full_conn *conn,
-                                                struct lsquic_stream *stream)
-{
-    struct lsquic_packet_out *packet_out;
-    unsigned need;
-    uint64_t off;
-    int sz;
-
-    off = lsquic_stream_combined_send_off(stream);
-    need = conn->ifc_conn.cn_pf->pf_stream_blocked_frame_size(stream->id, off);
-    packet_out = get_writeable_packet(conn, need);
-    if (!packet_out)
-        return 0;
-    sz = conn->ifc_conn.cn_pf->pf_gen_stream_blocked_frame(
-                         packet_out->po_data + packet_out->po_data_sz,
-                         lsquic_packet_out_avail(packet_out), stream->id, off);
-    if (sz < 0)
-    {
-        ABORT_ERROR("Generating STREAM_BLOCKED frame failed");
-        return 0;
-    }
-    LSQ_DEBUG("generated %d-byte STREAM_BLOCKED "
-        "frame; stream_id: %"PRIu64"; offset: %"PRIu64, sz, stream->id, off);
-    EV_LOG_CONN_EVENT(LSQUIC_LOG_CONN_ID, "generated %d-byte STREAM_BLOCKED "
-        "frame; stream_id: %"PRIu64"; offset: %"PRIu64, sz, stream->id, off);
-    if (0 != lsquic_packet_out_add_frame(packet_out, conn->ifc_pub.mm, 0,
-                        QUIC_FRAME_STREAM_BLOCKED, packet_out->po_data_sz, sz))
-    {
-        ABORT_ERROR("adding frame to packet failed: %d", errno);
-        return 0;
-    }
-    lsquic_send_ctl_incr_pack_sz(&conn->ifc_send_ctl, packet_out, sz);
-    packet_out->po_frame_types |= 1 << QUIC_FRAME_STREAM_BLOCKED;
-    lsquic_stream_blocked_frame_sent(stream);
-    return 1;
-}
-
-
 static int
 generate_stop_sending_frame_by_id (struct ietf_full_conn *conn,
                 lsquic_stream_id_t stream_id, enum http_error_code error_code)
@@ -2852,10 +2895,26 @@ process_stream_ready_to_send (struct ietf_full_conn *conn,
                                             struct lsquic_stream *stream)
 {
     int r = 1;
+
+    LSQ_DEBUG("process_stream_ready_to_send: stream: %"PRIu64", "
+              "sm_qflags: %d. stream_flags: %d, sm_bflags: %d, ", stream->id,
+              stream->sm_qflags, stream->stream_flags, stream->sm_bflags);
+
     if (stream->sm_qflags & SMQF_SEND_MAX_STREAM_DATA)
         r &= generate_max_stream_data_frame(conn, stream);
     if (stream->sm_qflags & SMQF_SEND_BLOCKED)
-        r &= generate_stream_blocked_frame(conn, stream);
+    {
+        if (lsquic_stream_is_write_reset(stream))
+        {
+            stream->sm_qflags &= ~SMQF_SEND_BLOCKED;
+            if (!(stream->sm_qflags & SMQF_SENDING_FLAGS))
+                TAILQ_REMOVE(&stream->conn_pub->sending_streams, stream,
+                             next_send_stream);
+        }
+        else
+            r &= lsquic_sendctl_gen_stream_blocked_frame(&conn->ifc_send_ctl,
+                                                         stream);
+    }
     if (stream->sm_qflags & SMQF_SEND_RST)
         r &= generate_rst_stream_frame(conn, stream);
     if (stream->sm_qflags & SMQF_SEND_STOP_SENDING)
@@ -2978,6 +3037,7 @@ retire_dcid (struct ietf_full_conn *conn, struct dcid_elem **dce)
     if ((*dce)->de_hash_el.qhe_flags & QHE_HASHED)
         lsquic_hash_erase(conn->ifc_enpub->enp_srst_hash, &(*dce)->de_hash_el);
     TAILQ_INSERT_TAIL(&conn->ifc_to_retire, *dce, de_next_to_ret);
+    ++conn->ifc_n_to_retire;
     LSQ_DEBUG("prepare to retire DCID seqno %"PRIu32"", (*dce)->de_seqno);
     *dce = NULL;
     conn->ifc_send_flags |= SF_SEND_RETIRE_CID;
@@ -2995,6 +3055,7 @@ retire_seqno (struct ietf_full_conn *conn, unsigned seqno)
         memset(dce, 0, sizeof(*dce));
         dce->de_seqno = seqno;
         TAILQ_INSERT_TAIL(&conn->ifc_to_retire, dce, de_next_to_ret);
+        ++conn->ifc_n_to_retire;
         LSQ_DEBUG("prepare to retire DCID seqno %"PRIu32, seqno);
         conn->ifc_send_flags |= SF_SEND_RETIRE_CID;
     }
@@ -3126,6 +3187,7 @@ ietf_full_conn_ci_destroy (struct lsquic_conn *lconn)
     while ((dce = TAILQ_FIRST(&conn->ifc_to_retire)))
     {
         TAILQ_REMOVE(&conn->ifc_to_retire, dce, de_next_to_ret);
+        --conn->ifc_n_to_retire;
         lsquic_malo_put(dce);
     }
     lsquic_send_ctl_cleanup(&conn->ifc_send_ctl);
@@ -3327,6 +3389,7 @@ retire_cid_from_tp (struct ietf_full_conn *conn,
                                                     sizeof(dce->de_srst));
     dce->de_flags = DE_SRST;
     TAILQ_INSERT_TAIL(&conn->ifc_to_retire, dce, de_next_to_ret);
+    ++conn->ifc_n_to_retire;
     LSQ_DEBUG("prepare to retire DCID seqno %"PRIu32, dce->de_seqno);
     conn->ifc_send_flags |= SF_SEND_RETIRE_CID;
 }
@@ -3799,6 +3862,29 @@ handshake_ok (struct lsquic_conn *lconn)
     else
         dce->de_flags = DE_ASSIGNED;
 
+    if (!(conn->ifc_flags & IFC_SERVER)
+        && (params->tp_set & (1 << TPI_VERSION_INFORMATION)))
+    {
+        LSQ_DEBUG("server chosen version %s",
+                  lsquic_ver2str[params->tp_chosen_version]);
+        if (((1 << params->tp_chosen_version)
+            & conn->ifc_settings->es_versions) == 0)
+        {
+            ABORT_QUIETLY(0, TEC_VERSION_NEGOTIATION_ERROR,
+                          "server chosen version %s is not supported",
+                          lsquic_ver2str[params->tp_chosen_version]
+                         );
+            return -1;
+        }
+//         if (conn->ifc_conn.cn_version != params->tp_chosen_version)
+//         {
+//             LSQ_DEBUG("version negociation: switch version from %s to %s",
+//                   lsquic_ver2str[conn->ifc_conn.cn_version],
+//                   lsquic_ver2str[params->tp_chosen_version]);
+//             conn->ifc_conn.cn_version = params->tp_chosen_version;
+//         }
+    }
+
     LSQ_INFO("applied peer transport parameters");
 
     if ((conn->ifc_flags & (IFC_HTTP|IFC_HTTP_INITED)) == IFC_HTTP)
@@ -4413,7 +4499,8 @@ generate_connection_close_packet (struct ietf_full_conn *conn)
     lsquic_send_ctl_incr_pack_sz(&conn->ifc_send_ctl, packet_out, sz);
     packet_out->po_frame_types |= 1 << QUIC_FRAME_CONNECTION_CLOSE;
     conn->ifc_mflags |= MF_CONN_CLOSE_PACK;
-    LSQ_DEBUG("generated CONNECTION_CLOSE frame in its own packet");
+    EV_LOG_CONN_EVENT(LSQUIC_LOG_CONN_ID,
+            "generated CONNECTION_CLOSE frame in its own packet");
     conn->ifc_send_flags &= ~SF_SEND_CONN_CLOSE;
 }
 
@@ -4431,12 +4518,30 @@ log_conn_flow_control (struct ietf_full_conn *conn)
 
 
 static void
-generate_ping_frame (struct ietf_full_conn *conn, lsquic_time_t unused)
+generate_ping_frame (struct ietf_full_conn *conn, lsquic_time_t now)
 {
     struct lsquic_packet_out *packet_out;
+    int pns;
     int sz;
 
-    packet_out = get_writeable_packet(conn, 1);
+    if (conn->ifc_conn.cn_flags & LSCONN_HANDSHAKE_DONE)
+        packet_out = get_writeable_packet(conn, 1);
+    else
+    {
+        conn->ifc_ping_period += HSK_PING_TIMEOUT;
+        lsquic_alarmset_set(&conn->ifc_alset, AL_PING,
+                            now + conn->ifc_ping_period);
+        if (iquic_esf_is_enc_level_ready(conn->ifc_conn.cn_enc_session,
+                                         ENC_LEV_HSK))
+            pns = PNS_HSK;
+        else
+            pns = PNS_INIT;
+        packet_out = lsquic_send_ctl_new_packet_out(&conn->ifc_send_ctl, 0, pns,
+                                                    CUR_NPATH(conn));
+        if (packet_out)
+            lsquic_send_ctl_scheduled_one(&conn->ifc_send_ctl, packet_out);
+
+    }
     if (!packet_out)
     {
         LSQ_DEBUG("cannot get writeable packet for PING frame");
@@ -4839,7 +4944,9 @@ static unsigned
 process_padding_frame (struct ietf_full_conn *conn,
     struct lsquic_packet_in *packet_in, const unsigned char *p, size_t len)
 {
-    return (unsigned) count_zero_bytes(p, len);
+    unsigned sz = (unsigned) count_zero_bytes(p, len);
+    EV_LOG_PADDING_FRAME_IN(LSQUIC_LOG_CONN_ID, sz);
+    return sz;
 }
 
 
@@ -5591,7 +5698,7 @@ process_crypto_frame_server (struct ietf_full_conn *conn,
                                                                     parsed_len);
         return (unsigned) parsed_len;
     }
-    if (enc_level < ENC_LEV_INIT)
+    if (enc_level < ENC_LEV_HSK)
     {   /* Must be dup */
         LSQ_DEBUG("discard %d-byte CRYPTO frame on level %s", parsed_len,
                                                 lsquic_enclev2str[enc_level]);
@@ -5613,6 +5720,11 @@ process_crypto_frame_server (struct ietf_full_conn *conn,
         LSQ_DEBUG("handshake confirmed: send HANDSHAKE_DONE");
         conn->ifc_flags &= ~IFC_PROC_CRYPTO;
         conn->ifc_send_flags |= SF_SEND_HANDSHAKE_DONE;
+
+        lsquic_alarmset_init_alarm(&conn->ifc_alset, AL_RET_CIDS,
+                                                ret_cids_alarm_expired, conn);
+        lsquic_alarmset_set(&conn->ifc_alset, AL_RET_CIDS,
+                                      lsquic_time_now() + RET_CID_TIMEOUT);
     }
 
     return (unsigned) parsed_len;
@@ -5653,7 +5765,7 @@ process_crypto_frame_client (struct ietf_full_conn *conn,
     EV_LOG_CRYPTO_FRAME_IN(LSQUIC_LOG_CONN_ID, stream_frame, enc_level);
     LSQ_DEBUG("Got CRYPTO frame for enc level #%u", enc_level);
     if ((conn->ifc_conn.cn_flags & LSCONN_HANDSHAKE_DONE)
-                                                && enc_level != ENC_LEV_FORW)
+                                                && enc_level != ENC_LEV_APP)
     {
         LSQ_DEBUG("handshake complete: ignore CRYPTO frames in "
             "non-forward-secure packets");
@@ -5743,7 +5855,6 @@ process_stream_frame (struct ietf_full_conn *conn,
         return 0;
     }
     EV_LOG_STREAM_FRAME_IN(LSQUIC_LOG_CONN_ID, stream_frame);
-    LSQ_DEBUG("Got stream frame for stream #%"PRIu64, stream_frame->stream_id);
     CONN_STATS(in.stream_frames, 1);
     CONN_STATS(in.stream_data_sz, stream_frame->data_frame.df_size);
 
@@ -5941,7 +6052,6 @@ process_ping_frame (struct ietf_full_conn *conn,
      * return the length of this frame.
      */
     EV_LOG_PING_FRAME_IN(LSQUIC_LOG_CONN_ID);
-    LSQ_DEBUG("received PING");
     if (conn->ifc_flags & IFC_SERVER)
         log_conn_flow_control(conn);
 
@@ -6037,7 +6147,7 @@ process_max_data_frame (struct ietf_full_conn *conn,
     if (parsed_len < 0)
         return 0;
 
-    EV_LOG_CONN_EVENT(LSQUIC_LOG_CONN_ID, "MAX_DATA frame in; offset: %"PRIu64,
+    EV_LOG_CONN_EVENT(LSQUIC_LOG_CONN_ID, "RX MAX_DATA frame; offset: %"PRIu64,
         max_data);
     if (max_data > conn->ifc_pub.conn_cap.cc_max)
     {
@@ -6067,7 +6177,7 @@ process_max_stream_data_frame (struct ietf_full_conn *conn,
     if (parsed_len < 0)
         return 0;
 
-    EV_LOG_CONN_EVENT(LSQUIC_LOG_CONN_ID, "MAX_STREAM_DATA frame in; "
+    EV_LOG_CONN_EVENT(LSQUIC_LOG_CONN_ID, "RX MAX_STREAM_DATA frame; "
         "stream_id: %"PRIu64"; offset: %"PRIu64, stream_id, max_data);
     if (conn_is_receive_only_stream(conn, stream_id))
     {
@@ -6480,7 +6590,7 @@ process_stream_blocked_frame (struct ietf_full_conn *conn,
     if (parsed_len < 0)
         return 0;
 
-    EV_LOG_CONN_EVENT(LSQUIC_LOG_CONN_ID, "STREAM_BLOCKED frame in: stream "
+    EV_LOG_CONN_EVENT(LSQUIC_LOG_CONN_ID, "RX STREAM_BLOCKED frame: stream "
         "%"PRIu64"; offset %"PRIu64, stream_id, peer_off);
     LSQ_DEBUG("received STREAM_BLOCKED frame: stream %"PRIu64
                                     "; offset %"PRIu64, stream_id, peer_off);
@@ -6546,7 +6656,7 @@ process_blocked_frame (struct ietf_full_conn *conn,
     if (parsed_len < 0)
         return 0;
 
-    EV_LOG_CONN_EVENT(LSQUIC_LOG_CONN_ID, "BLOCKED frame in: offset %"PRIu64,
+    EV_LOG_CONN_EVENT(LSQUIC_LOG_CONN_ID, "RX BLOCKED frame: offset %"PRIu64,
                                                                     peer_off);
     LSQ_DEBUG("received BLOCKED frame: offset %"PRIu64, peer_off);
 
@@ -6576,7 +6686,7 @@ process_handshake_done_frame (struct ietf_full_conn *conn,
     if (parsed_len < 0)
         return 0;
 
-    EV_LOG_CONN_EVENT(LSQUIC_LOG_CONN_ID, "HANDSHAKE_DONE frame in");
+    EV_LOG_CONN_EVENT(LSQUIC_LOG_CONN_ID, "RX HANDSHAKE_DONE frame");
     LSQ_DEBUG("received HANDSHAKE_DONE frame");
 
     if (conn->ifc_flags & IFC_SERVER)
@@ -6612,11 +6722,11 @@ process_ack_frequency_frame (struct ietf_full_conn *conn,
     if (parsed_len < 0)
         return 0;
 
-    EV_LOG_CONN_EVENT(LSQUIC_LOG_CONN_ID, "ACK_FREQUENCY(seqno: %"PRIu64"; "
-        "pack_tol: %"PRIu64"; upd: %"PRIu64"; ignore: %d) frame in", seqno,
+    EV_LOG_CONN_EVENT(LSQUIC_LOG_CONN_ID, "RX ACK_FREQUENCY frame: (seqno: %"PRIu64"; "
+        "pack_tol: %"PRIu64"; upd: %"PRIu64"; ignore: %d)", seqno,
         pack_tol, upd_mad, ignore);
-    LSQ_DEBUG("ACK_FREQUENCY(seqno: %"PRIu64"; pack_tol: %"PRIu64"; "
-        "upd: %"PRIu64"; ignore: %d) frame in", seqno, pack_tol, upd_mad,
+    LSQ_DEBUG("RX ACK_FREQUENCY frame: (seqno: %"PRIu64"; pack_tol: %"PRIu64"; "
+        "upd: %"PRIu64"; ignore: %d)", seqno, pack_tol, upd_mad,
         ignore);
 
     if (pack_tol == 0)
@@ -6764,6 +6874,7 @@ process_packet_frame (struct ietf_full_conn *conn,
     {
         LSQ_DEBUG("invalid frame %u (bytes: %s) at encryption level %s",
             type, HEXSTR(p, MIN(len, 8), str), lsquic_enclev2str[enc_level]);
+        ABORT_QUIETLY(0, TEC_FRAME_ENCODING_ERROR, "invalid frame");
         return 0;
     }
 }
@@ -7205,7 +7316,8 @@ on_dcid_change (struct ietf_full_conn *conn, const lsquic_cid_t *dcid_in)
     struct lsquic_conn *const lconn = &conn->ifc_conn;  /* Shorthand */
     struct conn_cid_elem *cce;
 
-    LSQ_DEBUG("peer switched its DCID, attempt to switch own SCID");
+    LSQ_DEBUGC("peer switched its DCID to %"CID_FMT
+              ", attempt to switch own SCID", CID_BITS(dcid_in));
 
     for (cce = lconn->cn_cces; cce < END_OF_CCES(lconn); ++cce)
         if (cce - lconn->cn_cces != lconn->cn_cur_cce_idx
@@ -7246,10 +7358,10 @@ ignore_init (struct ietf_full_conn *conn)
     lsquic_rechist_cleanup(&conn->ifc_rechist[PNS_INIT]);
     if (!(conn->ifc_flags & IFC_SERVER))
     {
-        if (conn->ifc_u.cli.crypto_streams[ENC_LEV_CLEAR])
+        if (conn->ifc_u.cli.crypto_streams[ENC_LEV_INIT])
         {
-            lsquic_stream_destroy(conn->ifc_u.cli.crypto_streams[ENC_LEV_CLEAR]);
-            conn->ifc_u.cli.crypto_streams[ENC_LEV_CLEAR] = NULL;
+            lsquic_stream_destroy(conn->ifc_u.cli.crypto_streams[ENC_LEV_INIT]);
+            conn->ifc_u.cli.crypto_streams[ENC_LEV_INIT] = NULL;
         }
         conn->ifc_conn.cn_if = ietf_full_conn_iface_ptr;
     }
@@ -7265,10 +7377,10 @@ ignore_hsk (struct ietf_full_conn *conn)
     lsquic_send_ctl_empty_pns(&conn->ifc_send_ctl, PNS_HSK);
     lsquic_rechist_cleanup(&conn->ifc_rechist[PNS_HSK]);
     if (!(conn->ifc_flags & IFC_SERVER))
-        if (conn->ifc_u.cli.crypto_streams[ENC_LEV_INIT])
+        if (conn->ifc_u.cli.crypto_streams[ENC_LEV_HSK])
         {
-            lsquic_stream_destroy(conn->ifc_u.cli.crypto_streams[ENC_LEV_INIT]);
-            conn->ifc_u.cli.crypto_streams[ENC_LEV_INIT] = NULL;
+            lsquic_stream_destroy(conn->ifc_u.cli.crypto_streams[ENC_LEV_HSK]);
+            conn->ifc_u.cli.crypto_streams[ENC_LEV_HSK] = NULL;
         }
 }
 
@@ -7316,6 +7428,7 @@ process_regular_packet (struct ietf_full_conn *conn,
     enum was_missing was_missing;
     int is_rechist_empty;
     unsigned char saved_path_id;
+    int is_dcid_changed;
 
     if (HETY_RETRY == packet_in->pi_header_type)
         return process_retry_packet(conn, packet_in);
@@ -7323,11 +7436,6 @@ process_regular_packet (struct ietf_full_conn *conn,
     CONN_STATS(in.packets, 1);
 
     pns = lsquic_hety2pns[ packet_in->pi_header_type ];
-    if (pns == PNS_INIT)
-        conn->ifc_conn.cn_esf.i->esfi_set_iscid(conn->ifc_conn.cn_enc_session,
-                                                                    packet_in);
-    else if (pns == PNS_HSK)
-        lsquic_send_ctl_maybe_calc_rough_rtt(&conn->ifc_send_ctl, pns - 1);
     if ((pns == PNS_INIT && (conn->ifc_flags & IFC_IGNORE_INIT))
                     || (pns == PNS_HSK  && (conn->ifc_flags & IFC_IGNORE_HSK)))
     {
@@ -7423,6 +7531,31 @@ process_regular_packet (struct ietf_full_conn *conn,
         }
     }
 
+    is_dcid_changed = !LSQUIC_CIDS_EQ(CN_SCID(&conn->ifc_conn),
+                                        &packet_in->pi_dcid);
+    if (pns == PNS_INIT)
+        conn->ifc_conn.cn_esf.i->esfi_set_iscid(conn->ifc_conn.cn_enc_session,
+                                                                    packet_in);
+    else
+    {
+        if (is_dcid_changed && HETY_0RTT != packet_in->pi_header_type)
+        {
+            if (LSQUIC_CIDS_EQ(&conn->ifc_conn.cn_cces[0].cce_cid,
+                            &packet_in->pi_dcid)
+                && !(conn->ifc_conn.cn_cces[0].cce_flags & CCE_SEQNO))
+            {
+                ABORT_QUIETLY(0, TEC_PROTOCOL_VIOLATION,
+                            "protocol violation detected bad dcid");
+                return -1;
+            }
+        }
+        if (pns == PNS_HSK)
+        {
+            if ((conn->ifc_flags & (IFC_SERVER | IFC_IGNORE_INIT)) == IFC_SERVER)
+                ignore_init(conn);
+            lsquic_send_ctl_maybe_calc_rough_rtt(&conn->ifc_send_ctl, pns - 1);
+        }
+    }
     EV_LOG_PACKET_IN(LSQUIC_LOG_CONN_ID, packet_in);
 
     is_rechist_empty = lsquic_rechist_is_empty(&conn->ifc_rechist[pns]);
@@ -7446,8 +7579,9 @@ process_regular_packet (struct ietf_full_conn *conn,
                                                     << packet_in->pi_path_id);
                 }
             }
-            else if (!LSQUIC_CIDS_EQ(CN_SCID(&conn->ifc_conn),
-                                                    &packet_in->pi_dcid))
+            else if (is_dcid_changed
+                && !LSQUIC_CIDS_EQ(CN_SCID(&conn->ifc_conn),
+                                   &packet_in->pi_dcid))
             {
                 if (0 != on_dcid_change(conn, &packet_in->pi_dcid))
                     return -1;
@@ -7534,7 +7668,7 @@ process_regular_packet (struct ietf_full_conn *conn,
         }
         conn->ifc_pub.bytes_in += packet_in->pi_data_sz;
         if ((conn->ifc_mflags & MF_VALIDATE_PATH) &&
-                (packet_in->pi_header_type == HETY_NOT_SET
+                (packet_in->pi_header_type == HETY_SHORT
               || packet_in->pi_header_type == HETY_HANDSHAKE))
         {
             conn->ifc_mflags &= ~MF_VALIDATE_PATH;
@@ -7562,7 +7696,24 @@ verneg_ok (const struct ietf_full_conn *conn)
     enum lsquic_version ver;
 
     ver = highest_bit_set(conn->ifc_u.cli.ifcli_ver_neg.vn_supp);
-    return (1 << ver) & LSQUIC_IETF_DRAFT_VERSIONS;
+    return (1 << ver) & LSQUIC_IETF_VERSIONS;
+}
+
+
+static void
+enable_ping_alarm_for_handshake (struct ietf_full_conn *conn)
+{
+    conn->ifc_ping_period = HSK_PING_TIMEOUT;
+    lsquic_alarmset_set(&conn->ifc_alset, AL_PING,
+                        lsquic_time_now() + conn->ifc_ping_period);
+}
+
+
+static int
+switch_version (struct ietf_full_conn *conn, enum lsquic_version version)
+{
+    conn->ifc_conn.cn_version = version;
+    return iquic_esfi_switch_version(conn->ifc_conn.cn_enc_session, NULL, 0);
 }
 
 
@@ -7650,19 +7801,39 @@ process_incoming_packet_verneg (struct ietf_full_conn *conn,
         return 0;
     }
 
+    if (packet_in->pi_version != conn->ifc_u.cli.ifcli_ver_neg.vn_ver)
+    {
+        if (!((1 << packet_in->pi_version)
+              & conn->ifc_u.cli.ifcli_ver_neg.vn_supp))
+        {
+            LSQ_DEBUG("server version doesn't match versions "
+                        "supported: ignore");
+            return 0;
+        }
+        LSQ_DEBUG("version negociation: server switched version from %s to %s",
+        lsquic_ver2str[conn->ifc_u.cli.ifcli_ver_neg.vn_ver],
+        lsquic_ver2str[packet_in->pi_version]);
+        switch_version(conn, packet_in->pi_version);
+    }
+    else
+        conn->ifc_conn.cn_version = conn->ifc_u.cli.ifcli_ver_neg.vn_ver;
     assert(conn->ifc_u.cli.ifcli_ver_neg.vn_tag);
     assert(conn->ifc_u.cli.ifcli_ver_neg.vn_state != VN_END);
     conn->ifc_u.cli.ifcli_ver_neg.vn_state = VN_END;
     conn->ifc_u.cli.ifcli_ver_neg.vn_tag = NULL;
-    conn->ifc_conn.cn_version = conn->ifc_u.cli.ifcli_ver_neg.vn_ver;
     conn->ifc_conn.cn_flags |= LSCONN_VER_SET;
     LSQ_DEBUG("end of version negotiation: agreed upon %s",
-                    lsquic_ver2str[conn->ifc_u.cli.ifcli_ver_neg.vn_ver]);
+                    lsquic_ver2str[conn->ifc_conn.cn_version]);
     EV_LOG_VER_NEG(LSQUIC_LOG_CONN_ID,
-            "agreed", lsquic_ver2str[conn->ifc_u.cli.ifcli_ver_neg.vn_ver]);
-    conn->ifc_process_incoming_packet = process_incoming_packet_fast;
+            "agreed", lsquic_ver2str[conn->ifc_conn.cn_version]);
+    conn->ifc_process_incoming_packet = process_regular_packet;
 
-    return process_regular_packet(conn, packet_in);
+    if (process_regular_packet(conn, packet_in) == 0)
+    {
+        enable_ping_alarm_for_handshake(conn);
+        return 0;
+    }
+    return -1;
 }
 
 
@@ -7785,7 +7956,7 @@ ietf_full_conn_ci_packet_sent (struct lsquic_conn *lconn,
     struct ietf_full_conn *const conn = (struct ietf_full_conn *) lconn;
     int s;
 
-    if (packet_out->po_frame_types & IQUIC_FRAME_RETX_MASK)
+    if (packet_out->po_frame_types & (IQUIC_FRAME_RETX_MASK))
         conn->ifc_n_cons_unretx = 0;
     else
         ++conn->ifc_n_cons_unretx;
@@ -7825,6 +7996,7 @@ static void (*const send_funcs[N_SEND])(
     [SEND_MAX_STREAMS_UNI]  = generate_max_streams_uni_frame,
     [SEND_MAX_STREAMS_BIDI] = generate_max_streams_bidi_frame,
     [SEND_STOP_SENDING] = generate_stop_sending_frames,
+    [SEND_NEW_TOKEN]    = generate_new_token_frame,
     [SEND_PATH_CHAL_PATH_0]    = generate_path_chal_0,
     [SEND_PATH_CHAL_PATH_1]    = generate_path_chal_1,
     [SEND_PATH_CHAL_PATH_2]    = generate_path_chal_2,
@@ -7849,7 +8021,7 @@ static void (*const send_funcs[N_SEND])(
     |SF_SEND_PATH_RESP_PATH_2|SF_SEND_PATH_RESP_PATH_3\
     |SF_SEND_PING|SF_SEND_HANDSHAKE_DONE\
     |SF_SEND_ACK_FREQUENCY\
-    |SF_SEND_STOP_SENDING)
+    |SF_SEND_STOP_SENDING|SF_SEND_NEW_TOKEN)
 
 
 /* This should be called before lsquic_alarmset_ring_expired() */
@@ -8417,7 +8589,8 @@ ietf_full_conn_ci_tick (struct lsquic_conn *lconn, lsquic_time_t now)
                     || 0 != lsquic_send_ctl_n_scheduled(&conn->ifc_send_ctl))
                 )
         {
-            RETURN_IF_OUT_OF_PACKETS();
+            /* CONNECTION_CLOSE frame should not be congestion controlled.
+            RETURN_IF_OUT_OF_PACKETS(); */
             generate_connection_close_packet(conn);
             tick |= TICK_SEND|TICK_CLOSE;
         }
@@ -8442,7 +8615,8 @@ ietf_full_conn_ci_tick (struct lsquic_conn *lconn, lsquic_time_t now)
             goto end;
         }
     }
-    else if (conn->ifc_ping_period)
+    else if (conn->ifc_ping_period
+            && (conn->ifc_conn.cn_flags & LSCONN_HANDSHAKE_DONE))
     {
         lsquic_alarmset_unset(&conn->ifc_alset, AL_PING);
         lsquic_send_ctl_sanity_check(&conn->ifc_send_ctl);
@@ -8649,25 +8823,6 @@ path_matches_local_sa (const struct network_path *path,
 }
 
 
-static const lsquic_cid_t *
-ietf_full_conn_ci_get_log_cid (const struct lsquic_conn *lconn)
-{
-    struct ietf_full_conn *const conn = (struct ietf_full_conn *) lconn;
-
-    if (lconn->cn_flags & LSCONN_SERVER)
-    {
-        if (CUR_DCID(conn)->len)
-            return CUR_DCID(conn);
-        else
-            return CN_SCID(lconn);
-    }
-    if (CN_SCID(lconn)->len)
-        return CN_SCID(lconn);
-    else
-        return CUR_DCID(conn);
-}
-
-
 static struct network_path *
 ietf_full_conn_ci_get_path (struct lsquic_conn *lconn,
                                                     const struct sockaddr *sa)
@@ -8797,7 +8952,7 @@ ietf_full_conn_ci_count_garbage (struct lsquic_conn *lconn, size_t garbage_sz)
 {
     struct ietf_full_conn *conn = (struct ietf_full_conn *) lconn;
 
-    conn->ifc_pub.bytes_in = garbage_sz;
+    conn->ifc_pub.bytes_in += garbage_sz;
     LSQ_DEBUG("count %zd bytes of garbage, new value: %u bytes", garbage_sz,
         conn->ifc_pub.bytes_in);
 }
@@ -8870,7 +9025,6 @@ ietf_full_conn_ci_log_stats (struct lsquic_conn *lconn)
     .ci_drop_crypto_streams  =  ietf_full_conn_ci_drop_crypto_streams, \
     .ci_early_data_failed    =  ietf_full_conn_ci_early_data_failed, \
     .ci_get_engine           =  ietf_full_conn_ci_get_engine, \
-    .ci_get_log_cid          =  ietf_full_conn_ci_get_log_cid, \
     .ci_get_min_datagram_size=  ietf_full_conn_ci_get_min_datagram_size, \
     .ci_get_path             =  ietf_full_conn_ci_get_path, \
     .ci_going_away           =  ietf_full_conn_ci_going_away, \
@@ -9481,6 +9635,7 @@ hcsi_on_new (void *stream_if_ctx, struct lsquic_stream *stream)
             break;
         case (0 << 8) | LSQVER_ID29:
         case (0 << 8) | LSQVER_I001:
+        case (0 << 8) | LSQVER_I002:
             callbacks = &hcsi_callbacks_client_29;
             break;
         default:
@@ -9488,6 +9643,7 @@ hcsi_on_new (void *stream_if_ctx, struct lsquic_stream *stream)
             /* fallthru */
         case (1 << 8) | LSQVER_ID29:
         case (1 << 8) | LSQVER_I001:
+        case (1 << 8) | LSQVER_I002:
             callbacks = &hcsi_callbacks_server_29;
             break;
     }

src/liblsquic/lsquic_handshake.c

@@ -767,7 +767,7 @@ gquic2_setup_handshake_keys (struct lsquic_enc_session *enc_session)
                                                     secret, sizeof(secret)))
             goto err;
         if (enc_session->es_flags & ES_LOG_SECRETS)
-            log_crypto_ctx(enc_session, ENC_LEV_CLEAR, i);
+            log_crypto_ctx(enc_session, ENC_LEV_INIT, i);
     }
 
     return 0;
@@ -3069,14 +3069,14 @@ decrypt_packet (struct lsquic_enc_session *enc_session, uint8_t path_id,
             key = enc_session->dec_ctx_f;
             memcpy(nonce, enc_session->dec_key_nonce_f, 4);
             LSQ_DEBUG("decrypt_packet using 'F' key...");
-            enc_level = ENC_LEV_FORW;
+            enc_level = ENC_LEV_APP;
         }
         else
         {
             key = enc_session->dec_ctx_i;
             memcpy(nonce, enc_session->dec_key_nonce_i, 4);
             LSQ_DEBUG("decrypt_packet using 'I' key...");
-            enc_level = ENC_LEV_INIT;
+            enc_level = ENC_LEV_HSK;
         }
         memcpy(nonce + 4, &path_id_packet_number,
                sizeof(path_id_packet_number));
@@ -3150,7 +3150,7 @@ lsquic_enc_session_decrypt (struct lsquic_enc_session *enc_session,
                         header_len, data_len, buf_out, max_out_len, out_len);
     else if (0 == verify_packet_hash(enc_session, version, buf, header_len,
                                      data_len, buf_out, max_out_len, out_len))
-        return ENC_LEV_CLEAR;
+        return ENC_LEV_INIT;
     else
         return -1;
 }
@@ -3247,7 +3247,7 @@ gquic_encrypt_buf (struct lsquic_enc_session *enc_session,
         memcpy(buf_out, header, header_len);
         memcpy(buf_out + header_len, md, HS_PKT_HASH_LENGTH);
         memcpy(buf_out + header_len + HS_PKT_HASH_LENGTH, data, data_len);
-        return ENC_LEV_CLEAR;
+        return ENC_LEV_INIT;
     }
     else
     {
@@ -3262,14 +3262,14 @@ gquic_encrypt_buf (struct lsquic_enc_session *enc_session,
             {
                 enc_session->server_start_use_final_key = 1;
             }
-            enc_level = ENC_LEV_INIT;
+            enc_level = ENC_LEV_HSK;
         }
         else
         {
             LSQ_DEBUG("lsquic_enc_session_encrypt using 'F' key...");
             key = enc_session->enc_ctx_f;
             memcpy(nonce, enc_session->enc_key_nonce_f, 4);
-            enc_level = ENC_LEV_FORW;
+            enc_level = ENC_LEV_APP;
         }
         path_id_packet_number = combine_path_id_pack_num(path_id, pack_num);
         memcpy(nonce + 4, &path_id_packet_number,
@@ -3964,9 +3964,9 @@ static const char *const gel2str[] =
 
 static const enum enc_level gel2el[] =
 {
-    [GEL_CLEAR] = ENC_LEV_CLEAR,
-    [GEL_EARLY] = ENC_LEV_EARLY,
-    [GEL_FORW]  = ENC_LEV_FORW,
+    [GEL_CLEAR] = ENC_LEV_INIT,
+    [GEL_EARLY] = ENC_LEV_0RTT,
+    [GEL_FORW]  = ENC_LEV_APP,
 };
 
 

src/liblsquic/lsquic_hkdf.h

@@ -14,6 +14,10 @@
 #define HSK_SALT ((unsigned char *) \
                      "\x38\x76\x2c\xf7\xf5\x59\x34\xb3\x4d\x17" \
                      "\x9a\xe6\xa4\xc8\x0c\xad\xcc\xbb\x7f\x0a")
+/* [draft-ietf-quic-v2] Section 3.3.1 */
+#define HSK_SALT_V2 ((unsigned char *) \
+                     "\x0d\xed\xe3\xde\xf7\x00\xa6\xdb\x81\x93" \
+                     "\x81\xbe\x6e\x26\x9d\xcb\xf9\xbd\x2e\xd9")
 #define HSK_SALT_SZ (sizeof(HSK_SALT_BUF) - 1)
 
 #define CLIENT_LABEL "client in"

src/liblsquic/lsquic_http1x_if.c

@@ -61,6 +61,9 @@ struct header_writer_ctx
 
 #define HWC_PSEH_VAL(hwc, ph) ((hwc)->pseh_bufs[ph])
 
+/* flags for frames with request headers */
+#define HWC_REQUEST_HANDLING_FLAGS (HWC_SERVER|HWC_PUSH_PROMISE)
+
 static void *
 h1h_create_header_set (void *ctx, lsquic_stream_t *stream, int is_push_promise)
 {
@@ -396,7 +399,15 @@ add_real_header (struct header_writer_ctx *hwc, struct lsxpack_header *xhdr)
     val_len = xhdr->val_len;
 
     if (4 == name_len && 0 == memcmp(name, "host", 4))
+    {
+        if(hwc->pseh_mask & BIT(PSEH_AUTHORITY))
+        {
+            LSQ_INFO("authority header was sent. ignoring host header with value `%.*s'", val_len, val);
+            return 0;
+        }
+
         hwc->hwc_flags |= HWC_SEEN_HOST;
+    }
 
     n_upper = 0;
     for (i = 0; i < name_len; ++i)
@@ -469,15 +480,22 @@ h1h_finish_hset (struct header_writer_ctx *hwc)
         return st;                                                      \
 } while (0)
 
-    if ((hwc->pseh_mask & BIT(PSEH_AUTHORITY)) &&
-                                0 == (hwc->hwc_flags & HWC_SEEN_HOST))
+    if(0 == (hwc->hwc_flags & HWC_SEEN_HOST) && 0 != (hwc->hwc_flags & HWC_REQUEST_HANDLING_FLAGS))
     {
-        LSQ_DEBUG("Setting 'Host: %.*s'", HWC_PSEH_LEN(hwc, PSEH_AUTHORITY),
+        if((hwc->pseh_mask & BIT(PSEH_AUTHORITY)))
+        {
+            LSQ_DEBUG("Setting 'Host: %.*s'", HWC_PSEH_LEN(hwc, PSEH_AUTHORITY),
                                             HWC_PSEH_VAL(hwc, PSEH_AUTHORITY));
-        HWC_UH_WRITE(hwc, "Host: ", 6);
-        HWC_UH_WRITE(hwc, HWC_PSEH_VAL(hwc, PSEH_AUTHORITY),
-                                        HWC_PSEH_LEN(hwc, PSEH_AUTHORITY));
-        HWC_UH_WRITE(hwc, "\r\n", 2);
+            HWC_UH_WRITE(hwc, "Host: ", 6);
+            HWC_UH_WRITE(hwc, HWC_PSEH_VAL(hwc, PSEH_AUTHORITY),
+                                            HWC_PSEH_LEN(hwc, PSEH_AUTHORITY));
+            HWC_UH_WRITE(hwc, "\r\n", 2);
+        }
+        else
+        {
+            LSQ_INFO("both host header and authority pseudo header were not found");
+            return 1;
+        }
     }
 
     if (hwc->cookie_val)

src/liblsquic/lsquic_ietf.h

@@ -24,6 +24,7 @@ enum trans_error_code
     TEC_KEY_UPDATE_ERROR           =  0xE,
     TEC_AEAD_LIMIT_REACHED         =  0xF,
     TEC_NO_VIABLE_PATH             = 0x10,
+    TEC_VERSION_NEGOTIATION_ERROR  = 0x11,
 };
 
 /* Must be at least two */
@@ -33,12 +34,12 @@ enum trans_error_code
 #define IETF_RETRY_KEY_SZ 16
 #define IETF_RETRY_NONCE_SZ 12
 
-#define N_IETF_RETRY_VERSIONS 3
+#define N_IETF_RETRY_VERSIONS 4
 extern const unsigned char *const lsquic_retry_key_buf[N_IETF_RETRY_VERSIONS];
 extern const unsigned char *const lsquic_retry_nonce_buf[N_IETF_RETRY_VERSIONS];
 #define lsquic_version_2_retryver(ver_) (                       \
     (ver_) <= LSQVER_ID27 ? 0 :                                 \
     (ver_) <  LSQVER_I001 ? 1 :                                 \
-    2)
+    (ver_) == LSQVER_I002 ? 3 : 2)
 
 #endif

src/liblsquic/lsquic_mini_conn.c

@@ -225,6 +225,7 @@ lsquic_mini_conn_new (struct lsquic_engine_public *enp,
     mc->mc_conn.cn_esf_c = select_esf_common_by_ver(version);
     mc->mc_conn.cn_esf.g = select_esf_gquic_by_ver(version);
     mc->mc_conn.cn_cid = packet_in->pi_conn_id;
+    mc->mc_conn.cn_logid = packet_in->pi_conn_id;
     mc->mc_conn.cn_flags = LSCONN_MINI | LSCONN_SERVER;
     mc->mc_conn.cn_if = conn_iface;
     LSQ_DEBUG("created mini connection object");
@@ -1054,7 +1055,11 @@ continue_handshake (struct mini_conn *mc)
      */
     TAILQ_FOREACH(packet_in, &mc->mc_packets_in, pi_next)
     {
-        assert(n_hsk_chunks < sizeof(hsk_chunks) / sizeof(hsk_chunks[0]));
+        if (n_hsk_chunks >= sizeof(hsk_chunks) / sizeof(hsk_chunks[0])) {
+            LSQ_WARN("too many handshake packets");
+            return -1;
+        }
+
         if (0 == (packet_in->pi_flags & PI_HSK_STREAM))
             continue;
         s = parse_frame(packet_in->pi_data + packet_in->pi_hsk_stream,

src/liblsquic/lsquic_mini_conn_ietf.c

@@ -58,10 +58,10 @@ ietf_mini_conn_ci_abort_error (struct lsquic_conn *lconn, int is_app,
 
 static const enum header_type el2hety[] =
 {
-    [ENC_LEV_INIT]  = HETY_HANDSHAKE,
-    [ENC_LEV_CLEAR] = HETY_INITIAL,
-    [ENC_LEV_FORW]  = HETY_NOT_SET,
-    [ENC_LEV_EARLY] = 0,    /* Invalid */
+    [ENC_LEV_HSK]  = HETY_HANDSHAKE,
+    [ENC_LEV_INIT] = HETY_INITIAL,
+    [ENC_LEV_APP]  = HETY_SHORT,
+    [ENC_LEV_0RTT] = 0,    /* Invalid */
 };
 
 
@@ -87,19 +87,13 @@ lsquic_mini_conn_ietf_ecn_ok (const struct ietf_mini_conn *conn)
 }
 
 
-#define imico_ecn_ok lsquic_mini_conn_ietf_ecn_ok
-
-
 static enum ecn
 imico_get_ecn (struct ietf_mini_conn *conn)
 {
     if (!conn->imc_enpub->enp_settings.es_ecn)
         return ECN_NOT_ECT;
-    else if (!conn->imc_sent_packnos /* We set ECT0 in first flight */
-                                                    || imico_ecn_ok(conn))
-        return ECN_ECT0;
     else
-        return ECN_NOT_ECT;
+        return ECN_ECT0;
 }
 
 
@@ -175,8 +169,54 @@ read_from_msg_ctx (void *ctx, void *buf, size_t len, int *fin)
 static int
 imico_chlo_has_been_consumed (const struct ietf_mini_conn *conn)
 {
-    return conn->imc_streams[ENC_LEV_CLEAR].mcs_read_off > 3
-        && conn->imc_streams[ENC_LEV_CLEAR].mcs_read_off >= conn->imc_ch_len;
+    return conn->imc_streams[ENC_LEV_INIT].mcs_read_off > 3
+        && conn->imc_streams[ENC_LEV_INIT].mcs_read_off >= conn->imc_ch_len;
+}
+
+
+static int
+imico_process_version_negociation (struct ietf_mini_conn *conn,
+                                   const struct transport_params *params)
+{
+    unsigned common_versions;
+    unsigned ver;
+    int i;
+    if (LSQ_LOG_ENABLED(LSQ_LOG_DEBUG))
+    {
+        LSQ_DEBUG("client chosen version %s",
+                    lsquic_ver2str[params->tp_chosen_version]);
+        for (i = 1; i < params->tp_version_cnt; ++i)
+        {
+            LSQ_DEBUG("client available version %s",
+                      lsquic_ver2str[params->tp_version_info[i]]);
+            //EV_LOG_VER_NEG(LSQUIC_LOG_CONN_ID, "supports",
+            //               lsquic_ver2str[params->tp_version_info[i]]);
+        }
+    }
+    if (params->tp_chosen_version != conn->imc_conn.cn_version)
+    {
+        LSQ_DEBUG("client chosen version does not match the version in use %s",
+                    lsquic_ver2str[conn->imc_conn.cn_version]);
+        ietf_mini_conn_ci_abort_error(&conn->imc_conn, 0,
+                                      TEC_VERSION_NEGOTIATION_ERROR,
+            "chosen version mismatch");
+        return -1;
+    }
+    common_versions = conn->imc_enpub->enp_settings.es_versions
+                & params->tp_versions;
+    ver = highest_bit_set(common_versions);
+    if (conn->imc_conn.cn_version != (enum lsquic_version)ver)
+    {
+        LSQ_DEBUG("version negociation: switch version from %s to %s",
+                  lsquic_ver2str[conn->imc_conn.cn_version],
+                  lsquic_ver2str[ver]);
+        conn->imc_conn.cn_version = ver;
+        conn->imc_conn.cn_flags |= LSCONN_VER_UPDATED;
+        iquic_esfi_switch_version(conn->imc_conn.cn_enc_session,
+                                  &conn->imc_cces[0].cce_cid, 1);
+    }
+
+    return 0;
 }
 
 
@@ -204,6 +244,15 @@ imico_maybe_process_params (struct ietf_mini_conn *conn)
             }
             LSQ_DEBUG("read transport params, packet size is set to %hu bytes",
                                                 conn->imc_path.np_pack_size);
+            if (params->tp_set & (1 << TPI_VERSION_INFORMATION))
+            {
+                if (imico_process_version_negociation(conn, params) == -1)
+                {
+                    conn->imc_flags |= IMC_VER_NEG_FAILED;
+                    return -1;
+                }
+            }
+            iquic_esfi_init_server_tp(conn->imc_conn.cn_enc_session);
         }
         else
         {
@@ -216,6 +265,10 @@ imico_maybe_process_params (struct ietf_mini_conn *conn)
 }
 
 
+static int
+imico_generate_ack (struct ietf_mini_conn *conn, enum packnum_space pns,
+                                                            lsquic_time_t now);
+
 static ssize_t
 imico_stream_write (void *stream, const void *bufp, size_t bufsz)
 {
@@ -239,19 +292,32 @@ imico_stream_write (void *stream, const void *bufp, size_t bufsz)
         return bufsz;
     }
 
+    if (cryst->mcs_enc_level == ENC_LEV_INIT
+        && (conn->imc_flags & IMC_QUEUED_ACK_INIT))
+    {
+        imico_generate_ack(conn, PNS_INIT, lsquic_time_now());
+    }
+
     while (msg_ctx.buf < msg_ctx.end)
     {
         header_sz = lconn->cn_pf->pf_calc_crypto_frame_header_sz(
                             cryst->mcs_write_off, msg_ctx.end - msg_ctx.buf);
-        need = header_sz + 1;
+        need = header_sz + 500;
         packet_out = imico_get_packet_out(conn,
                                         el2hety[ cryst->mcs_enc_level ], need);
         if (!packet_out)
             return -1;
-
+        // NOTE: reduce the size of first crypto frame to combine packets
+        int avail = lsquic_packet_out_avail(packet_out);
+        if (cryst->mcs_enc_level == ENC_LEV_HSK
+            && cryst->mcs_write_off == 0
+            && avail > conn->imc_hello_pkt_remain - conn->imc_long_header_sz)
+        {
+            avail = conn->imc_hello_pkt_remain - conn->imc_long_header_sz;
+        }
         p = msg_ctx.buf;
         len = pf->pf_gen_crypto_frame(packet_out->po_data + packet_out->po_data_sz,
-                    lsquic_packet_out_avail(packet_out), 0, cryst->mcs_write_off, 0,
+                    avail, 0, cryst->mcs_write_off, 0,
                     msg_ctx.end - msg_ctx.buf, read_from_msg_ctx, &msg_ctx);
         if (len < 0)
             return len;
@@ -261,6 +327,8 @@ imico_stream_write (void *stream, const void *bufp, size_t bufsz)
         packet_out->po_frame_types |= 1 << QUIC_FRAME_CRYPTO;
         packet_out->po_flags |= PO_HELLO;
         cryst->mcs_write_off += msg_ctx.buf - p;
+        if (cryst->mcs_enc_level == ENC_LEV_INIT)
+            conn->imc_hello_pkt_remain = avail - len;
     }
 
     assert(msg_ctx.buf == msg_ctx.end);
@@ -299,8 +367,8 @@ imico_read_chlo_size (struct ietf_mini_conn *conn, const unsigned char *buf,
 {
     const unsigned char *const end = buf + sz;
 
-    assert(conn->imc_streams[ENC_LEV_CLEAR].mcs_read_off < 4);
-    switch (conn->imc_streams[ENC_LEV_CLEAR].mcs_read_off)
+    assert(conn->imc_streams[ENC_LEV_INIT].mcs_read_off < 4);
+    switch (conn->imc_streams[ENC_LEV_INIT].mcs_read_off)
     {
     case 0:
         if (buf == end)
@@ -356,7 +424,7 @@ imico_stream_readf (void *stream,
         avail = DF_SIZE(frame) - frame->data_frame.df_read_off;
         buf = frame->data_frame.df_data + frame->data_frame.df_read_off;
         nread = readf(ctx, buf, avail, DF_FIN(frame));
-        if (cryst->mcs_enc_level == ENC_LEV_CLEAR && cryst->mcs_read_off < 4)
+        if (cryst->mcs_enc_level == ENC_LEV_INIT && cryst->mcs_read_off < 4)
             imico_read_chlo_size(conn, buf, nread);
         total_read += nread;
         cryst->mcs_read_off += nread;
@@ -468,7 +536,7 @@ imico_peer_addr_validated (struct ietf_mini_conn *conn, const char *how)
 
 struct lsquic_conn *
 lsquic_mini_conn_ietf_new (struct lsquic_engine_public *enpub,
-               const struct lsquic_packet_in *packet_in,
+                           struct lsquic_packet_in *packet_in,
            enum lsquic_version version, int is_ipv4, const lsquic_cid_t *odcid,
            size_t udp_payload_size)
 {
@@ -480,7 +548,7 @@ lsquic_mini_conn_ietf_new (struct lsquic_engine_public *enpub,
 
     if (!is_first_packet_ok(packet_in, udp_payload_size))
         return NULL;
-
+    packet_in->pi_flags |= PI_FIRST_INIT;
     conn = lsquic_malo_get(enpub->enp_mm.malo.mini_conn_ietf);
     if (!conn)
     {
@@ -497,15 +565,19 @@ lsquic_mini_conn_ietf_new (struct lsquic_engine_public *enpub,
     conn->imc_cces[0].cce_flags = CCE_USED;
     conn->imc_conn.cn_cces_mask = 1;
     lsquic_scid_from_packet_in(packet_in, &conn->imc_path.np_dcid);
-    LSQ_DEBUGC("recv SCID from client %"CID_FMT, CID_BITS(&conn->imc_cces[0].cce_cid));
-    LSQ_DEBUGC("recv DCID from client %"CID_FMT, CID_BITS(&conn->imc_path.np_dcid));
-
     /* Generate new SCID. Since is not the original SCID, it is given
      * a sequence number (0) and therefore can be retired by the client.
      */
     enpub->enp_generate_scid(enpub->enp_gen_scid_ctx, &conn->imc_conn,
         &conn->imc_conn.cn_cces[1].cce_cid, enpub->enp_settings.es_scid_len);
 
+    if (conn->imc_path.np_dcid.len)
+        conn->imc_conn.cn_logid = conn->imc_path.np_dcid;
+    else
+        conn->imc_conn.cn_logid = conn->imc_conn.cn_cces[1].cce_cid;
+
+    LSQ_DEBUGC("recv SCID from client %"CID_FMT, CID_BITS(&conn->imc_cces[0].cce_cid));
+    LSQ_DEBUGC("recv DCID from client %"CID_FMT, CID_BITS(&conn->imc_path.np_dcid));
     LSQ_DEBUGC("generated SCID %"CID_FMT" at index %u, switching to it",
                 CID_BITS(&conn->imc_conn.cn_cces[1].cce_cid), 1);
     conn->imc_conn.cn_cces[1].cce_flags = CCE_SEQNO | CCE_USED;
@@ -539,9 +611,16 @@ lsquic_mini_conn_ietf_new (struct lsquic_engine_public *enpub,
     }
 
     esfi = select_esf_iquic_by_ver(version);
-    enc_sess = esfi->esfi_create_server(enpub, &conn->imc_conn,
-                &packet_in->pi_dcid, conn->imc_stream_ps, &crypto_stream_if,
-                &conn->imc_cces[0].cce_cid, &conn->imc_path.np_dcid);
+    if (version > LSQVER_ID27)
+        enc_sess = esfi->esfi_create_server(enpub, &conn->imc_conn,
+                    &packet_in->pi_dcid, conn->imc_stream_ps, &crypto_stream_if,
+                    odcid ? odcid : &conn->imc_cces[0].cce_cid,
+                    &conn->imc_path.np_dcid,
+                    odcid ? &conn->imc_cces[0].cce_cid : NULL);
+    else
+        enc_sess = esfi->esfi_create_server(enpub, &conn->imc_conn,
+                    &packet_in->pi_dcid, conn->imc_stream_ps, &crypto_stream_if,
+                    odcid, &conn->imc_path.np_dcid, NULL);
     if (!enc_sess)
     {
         lsquic_malo_put(conn);
@@ -549,7 +628,8 @@ lsquic_mini_conn_ietf_new (struct lsquic_engine_public *enpub,
     }
 
     conn->imc_enpub = enpub;
-    conn->imc_created = packet_in->pi_received;
+    conn->imc_expire = packet_in->pi_received +
+                            enpub->enp_settings.es_handshake_to;
     if (enpub->enp_settings.es_base_plpmtu)
         conn->imc_path.np_pack_size = enpub->enp_settings.es_base_plpmtu;
     else if (is_ipv4)
@@ -576,8 +656,22 @@ lsquic_mini_conn_ietf_new (struct lsquic_engine_public *enpub,
     }
 #endif
 
+    conn->imc_long_header_sz = 1 /* Type */
+       + 4 /* Version */
+       + 1 /* DCIL */
+       + conn->imc_path.np_dcid.len
+       + 1 /* SCIL */
+       + CN_SCID(&conn->imc_conn)->len
+       + 0 /* HSK packet by server has no token */
+       + 2 /* Always use two bytes to encode payload length */
+       + 1 /* mini conn packet number < 256 */
+       + IQUIC_TAG_LEN
+       ;
+    conn->imc_hello_pkt_remain = conn->imc_path.np_pack_size
+                - conn->imc_long_header_sz - 1 /* token len */;
+
     LSQ_DEBUG("created mini connection object %p; max packet size=%hu",
-                                                conn, conn->imc_path.np_pack_size);
+                                     conn, conn->imc_path.np_pack_size);
     return &conn->imc_conn;
 }
 
@@ -679,7 +773,8 @@ ietf_mini_conn_ci_is_tickable (struct lsquic_conn *lconn)
     const struct lsquic_packet_out *packet_out;
     size_t packet_size;
 
-    if (conn->imc_enpub->enp_flags & ENPUB_CAN_SEND)
+    if ((conn->imc_enpub->enp_flags & ENPUB_CAN_SEND)
+        && !(conn->imc_flags & IMC_AMP_CAPPED))
         TAILQ_FOREACH(packet_out, &conn->imc_packets_out, po_next)
             if (!(packet_out->po_flags & PO_SENT))
             {
@@ -700,18 +795,117 @@ imico_can_send (const struct ietf_mini_conn *conn, size_t size)
 }
 
 
-static void
-imico_zero_pad (struct lsquic_packet_out *packet_out)
-{
-    size_t pad_size;
+// static void
+// imico_zero_pad (struct lsquic_packet_out *packet_out)
+// {
+//     size_t pad_size;
+//
+//     pad_size = lsquic_packet_out_avail(packet_out);
+//     memset(packet_out->po_data + packet_out->po_data_sz, 0, pad_size);
+//     packet_out->po_padding_sz = pad_size;
+//     packet_out->po_data_sz += pad_size;
+//     packet_out->po_frame_types |= QUIC_FTBIT_PADDING;
+// }
 
-    pad_size = lsquic_packet_out_avail(packet_out);
-    memset(packet_out->po_data + packet_out->po_data_sz, 0, pad_size);
-    packet_out->po_data_sz += pad_size;
-    packet_out->po_frame_types |= QUIC_FTBIT_PADDING;
+
+static lsquic_time_t
+imico_rechist_largest_recv (void *rechist_ctx);
+
+
+static int
+imico_build_ack_frame (struct ietf_mini_conn *conn, enum packnum_space pns,
+                       lsquic_time_t now, unsigned char *outbuf,
+                       size_t outbuf_sz, lsquic_packno_t *ack2ed)
+{
+    struct ietf_mini_rechist rechist;
+    uint64_t ecn_counts_buf[4];
+    const uint64_t *ecn_counts;
+    int not_used_has_missing, len;
+    if (conn->imc_incoming_ecn)
+    {
+        ecn_counts_buf[0]   = conn->imc_ecn_counts_in[pns][0];
+        ecn_counts_buf[1]   = conn->imc_ecn_counts_in[pns][1];
+        ecn_counts_buf[2]   = conn->imc_ecn_counts_in[pns][2];
+        ecn_counts_buf[3]   = conn->imc_ecn_counts_in[pns][3];
+        ecn_counts = ecn_counts_buf;
+    }
+    else
+        ecn_counts = NULL;
+    lsquic_imico_rechist_init(&rechist, conn, pns);
+    len = conn->imc_conn.cn_pf->pf_gen_ack_frame(outbuf, outbuf_sz,
+                lsquic_imico_rechist_first,
+                lsquic_imico_rechist_next, imico_rechist_largest_recv, &rechist,
+                now, &not_used_has_missing, ack2ed, ecn_counts);
+    if (len < 0)
+    {
+        LSQ_WARN("could not generate ACK frame");
+        return -1;
+    }
+    EV_LOG_GENERATED_ACK_FRAME(LSQUIC_LOG_CONN_ID, conn->imc_conn.cn_pf,
+                               outbuf, len);
+    return len;
 }
 
 
+static void
+remove_ack_frame (struct ietf_mini_conn *conn,
+                   struct lsquic_packet_out *packet_out)
+{
+    if (packet_out->po_regen_sz == 0)
+        return;
+
+    int l = packet_out->po_data_sz - packet_out->po_regen_sz
+            - packet_out->po_padding_sz;
+    memmove(packet_out->po_data ,
+            packet_out->po_data + packet_out->po_regen_sz, l);
+    memset(packet_out->po_data + l, 0, packet_out->po_regen_sz);
+    packet_out->po_padding_sz += packet_out->po_regen_sz;
+    packet_out->po_regen_sz = 0;
+}
+
+
+// static void
+// update_packet_ack (struct ietf_mini_conn *conn, enum packnum_space pns,
+//                    struct lsquic_packet_out *packet_out)
+// {
+//     int len, need;
+//     unsigned char outbuf[1024];
+//     size_t outbuf_sz = sizeof(outbuf);
+//     lsquic_packno_t ack2ed;
+//
+//     len = imico_build_ack_frame(conn, pns, lsquic_time_now(), outbuf,
+//                       outbuf_sz, &ack2ed);
+//     if (len > packet_out->po_regen_sz + packet_out->po_padding_sz)
+//         return;
+//     need = len + !!(packet_out->po_frame_types & (1 << QUIC_FRAME_PING));
+//     LSQ_DEBUG("update leading ACK frame for packet %"PRIu64
+//               ", regen_sz: %hd, padding_sz: %hd, need: %d, len: %d",
+//               packet_out->po_packno, packet_out->po_regen_sz,
+//               packet_out->po_padding_sz, need, len);
+//
+//     if (need != packet_out->po_regen_sz)
+//     {
+//         int l = packet_out->po_data_sz - packet_out->po_regen_sz
+//                 - packet_out->po_padding_sz;
+//         memmove(packet_out->po_data + need,
+//                 packet_out->po_data + packet_out->po_regen_sz, l);
+//         if (packet_out->po_padding_sz)
+//         {
+//             if (need < packet_out->po_regen_sz)
+//                 memset(packet_out->po_data + need + l, 0,
+//                     packet_out->po_regen_sz - need);
+//             packet_out->po_padding_sz += packet_out->po_regen_sz - need;
+//         }
+//         else
+//             packet_out->po_data_sz -= packet_out->po_regen_sz - need;
+//         packet_out->po_regen_sz = need;
+//     }
+//     memmove(packet_out->po_data, outbuf, len);
+//     packet_out->po_frame_types |= 1 << QUIC_FRAME_ACK;
+//     packet_out->po_ack2ed = ack2ed;
+// }
+
+
 static struct lsquic_packet_out *
 ietf_mini_conn_ci_next_packet_to_send (struct lsquic_conn *lconn,
                                             const struct to_coal *to_coal)
@@ -729,11 +923,31 @@ ietf_mini_conn_ci_next_packet_to_send (struct lsquic_conn *lconn,
          " ack-eliciting Initial packets to at least the smallest allowed
          " maximum datagram size of 1200 bytes.
          */
-        if (packet_out->po_header_type == HETY_INITIAL
-                && !(packet_out->po_frame_types & (1 << QUIC_FRAME_PADDING))
-                && (packet_out->po_frame_types & IQUIC_FRAME_ACKABLE_MASK)
-                && lsquic_packet_out_avail(packet_out) > 0)
-            imico_zero_pad(packet_out);
+        if (packet_out->po_header_type == HETY_INITIAL)
+        {
+            if (packet_out->po_frame_types & (1 << QUIC_FRAME_ACK)
+                && packet_out->po_retx_cnt > 0)
+            {
+                remove_ack_frame(conn, packet_out);
+            }
+
+            if (packet_out->po_retx_cnt > 0
+                && !imico_can_send(conn, IQUIC_MAX_IPv4_PACKET_SZ))
+            {
+                conn->imc_flags |= IMC_AMP_CAPPED;
+                LSQ_DEBUG("cannot send INIT packet #%"PRIu64" without "
+                          "enough quota", packet_out->po_packno);
+                return NULL;
+            }
+//             if (!(packet_out->po_frame_types & (1 << QUIC_FRAME_PADDING))
+//                 && (packet_out->po_frame_types & IQUIC_FRAME_ACKABLE_MASK)
+//                 && lsquic_packet_out_avail(packet_out) > 0)
+//             {
+//                 LSQ_DEBUG("generated PADDING frame: %hd bytes for packet %"PRIu64,
+//                         lsquic_packet_out_avail(packet_out), packet_out->po_packno);
+//                 imico_zero_pad(packet_out);
+//             }
+        }
         packet_size = lsquic_packet_out_total_sz(lconn, packet_out);
         if (!(to_coal
             && (packet_size + to_coal->prev_sz_sum
@@ -743,6 +957,7 @@ ietf_mini_conn_ci_next_packet_to_send (struct lsquic_conn *lconn,
         {
             if (!imico_can_send(conn, packet_size))
             {
+                conn->imc_flags |= IMC_AMP_CAPPED;
                 LSQ_DEBUG("cannot send packet %"PRIu64" of size %zu: client "
                     "address has not been validated", packet_out->po_packno,
                     packet_size);
@@ -778,7 +993,7 @@ imico_calc_retx_timeout (const struct ietf_mini_conn *conn)
     }
     else
         to = 300000;
-    return to << conn->imc_hsk_count;
+    return to;
 }
 
 
@@ -789,23 +1004,20 @@ ietf_mini_conn_ci_next_tick_time (struct lsquic_conn *lconn, unsigned *why)
     const struct lsquic_packet_out *packet_out;
     lsquic_time_t exp_time, retx_time;
 
-    exp_time = conn->imc_created +
-                        conn->imc_enpub->enp_settings.es_handshake_to;
+    exp_time = conn->imc_expire;
 
     TAILQ_FOREACH(packet_out, &conn->imc_packets_out, po_next)
         if (packet_out->po_flags & PO_SENT)
         {
-            retx_time = packet_out->po_sent + imico_calc_retx_timeout(conn);
+            retx_time = packet_out->po_sent + (imico_calc_retx_timeout(conn)
+                                            << (packet_out->po_retx_cnt >> 1));
             if (retx_time < exp_time)
             {
                 *why = N_AEWS + AL_RETX_HSK;
                 return retx_time;
             }
             else
-            {
-                *why = AEW_MINI_EXPIRE;
-                return exp_time;
-            }
+                break;
         }
 
     *why = AEW_MINI_EXPIRE;
@@ -1092,6 +1304,7 @@ static unsigned
 imico_process_ping_frame (IMICO_PROC_FRAME_ARGS)
 {
     LSQ_DEBUG("got a PING frame, do nothing");
+    EV_LOG_PING_FRAME_IN(LSQUIC_LOG_CONN_ID);
     return 1;
 }
 
@@ -1136,6 +1349,17 @@ imico_process_invalid_frame (IMICO_PROC_FRAME_ARGS)
 }
 
 
+static unsigned
+imico_process_invalid_frame_pv (IMICO_PROC_FRAME_ARGS)
+{
+    LSQ_DEBUG("invalid frame %u (%s)", p[0],
+        frame_type_2_str[ conn->imc_conn.cn_pf->pf_parse_frame_type(p, len) ]);
+    ietf_mini_conn_ci_abort_error(&conn->imc_conn, 0, TEC_PROTOCOL_VIOLATION,
+                "protocol violation detected while processing frame");
+    return 0;
+}
+
+
 static unsigned (*const imico_process_frames[N_QUIC_FRAMES])
                                                 (IMICO_PROC_FRAME_ARGS) =
 {
@@ -1154,13 +1378,15 @@ static unsigned (*const imico_process_frames[N_QUIC_FRAMES])
     [QUIC_FRAME_BLOCKED]            =  imico_process_invalid_frame,
     [QUIC_FRAME_STREAM_BLOCKED]     =  imico_process_invalid_frame,
     [QUIC_FRAME_STREAMS_BLOCKED]    =  imico_process_invalid_frame,
-    [QUIC_FRAME_NEW_CONNECTION_ID]  =  imico_process_invalid_frame,
+    [QUIC_FRAME_NEW_CONNECTION_ID]  =  imico_process_invalid_frame_pv,
     [QUIC_FRAME_STOP_SENDING]       =  imico_process_invalid_frame,
     [QUIC_FRAME_PATH_CHALLENGE]     =  imico_process_invalid_frame,
     [QUIC_FRAME_PATH_RESPONSE]      =  imico_process_invalid_frame,
     /* STREAM frame can only come in the App PNS and we delay those packets: */
     [QUIC_FRAME_STREAM]             =  imico_process_invalid_frame,
-    [QUIC_FRAME_HANDSHAKE_DONE]     =  imico_process_invalid_frame,
+    [QUIC_FRAME_RETIRE_CONNECTION_ID] =  imico_process_invalid_frame_pv,
+    [QUIC_FRAME_NEW_TOKEN]          =  imico_process_invalid_frame_pv,
+    [QUIC_FRAME_HANDSHAKE_DONE]     =  imico_process_invalid_frame_pv,
     [QUIC_FRAME_ACK_FREQUENCY]      =  imico_process_invalid_frame,
     [QUIC_FRAME_TIMESTAMP]          =  imico_process_invalid_frame,
 };
@@ -1185,6 +1411,16 @@ imico_process_packet_frame (struct ietf_mini_conn *conn,
     {
         LSQ_DEBUG("invalid frame %u at encryption level %s", type,
                                                 lsquic_enclev2str[enc_level]);
+        if (type == QUIC_FRAME_INVALID)
+            ietf_mini_conn_ci_abort_error(&conn->imc_conn, 0,
+                                          TEC_FRAME_ENCODING_ERROR,
+                                    "invalid frame %u at encryption level %s",
+                                          type, lsquic_enclev2str[enc_level]);
+        else
+            ietf_mini_conn_ci_abort_error(&conn->imc_conn, 0,
+                                          TEC_PROTOCOL_VIOLATION,
+                "protcol violation for invalid frame %u at encryption level %s",
+                                          type, lsquic_enclev2str[enc_level]);
         return 0;
     }
 }
@@ -1200,6 +1436,14 @@ imico_parse_regular_packet (struct ietf_mini_conn *conn,
     p = packet_in->pi_data + packet_in->pi_header_sz;
     pend = packet_in->pi_data + packet_in->pi_data_sz;
 
+    if (p >= pend)
+    {
+        ietf_mini_conn_ci_abort_error(&conn->imc_conn, 0,
+                                      TEC_PROTOCOL_VIOLATION,
+                                      "packet %"PRIu64" has no frames",
+                                      packet_in->pi_packno);
+        return -1;
+    }
     while (p < pend)
     {
         len = imico_process_packet_frame(conn, packet_in, p, pend - p);
@@ -1258,6 +1502,15 @@ ignore_init (struct ietf_mini_conn *conn)
 
     LSQ_DEBUG("henceforth, no Initial packets shall be sent or received; "
         "destroyed %u packet%.*s", count, count != 1, "s");
+
+    int ext_to = conn->imc_enpub->enp_settings.es_idle_timeout * 1000000
+            - conn->imc_enpub->enp_settings.es_handshake_to;
+    if (ext_to > 0)
+    {
+        conn->imc_expire += ext_to;
+        LSQ_DEBUG("extend mini-conn expire time to %d seconds",
+                conn->imc_enpub->enp_settings.es_idle_timeout);
+    }
 }
 
 
@@ -1397,6 +1650,10 @@ imico_switch_to_trechist (struct ietf_mini_conn *conn)
 }
 
 
+static int
+imico_generate_handshake_done (struct ietf_mini_conn *conn);
+
+
 /* Only a single packet is supported */
 static void
 ietf_mini_conn_ci_packet_in (struct lsquic_conn *lconn,
@@ -1416,6 +1673,7 @@ ietf_mini_conn_ci_packet_in (struct lsquic_conn *lconn,
      " that contain packets that are all discarded.
      */
     conn->imc_bytes_in += packet_in->pi_data_sz;
+    conn->imc_flags &= ~IMC_AMP_CAPPED;
 
     if (conn->imc_flags & IMC_ERROR)
     {
@@ -1427,10 +1685,20 @@ ietf_mini_conn_ci_packet_in (struct lsquic_conn *lconn,
         imico_maybe_validate_by_dcid(conn, &packet_in->pi_dcid);
 
     pns = lsquic_hety2pns[ packet_in->pi_header_type ];
-    if (pns == PNS_INIT && (conn->imc_flags & IMC_IGNORE_INIT))
+    if (pns == PNS_INIT)
     {
-        LSQ_DEBUG("ignore init packet");    /* Don't bother decrypting */
-        return;
+        if (conn->imc_flags & IMC_IGNORE_INIT)
+        {
+            LSQ_DEBUG("ignore init packet");    /* Don't bother decrypting */
+            return;
+        }
+        if (packet_in->pi_pkt_size
+            && packet_in->pi_pkt_size < IQUIC_MIN_INIT_PACKET_SZ)
+        {
+            LSQ_DEBUG("ignore init packet smaller than minimum size required");
+            /* Don't bother decrypting */
+            return;
+        }
     }
 
     dec_packin = lconn->cn_esf_c->esf_decrypt_packet(lconn->cn_enc_session,
@@ -1446,6 +1714,15 @@ ietf_mini_conn_ci_packet_in (struct lsquic_conn *lconn,
     case DECPI_NOT_YET:
         imico_maybe_delay_processing(conn, packet_in);
         return;
+    case DECPI_BADCRYPT:
+        if (packet_in->pi_flags & PI_FIRST_INIT)
+        {
+            LSQ_DEBUG("possible packet corruption, destroy mini-conn.");
+            conn->imc_flags |= IMC_ERROR | IMC_CLOSE_RECVD;
+            /* avoid adding CID to black list */
+            conn->imc_conn.cn_flags |= LSCONN_NO_BL;
+        }
+        //fall through
     default:
         LSQ_DEBUG("could not decrypt packet");
         return;
@@ -1459,7 +1736,15 @@ ietf_mini_conn_ci_packet_in (struct lsquic_conn *lconn,
         return;
     }
     else if (pns == PNS_HSK)
+    {
+        if (!LSQUIC_CIDS_EQ(CN_SCID(&conn->imc_conn), &packet_in->pi_dcid))
+        {
+            ietf_mini_conn_ci_abort_error(lconn, 0, TEC_PROTOCOL_VIOLATION,
+                    "protocol violation detected bad dcid for HSK pns");
+            return;
+        }
         imico_peer_addr_validated(conn, "handshake PNS");
+    }
 
     if (((conn->imc_flags >> IMCBIT_PNS_BIT_SHIFT) & 3) < pns)
     {
@@ -1491,19 +1776,28 @@ ietf_mini_conn_ci_packet_in (struct lsquic_conn *lconn,
         conn->imc_largest_recvd[pns] = packet_in->pi_received;
     imico_record_recvd_packno(conn, pns, packet_in->pi_packno);
 
+    if (!(conn->imc_flags & (IMC_QUEUED_ACK_INIT << pns)))
+    {
+        LSQ_DEBUG("queued ACK in %s", lsquic_pns2str[pns]);
+        conn->imc_flags |= IMC_QUEUED_ACK_INIT << pns;
+    }
+    ++conn->imc_ecn_counts_in[pns][ lsquic_packet_in_ecn(packet_in) ];
+    conn->imc_incoming_ecn <<= 1;
+    conn->imc_incoming_ecn |= lsquic_packet_in_ecn(packet_in) != ECN_NOT_ECT;
+
     if (0 != imico_parse_regular_packet(conn, packet_in))
     {
         LSQ_DEBUG("connection is now in error state");
         conn->imc_flags |= IMC_ERROR;
         return;
     }
-
-    if (!(conn->imc_flags & (IMC_QUEUED_ACK_INIT << pns)))
-        LSQ_DEBUG("queued ACK in %s", lsquic_pns2str[pns]);
-    conn->imc_flags |= IMC_QUEUED_ACK_INIT << pns;
-    ++conn->imc_ecn_counts_in[pns][ lsquic_packet_in_ecn(packet_in) ];
-    conn->imc_incoming_ecn <<= 1;
-    conn->imc_incoming_ecn |= lsquic_packet_in_ecn(packet_in) != ECN_NOT_ECT;
+    if (conn->imc_flags & IMC_HSK_OK)
+    {
+        if (lconn->cn_esf.i->esfi_in_init(lconn->cn_enc_session))
+            LSQ_DEBUG("still in init, defer HANDSHAKE_DONE");
+        else if (0 != imico_generate_handshake_done(conn))
+            conn->imc_flags |= IMC_ERROR;
+    }
 }
 
 
@@ -1563,12 +1857,11 @@ imico_repackage_packet (struct ietf_mini_conn *conn,
     if (packno > MAX_PACKETS)
         return -1;
 
-    LSQ_DEBUG("Packet %"PRIu64" repackaged for resending as packet %"PRIu64,
-                                                        oldno, packno);
-    EV_LOG_CONN_EVENT(LSQUIC_LOG_CONN_ID, "packet %"PRIu64" repackaged for "
-        "resending as packet %"PRIu64, oldno, packno);
+    EV_LOG_CONN_EVENT(LSQUIC_LOG_CONN_ID, "schedule resend, repackage packet "
+                      "#%"PRIu64" -> #%"PRIu64, oldno, packno);
     packet_out->po_packno = packno;
     packet_out->po_flags &= ~PO_SENT;
+    ++packet_out->po_retx_cnt;
     lsquic_packet_out_set_ecn(packet_out, imico_get_ecn(conn));
     if (packet_out->po_flags & PO_ENCRYPTED)
         imico_return_enc_data(conn, packet_out);
@@ -1595,14 +1888,26 @@ imico_handle_losses_and_have_unsent (struct ietf_mini_conn *conn,
         next = TAILQ_NEXT(packet_out, po_next);
         if (packet_out->po_flags & PO_SENT)
         {
-            if (0 == retx_to)
-                retx_to = imico_calc_retx_timeout(conn);
-            if (packet_out->po_sent + retx_to < now)
+            if (packet_out->po_frame_types & IQUIC_FRAME_RETX_MASK)
+            {
+                if (0 == retx_to)
+                    retx_to = imico_calc_retx_timeout(conn);
+                if (conn->imc_hsk_count == 0)
+                    packet_out->po_retx_cnt = 0;
+                if (packet_out->po_sent
+                    + (retx_to << (packet_out->po_retx_cnt >> 1)) < now)
+                {
+                    LSQ_DEBUG("packet %"PRIu64" has been lost (rto: %"PRIu64")",
+                              packet_out->po_packno,
+                              retx_to << (packet_out->po_retx_cnt >> 1));
+                    TAILQ_REMOVE(&conn->imc_packets_out, packet_out, po_next);
+                    TAILQ_INSERT_TAIL(&lost_packets, packet_out, po_next);
+                }
+            }
+            else
             {
-                LSQ_DEBUG("packet %"PRIu64" has been lost (rto: %"PRIu64")",
-                                                packet_out->po_packno, retx_to);
                 TAILQ_REMOVE(&conn->imc_packets_out, packet_out, po_next);
-                TAILQ_INSERT_TAIL(&lost_packets, packet_out, po_next);
+                imico_destroy_packet(conn, packet_out);
             }
         }
         else if (packet_size = lsquic_packet_out_total_sz(lconn, packet_out),
@@ -1617,8 +1922,7 @@ imico_handle_losses_and_have_unsent (struct ietf_mini_conn *conn,
     while ((packet_out = TAILQ_FIRST(&lost_packets)))
     {
         TAILQ_REMOVE(&lost_packets, packet_out, po_next);
-        if ((packet_out->po_frame_types & IQUIC_FRAME_RETX_MASK)
-                            && 0 == imico_repackage_packet(conn, packet_out))
+        if (0 == imico_repackage_packet(conn, packet_out))
         {
             packet_size = lsquic_packet_out_total_sz(lconn, packet_out);
             if (imico_can_send(conn, packet_size))
@@ -1738,57 +2042,58 @@ static const enum header_type pns2hety[] =
 {
     [PNS_INIT]  = HETY_INITIAL,
     [PNS_HSK]   = HETY_HANDSHAKE,
-    [PNS_APP]   = HETY_NOT_SET,
+    [PNS_APP]   = HETY_SHORT,
 };
 
 
+static int
+imico_generate_ping (struct ietf_mini_conn *conn,
+                     struct lsquic_packet_out *packet_out)
+{
+    int len;
+    len = conn->imc_conn.cn_pf->pf_gen_ping_frame(
+                packet_out->po_data + packet_out->po_data_sz,
+                lsquic_packet_out_avail(packet_out));
+    if (len > 0)
+    {
+        packet_out->po_frame_types |= 1 << QUIC_FRAME_PING;
+        packet_out->po_data_sz += len;
+        packet_out->po_regen_sz += len;
+        LSQ_DEBUG("wrote PING frame of size %d to packet %" PRIu64,
+                len,  packet_out->po_packno);
+    }
+    return 0;
+}
+
+
 static int
 imico_generate_ack (struct ietf_mini_conn *conn, enum packnum_space pns,
                                                             lsquic_time_t now)
 {
     struct lsquic_packet_out *packet_out;
     enum header_type header_type;
-    struct ietf_mini_rechist rechist;
-    int not_used_has_missing, len;
-    uint64_t ecn_counts_buf[4];
-    const uint64_t *ecn_counts;
+    int len;
 
     header_type = pns2hety[pns];
-
-    if (conn->imc_incoming_ecn)
-    {
-        ecn_counts_buf[0]   = conn->imc_ecn_counts_in[pns][0];
-        ecn_counts_buf[1]   = conn->imc_ecn_counts_in[pns][1];
-        ecn_counts_buf[2]   = conn->imc_ecn_counts_in[pns][2];
-        ecn_counts_buf[3]   = conn->imc_ecn_counts_in[pns][3];
-        ecn_counts = ecn_counts_buf;
-    }
-    else
-        ecn_counts = NULL;
-
     packet_out = imico_get_packet_out(conn, header_type, 0);
     if (!packet_out)
         return -1;
 
-    /* Generate ACK frame */
-    lsquic_imico_rechist_init(&rechist, conn, pns);
-    len = conn->imc_conn.cn_pf->pf_gen_ack_frame(
-                packet_out->po_data + packet_out->po_data_sz,
-                lsquic_packet_out_avail(packet_out), lsquic_imico_rechist_first,
-                lsquic_imico_rechist_next, imico_rechist_largest_recv, &rechist,
-                now, &not_used_has_missing, &packet_out->po_ack2ed, ecn_counts);
+    len = imico_build_ack_frame(conn, pns, now,
+            packet_out->po_data + packet_out->po_data_sz,
+            lsquic_packet_out_avail(packet_out),
+            &packet_out->po_ack2ed);
     if (len < 0)
-    {
-        LSQ_WARN("could not generate ACK frame");
         return -1;
-    }
-    EV_LOG_GENERATED_ACK_FRAME(LSQUIC_LOG_CONN_ID, conn->imc_conn.cn_pf,
-                        packet_out->po_data + packet_out->po_data_sz, len);
+
     packet_out->po_frame_types |= 1 << QUIC_FRAME_ACK;
     packet_out->po_data_sz += len;
     packet_out->po_regen_sz += len;
     conn->imc_flags &= ~(IMC_QUEUED_ACK_INIT << pns);
-    LSQ_DEBUG("wrote ACK frame of size %d in %s", len, lsquic_pns2str[pns]);
+    LSQ_DEBUG("wrote ACK frame of size %d in %s to packet %" PRIu64,
+              len, lsquic_pns2str[pns], packet_out->po_packno);
+    if (pns == PNS_INIT && conn->imc_hsk_count > 0)
+        imico_generate_ping(conn, packet_out);
     return 0;
 }
 
@@ -1801,9 +2106,26 @@ imico_generate_acks (struct ietf_mini_conn *conn, lsquic_time_t now)
     for (pns = PNS_INIT; pns < IMICO_N_PNS; ++pns)
         if (conn->imc_flags & (IMC_QUEUED_ACK_INIT << pns)
                 && !(pns == PNS_INIT && (conn->imc_flags & IMC_IGNORE_INIT)))
+        {
             if (0 != imico_generate_ack(conn, pns, now))
                 return -1;
+        }
+    return 0;
+}
 
+
+int
+lsquic_mini_conn_ietf_pre_promote(struct ietf_mini_conn *conn,
+                                  lsquic_time_t now)
+{
+    if (conn->imc_flags & (IMC_QUEUED_ACK_INIT|IMC_QUEUED_ACK_HSK))
+    {
+        if (0 != imico_generate_acks(conn, now))
+        {
+            conn->imc_flags |= IMC_ERROR;
+            return -1;
+        }
+    }
     return 0;
 }
 
@@ -1848,6 +2170,13 @@ imico_generate_conn_close (struct ietf_mini_conn *conn)
         reason = "bad transport parameters";
         rlen = 24;
     }
+    else if (conn->imc_flags & IMC_VER_NEG_FAILED)
+    {
+        is_app = 0;
+        error_code = TEC_VERSION_NEGOTIATION_ERROR;
+        reason = "version negociation failed";
+        rlen = 26;
+    }
     else if (conn->imc_flags & IMC_HSK_FAILED)
     {
         is_app = 0;
@@ -1953,7 +2282,7 @@ imico_generate_handshake_done (struct ietf_mini_conn *conn)
     int sz;
 
     need = conn->imc_conn.cn_pf->pf_handshake_done_frame_size();
-    packet_out = imico_get_packet_out(conn, HETY_NOT_SET, need);
+    packet_out = imico_get_packet_out(conn, HETY_SHORT, need);
     if (!packet_out)
         return -1;
     sz = conn->imc_conn.cn_pf->pf_gen_handshake_done_frame(
@@ -1980,12 +2309,20 @@ ietf_mini_conn_ci_tick (struct lsquic_conn *lconn, lsquic_time_t now)
     struct ietf_mini_conn *conn = (struct ietf_mini_conn *) lconn;
     enum tick_st tick;
 
-    if (conn->imc_created + conn->imc_enpub->enp_settings.es_handshake_to < now)
+    if (conn->imc_expire < now)
     {
         LSQ_DEBUG("connection expired: closing");
         return TICK_CLOSE;
     }
 
+    if (!(conn->imc_flags &
+                    (IMC_HAVE_TP|IMC_ADDR_VALIDATED|IMC_BAD_TRANS_PARAMS))
+                            && conn->imc_enpub->enp_settings.es_support_srej)
+    {
+        LSQ_DEBUG("Peer not validated and do not have transport parameters "
+            "on the first tick: retry");
+        return TICK_RETRY|TICK_CLOSE;
+    }
 
     if (conn->imc_flags & (IMC_QUEUED_ACK_INIT|IMC_QUEUED_ACK_HSK))
     {
@@ -1997,6 +2334,9 @@ ietf_mini_conn_ci_tick (struct lsquic_conn *lconn, lsquic_time_t now)
     }
 
 
+    if (lconn->cn_flags & (LSCONN_PROMOTED|LSCONN_PROMOTE_FAIL))
+        return TICK_CLOSE;
+
     tick = 0;
 
     if (conn->imc_flags & IMC_ERROR)
@@ -2073,18 +2413,6 @@ ietf_mini_conn_ci_get_path (struct lsquic_conn *lconn,
 }
 
 
-static const lsquic_cid_t *
-ietf_mini_conn_ci_get_log_cid (const struct lsquic_conn *lconn)
-{
-    struct ietf_mini_conn *conn = (struct ietf_mini_conn *) lconn;
-
-    if (conn->imc_path.np_dcid.len)
-        return &conn->imc_path.np_dcid;
-    else
-        return CN_SCID(lconn);
-}
-
-
 static unsigned char
 ietf_mini_conn_ci_record_addrs (struct lsquic_conn *lconn, void *peer_ctx,
             const struct sockaddr *local_sa, const struct sockaddr *peer_sa)
@@ -2131,6 +2459,7 @@ ietf_mini_conn_ci_count_garbage (struct lsquic_conn *lconn, size_t garbage_sz)
     struct ietf_mini_conn *conn = (struct ietf_mini_conn *) lconn;
 
     conn->imc_bytes_in += garbage_sz;
+    conn->imc_flags &= ~IMC_AMP_CAPPED;
     LSQ_DEBUG("count %zd bytes of garbage, new value: %u bytes", garbage_sz,
         conn->imc_bytes_in);
 }
@@ -2142,7 +2471,6 @@ static const struct conn_iface mini_conn_ietf_iface = {
     .ci_count_garbage        =  ietf_mini_conn_ci_count_garbage,
     .ci_destroy              =  ietf_mini_conn_ci_destroy,
     .ci_get_engine           =  ietf_mini_conn_ci_get_engine,
-    .ci_get_log_cid          =  ietf_mini_conn_ci_get_log_cid,
     .ci_get_path             =  ietf_mini_conn_ci_get_path,
     .ci_hsk_done             =  ietf_mini_conn_ci_hsk_done,
     .ci_internal_error       =  ietf_mini_conn_ci_internal_error,

src/liblsquic/lsquic_mini_conn_ietf.h

@@ -6,6 +6,8 @@
 #ifndef LSQUIC_MINI_CONN_IETF_H
 #define LSQUIC_MINI_CONN_IETF_H 1
 
+#include "lsquic_ietf.h"
+
 struct lsquic_conn;
 struct lsquic_engine_public;
 struct lsquic_packet_in;
@@ -32,14 +34,13 @@ typedef uint64_t packno_set_t;
  * connection is promoted.  This means we do not have to have data
  * structures to track the App PNS.
  */
-#define IMICO_N_PNS (N_PNS - 1)
 
 struct ietf_mini_conn
 {
     struct lsquic_conn              imc_conn;
     struct conn_cid_elem            imc_cces[3];
     struct lsquic_engine_public    *imc_enpub;
-    lsquic_time_t                   imc_created;
+    lsquic_time_t                   imc_expire;
     enum {
         IMC_ENC_SESS_INITED     = 1 << 0,
         IMC_QUEUED_ACK_INIT     = 1 << 1,
@@ -66,6 +67,8 @@ struct ietf_mini_conn
         IMC_PATH_CHANGED        = 1 << 21,
         IMC_HSK_DONE_SENT       = 1 << 22,
         IMC_TRECHIST            = 1 << 23,
+        IMC_VER_NEG_FAILED      = 1 << 24,
+        IMC_AMP_CAPPED          = 1 << 25,
     }                               imc_flags;
     struct mini_crypto_stream       imc_streams[N_ENC_LEVS];
     void                           *imc_stream_ps[N_ENC_LEVS];
@@ -87,7 +90,7 @@ struct ietf_mini_conn
     packno_set_t                    imc_acked_packnos[IMICO_N_PNS];
     lsquic_time_t                   imc_largest_recvd[IMICO_N_PNS];
     struct lsquic_rtt_stats         imc_rtt_stats;
-    unsigned                        imc_error_code;
+    enum trans_error_code           imc_error_code;
     unsigned                        imc_bytes_in;
     unsigned                        imc_bytes_out;
     unsigned short                  imc_crypto_frames_sz;
@@ -112,6 +115,8 @@ struct ietf_mini_conn
     unsigned char                   imc_delayed_packets_count;
 #define IMICO_MAX_STASHED_FRAMES 10u
     unsigned char                   imc_n_crypto_frames;
+    unsigned short                  imc_hello_pkt_remain;
+    unsigned char                   imc_long_header_sz;
     struct network_path             imc_path;
 };
 
@@ -128,7 +133,7 @@ struct ietf_mini_conn
 
 struct lsquic_conn *
 lsquic_mini_conn_ietf_new (struct lsquic_engine_public *,
-               const struct lsquic_packet_in *,
+               struct lsquic_packet_in *,
                enum lsquic_version, int is_ipv4, const struct lsquic_cid *,
                size_t udp_payload_size);
 
@@ -159,4 +164,8 @@ lsquic_imico_rechist_first (void *rechist_ctx);
 const struct lsquic_packno_range *
 lsquic_imico_rechist_next (void *rechist_ctx);
 
+int
+lsquic_mini_conn_ietf_pre_promote(struct ietf_mini_conn *conn,
+                                  lsquic_time_t now);
+
 #endif

src/liblsquic/lsquic_mm.c

@@ -364,6 +364,7 @@ maybe_shrink_packet_out_bufs (struct lsquic_mm *mm, unsigned idx)
 #endif
 
 
+#if LSQUIC_USE_POOLS
 /* If average maximum falls under 1/4 of all objects allocated, release
  * half of the objects allocated.
  */
@@ -398,6 +399,7 @@ maybe_shrink_packet_in_bufs (struct lsquic_mm *mm, unsigned idx)
     }
 #endif
 }
+#endif
 
 
 void

src/liblsquic/lsquic_packet_common.c

@@ -49,11 +49,11 @@ lsquic_frame_types_to_str (char *buf, size_t bufsz,
 
 const char *const lsquic_hety2str[] =
 {
-    [HETY_NOT_SET]      = "Short",
-    [HETY_VERNEG]       = "Version Negotiation",
-    [HETY_INITIAL]      = "Initial",
-    [HETY_RETRY]        = "Retry",
-    [HETY_HANDSHAKE]    = "Handshake",
+    [HETY_SHORT]        = "SHORT",
+    [HETY_VERNEG]       = "VERNEG",
+    [HETY_INITIAL]      = "INIT",
+    [HETY_RETRY]        = "RETRY",
+    [HETY_HANDSHAKE]    = "HSK",
     [HETY_0RTT]         = "0-RTT",
 };
 
@@ -61,7 +61,7 @@ const char *const lsquic_hety2str[] =
 /* [draft-ietf-quic-tls-14], Section 4 */
 const enum packnum_space lsquic_hety2pns[] =
 {
-    [HETY_NOT_SET]      = PNS_APP,
+    [HETY_SHORT]        = PNS_APP,
     [HETY_VERNEG]       = 0,
     [HETY_INITIAL]      = PNS_INIT,
     [HETY_RETRY]        = 0,
@@ -73,16 +73,16 @@ const enum packnum_space lsquic_hety2pns[] =
 /* [draft-ietf-quic-tls-14], Section 4 */
 const enum packnum_space lsquic_enclev2pns[] =
 {
-    [ENC_LEV_CLEAR]      = PNS_INIT,
-    [ENC_LEV_INIT]       = PNS_HSK,
-    [ENC_LEV_EARLY]      = PNS_APP,
-    [ENC_LEV_FORW]       = PNS_APP,
+    [ENC_LEV_INIT]      = PNS_INIT,
+    [ENC_LEV_HSK]       = PNS_HSK,
+    [ENC_LEV_0RTT]      = PNS_APP,
+    [ENC_LEV_APP]       = PNS_APP,
 };
 
 
 const char *const lsquic_pns2str[] =
 {
-    [PNS_INIT]  = "Init PNS",
-    [PNS_HSK]   = "Handshake PNS",
-    [PNS_APP]   = "App PNS",
+    [PNS_INIT]  = "INIT pns",
+    [PNS_HSK]   = "HSK pns",
+    [PNS_APP]   = "APP pns",
 };

src/liblsquic/lsquic_packet_common.h

@@ -148,35 +148,24 @@ lsquic_frame_types_to_str (char *buf, size_t bufsz, enum quic_ft_bit);
 enum packno_bits
 {
     PACKNO_BITS_0 = 0,
-    PACKNO_BITS_1 = 1,
-    PACKNO_BITS_2 = 2,
-    PACKNO_BITS_3 = 3,
-};
-
-
-/* GQUIC maps 0, 1, 2, 3 -> 1, 2, 4, 6 */
-enum
-{
     GQUIC_PACKNO_LEN_1 = PACKNO_BITS_0,
-    GQUIC_PACKNO_LEN_2 = PACKNO_BITS_1,
-    GQUIC_PACKNO_LEN_4 = PACKNO_BITS_2,
-    GQUIC_PACKNO_LEN_6 = PACKNO_BITS_3,
-};
-
-
-/* IQUIC maps 0, 1, 2, 3 -> 1, 2, 3, 4 (as of ID-17) */
-enum
-{
     IQUIC_PACKNO_LEN_1 = PACKNO_BITS_0,
+    PACKNO_BITS_1 = 1,
+    GQUIC_PACKNO_LEN_2 = PACKNO_BITS_1,
     IQUIC_PACKNO_LEN_2 = PACKNO_BITS_1,
+    PACKNO_BITS_2 = 2,
+    GQUIC_PACKNO_LEN_4 = PACKNO_BITS_2,
     IQUIC_PACKNO_LEN_3 = PACKNO_BITS_2,
+    PACKNO_BITS_3 = 3,
+    GQUIC_PACKNO_LEN_6 = PACKNO_BITS_3,
     IQUIC_PACKNO_LEN_4 = PACKNO_BITS_3,
 };
 
 
 enum header_type
 {
-    HETY_NOT_SET,       /* This value must be zero */
+    HETY_NOT_SET,                   /* This value must be zero */
+    HETY_SHORT = HETY_NOT_SET,      /* This value must be zero */
     HETY_VERNEG,
     HETY_INITIAL,
     HETY_RETRY,
@@ -197,6 +186,7 @@ enum packnum_space
     PNS_INIT,
     PNS_HSK,
     PNS_APP,
+    IMICO_N_PNS = PNS_APP,
     N_PNS
 };
 
@@ -240,8 +230,9 @@ extern const char *const lsquic_pns2str[];
  * regenerating them.  This keeps the code simple(r).
  */
 #define IQUIC_FRAME_RETX_MASK (  \
-    ALL_IQUIC_FRAMES & ~(QUIC_FTBIT_PADDING|QUIC_FTBIT_PATH_RESPONSE    \
-            |QUIC_FTBIT_PATH_CHALLENGE|QUIC_FTBIT_ACK|QUIC_FTBIT_TIMESTAMP))
+    ALL_IQUIC_FRAMES & ~(QUIC_FTBIT_PADDING | QUIC_FTBIT_PATH_RESPONSE    \
+            | QUIC_FTBIT_PATH_CHALLENGE | QUIC_FTBIT_ACK \
+            | QUIC_FTBIT_TIMESTAMP | QUIC_FTBIT_PING))
 
 extern const enum quic_ft_bit lsquic_legal_frames_by_level[][4];
 

src/liblsquic/lsquic_packet_in.h

@@ -88,9 +88,12 @@ typedef struct lsquic_packet_in
         PI_LOG_QL_BITS  = (1 <<14),
         PI_SQUARE_BIT   = (1 <<15),
         PI_LOSS_BIT     = (1 <<16),
+        PI_VER_PARSED   = (1 <<17),
+        PI_FIRST_INIT   = (1 <<18),
     }                               pi_flags;
     /* pi_token and pi_token_size are set in Initial and Retry packets */
     unsigned short                  pi_token_size; /* Size of the token */
+    unsigned short                  pi_pkt_size;   /* Size of the whole packet */
     unsigned char                   pi_token;      /* Offset to token */
     /* pi_odcid and pi_odcid_len are only set in Retry packets for I-D < 25 */
     unsigned char                   pi_odcid;      /* Offset to Original DCID */
@@ -101,6 +104,7 @@ typedef struct lsquic_packet_in
     unsigned char                   pi_nonce;      /* Offset to nonce */
     enum header_type                pi_header_type:8;
     unsigned char                   pi_path_id;
+    unsigned char                   pi_version;    /* parsed enum lsquic_version */
     /* If PI_OWN_DATA flag is not set, `pi_data' points to user-supplied
      * packet data, which is NOT TO BE MODIFIED.
      */

src/liblsquic/lsquic_packet_out.c

@@ -382,7 +382,7 @@ lsquic_packet_out_chop_regen (lsquic_packet_out_t *packet_out)
         }
     }
 
-    assert(adj);    /* Otherwise why are we called? */
+    //assert(adj);    /* Otherwise why are we called? */
     packet_out->po_regen_sz = 0;
     packet_out->po_frame_types &= ~BQUIC_FRAME_REGEN_MASK;
 }
@@ -479,7 +479,7 @@ lsquic_packet_out_turn_on_fin (struct lsquic_packet_out *packet_out,
 static unsigned
 offset_to_dcid (const struct lsquic_packet_out *packet_out)
 {
-    if (packet_out->po_header_type == HETY_NOT_SET)
+    if (packet_out->po_header_type == HETY_SHORT)
         return 1;
     else
     {

src/liblsquic/lsquic_packet_out.h

@@ -111,6 +111,8 @@ typedef struct lsquic_packet_out
         PO_SCHED    = (1 <<14),         /* On scheduled queue */
         PO_SENT_SZ  = (1 <<15),
         PO_LONGHEAD = (1 <<16),
+        PO_ACKED_LOSS_CHAIN = (1<<17),
+
 #define POIPv6_SHIFT 20
         PO_IPv6     = (1 <<20),         /* Set if pmi_allocate was passed is_ipv6=1,
                                          *   otherwise unset.
@@ -185,6 +187,8 @@ typedef struct lsquic_packet_out
     unsigned char     *po_enc_data;
 
     lsquic_ver_tag_t   po_ver_tag;      /* Set if PO_VERSION is set */
+    unsigned short     po_retx_cnt;
+    unsigned short     po_padding_sz;
     unsigned char     *po_nonce;        /* Use to generate header if PO_NONCE is set */
     const struct network_path
                       *po_path;

src/liblsquic/lsquic_parse_Q050.c

@@ -486,7 +486,7 @@ gquic_Q050_packout_size (const struct lsquic_conn *lconn,
     size_t sz;
 
     if ((lconn->cn_flags & LSCONN_HANDSHAKE_DONE)
-                                && packet_out->po_header_type == HETY_NOT_SET)
+                                && packet_out->po_header_type == HETY_SHORT)
         sz = gquic_Q050_packout_header_size_short(lconn, packet_out->po_flags);
     else
         sz = gquic_Q050_packout_header_size_long_by_packet(lconn, packet_out);

src/liblsquic/lsquic_parse_common.c

@@ -21,26 +21,27 @@ parse_ietf_v1_or_Q046plus_long_begin (struct lsquic_packet_in *packet_in,
                 size_t length, int is_server, unsigned cid_len,
                 struct packin_parse_state *state)
 {
-    lsquic_ver_tag_t tag;
-
-    if (length >= 5)
-    {
-        memcpy(&tag, packet_in->pi_data + 1, 4);
-        switch (tag)
-        {
-        case TAG('Q', '0', '4', '6'):
-            return lsquic_Q046_parse_packet_in_long_begin(packet_in, length,
-                                                    is_server, cid_len, state);
-        case TAG('Q', '0', '5', '0'):
-            return lsquic_Q050_parse_packet_in_long_begin(packet_in, length,
-                                                    is_server, cid_len, state);
-        default:
-            return lsquic_ietf_v1_parse_packet_in_long_begin(packet_in, length,
-                                                    is_server, cid_len, state);
-        }
-    }
-    else
+    enum lsquic_version version;
+    if (length < 6)
         return -1;
+    version = lsquic_tag2ver_fast(packet_in->pi_data + 1);
+    if (version != N_LSQVER)
+    {
+        packet_in->pi_version = version;
+        packet_in->pi_flags |= PI_VER_PARSED;
+    }
+    switch (version)
+    {
+    case LSQVER_046:
+        return lsquic_Q046_parse_packet_in_long_begin(packet_in, length,
+                                                is_server, cid_len, state);
+    case LSQVER_050:
+        return lsquic_Q050_parse_packet_in_long_begin(packet_in, length,
+                                                is_server, cid_len, state);
+    default:
+        return lsquic_ietf_v1_parse_packet_in_long_begin(packet_in, length,
+                                                is_server, cid_len, state);
+    }
 }
 
 
@@ -51,10 +52,10 @@ static int (* const parse_begin_funcs[32]) (struct lsquic_packet_in *,
     /* Xs vary, Gs are iGnored: */
 #define PBEL(mask) [(mask) >> 3]
     /* 1X11 XGGG: */
-    PBEL(0x80|0x40|0x20|0x10|0x08)  = lsquic_Q046_parse_packet_in_long_begin,
-    PBEL(0x80|0x00|0x20|0x10|0x08)  = lsquic_Q046_parse_packet_in_long_begin,
-    PBEL(0x80|0x40|0x20|0x10|0x00)  = lsquic_Q046_parse_packet_in_long_begin,
-    PBEL(0x80|0x00|0x20|0x10|0x00)  = lsquic_Q046_parse_packet_in_long_begin,
+    PBEL(0x80|0x40|0x20|0x10|0x08)  = parse_ietf_v1_or_Q046plus_long_begin,
+    PBEL(0x80|0x00|0x20|0x10|0x08)  = parse_ietf_v1_or_Q046plus_long_begin,
+    PBEL(0x80|0x40|0x20|0x10|0x00)  = parse_ietf_v1_or_Q046plus_long_begin,
+    PBEL(0x80|0x00|0x20|0x10|0x00)  = parse_ietf_v1_or_Q046plus_long_begin,
     /* 1X00 XGGG: */
     PBEL(0x80|0x40|0x00|0x00|0x08)  = parse_ietf_v1_or_Q046plus_long_begin,
     PBEL(0x80|0x00|0x00|0x00|0x08)  = parse_ietf_v1_or_Q046plus_long_begin,
@@ -231,10 +232,12 @@ lsquic_dcid_from_packet (const unsigned char *buf, size_t bufsz,
     {
     /* Xs vary, Gs are iGnored: */
     /* 1X11 XGGG: */
+    /*
     case (0x80|0x40|0x20|0x10|0x08) >> 3:
     case (0x80|0x00|0x20|0x10|0x08) >> 3:
     case (0x80|0x40|0x20|0x10|0x00) >> 3:
     case (0x80|0x00|0x20|0x10|0x00) >> 3:
+    */
   Q046_long:
         /* lsquic_Q046_parse_packet_in_long_begin */
         if (bufsz < 14)
@@ -336,10 +339,10 @@ lsquic_dcid_from_packet (const unsigned char *buf, size_t bufsz,
 /* See [draft-ietf-quic-transport-28], Section 12.4 (Table 3) */
 const enum quic_ft_bit lsquic_legal_frames_by_level[N_LSQVER][N_ENC_LEVS] =
 {
-    [LSQVER_I001] = {
-    [ENC_LEV_CLEAR] = QUIC_FTBIT_CRYPTO | QUIC_FTBIT_PADDING | QUIC_FTBIT_PING
+    [LSQVER_I002] = {
+    [ENC_LEV_INIT] = QUIC_FTBIT_CRYPTO | QUIC_FTBIT_PADDING | QUIC_FTBIT_PING
                     | QUIC_FTBIT_ACK | QUIC_FTBIT_CONNECTION_CLOSE,
-    [ENC_LEV_EARLY] = QUIC_FTBIT_PADDING | QUIC_FTBIT_PING
+    [ENC_LEV_0RTT] = QUIC_FTBIT_PADDING | QUIC_FTBIT_PING
                     | QUIC_FTBIT_STREAM | QUIC_FTBIT_RST_STREAM
                     | QUIC_FTBIT_BLOCKED | QUIC_FTBIT_CONNECTION_CLOSE
                     | QUIC_FTBIT_MAX_DATA | QUIC_FTBIT_MAX_STREAM_DATA
@@ -349,9 +352,39 @@ const enum quic_ft_bit lsquic_legal_frames_by_level[N_LSQVER][N_ENC_LEVS] =
                     | QUIC_FTBIT_PATH_CHALLENGE
                     | QUIC_FTBIT_DATAGRAM
                     | QUIC_FTBIT_RETIRE_CONNECTION_ID,
-    [ENC_LEV_INIT]  = QUIC_FTBIT_CRYPTO | QUIC_FTBIT_PADDING | QUIC_FTBIT_PING
+    [ENC_LEV_HSK]  = QUIC_FTBIT_CRYPTO | QUIC_FTBIT_PADDING | QUIC_FTBIT_PING
                     | QUIC_FTBIT_ACK| QUIC_FTBIT_CONNECTION_CLOSE,
-    [ENC_LEV_FORW]  = QUIC_FTBIT_CRYPTO | QUIC_FTBIT_PADDING | QUIC_FTBIT_PING
+    [ENC_LEV_APP]  = QUIC_FTBIT_CRYPTO | QUIC_FTBIT_PADDING | QUIC_FTBIT_PING
+                    | QUIC_FTBIT_ACK | QUIC_FTBIT_CONNECTION_CLOSE
+                    | QUIC_FTBIT_STREAM | QUIC_FTBIT_RST_STREAM
+                    | QUIC_FTBIT_BLOCKED
+                    | QUIC_FTBIT_MAX_DATA | QUIC_FTBIT_MAX_STREAM_DATA
+                    | QUIC_FTBIT_MAX_STREAMS | QUIC_FTBIT_STREAM_BLOCKED
+                    | QUIC_FTBIT_STREAMS_BLOCKED
+                    | QUIC_FTBIT_NEW_CONNECTION_ID | QUIC_FTBIT_STOP_SENDING
+                    | QUIC_FTBIT_PATH_CHALLENGE | QUIC_FTBIT_PATH_RESPONSE
+                    | QUIC_FTBIT_HANDSHAKE_DONE | QUIC_FTBIT_ACK_FREQUENCY
+                    | QUIC_FTBIT_RETIRE_CONNECTION_ID | QUIC_FTBIT_NEW_TOKEN
+                    | QUIC_FTBIT_TIMESTAMP
+                    | QUIC_FTBIT_DATAGRAM
+                    ,
+    },
+    [LSQVER_I001] = {
+    [ENC_LEV_INIT] = QUIC_FTBIT_CRYPTO | QUIC_FTBIT_PADDING | QUIC_FTBIT_PING
+                    | QUIC_FTBIT_ACK | QUIC_FTBIT_CONNECTION_CLOSE,
+    [ENC_LEV_0RTT] = QUIC_FTBIT_PADDING | QUIC_FTBIT_PING
+                    | QUIC_FTBIT_STREAM | QUIC_FTBIT_RST_STREAM
+                    | QUIC_FTBIT_BLOCKED | QUIC_FTBIT_CONNECTION_CLOSE
+                    | QUIC_FTBIT_MAX_DATA | QUIC_FTBIT_MAX_STREAM_DATA
+                    | QUIC_FTBIT_MAX_STREAMS | QUIC_FTBIT_STREAM_BLOCKED
+                    | QUIC_FTBIT_STREAMS_BLOCKED
+                    | QUIC_FTBIT_NEW_CONNECTION_ID | QUIC_FTBIT_STOP_SENDING
+                    | QUIC_FTBIT_PATH_CHALLENGE
+                    | QUIC_FTBIT_DATAGRAM
+                    | QUIC_FTBIT_RETIRE_CONNECTION_ID,
+    [ENC_LEV_HSK]  = QUIC_FTBIT_CRYPTO | QUIC_FTBIT_PADDING | QUIC_FTBIT_PING
+                    | QUIC_FTBIT_ACK| QUIC_FTBIT_CONNECTION_CLOSE,
+    [ENC_LEV_APP]  = QUIC_FTBIT_CRYPTO | QUIC_FTBIT_PADDING | QUIC_FTBIT_PING
                     | QUIC_FTBIT_ACK | QUIC_FTBIT_CONNECTION_CLOSE
                     | QUIC_FTBIT_STREAM | QUIC_FTBIT_RST_STREAM
                     | QUIC_FTBIT_BLOCKED
@@ -367,9 +400,9 @@ const enum quic_ft_bit lsquic_legal_frames_by_level[N_LSQVER][N_ENC_LEVS] =
                     ,
     },
     [LSQVER_ID29] = {
-    [ENC_LEV_CLEAR] = QUIC_FTBIT_CRYPTO | QUIC_FTBIT_PADDING | QUIC_FTBIT_PING
+    [ENC_LEV_INIT] = QUIC_FTBIT_CRYPTO | QUIC_FTBIT_PADDING | QUIC_FTBIT_PING
                     | QUIC_FTBIT_ACK | QUIC_FTBIT_CONNECTION_CLOSE,
-    [ENC_LEV_EARLY] = QUIC_FTBIT_PADDING | QUIC_FTBIT_PING
+    [ENC_LEV_0RTT] = QUIC_FTBIT_PADDING | QUIC_FTBIT_PING
                     | QUIC_FTBIT_STREAM | QUIC_FTBIT_RST_STREAM
                     | QUIC_FTBIT_BLOCKED | QUIC_FTBIT_CONNECTION_CLOSE
                     | QUIC_FTBIT_MAX_DATA | QUIC_FTBIT_MAX_STREAM_DATA
@@ -379,9 +412,9 @@ const enum quic_ft_bit lsquic_legal_frames_by_level[N_LSQVER][N_ENC_LEVS] =
                     | QUIC_FTBIT_PATH_CHALLENGE | QUIC_FTBIT_PATH_RESPONSE
                     | QUIC_FTBIT_DATAGRAM
                     | QUIC_FTBIT_RETIRE_CONNECTION_ID,
-    [ENC_LEV_INIT]  = QUIC_FTBIT_CRYPTO | QUIC_FTBIT_PADDING | QUIC_FTBIT_PING
+    [ENC_LEV_HSK]  = QUIC_FTBIT_CRYPTO | QUIC_FTBIT_PADDING | QUIC_FTBIT_PING
                     | QUIC_FTBIT_ACK| QUIC_FTBIT_CONNECTION_CLOSE,
-    [ENC_LEV_FORW]  = QUIC_FTBIT_CRYPTO | QUIC_FTBIT_PADDING | QUIC_FTBIT_PING
+    [ENC_LEV_APP]  = QUIC_FTBIT_CRYPTO | QUIC_FTBIT_PADDING | QUIC_FTBIT_PING
                     | QUIC_FTBIT_ACK | QUIC_FTBIT_CONNECTION_CLOSE
                     | QUIC_FTBIT_STREAM | QUIC_FTBIT_RST_STREAM
                     | QUIC_FTBIT_BLOCKED
@@ -397,9 +430,9 @@ const enum quic_ft_bit lsquic_legal_frames_by_level[N_LSQVER][N_ENC_LEVS] =
                     ,
     },
     [LSQVER_ID27] = {
-    [ENC_LEV_CLEAR] = QUIC_FTBIT_CRYPTO | QUIC_FTBIT_PADDING | QUIC_FTBIT_PING
+    [ENC_LEV_INIT] = QUIC_FTBIT_CRYPTO | QUIC_FTBIT_PADDING | QUIC_FTBIT_PING
                     | QUIC_FTBIT_ACK | QUIC_FTBIT_CONNECTION_CLOSE,
-    [ENC_LEV_EARLY] = QUIC_FTBIT_PADDING | QUIC_FTBIT_PING
+    [ENC_LEV_0RTT] = QUIC_FTBIT_PADDING | QUIC_FTBIT_PING
                     | QUIC_FTBIT_STREAM | QUIC_FTBIT_RST_STREAM
                     | QUIC_FTBIT_BLOCKED
                     | QUIC_FTBIT_MAX_DATA | QUIC_FTBIT_MAX_STREAM_DATA
@@ -410,9 +443,9 @@ const enum quic_ft_bit lsquic_legal_frames_by_level[N_LSQVER][N_ENC_LEVS] =
                     | QUIC_FTBIT_RETIRE_CONNECTION_ID
                     | QUIC_FTBIT_DATAGRAM
                     ,
-    [ENC_LEV_INIT]  = QUIC_FTBIT_CRYPTO | QUIC_FTBIT_PADDING | QUIC_FTBIT_PING
+    [ENC_LEV_HSK]  = QUIC_FTBIT_CRYPTO | QUIC_FTBIT_PADDING | QUIC_FTBIT_PING
                     | QUIC_FTBIT_ACK| QUIC_FTBIT_CONNECTION_CLOSE,
-    [ENC_LEV_FORW]  = QUIC_FTBIT_CRYPTO | QUIC_FTBIT_PADDING | QUIC_FTBIT_PING
+    [ENC_LEV_APP]  = QUIC_FTBIT_CRYPTO | QUIC_FTBIT_PADDING | QUIC_FTBIT_PING
                     | QUIC_FTBIT_ACK | QUIC_FTBIT_CONNECTION_CLOSE
                     | QUIC_FTBIT_STREAM | QUIC_FTBIT_RST_STREAM
                     | QUIC_FTBIT_BLOCKED

src/liblsquic/lsquic_parse_common.h

@@ -68,6 +68,11 @@ int
 lsquic_ietf_v1_gen_ver_nego_pkt (unsigned char *buf, size_t bufsz,
     const lsquic_cid_t *scid, const lsquic_cid_t *dcid, unsigned versions,
     uint8_t);
+int
+lsquic_iquic_gen_retry_pkt (unsigned char *buf, size_t bufsz,
+    const struct lsquic_engine_public *, const lsquic_cid_t *scid,
+    const lsquic_cid_t *dcid, enum lsquic_version, const struct sockaddr *,
+    uint8_t random_nybble);
 
 #define GQUIC_RESET_SZ 33
 ssize_t

src/liblsquic/lsquic_parse_ietf_v1.c

@@ -142,7 +142,7 @@ ietf_v1_packout_max_header_size (const struct lsquic_conn *lconn,
     enum packet_out_flags flags, size_t dcid_len, enum header_type header_type)
 {
     if ((lconn->cn_flags & LSCONN_HANDSHAKE_DONE)
-                                                && header_type == HETY_NOT_SET)
+                                                && header_type == HETY_SHORT)
         return ietf_v1_packout_header_size_short(flags, dcid_len);
     else
         return ietf_v1_packout_header_size_long_by_flags(lconn, header_type,
@@ -159,6 +159,15 @@ static const unsigned char header_type_to_bin[] = {
 };
 
 
+/* [draft-ietf-quic-v2] Section-3.2 */
+static const unsigned char header_type_to_bin_v2[] = {
+    [HETY_INITIAL]      = 0x1,
+    [HETY_0RTT]         = 0x2,
+    [HETY_HANDSHAKE]    = 0x3,
+    [HETY_RETRY]        = 0x0,
+};
+
+
 static unsigned
 write_packno (unsigned char *p, lsquic_packno_t packno,
                                                 enum packno_bits bits)
@@ -205,10 +214,16 @@ gen_long_pkt_header (const struct lsquic_conn *lconn,
 
     packno_bits = lsquic_packet_out_packno_bits(packet_out);
     p = buf;
-    *p++ = 0xC0
-         | ( header_type_to_bin[ packet_out->po_header_type ] << 4)
-         | packno_bits
-         ;
+    if (lconn->cn_version == LSQVER_I002)
+        *p++ = 0xC0
+            | ( header_type_to_bin_v2[ packet_out->po_header_type ] << 4)
+            | packno_bits
+            ;
+    else
+        *p++ = 0xC0
+            | ( header_type_to_bin[ packet_out->po_header_type ] << 4)
+            | packno_bits
+            ;
     ver_tag = lsquic_ver2tag(lconn->cn_version);
     memcpy(p, &ver_tag, sizeof(ver_tag));
     p += sizeof(ver_tag);
@@ -285,7 +300,7 @@ ietf_v1_gen_reg_pkt_header (const struct lsquic_conn *lconn,
             const struct lsquic_packet_out *packet_out, unsigned char *buf,
             size_t bufsz, unsigned *packno_off, unsigned *packno_len)
 {
-    if (packet_out->po_header_type == HETY_NOT_SET)
+    if (packet_out->po_header_type == HETY_SHORT)
         return gen_short_pkt_header(lconn, packet_out, buf, bufsz, packno_off,
                                                                     packno_len);
     else
@@ -301,7 +316,7 @@ ietf_v1_packout_size (const struct lsquic_conn *lconn,
     size_t sz;
 
     if ((lconn->cn_flags & LSCONN_HANDSHAKE_DONE)
-                                && packet_out->po_header_type == HETY_NOT_SET)
+                                && packet_out->po_header_type == HETY_SHORT)
         sz = ietf_v1_packout_header_size_short(packet_out->po_flags,
                                             packet_out->po_path->np_dcid.len);
     else
@@ -1700,16 +1715,21 @@ static const enum header_type bits2ht[4] =
 };
 
 
+static const enum header_type bits2ht_v2[4] =
+{
+    [0] = HETY_RETRY,
+    [1] = HETY_INITIAL,
+    [2] = HETY_0RTT,
+    [3] = HETY_HANDSHAKE,
+};
+
 #if LSQUIC_QIR
 /* Return true if the parsing function is to enforce the minimum DCID
  * length requirement as specified in IETF v1 and the I-Ds.
  */
 static int
-enforce_initial_dcil (lsquic_ver_tag_t tag)
+enforce_initial_dcil (enum lsquic_version version)
 {
-    enum lsquic_version version;
-
-    version = lsquic_tag2ver(tag);
     return version != (enum lsquic_version) -1
         && ((1 << version) & LSQUIC_IETF_VERSIONS);
 }
@@ -1723,22 +1743,28 @@ lsquic_ietf_v1_parse_packet_in_long_begin (struct lsquic_packet_in *packet_in,
 {
     const unsigned char *p = packet_in->pi_data;
     const unsigned char *const end = p + length;
-    lsquic_ver_tag_t tag;
     enum header_type header_type;
     unsigned dcil, scil;
-    int verneg, r;
+    int r;
     unsigned char first_byte;
     uint64_t payload_len, token_len;
 
     if (length < 6)
         return -1;
     first_byte = *p++;
-
-    memcpy(&tag, p, 4);
+    if ((packet_in->pi_flags & PI_VER_PARSED) == 0)
+    {
+        packet_in->pi_version = lsquic_tag2ver_fast(p);
+        packet_in->pi_flags |= PI_VER_PARSED;
+    }
     p += 4;
-    verneg = 0 == tag;
-    if (!verneg)
-        header_type = bits2ht[ (first_byte >> 4) & 3 ];
+    if (packet_in->pi_version != LSQVER_VERNEG)
+    {
+        if (packet_in->pi_version == LSQVER_I002)
+            header_type = bits2ht_v2[ (first_byte >> 4) & 3 ];
+        else
+            header_type = bits2ht[ (first_byte >> 4) & 3 ];
+    }
     else
         header_type = HETY_VERNEG;
 
@@ -1769,7 +1795,7 @@ lsquic_ietf_v1_parse_packet_in_long_begin (struct lsquic_packet_in *packet_in,
     {
     case HETY_INITIAL:
 #if LSQUIC_QIR
-        if (!enforce_initial_dcil(tag))
+        if (!enforce_initial_dcil(packet_in->pi_version))
         {
             /* Count even zero-length DCID as having DCID */
             packet_in->pi_flags |= PI_CONN_ID;
@@ -1893,7 +1919,10 @@ lsquic_is_valid_ietf_v1_or_Q046plus_hs_packet (const unsigned char *buf,
         return 0;
     first_byte = *p++;
 
-    header_type = bits2ht[ (first_byte >> 4) & 3 ];
+    if (*p != 0x6B)
+        header_type = bits2ht[ (first_byte >> 4) & 3 ];
+    else
+        header_type = bits2ht_v2[ (first_byte >> 4) & 3 ];
     if (header_type != HETY_INITIAL)
         return 0;
 

src/liblsquic/lsquic_parse_iquic_common.c

@@ -11,6 +11,7 @@
 #include <sys/types.h>
 
 #include <openssl/rand.h>
+#include <openssl/aead.h>
 
 #include "lsquic_types.h"
 #include "lsquic_int_types.h"
@@ -181,7 +182,7 @@ lsquic_Q046_parse_packet_in_short_begin (lsquic_packet_in_t *packet_in,
     packet_in->pi_packno = packno;
     p += packet_len;
 
-    packet_in->pi_header_type  = HETY_NOT_SET;
+    packet_in->pi_header_type  = HETY_SHORT;
     packet_in->pi_quic_ver     = 0;
     packet_in->pi_nonce        = 0;
     packet_in->pi_header_sz    = p - packet_in->pi_data;
@@ -320,8 +321,6 @@ popcount (unsigned v)
             ++count;
     return count;
 }
-
-
 #endif
 
 
@@ -379,3 +378,76 @@ lsquic_Q046_gen_ver_nego_pkt (unsigned char *buf, size_t bufsz,
 }
 
 
+int
+lsquic_iquic_gen_retry_pkt (unsigned char *buf, size_t bufsz,
+        const struct lsquic_engine_public *enpub,
+        const lsquic_cid_t *scid, const lsquic_cid_t *dcid,
+        enum lsquic_version version, const struct sockaddr *sockaddr,
+        uint8_t random_nybble)
+{
+    struct token_generator *const tokgen = enpub->enp_tokgen;
+    const unsigned our_scid_len = enpub->enp_settings.es_scid_len;
+    unsigned char *const end = buf + bufsz;
+    unsigned char *p = buf;
+    lsquic_ver_tag_t ver_tag;
+    size_t ad_len, out_len;
+    unsigned ret_ver;
+    ssize_t sz;
+#define INTEGRITY_TAG_LEN 16
+
+    /* [draft-ietf-quic-tls-25] Section 5.8 specifies the layout of the
+     * Retry Pseudo-Packet:
+     */
+    unsigned char ad_buf[
+        1 + MAX_CID_LEN     /* ODCID */
+      + 1 + 4               /* Type and version */
+      + 1 + MAX_CID_LEN     /* DCID */
+      + 1 + MAX_CID_LEN     /* SCID */
+      + MAX_RETRY_TOKEN_LEN /* Retry token */
+    ];
+
+    unsigned char tag[INTEGRITY_TAG_LEN];
+
+    /* See [draft-ietf-quic-transport-25], Section 17.2.5 */
+
+    if (bufsz < 1 + sizeof(ver_tag) + 1 + our_scid_len + 1 + dcid->len
+                        + MAX_RETRY_TOKEN_LEN + INTEGRITY_TAG_LEN)
+        return -1;
+
+    p = ad_buf;
+    *p++ = dcid->len;
+    memcpy(p, dcid->idbuf, dcid->len);
+    p += dcid->len;
+    *p++ = 0xC0
+         | (3 << 4)
+         | random_nybble
+         ;
+    ver_tag = lsquic_ver2tag(version);
+    memcpy(p, &ver_tag, sizeof(ver_tag));
+    p += sizeof(ver_tag);
+
+    *p++ = scid->len;
+    memcpy(p, scid->idbuf, scid->len);
+    p += scid->len;
+    *p++ = our_scid_len;
+    RAND_bytes(p, our_scid_len);
+    p += our_scid_len;
+    sz = lsquic_tg_generate_retry(tokgen, p, end - p,
+                        p - our_scid_len, our_scid_len, sockaddr, dcid);
+    if (sz < 0)
+        return -1;
+    p += sz;
+    ad_len = p - ad_buf;
+
+    ret_ver = lsquic_version_2_retryver(version);
+    out_len = sizeof(tag);
+    if (!(1 == EVP_AEAD_CTX_seal(&enpub->enp_retry_aead_ctx[ret_ver], tag,
+                &out_len, out_len, lsquic_retry_nonce_buf[ret_ver],
+                IETF_RETRY_NONCE_SZ,
+                NULL, 0, ad_buf, ad_len) && out_len == sizeof(tag)))
+        return -1;
+
+    memcpy(buf, ad_buf + 1 + dcid->len, ad_len - 1 - dcid->len);
+    memcpy(buf + ad_len - 1 - dcid->len, tag, sizeof(tag));
+    return ad_len - 1 - dcid->len + sizeof(tag);
+}

src/liblsquic/lsquic_pr_queue.c

@@ -84,6 +84,13 @@ struct evanescent_conn
                 + 1 /* DCIL */ + MAX_CID_LEN + 1 /* SCIL */ + MAX_CID_LEN + \
                 4 * N_LSQVER)
 
+/* [draft-ietf-quic-transport-22], Section 17.2.5 */
+#define IQUIC_RETRY_SIZE (1 /* Type */ + 4 /* Version */ + \
+                + 1 /* DCIL */ + MAX_CID_LEN + 1 /* SCIL */ + MAX_CID_LEN + \
+                + 1 /* ODCIL */ + MAX_CID_LEN + MAX_RETRY_TOKEN_LEN)
+
+/* GQUIC retry is dynamically size, this is a reasonable high bound */
+#define GQUIC_RETRY_SIZE 512
 
 struct pr_queue
 {
@@ -262,7 +269,7 @@ put_req (struct pr_queue *prq, struct packet_req *req)
 }
 
 
-static int
+int
 lsquic_prq_new_req_ext (struct pr_queue *prq, enum packet_req_type type,
     unsigned flags, enum lsquic_version version, unsigned short data_sz,
     const lsquic_cid_t *dcid, const lsquic_cid_t *scid, void *peer_ctx,
@@ -370,10 +377,11 @@ lsquic_prq_new_req (struct pr_queue *prq, enum packet_req_type type,
 static size_t
 max_bufsz (const struct pr_queue *prq)
 {
-    return  MAX(MAX(MAX(IQUIC_VERNEG_SIZE,
-                        IQUIC_MIN_SRST_SIZE),
-                        sizeof(prq->prq_verneg_g_buf)),
-                        sizeof(prq->prq_pubres_g_buf));
+    return  MAX(MAX(MAX(MAX(IQUIC_VERNEG_SIZE,
+                            IQUIC_RETRY_SIZE),
+                            IQUIC_MAX_SRST_SIZE),
+                            sizeof(prq->prq_verneg_g_buf)),
+                            sizeof(prq->prq_pubres_g_buf));
 }
 
 
@@ -484,6 +492,17 @@ lsquic_prq_next_conn (struct pr_queue *prq)
         else
             packet_out->po_data_sz = 0;
         break;
+    case (PACKET_REQ_RETRY << 29) | 0:
+        packet_out->po_flags |= PO_RETRY;
+        len = lsquic_iquic_gen_retry_pkt(packet_out->po_data, max_bufsz(prq),
+                    prq->prq_enpub, &req->pr_scid, &req->pr_dcid,
+                    req->pr_version, NP_PEER_SA(&req->pr_path),
+                    lsquic_crand_get_nybble(prq->prq_enpub->enp_crand));
+        if (len > 0)
+            packet_out->po_data_sz = len;
+        else
+            packet_out->po_data_sz = 0;
+        break;
     default:
         packet_out->po_flags &= ~PO_VERNEG;
         packet_out->po_data_sz = req->pr_rst_sz;

src/liblsquic/lsquic_pr_queue.h

@@ -51,6 +51,7 @@ struct sockaddr;
 enum packet_req_type {
     PACKET_REQ_VERNEG,
     PACKET_REQ_PUBRES,
+    PACKET_REQ_RETRY,
     N_PREQ_TYPES,
 };
 
@@ -75,6 +76,12 @@ lsquic_prq_next_conn (struct pr_queue *);
 int
 lsquic_prq_have_pending (const struct pr_queue *);
 
+int
+lsquic_prq_new_req_ext (struct pr_queue *prq, enum packet_req_type type,
+    unsigned flags, enum lsquic_version version, unsigned short data_sz,
+    const lsquic_cid_t *dcid, const lsquic_cid_t *scid, void *peer_ctx,
+    const struct sockaddr *local_addr, const struct sockaddr *peer_addr);
+
 void
 lsquic_prq_drop (struct lsquic_conn *);
 

src/liblsquic/lsquic_send_ctl.c

@@ -68,6 +68,7 @@
 #define DEFAULT_RETX_DELAY      500000      /* Microseconds */
 #define MAX_RTO_DELAY           60000000    /* Microseconds */
 #define MIN_RTO_DELAY           200000      /* Microseconds */
+#define INITIAL_RTT             333333      /* Microseconds */
 #define N_NACKS_BEFORE_RETX     3
 
 #define CGP(ctl) ((struct cong_ctl *) (ctl)->sc_cong_ctl)
@@ -260,6 +261,10 @@ get_retx_delay (const struct lsquic_rtt_stats *rtt_stats)
 }
 
 
+static lsquic_time_t
+calculate_packet_rto (lsquic_send_ctl_t *ctl);
+
+
 static void
 retx_alarm_rings (enum alarm_id al_id, void *ctx, lsquic_time_t expiry, lsquic_time_t now)
 {
@@ -275,7 +280,7 @@ retx_alarm_rings (enum alarm_id al_id, void *ctx, lsquic_time_t expiry, lsquic_t
     assert(!lsquic_alarmset_is_set(ctl->sc_alset, AL_RETX_INIT + pns));
 
     rm = get_retx_mode(ctl);
-    LSQ_INFO("retx timeout, mode %s", retx2str[rm]);
+    LSQ_INFO("%s timeout, mode %s", lsquic_alid2str[al_id], retx2str[rm]);
 
     switch (rm)
     {
@@ -287,18 +292,24 @@ retx_alarm_rings (enum alarm_id al_id, void *ctx, lsquic_time_t expiry, lsquic_t
         send_ctl_detect_losses(ctl, pns, now);
         break;
     case RETX_MODE_TLP:
+        ctl->sc_last_rto_time = now;
         ++ctl->sc_n_tlp;
         send_ctl_expire(ctl, pns, EXFI_LAST);
         break;
     case RETX_MODE_RTO:
-        ctl->sc_last_rto_time = now;
-        ++ctl->sc_n_consec_rtos;
-        ctl->sc_next_limit = 2;
-        LSQ_DEBUG("packet RTO is %"PRIu64" usec", expiry);
+        if ((ctl->sc_flags & SC_1RTT_ACKED)
+            || now - ctl->sc_last_rto_time >= calculate_packet_rto(ctl))
+        {
+            ctl->sc_last_rto_time = now;
+            ++ctl->sc_n_consec_rtos;
+            ctl->sc_next_limit = 2;
+            ctl->sc_ci->cci_timeout(CGP(ctl));
+            if (lconn->cn_if->ci_retx_timeout)
+                lconn->cn_if->ci_retx_timeout(lconn);
+        }
+        LSQ_DEBUG("packet RTO is %"PRIu64" (+%"PRIu64") usec, consec RTOs: %d",
+                  expiry, now - expiry, ctl->sc_n_consec_rtos);
         send_ctl_expire(ctl, pns, EXFI_ALL);
-        ctl->sc_ci->cci_timeout(CGP(ctl));
-        if (lconn->cn_if->ci_retx_timeout)
-            lconn->cn_if->ci_retx_timeout(lconn);
         break;
     }
 
@@ -450,18 +461,14 @@ calculate_tlp_delay (lsquic_send_ctl_t *ctl)
     lsquic_time_t srtt, delay;
 
     srtt = lsquic_rtt_stats_get_srtt(&ctl->sc_conn_pub->rtt_stats);
+    if (!srtt)
+        srtt = INITIAL_RTT;
     if (ctl->sc_n_in_flight_all > 1)
-    {
         delay = 10000;  /* 10 ms is the minimum tail loss probe delay */
-        if (delay < 2 * srtt)
-            delay = 2 * srtt;
-    }
     else
-    {
         delay = srtt + srtt / 2 + ctl->sc_conn_pub->max_peer_ack_usec;
-        if (delay < 2 * srtt)
-            delay = 2 * srtt;
-    }
+    if (delay < 2 * srtt)
+        delay = 2 * srtt;
 
     return delay;
 }
@@ -519,8 +526,9 @@ set_retx_alarm (struct lsquic_send_ctl *ctl, enum packnum_space pns,
     if (delay > MAX_RTO_DELAY)
         delay = MAX_RTO_DELAY;
 
-    LSQ_DEBUG("set retx alarm to %"PRIu64", which is %"PRIu64
-        " usec from now, mode %s", now + delay, delay, retx2str[rm]);
+    LSQ_DEBUG("set RETX_%s alarm to %"PRIu64" (%"PRIu64
+        "), mode %s", lsquic_pns2str[pns],
+              now + delay, delay, retx2str[rm]);
     lsquic_alarmset_set(ctl->sc_alset, AL_RETX_INIT + pns, now + delay);
 
     if (PNS_APP == pns
@@ -656,7 +664,7 @@ send_ctl_add_poison (struct lsquic_send_ctl *ctl)
     poison->po_packno     = ctl->sc_gap;
     poison->po_loss_chain = poison; /* Won't be used, but just in case */
     TAILQ_INSERT_TAIL(&ctl->sc_unacked_packets[PNS_APP], poison, po_next);
-    LSQ_DEBUG("insert poisoned packet %"PRIu64, poison->po_packno);
+    LSQ_DEBUG("insert poisoned packet #%"PRIu64, poison->po_packno);
     ctl->sc_flags |= SC_POISON;
     return 0;
 }
@@ -672,7 +680,7 @@ send_ctl_reschedule_poison (struct lsquic_send_ctl *ctl)
     TAILQ_FOREACH(poison, &ctl->sc_unacked_packets[PNS_APP], po_next)
         if (poison->po_flags & PO_POISON)
         {
-            LSQ_DEBUG("remove poisoned packet %"PRIu64, poison->po_packno);
+            LSQ_DEBUG("remove poisoned packet #%"PRIu64, poison->po_packno);
             TAILQ_REMOVE(&ctl->sc_unacked_packets[PNS_APP], poison, po_next);
             lsquic_malo_put(poison);
             lsquic_send_ctl_begin_optack_detection(ctl);
@@ -690,7 +698,7 @@ send_ctl_reschedule_poison (struct lsquic_send_ctl *ctl)
     }
     else
         log_level = LSQ_LOG_DEBUG;
-    LSQ_LOG(log_level, "odd: poisoned packet %"PRIu64" not found during "
+    LSQ_LOG(log_level, "odd: poisoned packet #%"PRIu64" not found during "
         "reschedule, flag: %d", ctl->sc_gap, !!(ctl->sc_flags & SC_POISON));
 }
 
@@ -734,7 +742,7 @@ lsquic_send_ctl_sent_packet (lsquic_send_ctl_t *ctl,
     pns = lsquic_packet_out_pns(packet_out);
     if (0 != send_ctl_update_poison_hist(ctl, packet_out->po_packno))
         return -1;
-    LSQ_DEBUG("packet %"PRIu64" has been sent (frame types: %s)",
+    LSQ_DEBUG("sent packet #%"PRIu64" (%s)",
         packet_out->po_packno, lsquic_frame_types_to_str(frames,
             sizeof(frames), packet_out->po_frame_types));
     lsquic_senhist_add(&ctl->sc_senhist, packet_out->po_packno);
@@ -795,7 +803,7 @@ take_rtt_sample (lsquic_send_ctl_t *ctl,
     const lsquic_packno_t packno = ctl->sc_largest_acked_packno;
     const lsquic_time_t sent = ctl->sc_largest_acked_sent_time;
     const lsquic_time_t measured_rtt = now - sent;
-    if (packno > ctl->sc_max_rtt_packno && lack_delta < measured_rtt)
+    if ((!packno || packno > ctl->sc_max_rtt_packno) && lack_delta < measured_rtt)
     {
         if (UNLIKELY(ctl->sc_flags & SC_ROUGH_RTT))
         {
@@ -860,6 +868,85 @@ send_ctl_maybe_renumber_sched_to_right (struct lsquic_send_ctl *ctl,
 }
 
 
+static void
+send_ctl_process_loss_chain_pkt (struct lsquic_send_ctl *ctl,
+                        struct lsquic_packet_out *const chain_cur,
+                        struct lsquic_packet_out **next)
+{
+    unsigned packet_sz;
+    const char *state;
+    enum packnum_space pns;
+    switch (chain_cur->po_flags & (PO_SCHED|PO_UNACKED|PO_LOST))
+    {
+    case PO_SCHED:
+        send_ctl_maybe_renumber_sched_to_right(ctl, chain_cur);
+        send_ctl_sched_remove(ctl, chain_cur);
+        state = "scheduled";
+        break;
+    case PO_UNACKED:
+        if (chain_cur->po_flags & PO_LOSS_REC)
+        {
+            pns = lsquic_packet_out_pns(chain_cur);
+            TAILQ_REMOVE(&ctl->sc_unacked_packets[pns], chain_cur, po_next);
+            state = "loss record";
+        }
+        else
+        {
+            packet_sz = packet_out_sent_sz(chain_cur);
+            send_ctl_unacked_remove(ctl, chain_cur, packet_sz);
+            state = "unacked";
+        }
+        break;
+    case PO_LOST:
+        TAILQ_REMOVE(&ctl->sc_lost_packets, chain_cur, po_next);
+        state = "lost";
+        break;
+    case 0:
+        /* This is also weird, but let it pass */
+        state = "unknown";
+        break;
+    default:
+        assert(0);
+        state = "bad";
+        break;
+    }
+    if (next && *next == chain_cur)
+        *next = TAILQ_NEXT(*next, po_next);
+    if (0 == (chain_cur->po_flags & PO_LOSS_REC))
+        lsquic_packet_out_ack_streams(chain_cur);
+    LSQ_DEBUG("loss chain, destroy %s packet #%"PRIu64, state,
+                chain_cur->po_packno);
+    send_ctl_destroy_packet(ctl, chain_cur);
+}
+
+
+static void
+send_ctl_acked_loss_chain (struct lsquic_send_ctl *ctl,
+                        struct lsquic_packet_out *const packet_out,
+                        struct lsquic_packet_out **next,
+                        lsquic_packno_t largest_acked,
+                        signed char *do_rtt)
+{
+    struct lsquic_packet_out *chain_cur, *chain_next;
+    unsigned count;
+    count = 0;
+    for (chain_cur = packet_out->po_loss_chain; chain_cur != packet_out;
+                                                    chain_cur = chain_next)
+    {
+        chain_next = chain_cur->po_loss_chain;
+        if (chain_cur->po_packno == largest_acked)
+            *do_rtt = 1;
+        send_ctl_process_loss_chain_pkt(ctl, chain_cur, next);
+        ++count;
+    }
+    packet_out->po_loss_chain = packet_out;
+
+    if (count)
+        LSQ_DEBUG("destroyed %u packet%.*s in chain of packet #%"PRIu64,
+            count, count != 1, "s", packet_out->po_packno);
+}
+
+
 /* The third argument to advance `next' pointer when modifying the unacked
  * queue.  This is because the unacked queue may contain several elements
  * of the same chain.  This is not true of the lost and scheduled packet
@@ -871,50 +958,19 @@ send_ctl_destroy_chain (struct lsquic_send_ctl *ctl,
                         struct lsquic_packet_out **next)
 {
     struct lsquic_packet_out *chain_cur, *chain_next;
-    unsigned packet_sz, count;
-    enum packnum_space pns = lsquic_packet_out_pns(packet_out);
-
+    unsigned count;
     count = 0;
     for (chain_cur = packet_out->po_loss_chain; chain_cur != packet_out;
                                                     chain_cur = chain_next)
     {
         chain_next = chain_cur->po_loss_chain;
-        switch (chain_cur->po_flags & (PO_SCHED|PO_UNACKED|PO_LOST))
-        {
-        case PO_SCHED:
-            send_ctl_maybe_renumber_sched_to_right(ctl, chain_cur);
-            send_ctl_sched_remove(ctl, chain_cur);
-            break;
-        case PO_UNACKED:
-            if (chain_cur->po_flags & PO_LOSS_REC)
-                TAILQ_REMOVE(&ctl->sc_unacked_packets[pns], chain_cur, po_next);
-            else
-            {
-                packet_sz = packet_out_sent_sz(chain_cur);
-                send_ctl_unacked_remove(ctl, chain_cur, packet_sz);
-            }
-            break;
-        case PO_LOST:
-            TAILQ_REMOVE(&ctl->sc_lost_packets, chain_cur, po_next);
-            break;
-        case 0:
-            /* This is also weird, but let it pass */
-            break;
-        default:
-            assert(0);
-            break;
-        }
-        if (next && *next == chain_cur)
-            *next = TAILQ_NEXT(*next, po_next);
-        if (0 == (chain_cur->po_flags & PO_LOSS_REC))
-            lsquic_packet_out_ack_streams(chain_cur);
-        send_ctl_destroy_packet(ctl, chain_cur);
+        send_ctl_process_loss_chain_pkt(ctl, chain_cur, next);
         ++count;
     }
     packet_out->po_loss_chain = packet_out;
 
     if (count)
-        LSQ_DEBUG("destroyed %u packet%.*s in chain of packet %"PRIu64,
+        LSQ_DEBUG("destroyed %u packet%.*s in chain of packet #%"PRIu64,
             count, count != 1, "s", packet_out->po_packno);
 }
 
@@ -972,7 +1028,7 @@ send_ctl_handle_regular_lost_packet (struct lsquic_send_ctl *ctl,
     if (packet_out->po_frame_types & (1 << QUIC_FRAME_ACK))
     {
         ctl->sc_flags |= SC_LOST_ACK_INIT << lsquic_packet_out_pns(packet_out);
-        LSQ_DEBUG("lost ACK in packet %"PRIu64, packet_out->po_packno);
+        LSQ_DEBUG("lost ACK in packet #%"PRIu64, packet_out->po_packno);
     }
 
     if (ctl->sc_ci->cci_lost)
@@ -990,7 +1046,7 @@ send_ctl_handle_regular_lost_packet (struct lsquic_send_ctl *ctl,
 
     if (packet_out->po_frame_types & ctl->sc_retx_frames)
     {
-        LSQ_DEBUG("lost retransmittable packet %"PRIu64,
+        LSQ_DEBUG("lost retransmittable packet #%"PRIu64,
                                                     packet_out->po_packno);
         loss_record = send_ctl_record_loss(ctl, packet_out);
         send_ctl_unacked_remove(ctl, packet_out, packet_sz);
@@ -1000,7 +1056,7 @@ send_ctl_handle_regular_lost_packet (struct lsquic_send_ctl *ctl,
     }
     else
     {
-        LSQ_DEBUG("lost unretransmittable packet %"PRIu64,
+        LSQ_DEBUG("lost unretransmittable packet #%"PRIu64,
                                                     packet_out->po_packno);
         send_ctl_unacked_remove(ctl, packet_out, packet_sz);
         send_ctl_destroy_chain(ctl, packet_out, next);
@@ -1016,7 +1072,7 @@ send_ctl_handle_lost_mtu_probe (struct lsquic_send_ctl *ctl,
 {
     unsigned packet_sz;
 
-    LSQ_DEBUG("lost MTU probe in packet %"PRIu64, packet_out->po_packno);
+    LSQ_DEBUG("lost MTU probe in packet #%"PRIu64, packet_out->po_packno);
     packet_sz = packet_out_sent_sz(packet_out);
     send_ctl_unacked_remove(ctl, packet_out, packet_sz);
     assert(packet_out->po_loss_chain == packet_out);
@@ -1090,7 +1146,7 @@ send_ctl_detect_losses (struct lsquic_send_ctl *ctl, enum packnum_space pns,
         if (packet_out->po_packno + ctl->sc_reord_thresh <
                                                 ctl->sc_largest_acked_packno)
         {
-            LSQ_DEBUG("loss by FACK detected (dist: %"PRIu64"), packet %"PRIu64,
+            LSQ_DEBUG("loss by FACK detected (dist: %"PRIu64"), packet #%"PRIu64,
                 ctl->sc_largest_acked_packno - packet_out->po_packno,
                                                     packet_out->po_packno);
             if (0 == (packet_out->po_flags & PO_MTU_PROBE))
@@ -1111,12 +1167,12 @@ send_ctl_detect_losses (struct lsquic_send_ctl *ctl, enum packnum_space pns,
             && 0 == (packet_out->po_flags & PO_MTU_PROBE)
             && largest_retx_packno <= ctl->sc_largest_acked_packno)
         {
-            LSQ_DEBUG("loss by early retransmit detected, packet %"PRIu64,
+            LSQ_DEBUG("loss by early retransmit detected, packet #%"PRIu64,
                                                     packet_out->po_packno);
             largest_lost_packno = packet_out->po_packno;
             ctl->sc_loss_to =
                 lsquic_rtt_stats_get_srtt(&ctl->sc_conn_pub->rtt_stats) / 4;
-            LSQ_DEBUG("set sc_loss_to to %"PRIu64", packet %"PRIu64,
+            LSQ_DEBUG("set sc_loss_to to %"PRIu64", packet #%"PRIu64,
                                     ctl->sc_loss_to, packet_out->po_packno);
             (void) send_ctl_handle_lost_packet(ctl, packet_out, &next);
             continue;
@@ -1125,7 +1181,7 @@ send_ctl_detect_losses (struct lsquic_send_ctl *ctl, enum packnum_space pns,
         if (ctl->sc_largest_acked_sent_time > packet_out->po_sent +
                     lsquic_rtt_stats_get_srtt(&ctl->sc_conn_pub->rtt_stats))
         {
-            LSQ_DEBUG("loss by sent time detected: packet %"PRIu64,
+            LSQ_DEBUG("loss by sent time detected: packet #%"PRIu64,
                                                     packet_out->po_packno);
             if ((packet_out->po_frame_types & ctl->sc_retx_frames)
                             && 0 == (packet_out->po_flags & PO_MTU_PROBE))
@@ -1138,7 +1194,7 @@ send_ctl_detect_losses (struct lsquic_send_ctl *ctl, enum packnum_space pns,
 
     if (largest_lost_packno > ctl->sc_largest_sent_at_cutback)
     {
-        LSQ_DEBUG("detected new loss: packet %"PRIu64"; new lsac: "
+        LSQ_DEBUG("detected new loss: packet #%"PRIu64"; new lsac: "
             "%"PRIu64, largest_lost_packno, ctl->sc_largest_sent_at_cutback);
         send_ctl_loss_event(ctl);
     }
@@ -1147,7 +1203,7 @@ send_ctl_detect_losses (struct lsquic_send_ctl *ctl, enum packnum_space pns,
          * number sent at the time of the last loss event indicate the same
          * loss event.  This follows NewReno logic, see RFC 6582.
          */
-        LSQ_DEBUG("ignore loss of packet %"PRIu64" smaller than lsac "
+        LSQ_DEBUG("ignore loss of packet #%"PRIu64" smaller than lsac "
             "%"PRIu64, largest_lost_packno, ctl->sc_largest_sent_at_cutback);
 
     return largest_lost_packno > ctl->sc_largest_sent_at_cutback;
@@ -1160,7 +1216,7 @@ send_ctl_mtu_probe_acked (struct lsquic_send_ctl *ctl,
 {
     struct lsquic_conn *const lconn = ctl->sc_conn_pub->lconn;
 
-    LSQ_DEBUG("MTU probe in packet %"PRIu64" has been ACKed",
+    LSQ_DEBUG("MTU probe in packet #%"PRIu64" has been ACKed",
                                                         packet_out->po_packno);
     assert(lconn->cn_if->ci_mtu_probe_acked);
     if (lconn->cn_if->ci_mtu_probe_acked)
@@ -1181,7 +1237,7 @@ send_ctl_maybe_increase_reord_thresh (struct lsquic_send_ctl *ctl,
                 < prev_largest_acked)
     {
         ctl->sc_reord_thresh = prev_largest_acked - loss_record->po_packno;
-        LSQ_DEBUG("packet %"PRIu64" was a spurious loss by FACK, increase "
+        LSQ_DEBUG("packet #%"PRIu64" was a spurious loss by FACK, increase "
             "reordering threshold to %u", loss_record->po_packno,
             ctl->sc_reord_thresh);
     }
@@ -1294,14 +1350,14 @@ lsquic_send_ctl_got_ack (lsquic_send_ctl_t *ctl,
             ctl->sc_largest_acked_sent_time = packet_out->po_sent;
             ecn_total_acked += lsquic_packet_out_ecn(packet_out) != ECN_NOT_ECT;
             ecn_ce_cnt += lsquic_packet_out_ecn(packet_out) == ECN_CE;
-            one_rtt_cnt += lsquic_packet_out_enc_level(packet_out) == ENC_LEV_FORW;
+            one_rtt_cnt += lsquic_packet_out_enc_level(packet_out) == ENC_LEV_APP;
             if (0 == (packet_out->po_flags
                                         & (PO_LOSS_REC|PO_POISON|PO_MTU_PROBE)))
             {
                 packet_sz = packet_out_sent_sz(packet_out);
                 send_ctl_unacked_remove(ctl, packet_out, packet_sz);
                 lsquic_packet_out_ack_streams(packet_out);
-                LSQ_DEBUG("acking via regular record %"PRIu64,
+                LSQ_DEBUG("acking via regular record #%"PRIu64,
                                                         packet_out->po_packno);
             }
             else if (packet_out->po_flags & PO_LOSS_REC)
@@ -1309,7 +1365,7 @@ lsquic_send_ctl_got_ack (lsquic_send_ctl_t *ctl,
                 packet_sz = packet_out->po_sent_sz;
                 TAILQ_REMOVE(&ctl->sc_unacked_packets[pns], packet_out,
                                                                     po_next);
-                LSQ_DEBUG("acking via loss record %"PRIu64,
+                LSQ_DEBUG("acking via loss record #%"PRIu64,
                                                         packet_out->po_packno);
                 send_ctl_maybe_increase_reord_thresh(ctl, packet_out,
                                                             prev_largest_acked);
@@ -1325,7 +1381,7 @@ lsquic_send_ctl_got_ack (lsquic_send_ctl_t *ctl,
             }
             else
             {
-                LSQ_WARN("poisoned packet %"PRIu64" acked",
+                LSQ_WARN("poisoned packet #%"PRIu64" acked",
                                                         packet_out->po_packno);
                 return -1;
             }
@@ -1334,7 +1390,8 @@ lsquic_send_ctl_got_ack (lsquic_send_ctl_t *ctl,
             do_rtt |= packet_out->po_packno == largest_acked(acki);
             ctl->sc_ci->cci_ack(CGP(ctl), packet_out, packet_sz, now,
                                                              app_limited);
-            send_ctl_destroy_chain(ctl, packet_out, &next);
+            send_ctl_acked_loss_chain(ctl, packet_out, &next,
+                                      largest_acked(acki), &do_rtt);
             send_ctl_destroy_packet(ctl, packet_out);
         }
         packet_out = next;
@@ -1344,6 +1401,7 @@ lsquic_send_ctl_got_ack (lsquic_send_ctl_t *ctl,
     if (do_rtt)
     {
         take_rtt_sample(ctl, ack_recv_time, acki->lack_delta);
+        LSQ_DEBUG("clear sc_n_consec_rtos, sc_n_hsk, sc_ntlp");
         ctl->sc_n_consec_rtos = 0;
         ctl->sc_n_hsk = 0;
         ctl->sc_n_tlp = 0;
@@ -1352,7 +1410,11 @@ lsquic_send_ctl_got_ack (lsquic_send_ctl_t *ctl,
   detect_losses:
     losses_detected = send_ctl_detect_losses(ctl, pns, ack_recv_time);
     if (send_ctl_first_unacked_retx_packet(ctl, pns))
-        set_retx_alarm(ctl, pns, now);
+    {
+        if ((ctl->sc_flags & SC_1RTT_ACKED)
+            || (!lsquic_alarmset_is_set(ctl->sc_alset, pns) && losses_detected))
+            set_retx_alarm(ctl, pns, now);
+    }
     else
     {
         LSQ_DEBUG("No retransmittable packets: clear alarm");
@@ -1469,7 +1531,7 @@ send_ctl_next_lost (lsquic_send_ctl_t *ctl)
                                                                     UINT64_MAX);
                 if (lost_packet->po_regen_sz >= lost_packet->po_data_sz)
                 {
-                    LSQ_DEBUG("Dropping packet %"PRIu64" from lost queue",
+                    LSQ_DEBUG("Dropping packet #%"PRIu64" from lost queue",
                         lost_packet->po_packno);
                     TAILQ_REMOVE(&ctl->sc_lost_packets, lost_packet, po_next);
                     lost_packet->po_flags &= ~PO_LOST;
@@ -1544,7 +1606,7 @@ lsquic_send_ctl_cleanup (lsquic_send_ctl_t *ctl)
         send_ctl_destroy_packet(ctl, packet_out);
     }
     assert(0 == ctl->sc_n_scheduled);
-    assert(0 == ctl->sc_bytes_scheduled);
+    //assert(0 == ctl->sc_bytes_scheduled);
     for (pns = PNS_INIT; pns < N_PNS; ++pns)
         while ((packet_out = TAILQ_FIRST(&ctl->sc_unacked_packets[pns])))
         {
@@ -1642,16 +1704,23 @@ lsquic_send_ctl_pacer_blocked (struct lsquic_send_ctl *ctl)
 static int
 send_ctl_can_send (struct lsquic_send_ctl *ctl)
 {
+    uint64_t cwnd = ctl->sc_ci->cci_get_cwnd(CGP(ctl));
     const unsigned n_out = send_ctl_all_bytes_out(ctl);
-    LSQ_DEBUG("%s: n_out: %u (unacked_all: %u); cwnd: %"PRIu64
-        "; ccfc: %"PRIu64"/%"PRIu64, __func__,
-        n_out, ctl->sc_bytes_unacked_all,
-        ctl->sc_ci->cci_get_cwnd(CGP(ctl)),
+    LSQ_DEBUG("%s: sc_flags: 0x%X, b_out: %u = (%u + %u); b_retx: %u; cwnd: %"PRIu64
+        "; ccfc: %"PRIu64"/%"PRIu64"; n_scheduled: %d, n_in_flight_all: %d"
+        "; pa_burst: %d; pa_next: %lu; pa_now: %lu"
+        , __func__, ctl->sc_flags,
+        n_out, ctl->sc_bytes_unacked_all, ctl->sc_bytes_scheduled,
+        ctl->sc_bytes_unacked_retx, cwnd,
         ctl->sc_conn_pub->conn_cap.cc_sent,
-        ctl->sc_conn_pub->conn_cap.cc_max);
+        ctl->sc_conn_pub->conn_cap.cc_max,
+        ctl->sc_n_scheduled, ctl->sc_n_in_flight_all,
+        ctl->sc_pacer.pa_burst_tokens,
+        ctl->sc_pacer.pa_next_sched, ctl->sc_pacer.pa_now);
+
     if (ctl->sc_flags & SC_PACE)
     {
-        if (n_out >= ctl->sc_ci->cci_get_cwnd(CGP(ctl)))
+        if (n_out >= cwnd)
             return 0;
         if (lsquic_pacer_can_schedule(&ctl->sc_pacer,
                                ctl->sc_n_scheduled + ctl->sc_n_in_flight_all))
@@ -1666,7 +1735,7 @@ send_ctl_can_send (struct lsquic_send_ctl *ctl)
         return 0;
     }
     else
-        return n_out < ctl->sc_ci->cci_get_cwnd(CGP(ctl));
+        return n_out < cwnd;
 }
 
 
@@ -1927,7 +1996,7 @@ send_ctl_maybe_zero_pad (struct lsquic_send_ctl *ctl,
         if (cum_size + size > limit)
             break;
         cum_size += size;
-        if (HETY_NOT_SET == packet_out->po_header_type)
+        if (HETY_SHORT == packet_out->po_header_type)
             break;
     }
 
@@ -1942,7 +2011,7 @@ send_ctl_maybe_zero_pad (struct lsquic_send_ctl *ctl,
         initial_packet->po_data_sz += size;
         initial_packet->po_frame_types |= QUIC_FTBIT_PADDING;
     }
-    LSQ_DEBUG("Added %zu bytes of PADDING to packet %"PRIu64, size,
+    LSQ_DEBUG("Added %zu bytes of PADDING to packet #%"PRIu64, size,
                                                 initial_packet->po_packno);
 }
 
@@ -1960,7 +2029,8 @@ lsquic_send_ctl_next_packet_to_send_predict (struct lsquic_send_ctl *ctl)
     n_rtos = ~0u;
     TAILQ_FOREACH(packet_out, &ctl->sc_scheduled_packets, po_next)
     {
-        if (!(packet_out->po_frame_types & (1 << QUIC_FRAME_ACK))
+        if (!(packet_out->po_frame_types
+                    & ((1 << QUIC_FRAME_ACK) | (1 << QUIC_FRAME_CRYPTO)))
             && 0 == ctl->sc_next_limit
             && 0 != (n_rtos == ~0u ? /* Initialize once */
                     (n_rtos = send_ctl_get_n_consec_rtos(ctl)) : n_rtos))
@@ -1972,11 +2042,11 @@ lsquic_send_ctl_next_packet_to_send_predict (struct lsquic_send_ctl *ctl)
                     && packet_out->po_regen_sz == packet_out->po_data_sz
                     && packet_out->po_frame_types != QUIC_FTBIT_PATH_CHALLENGE)
         {
-            LSQ_DEBUG("send prediction: packet %"PRIu64" would be dropped, "
+            LSQ_DEBUG("send prediction: packet #%"PRIu64" would be dropped, "
                 "continue", packet_out->po_packno);
             continue;
         }
-        LSQ_DEBUG("send prediction: yes, packet %"PRIu64", flags %u, frames 0x%X",
+        LSQ_DEBUG("send prediction: yes, packet #%"PRIu64", flags %u, frames 0x%X",
             packet_out->po_packno, (unsigned) packet_out->po_flags,
             (unsigned) packet_out->po_frame_types);
         return 1;
@@ -1998,18 +2068,26 @@ lsquic_send_ctl_next_packet_to_send (struct lsquic_send_ctl *ctl,
   get_packet:
     packet_out = TAILQ_FIRST(&ctl->sc_scheduled_packets);
     if (!packet_out)
+    {
+        LSQ_DEBUG("no more scheduled packets");
         return NULL;
+    }
 
     /* Note: keep logic in this function and in
      * lsquic_send_ctl_next_packet_to_send_predict() in synch.
      */
-    if (!(packet_out->po_frame_types & (1 << QUIC_FRAME_ACK))
-                                        && send_ctl_get_n_consec_rtos(ctl))
+    if (!(packet_out->po_frame_types
+                & ((1 << QUIC_FRAME_ACK) | (1 << QUIC_FRAME_CRYPTO)))
+            && send_ctl_get_n_consec_rtos(ctl))
     {
         if (ctl->sc_next_limit)
             dec_limit = 1;
         else
+        {
+            LSQ_DEBUG("sc_n_consec_rtos: %d, sc_next_limit is 0",
+                      ctl->sc_n_consec_rtos);
             return NULL;
+        }
     }
     else
         dec_limit = 0;
@@ -2024,7 +2102,7 @@ lsquic_send_ctl_next_packet_to_send (struct lsquic_send_ctl *ctl,
         }
         else
         {
-            LSQ_DEBUG("Dropping packet %"PRIu64" from scheduled queue",
+            LSQ_DEBUG("Dropping packet #%"PRIu64" from scheduled queue",
                 packet_out->po_packno);
             send_ctl_sched_remove(ctl, packet_out);
             send_ctl_destroy_chain(ctl, packet_out, NULL);
@@ -2043,7 +2121,7 @@ lsquic_send_ctl_next_packet_to_send (struct lsquic_send_ctl *ctl,
                                                         > SC_PACK_SIZE(ctl)
             || !lsquic_packet_out_equal_dcids(to_coal->prev_packet, packet_out))
             return NULL;
-        LSQ_DEBUG("packet %"PRIu64" (%zu bytes) will be tacked on to "
+        LSQ_DEBUG("packet #%"PRIu64" (%zu bytes) will be tacked on to "
             "previous packet(s) (%zu bytes) (coalescing)",
             packet_out->po_packno, packet_out_total_sz(packet_out),
             to_coal->prev_sz_sum);
@@ -2080,7 +2158,7 @@ lsquic_send_ctl_next_packet_to_send (struct lsquic_send_ctl *ctl,
         }
         else
             packet_out->po_lflags &= ~POL_LOSS_BIT;
-        if (packet_out->po_header_type == HETY_NOT_SET)
+        if (packet_out->po_header_type == HETY_SHORT)
         {
             if (ctl->sc_gap + 1 == packet_out->po_packno)
                 ++ctl->sc_square_count;
@@ -2102,14 +2180,14 @@ lsquic_send_ctl_delayed_one (lsquic_send_ctl_t *ctl,
     send_ctl_sched_prepend(ctl, packet_out);
     if (packet_out->po_lflags & POL_LIMITED)
         ++ctl->sc_next_limit;
-    LSQ_DEBUG("packet %"PRIu64" has been delayed", packet_out->po_packno);
+    LSQ_DEBUG("packet #%"PRIu64" has been delayed", packet_out->po_packno);
 #if LSQUIC_SEND_STATS
     ++ctl->sc_stats.n_delayed;
 #endif
     if (packet_out->po_lflags & POL_LOSS_BIT)
         ++ctl->sc_loss_count;
     if ((ctl->sc_flags & SC_QL_BITS)
-                            && packet_out->po_header_type == HETY_NOT_SET)
+                            && packet_out->po_header_type == HETY_SHORT)
         ctl->sc_square_count -= 1 + (ctl->sc_gap + 1 == packet_out->po_packno);
 }
 
@@ -2168,7 +2246,7 @@ send_ctl_allocate_packet (struct lsquic_send_ctl *ctl, enum packno_bits bits,
     {
         [PNS_INIT]  = HETY_INITIAL,
         [PNS_HSK]   = HETY_HANDSHAKE,
-        [PNS_APP]   = HETY_NOT_SET,
+        [PNS_APP]   = HETY_SHORT,
     };
     lsquic_packet_out_t *packet_out;
 
@@ -2232,7 +2310,7 @@ lsquic_send_ctl_new_packet_out (lsquic_send_ctl_t *ctl, unsigned need_at_least,
         return NULL;
 
     packet_out->po_packno = send_ctl_next_packno(ctl);
-    LSQ_DEBUG("created packet %"PRIu64, packet_out->po_packno);
+    LSQ_DEBUG("created packet #%"PRIu64, packet_out->po_packno);
     EV_LOG_PACKET_CREATED(LSQUIC_LOG_CONN_ID, packet_out);
     return packet_out;
 }
@@ -2367,10 +2445,8 @@ update_for_resending (lsquic_send_ctl_t *ctl, lsquic_packet_out_t *packet_out)
             ctl->sc_bytes_scheduled -= packet_out->po_regen_sz;
         lsquic_packet_out_chop_regen(packet_out);
     }
-    LSQ_DEBUG("Packet %"PRIu64" repackaged for resending as packet %"PRIu64,
-                                                            oldno, packno);
-    EV_LOG_CONN_EVENT(LSQUIC_LOG_CONN_ID, "packet %"PRIu64" repackaged for "
-        "resending as packet %"PRIu64, oldno, packno);
+    EV_LOG_CONN_EVENT(LSQUIC_LOG_CONN_ID, "schedule resend, repackage packet "
+                      "#%"PRIu64" -> #%"PRIu64, oldno, packno);
 }
 
 
@@ -2447,7 +2523,7 @@ lsquic_send_ctl_elide_stream_frames (lsquic_send_ctl_t *ctl,
             ctl->sc_bytes_scheduled -= adj;
             if (0 == packet_out->po_frame_types)
             {
-                LSQ_DEBUG("cancel packet %"PRIu64" after eliding frames for "
+                LSQ_DEBUG("cancel packet #%"PRIu64" after eliding frames for "
                     "stream %"PRIu64, packet_out->po_packno, stream_id);
                 send_ctl_sched_remove(ctl, packet_out);
                 send_ctl_destroy_chain(ctl, packet_out, NULL);
@@ -2587,7 +2663,7 @@ lsquic_send_ctl_squeeze_sched (lsquic_send_ctl_t *ctl)
                                         "scheduled packets before squeezing");
 #endif
             send_ctl_sched_remove(ctl, packet_out);
-            LSQ_DEBUG("Dropping packet %"PRIu64" from scheduled queue",
+            LSQ_DEBUG("Dropping packet #%"PRIu64" from scheduled queue",
                 packet_out->po_packno);
             send_ctl_destroy_chain(ctl, packet_out, NULL);
             send_ctl_destroy_packet(ctl, packet_out);
@@ -2897,6 +2973,52 @@ lsquic_send_ctl_get_packet_for_stream (lsquic_send_ctl_t *ctl,
 }
 
 
+/* Return true if generated, false otherwise */
+int
+lsquic_sendctl_gen_stream_blocked_frame (struct lsquic_send_ctl *ctl,
+                                         struct lsquic_stream *stream)
+{
+    struct lsquic_packet_out *packet_out;
+    const struct parse_funcs *const pf = stream->conn_pub->lconn->cn_pf;
+    unsigned need;
+    uint64_t off;
+    int sz;
+    int is_err;
+
+    off = lsquic_stream_combined_send_off(stream);
+    need = pf->pf_stream_blocked_frame_size(stream->id, off);
+    packet_out = lsquic_send_ctl_get_packet_for_stream(ctl, need,
+                       stream->conn_pub->path, stream);
+    if (!packet_out)
+    {
+        LSQ_DEBUG("failed to get packet_out with lsquic_send_ctl_get_packet_for_stream");
+        packet_out = lsquic_send_ctl_get_writeable_packet(ctl,
+                            PNS_APP, need, stream->conn_pub->path, 0, &is_err);
+        if (!packet_out)
+        {
+            LSQ_DEBUG("cannot get writeable packet for STREAM_BLOCKED frame");
+            return 0;
+        }
+    }
+    sz = pf->pf_gen_stream_blocked_frame(
+                         packet_out->po_data + packet_out->po_data_sz,
+                         lsquic_packet_out_avail(packet_out), stream->id, off);
+    if (sz < 0)
+        return 0;
+    LSQ_DEBUG("generated %d-byte STREAM_BLOCKED "
+        "frame; stream_id: %"PRIu64"; offset: %"PRIu64, sz, stream->id, off);
+    EV_LOG_CONN_EVENT(LSQUIC_LOG_CONN_ID, "generated %d-byte STREAM_BLOCKED "
+        "frame; stream_id: %"PRIu64"; offset: %"PRIu64, sz, stream->id, off);
+    if (0 != lsquic_packet_out_add_frame(packet_out, &ctl->sc_enpub->enp_mm, 0,
+                        QUIC_FRAME_STREAM_BLOCKED, packet_out->po_data_sz, sz))
+        return 0;
+    lsquic_send_ctl_incr_pack_sz(ctl, packet_out, sz);
+    packet_out->po_frame_types |= 1 << QUIC_FRAME_STREAM_BLOCKED;
+    lsquic_stream_blocked_frame_sent(stream);
+    return 1;
+}
+
+
 #ifdef NDEBUG
 static
 #elif __GNUC__
@@ -3107,7 +3229,7 @@ lsquic_send_ctl_schedule_buffered (lsquic_send_ctl_t *ctl,
         --packet_q->bpq_count;
         packet_out->po_packno = send_ctl_next_packno(ctl);
         LSQ_DEBUG("Remove packet from buffered queue #%u; count: %u.  "
-            "It becomes packet %"PRIu64, packet_type, packet_q->bpq_count,
+            "It becomes packet #%"PRIu64, packet_type, packet_q->bpq_count,
             packet_out->po_packno);
         lsquic_send_ctl_scheduled_one(ctl, packet_out);
     }
@@ -3360,6 +3482,8 @@ lsquic_send_ctl_maybe_calc_rough_rtt (struct lsquic_send_ctl *ctl,
     if (min_sent < UINT64_MAX)
     {
         rtt = lsquic_time_now() - min_sent;
+        if (rtt > 500000)
+            rtt = 500000;
         lsquic_rtt_stats_update(&ctl->sc_conn_pub->rtt_stats, rtt, 0);
         ctl->sc_flags |= SC_ROUGH_RTT;
         LSQ_DEBUG("set rough RTT to %"PRIu64" usec", rtt);
@@ -3557,7 +3681,7 @@ send_ctl_resize_q (struct lsquic_send_ctl *ctl, struct lsquic_packets_tailq *q,
             ++count_dst;
             packet_out->po_packno = send_ctl_next_packno(ctl);
             send_ctl_sched_append(ctl, packet_out);
-            LSQ_DEBUG("created packet %"PRIu64, packet_out->po_packno);
+            LSQ_DEBUG("created packet #%"PRIu64, packet_out->po_packno);
             EV_LOG_PACKET_CREATED(LSQUIC_LOG_CONN_ID, packet_out);
         }
     else
@@ -3704,9 +3828,9 @@ lsquic_send_ctl_resize (struct lsquic_send_ctl *ctl)
                 size = lsquic_packet_out_total_sz(lconn, packet_out);
                 if (size > packet_out->po_path->np_pack_size)
                 {
-                    send_ctl_resize_q(ctl, *q, packet_out->po_path);
                     path_ids |= 1 << packet_out->po_path->np_path_id;
                     q_idxs |= 1 << (q - queues);
+                    send_ctl_resize_q(ctl, *q, packet_out->po_path);
                     goto redo_q;
                 }
             }
@@ -4040,13 +4164,14 @@ lsquic_send_ctl_0rtt_to_1rtt (struct lsquic_send_ctl *ctl)
             if (packet_out->po_header_type == HETY_0RTT)
             {
                 ++count;
-                packet_out->po_header_type = HETY_NOT_SET;
+                packet_out->po_header_type = HETY_SHORT;
                 if (packet_out->po_flags & PO_ENCRYPTED)
                     send_ctl_return_enc_data(ctl, packet_out);
             }
 
     LSQ_DEBUG("handshake ok: changed %u packet%.*s from 0-RTT to 1-RTT",
                                                     count, count != 1, "s");
+    ctl->sc_n_consec_rtos >>= 1;
 }
 
 

src/liblsquic/lsquic_send_ctl.h

@@ -236,6 +236,10 @@ lsquic_send_ctl_get_packet_for_stream (lsquic_send_ctl_t *,
                     unsigned need_at_least, const struct network_path *,
                     const struct lsquic_stream *);
 
+int
+lsquic_sendctl_gen_stream_blocked_frame (struct lsquic_send_ctl *ctl,
+                                         struct lsquic_stream *stream);
+
 struct lsquic_packet_out *
 lsquic_send_ctl_get_packet_for_crypto (struct lsquic_send_ctl *ctl,
     unsigned need_at_least, enum packnum_space, const struct network_path *);
@@ -281,32 +285,41 @@ lsquic_send_ctl_ack_to_front (struct lsquic_send_ctl *, unsigned n_acks);
 #define lsquic_send_ctl_n_stop_waiting(ctl) \
                                     (+(ctl)->sc_n_stop_waiting)
 
-#define lsquic_send_ctl_n_stop_waiting_reset(ctl) do {      \
-    (ctl)->sc_n_stop_waiting = 0;                           \
-} while (0)
+static inline void
+lsquic_send_ctl_n_stop_waiting_reset(lsquic_send_ctl_t *ctl)
+{
+    ctl->sc_n_stop_waiting = 0;
+}
 
 void
 lsquic_send_ctl_drop_scheduled (lsquic_send_ctl_t *);
 
-#define lsquic_send_ctl_tick_in(ctl, now) do {              \
-    if ((ctl)->sc_flags & SC_PACE)                          \
-    {                                                       \
-        (ctl)->sc_flags |= SC_SCHED_TICK;                   \
-        lsquic_pacer_tick_in(&(ctl)->sc_pacer, now);        \
-    }                                                       \
-    (ctl)->sc_flags &= ~SC_APP_LIMITED;                     \
-} while (0)
+static inline void
+lsquic_send_ctl_tick_in(lsquic_send_ctl_t *ctl, lsquic_time_t now)
+{
+    if ((ctl)->sc_flags & SC_PACE)
+    {
+        (ctl)->sc_flags |= SC_SCHED_TICK;
+        lsquic_pacer_tick_in(&(ctl)->sc_pacer, now);
+    }
+    (ctl)->sc_flags &= ~SC_APP_LIMITED;
+}
 
-#define lsquic_send_ctl_tick_out(ctl) do {                  \
-    if ((ctl)->sc_flags & SC_PACE)                          \
-        lsquic_pacer_tick_out(&(ctl)->sc_pacer);            \
-} while (0)
+static inline void
+lsquic_send_ctl_tick_out(lsquic_send_ctl_t *ctl)
+{
+    if ((ctl)->sc_flags & SC_PACE)
+        lsquic_pacer_tick_out(&(ctl)->sc_pacer);
+}
 
-#define lsquic_send_ctl_next_pacer_time(ctl) (              \
-    ((ctl)->sc_flags & SC_PACE)                             \
-        && lsquic_pacer_delayed(&(ctl)->sc_pacer)           \
-        ? lsquic_pacer_next_sched(&(ctl)->sc_pacer)         \
-        : 0 )
+static inline lsquic_time_t
+lsquic_send_ctl_next_pacer_time(lsquic_send_ctl_t *ctl)
+{
+    return ((ctl)->sc_flags & SC_PACE)
+            && lsquic_pacer_delayed(&(ctl)->sc_pacer)
+            ? lsquic_pacer_next_sched(&(ctl)->sc_pacer)
+            : 0;
+}
 
 enum packno_bits
 lsquic_send_ctl_packno_bits (struct lsquic_send_ctl *, enum packnum_space);

src/liblsquic/lsquic_stream.c

@@ -1334,6 +1334,15 @@ lsquic_stream_stop_sending_in (struct lsquic_stream *stream,
                                     && !(stream->sm_qflags & SMQF_SEND_RST))
         stream_reset(stream, 0, 0);
 
+    if (stream->sm_qflags & (SMQF_SEND_WUF | SMQF_SEND_BLOCKED \
+                             | SMQF_SEND_STOP_SENDING))
+    {
+        stream->sm_qflags &= ~(SMQF_SEND_WUF | SMQF_SEND_BLOCKED \
+                               | SMQF_SEND_STOP_SENDING);
+        if (!(stream->sm_qflags & SMQF_SENDING_FLAGS))
+            TAILQ_REMOVE(&stream->conn_pub->sending_streams, stream, next_send_stream);
+    }
+
     maybe_finish_stream(stream);
     maybe_schedule_call_on_close(stream);
 }
@@ -1879,6 +1888,11 @@ stream_shutdown_write (lsquic_stream_t *stream)
             {
                 LSQ_DEBUG("turned on FIN flag in the yet-unsent STREAM frame");
                 stream->stream_flags |= STREAM_FIN_SENT;
+                if (stream->sm_qflags & SMQF_WANT_FLUSH)
+                {
+                    LSQ_DEBUG("turned off SMQF_WANT_FLUSH flag as FIN flag is turned on.");
+                    maybe_remove_from_write_q(stream, SMQF_WANT_FLUSH);
+                }
             }
             else
             {
@@ -2408,7 +2422,10 @@ lsquic_stream_dispatch_write_events (lsquic_stream_t *stream)
     unsigned short n_buffered;
     enum stream_q_flags q_flags;
 
-    LSQ_DEBUG("dispatch_write_events");
+    LSQ_DEBUG("dispatch_write_events, sm_qflags: %d. stream_flags: %d, sm_bflags: %d, "
+                "max_send_off: %" PRIu64 ", tosend_off: %" PRIu64 ", sm_n_buffered: %u",
+              stream->sm_qflags, stream->stream_flags, stream->sm_bflags,
+              stream->max_send_off, stream->tosend_off, stream->sm_n_buffered);
 
     if (!(stream->sm_qflags & SMQF_WRITE_Q_FLAGS)
         || (stream->stream_flags & STREAM_FINISHED))
@@ -2434,6 +2451,12 @@ lsquic_stream_dispatch_write_events (lsquic_stream_t *stream)
     else
         stream_dispatch_write_events_loop(stream);
 
+    if ((stream->sm_qflags & SMQF_SEND_BLOCKED) &&
+        (stream->sm_bflags & SMBF_IETF))
+    {
+        lsquic_sendctl_gen_stream_blocked_frame(stream->conn_pub->send_ctl, stream);
+    }
+
     /* Progress means either flags or offsets changed: */
     progress = !((stream->sm_qflags & SMQF_WRITE_Q_FLAGS) == q_flags &&
                         stream->tosend_off == tosend_off &&
@@ -2561,7 +2584,7 @@ lsquic_stream_flush_threshold (const struct lsquic_stream *stream,
         flags |= PO_LONGHEAD;
 
     packet_header_sz = lsquic_po_header_length(stream->conn_pub->lconn, flags,
-                            stream->conn_pub->path->np_dcid.len, HETY_NOT_SET);
+                            stream->conn_pub->path->np_dcid.len, HETY_SHORT);
     stream_header_sz = stream->sm_frame_header_sz(stream, data_sz);
     tag_len = stream->conn_pub->lconn->cn_esf_c->esf_tag_len;
 
@@ -3441,6 +3464,11 @@ stream_write_to_packets (lsquic_stream_t *stream, struct lsquic_reader *reader,
                 if (use_framing && seen_ok)
                     maybe_close_varsize_hq_frame(stream);
                 stream->stream_flags |= STREAM_FIN_SENT;
+                if (stream->sm_qflags & SMQF_WANT_FLUSH)
+                {
+                    LSQ_DEBUG("turned off SMQF_WANT_FLUSH flag as FIN has been sent.");
+                    maybe_remove_from_write_q(stream, SMQF_WANT_FLUSH);
+                }
                 goto end;
             }
             else
@@ -3695,10 +3723,15 @@ stream_write (lsquic_stream_t *stream, struct lsquic_reader *reader,
         }
         while (nwritten < len
                         && stream->sm_n_buffered < stream->sm_n_allocated);
-        return nwritten;
     }
     else
-        return stream_write_to_packets(stream, reader, thresh, swo);
+        nwritten = stream_write_to_packets(stream, reader, thresh, swo);
+    if ((stream->sm_qflags & SMQF_SEND_BLOCKED) &&
+        (stream->sm_bflags & SMBF_IETF))
+    {
+        lsquic_sendctl_gen_stream_blocked_frame(stream->conn_pub->send_ctl, stream);
+    }
+    return nwritten;
 }
 
 

src/liblsquic/lsquic_tokgen.c

@@ -7,8 +7,12 @@
 #include <time.h>
 
 #ifndef WIN32
+#include <arpa/inet.h>
 #include <netinet/in.h>
 #include <sys/socket.h>
+#else
+#include "vc_compat.h"
+#include <Ws2tcpip.h>
 #endif
 
 #include <openssl/aead.h>
@@ -49,13 +53,22 @@ struct tokgen_shm_state
 {
     uint8_t     tgss_version;
     uint8_t     tgss_magic_top[sizeof(TOKGEN_SHM_MAGIC_TOP) - 1];
-    uint8_t     tgss_padding[2 * CRYPTER_KEY_SIZE];
+    uint8_t     tgss_crypter_key[N_TOKEN_TYPES][CRYPTER_KEY_SIZE];
     uint8_t     tgss_srst_prk_size;
     uint8_t     tgss_srst_prk[SRST_MAX_PRK_SIZE];
     uint8_t     tgss_magic_bottom[sizeof(TOKGEN_SHM_MAGIC_BOTTOM) - 1];
 };
 
 
+/* The various salt values below were obtained by reading from /dev/random
+ * when the code was first written.
+ */
+
+static const uint64_t salts[N_TOKEN_TYPES] =
+{
+    [TOKEN_RETRY]  = 0xa49c3ef763a6243f,
+    [TOKEN_RESUME] = 0x0b3664549086b8ca,
+};
 
 static const uint8_t srst_salt[8] = "\x28\x6e\x81\x02\x40\x5b\x2c\x2b";
 
@@ -68,19 +81,56 @@ struct crypter
 };
 
 
+/* Bloom filter of Resume tokens.  See below. */
+struct resumed_token_page
+{
+    TAILQ_ENTRY(resumed_token_page)     next;
+    time_t                              begin,  /* Oldest entry */
+                                        end;    /* Newest entry */
+    unsigned                            count;  /* Number of entries */
+    uintptr_t                           masks[];
+};
 
 
 struct token_generator
 {
+    /* We encrypt different token types using different keys. */
+    struct crypter  tg_crypters[N_TOKEN_TYPES];
 
     /* Stateless reset token is generated using HKDF with CID as the
      * `info' parameter to HKDF-Expand.
      */
     size_t          tg_srst_prk_sz;
     uint8_t         tg_srst_prk_buf[SRST_MAX_PRK_SIZE];
+    unsigned        tg_retry_token_duration;
+    TAILQ_HEAD(resumed_token_pages_head, resumed_token_page)
+                    tg_resume_token_pages;
 };
 
 
+static int
+setup_nonce_prk (unsigned char *nonce_prk_buf, size_t *nonce_prk_sz,
+                                                    unsigned i, time_t now)
+{
+    struct {
+        time_t          now;
+        enum token_type tt;
+        uint8_t         buf[16];
+    } ikm;
+
+    ikm.now = now;
+    ikm.tt  = i;
+    RAND_bytes(ikm.buf, sizeof(ikm.buf));
+    if (HKDF_extract(nonce_prk_buf, nonce_prk_sz,
+                     EVP_sha256(), (uint8_t *) &ikm, sizeof(ikm),
+                     (void *) &salts[i], sizeof(salts[i])))
+        return 0;
+    else
+    {
+        LSQ_ERROR("HKDF_extract failed");
+        return -1;
+    }
+}
 
 
 static int
@@ -149,10 +199,14 @@ get_or_generate_state (struct lsquic_engine_public *enpub, time_t now,
     if (getenv("LSQUIC_NULL_TOKGEN"))
     {
         LSQ_NOTICE("using NULL tokgen");
+        memset(shm_state->tgss_crypter_key, 0,
+                                        sizeof(shm_state->tgss_crypter_key));
         memset(&srst_ikm, 0, sizeof(srst_ikm));
     }
     else
     {
+        RAND_bytes((void *) shm_state->tgss_crypter_key,
+                                        sizeof(shm_state->tgss_crypter_key));
         srst_ikm.now = now;
         RAND_bytes(srst_ikm.buf, sizeof(srst_ikm.buf));
     }
@@ -205,6 +259,26 @@ lsquic_tg_new (struct lsquic_engine_public *enpub)
     if (0 != get_or_generate_state(enpub, now, &shm_state))
         goto err;
 
+    TAILQ_INIT(&tokgen->tg_resume_token_pages);
+    unsigned i;
+    for (i = 0; i < sizeof(tokgen->tg_crypters)
+                                    / sizeof(tokgen->tg_crypters[0]); ++i)
+    {
+        struct crypter *crypter;
+        crypter = tokgen->tg_crypters + i;
+        if (0 != setup_nonce_prk(crypter->nonce_prk_buf,
+                                        &crypter->nonce_prk_sz, i, now))
+            goto err;
+        if (1 != EVP_AEAD_CTX_init(&crypter->ctx, EVP_aead_aes_128_gcm(),
+            shm_state.tgss_crypter_key[i],
+            sizeof(shm_state.tgss_crypter_key[i]), RETRY_TAG_LEN, 0))
+            goto err;
+    }
+
+    tokgen->tg_retry_token_duration
+                            = enpub->enp_settings.es_retry_token_duration;
+    if (tokgen->tg_retry_token_duration == 0)
+        tokgen->tg_retry_token_duration = LSQUIC_DF_RETRY_TOKEN_DURATION;
 
     tokgen->tg_srst_prk_sz = shm_state.tgss_srst_prk_size;
     if (tokgen->tg_srst_prk_sz > sizeof(tokgen->tg_srst_prk_buf))
@@ -228,11 +302,513 @@ lsquic_tg_new (struct lsquic_engine_public *enpub)
 void
 lsquic_tg_destroy (struct token_generator *tokgen)
 {
+    struct resumed_token_page *page;
+    struct crypter *crypter;
+    unsigned i;
+
+    while ((page = TAILQ_FIRST(&tokgen->tg_resume_token_pages)))
+    {
+        TAILQ_REMOVE(&tokgen->tg_resume_token_pages, page, next);
+        free(page);
+    }
+    for (i = 0; i < sizeof(tokgen->tg_crypters)
+                                    / sizeof(tokgen->tg_crypters[0]); ++i)
+    {
+        crypter = tokgen->tg_crypters + i;
+        EVP_AEAD_CTX_cleanup(&crypter->ctx);
+    }
     free(tokgen);
     LSQ_DEBUG("destroyed");
 }
 
 
+/* To limit reuse of Resume tokens, used Resume tokens are inserted into a
+ * list of Bloom filters with very low false positive rate.  Before a Resume
+ * token is used, we check in the Bloom filter.  If this token has already
+ * been used, it fails validation.
+ *
+ * There are three ways when this check will fail:
+ *  1. Bloom filter false positive.  In this case, Resume token fails
+ *     validation, which may cause the server may issue a Retry.  This
+ *     should happen very infrequently (see below).
+ *  2. Server restart.  Because the Bloom filter is stored in process
+ *     memory, this will result in false negative and a Resume token can be
+ *     reused.
+ *  3. Different working process.  Similar to (2).
+ *
+ * Bloom filters are on a linked list.  Each filter is used up to MAX_PER_PAGE
+ * values or RESUME_MAX_SECS seconds, after which a new Bloom filter is inserted.
+ * Bloom filters are removed once the most recent element is older than
+ * RESUME_MAX_HOURS hours.
+ */
+
+#define RESUME_MAX_SECS (24 * 3600)
+
+#define N_BLOOM_FUNCS 10
+
+/* We need 30 bytes to generate 10 24-bit Bloom filter values */
+typedef char enough_blooms[MIN_RESUME_TOKEN_LEN >= 3 * N_BLOOM_FUNCS ? 1 : -1];
+
+typedef uint32_t bloom_vals_t[N_BLOOM_FUNCS];
+
+#define RESUME_TOKEN_PAGE_SIZE (1u << 21)
+
+/* For memory efficiency, we allocate 2MB chunks of memory,
+ * not 2MB + 28 bytes.  Thus, we can't use the whole 24-bit range.
+ */
+#define MAX_BLOOM_VALUE ((RESUME_TOKEN_PAGE_SIZE - \
+    sizeof(struct resumed_token_page)) * 8 - 1)
+
+#define MAX_PER_PAGE 500000
+/* This works out to 0.00012924% false positive rate:
+ * perl -E '$k=10;$m=1<<24;$n=500000;printf("%.10lf",(1-exp(1)**-($k*$n/$m))**$k)'
+ */
+
+static int
+tokgen_seen_resumed_token (struct token_generator *tokgen,
+        const unsigned char *token, size_t token_sz, bloom_vals_t bloom_vals)
+{
+    const struct resumed_token_page *page;
+    unsigned n, idx;
+    uintptr_t slot;
+
+    if (1 + N_BLOOM_FUNCS * 3 > token_sz)
+        return 0;
+
+    ++token;
+    for (n = 0; n < N_BLOOM_FUNCS; ++n)
+    {
+        bloom_vals[n] = *token++;
+        bloom_vals[n] |= *token++ << 8;
+        bloom_vals[n] |= *token++ << 16;
+        if (bloom_vals[n] > MAX_BLOOM_VALUE)
+            bloom_vals[n] = MAX_BLOOM_VALUE;
+    }
+
+    page = TAILQ_FIRST(&tokgen->tg_resume_token_pages);
+    while (page)
+    {
+        for (n = 0; n < N_BLOOM_FUNCS; ++n)
+        {
+            idx = bloom_vals[n] / (sizeof(page->masks[0]) * 8);
+            slot = 1;
+            slot <<= bloom_vals[n] % (sizeof(page->masks[0]) * 8);
+            if (!(page->masks[idx] & slot))
+                goto next_page;
+        }
+        return 1;
+  next_page:
+        page = TAILQ_NEXT(page, next);
+    }
+
+    return 0;
+}
+
+
+static void /* void: if it fails, there is nothing to do */
+tokgen_record_resumed_token (struct token_generator *tokgen, time_t now,
+                                                        bloom_vals_t bloom_vals)
+{
+    struct resumed_token_page *page;
+    unsigned n, idx;
+    uintptr_t slot;
+
+    /* Expunge old pages at insertion time only to save on time() syscall */
+    while ((page = TAILQ_FIRST(&tokgen->tg_resume_token_pages)))
+        if (page->end + RESUME_MAX_SECS < now)
+        {
+            LSQ_DEBUG("drop resumed cache page");
+            TAILQ_REMOVE(&tokgen->tg_resume_token_pages, page, next);
+            free(page);
+        }
+        else
+            break;
+
+    page = TAILQ_LAST(&tokgen->tg_resume_token_pages, resumed_token_pages_head);
+    if (!(page && page->count < MAX_PER_PAGE && now
+                                            < page->begin + RESUME_MAX_SECS))
+    {
+        page = calloc(1, RESUME_TOKEN_PAGE_SIZE);
+        if (!page)
+        {
+            LSQ_WARN("cannot allocate resumed cache page");
+            return;
+        }
+        LSQ_DEBUG("allocate resumed cache page");
+        TAILQ_INSERT_TAIL(&tokgen->tg_resume_token_pages, page, next);
+        page->begin = now;
+        page->end = now;
+        page->count = 0;
+    }
+
+    page->end = now;
+    ++page->count;
+
+    for (n = 0; n < N_BLOOM_FUNCS; ++n)
+    {
+        idx = bloom_vals[n] / (sizeof(page->masks[0]) * 8);
+        slot = 1;
+        slot <<= bloom_vals[n] % (sizeof(page->masks[0]) * 8);
+        page->masks[idx] |= slot;
+    }
+}
+
+
+static const char *const tt2str[N_TOKEN_TYPES] = {
+    [TOKEN_RESUME] = "resume",
+    [TOKEN_RETRY]  = "retry",
+};
+
+
+int
+lsquic_tg_validate_token (struct token_generator *tokgen,
+    const struct lsquic_packet_in *packet_in, const struct sockaddr *sa_peer,
+    lsquic_cid_t *odcid)
+{
+    size_t decr_token_len, encr_token_len, ad_len;
+    const unsigned char *nonce, *encr_token, *p, *end, *ad;
+    struct crypter *crypter;
+    enum token_type token_type;
+    time_t issued_at, ttl, now;
+    int is_ipv6;
+    unsigned version;
+    bloom_vals_t bloom_vals;
+    unsigned char decr_token[MAX_RETRY_TOKEN_LEN - RETRY_TAG_LEN
+                                                        - RETRY_NONCE_LEN];
+    char token_str[MAX_RETRY_TOKEN_LEN * 2 + 1];
+    char addr_str[2][INET6_ADDRSTRLEN];
+
+    if (!(packet_in->pi_token && packet_in->pi_token_size))
+    {
+        LSQ_DEBUGC("packet for connection %"CID_FMT" has no token: "
+            "validation failed", CID_BITS(&packet_in->pi_dcid));
+        return -1;
+    }
+
+    if (packet_in->pi_token_size < RETRY_TAG_LEN + RETRY_NONCE_LEN)
+    {
+        LSQ_DEBUGC("packet for connection %"CID_FMT" has too-short token "
+            "(%hu bytes): validation failed", CID_BITS(&packet_in->pi_dcid),
+            packet_in->pi_token_size);
+        return -1;
+    }
+
+    token_type = packet_in->pi_data[packet_in->pi_token];
+    switch (token_type)
+    {
+    case TOKEN_RETRY:
+        ttl = tokgen->tg_retry_token_duration;
+        ad = packet_in->pi_dcid.idbuf;
+        ad_len = packet_in->pi_dcid.len;
+        break;
+    case TOKEN_RESUME:
+        if (tokgen_seen_resumed_token(tokgen, packet_in->pi_data
+                + packet_in->pi_token, packet_in->pi_token_size, bloom_vals))
+        {
+            LSQ_DEBUGC("%s token for connection %"CID_FMT" has already "
+                "been used: validation failed", tt2str[token_type],
+                CID_BITS(&packet_in->pi_dcid));
+            return -1;
+        }
+        ttl = RESUME_MAX_SECS;
+        ad = NULL;
+        ad_len = 0;
+        break;
+    default:
+        LSQ_DEBUGC("packet for connection %"CID_FMT" has unknown token "
+            "type (%u): validation failed", CID_BITS(&packet_in->pi_dcid),
+            token_type);
+        return -1;
+    }
+    crypter = &tokgen->tg_crypters[ token_type ];
+
+    nonce = packet_in->pi_data + packet_in->pi_token;
+    encr_token = nonce + RETRY_NONCE_LEN;
+    encr_token_len = packet_in->pi_token_size - RETRY_NONCE_LEN;
+    decr_token_len = sizeof(decr_token);
+    if (!EVP_AEAD_CTX_open(&crypter->ctx, decr_token, &decr_token_len,
+                           decr_token_len, nonce, RETRY_NONCE_LEN,
+                           encr_token, encr_token_len, ad, ad_len))
+    {
+        LSQ_DEBUGC("packet for connection %"CID_FMT" has undecryptable %s "
+            "token %s: validation failed", CID_BITS(&packet_in->pi_dcid),
+            tt2str[token_type],
+            HEXSTR(packet_in->pi_data + packet_in->pi_token,
+                                    packet_in->pi_token_size, token_str));
+        return -1;
+    }
+
+    /* From here on, we begin to warn: this is because we were able to
+     * decrypt it, so this is our token.  We should be able to parse it.
+     */
+
+    p = decr_token;
+    end = p + decr_token_len;
+
+    if (p + 1 > end)
+        goto too_short;
+    version = *p++;
+    if (version != TOKGEN_VERSION)
+    {
+        LSQ_DEBUGC("packet for connection %"CID_FMT" has %s token with "
+            "wrong version %u (expected %u): validation failed",
+            CID_BITS(&packet_in->pi_dcid), tt2str[token_type],
+            version, TOKGEN_VERSION);
+        return -1;
+    }
+
+    if (p + sizeof(issued_at) > end)
+        goto too_short;
+    memcpy(&issued_at, p, sizeof(issued_at));
+    now = time(NULL);
+    if (issued_at + ttl < now)
+    {
+        LSQ_DEBUGC("%s token for connection %"CID_FMT" expired %lu "
+            "seconds ago", tt2str[token_type], CID_BITS(&packet_in->pi_dcid),
+            (unsigned long) (now - issued_at - ttl));
+        return -1;
+    }
+    p += sizeof(issued_at);
+
+    if (p + 1 > end)
+        goto too_short;
+    is_ipv6 = *p++;
+    if (is_ipv6)
+    {
+        if (p + 16 > end)
+            goto too_short;
+        if (!(AF_INET6 == sa_peer->sa_family &&
+            0 == memcmp(p, &((struct sockaddr_in6 *) sa_peer)->sin6_addr, 16)))
+                goto ip_mismatch;
+        p += 16;
+    }
+    else
+    {
+        if (p + 4 > end)
+            goto too_short;
+        if (!(AF_INET == sa_peer->sa_family &&
+            0 == memcmp(p, &((struct sockaddr_in *)
+                                            sa_peer)->sin_addr.s_addr, 4)))
+                goto ip_mismatch;
+        p += 4;
+    }
+
+    if (TOKEN_RETRY == token_type)
+    {
+        if (p + 2 >= end)
+            goto too_short;
+        if (AF_INET == sa_peer->sa_family)
+        {
+            if (memcmp(p, &((struct sockaddr_in *) sa_peer)->sin_port, 2))
+                goto port_mismatch;
+        }
+        else if (memcmp(p, &((struct sockaddr_in6 *) sa_peer)->sin6_port, 2))
+            goto port_mismatch;
+        if (0 && LSQ_LOG_ENABLED(LSQ_LOG_DEBUG))
+        {
+            uint16_t port;
+            memcpy(&port, p, sizeof(port));
+            port = ntohs(port);
+            LSQ_DEBUG("port %hu in Retry token matches", port);
+        }
+        p += 2;
+        if (end - p > MAX_CID_LEN)
+            goto too_long;
+        if (odcid)
+        {
+            memcpy(odcid->idbuf, p, end - p);
+            odcid->len = end - p;
+            LSQ_DEBUGC("ODCID: %"CID_FMT, CID_BITS(odcid));
+        }
+    }
+    else
+    {
+        if (p != end)
+        {
+            assert(p < end);
+            goto too_long;
+        }
+        tokgen_record_resumed_token(tokgen, now, bloom_vals);
+    }
+
+    LSQ_DEBUGC("validated %lu-second-old %s token %s for connection "
+        "%"CID_FMT, (unsigned long) (now - issued_at), tt2str[token_type],
+        HEXSTR(packet_in->pi_data + packet_in->pi_token,
+                                    packet_in->pi_token_size, token_str),
+        CID_BITS(&packet_in->pi_dcid));
+    return 0;
+
+  too_short:
+    LSQ_INFOC("decrypted %s token for connection %"CID_FMT" is too short "
+            "(%zu bytes): validation failed", tt2str[token_type],
+            CID_BITS(&packet_in->pi_dcid), decr_token_len);
+    return -1;
+
+  ip_mismatch:
+    addr_str[0][0] = '\0';
+    addr_str[1][0] = '\0';
+    (void) inet_ntop(is_ipv6 ? AF_INET6 : AF_INET, p, addr_str[0],
+                                                    sizeof(addr_str[0]));
+    if (AF_INET6 == sa_peer->sa_family)
+        (void) inet_ntop(AF_INET6, &((struct sockaddr_in6 *) sa_peer
+                    )->sin6_addr, addr_str[1], sizeof(addr_str[1]));
+    else
+        (void) inet_ntop(AF_INET, &((struct sockaddr_in *) sa_peer
+                    )->sin_addr.s_addr, addr_str[1], sizeof(addr_str[1]));
+    LSQ_INFOC("IP address %s in %s token for connection %"CID_FMT" does not "
+        "match peer IP address %s: validation failed", addr_str[0],
+        tt2str[token_type], CID_BITS(&packet_in->pi_dcid), addr_str[1]);
+    return -1;
+
+  too_long:
+    LSQ_INFOC("decrypted %s token for connection %"CID_FMT" is too long "
+            "(%zu bytes): validation failed", tt2str[token_type],
+            CID_BITS(&packet_in->pi_dcid), decr_token_len);
+    return -1;
+
+  port_mismatch:
+  {
+    uint16_t ports[2];
+    ports[0] = AF_INET6 == sa_peer->sa_family
+             ? ((struct sockaddr_in6 *) sa_peer)->sin6_port
+             : ((struct sockaddr_in *) sa_peer)->sin_port;
+    ports[0] = ntohs(ports[0]);
+    memcpy(&ports[1], p, sizeof(ports[1]));
+    ports[1] = ntohs(ports[1]);
+    LSQ_INFOC("port %hu in %s token for connection %"CID_FMT" does not "
+        "match peer port %hu: validation failed", ports[1], tt2str[token_type],
+        CID_BITS(&packet_in->pi_dcid), ports[0]);
+    return -1;
+  }
+}
+
+
+#define LABEL_PREFIX_SZ 8
+
+static const uint8_t *labels[N_TOKEN_TYPES] =
+{
+    [TOKEN_RETRY]  = (uint8_t *) "retry me",
+    [TOKEN_RESUME] = (uint8_t *) "resume m",
+};
+
+
+static ssize_t
+tokgen_generate_token (struct token_generator *tokgen,
+            enum token_type token_type, unsigned char *buf, size_t bufsz,
+            const unsigned char *ad_buf, size_t ad_len,
+            const struct sockaddr *sa_peer, const lsquic_cid_t *odcid)
+{
+    struct crypter *crypter;
+    unsigned char *p, *in;
+    time_t now;
+    size_t len, in_len;
+    unsigned char label[ LABEL_PREFIX_SZ + sizeof(crypter->nonce_counter) ];
+    char in_str[(MAX_RETRY_TOKEN_LEN - RETRY_NONCE_LEN
+                                                    - RETRY_TAG_LEN) * 2 + 1],
+         ad_str[MAX_CID_LEN * 2 + 1],
+         token_str[MAX_RETRY_TOKEN_LEN * 2 + 1];
+
+    if (bufsz < MAX_RETRY_TOKEN_LEN)
+        return -1;
+
+    crypter = &tokgen->tg_crypters[ token_type ];
+    p = buf;
+
+    *p = token_type;
+    memcpy(label, labels[token_type], LABEL_PREFIX_SZ);
+    memcpy(label + LABEL_PREFIX_SZ, &crypter->nonce_counter,
+                                        sizeof(crypter->nonce_counter));
+    (void) HKDF_expand(p + 1, RETRY_NONCE_LEN - 1, EVP_sha256(),
+        crypter->nonce_prk_buf, crypter->nonce_prk_sz, label, sizeof(label));
+    p += RETRY_NONCE_LEN;
+    *p++ = TOKGEN_VERSION;
+    now = time(NULL);
+    memcpy(p, &now, sizeof(now));
+    p += sizeof(now);
+
+    if (AF_INET == sa_peer->sa_family)
+    {
+        *p++ = 0;
+        memcpy(p, &((struct sockaddr_in *) sa_peer)->sin_addr.s_addr, 4);
+        p += 4;
+    }
+    else
+    {
+        *p++ = 1;
+        memcpy(p, &((struct sockaddr_in6 *) sa_peer)->sin6_addr, 16);
+        p += 16;
+    }
+
+    if (token_type == TOKEN_RETRY)
+    {
+        if (AF_INET == sa_peer->sa_family)
+            memcpy(p, &((struct sockaddr_in *) sa_peer)->sin_port, 2);
+        else
+            memcpy(p, &((struct sockaddr_in6 *) sa_peer)->sin6_port, 2);
+        p += 2;
+    }
+
+    if (odcid)
+    {
+        assert(odcid->len <= MAX_CID_LEN);
+        memcpy(p, odcid->idbuf, odcid->len);
+        p += odcid->len;
+    }
+
+    len = bufsz - RETRY_NONCE_LEN;
+    in = buf + RETRY_NONCE_LEN;
+    in_len = p - buf - RETRY_NONCE_LEN;
+    if (LSQ_LOG_ENABLED(LSQ_LOG_DEBUG))
+        lsquic_hexstr(in, in_len, in_str, sizeof(in_str));
+    if (EVP_AEAD_CTX_seal(&crypter->ctx, in, &len, len,
+                buf, RETRY_NONCE_LEN, in, in_len, ad_buf, ad_len))
+    {
+        ++crypter->nonce_counter;
+        LSQ_DEBUG("in: %s, ad: %s -> %s token: %s (%zu bytes)",
+            in_str,
+            HEXSTR(ad_buf, ad_len, ad_str),
+            tt2str[token_type],
+            HEXSTR(buf, RETRY_NONCE_LEN + len, token_str),
+            RETRY_NONCE_LEN + len);
+        return RETRY_NONCE_LEN + len;
+    }
+    else
+    {
+        LSQ_WARN("could not seal retry token");
+        return -1;
+    }
+}
+
+
+ssize_t
+lsquic_tg_generate_retry (struct token_generator *tokgen,
+            unsigned char *buf, size_t bufsz, const unsigned char *scid_buf,
+            size_t scid_len, const struct sockaddr *sa_peer,
+            const lsquic_cid_t *odcid)
+{
+    return tokgen_generate_token(tokgen, TOKEN_RETRY, buf, bufsz, scid_buf,
+                    scid_len, sa_peer, odcid);
+}
+
+
+ssize_t
+lsquic_tg_generate_resume (struct token_generator *tokgen,
+            unsigned char *buf, size_t bufsz, const struct sockaddr *sa_peer)
+{
+    return tokgen_generate_token(tokgen, TOKEN_RESUME, buf, bufsz, NULL,
+                    0, sa_peer, NULL);
+}
+
+
+size_t
+lsquic_tg_token_size (const struct token_generator *tokgen,
+                enum token_type token_type, const struct sockaddr *sa_peer)
+{
+    return MAX_RETRY_TOKEN_LEN - 16
+        + (AF_INET == sa_peer->sa_family ? 4 : 16);
+}
+
+
 void
 lsquic_tg_generate_sreset (struct token_generator *tokgen,
         const struct lsquic_cid *cid, unsigned char *reset_token)

src/liblsquic/lsquic_tokgen.h

@@ -7,6 +7,8 @@ struct sockaddr;
 struct lsquic_packet_in;
 struct lsquic_cid;
 
+enum token_type { TOKEN_RETRY, TOKEN_RESUME, N_TOKEN_TYPES, };
+
 struct token_generator;
 
 struct token_generator *
@@ -20,4 +22,44 @@ void
 lsquic_tg_generate_sreset (struct token_generator *,
         const struct lsquic_cid *cid, unsigned char *reset_token);
 
+
+/* Retry and Resume tokens have identical sizes.  Use *RETRY* macros
+ * for both.
+ */
+#define RETRY_TAG_LEN 16
+
+/* Type is encoded in the nonce */
+#define RETRY_NONCE_LEN 12
+
+#define MAX_RETRY_TOKEN_LEN (RETRY_NONCE_LEN + 1 /* version */ + \
+    sizeof(time_t) /* time */ + 1 /* IPv4 or IPv6 */ + \
+    16 /* IPv6 or IPv4 address */ + 2 /* Port number */ + MAX_CID_LEN + \
+    RETRY_TAG_LEN)
+
+/* Need this to make sure we have enough bytes for Bloom filter functions */
+#define MIN_RESUME_TOKEN_LEN (RETRY_NONCE_LEN + 1 /* version */ + \
+    sizeof(time_t) /* time */ + 1 /* IPv4 or IPv6 */ + \
+    4 /* IPv4 address */ + 0 /* No port number */ + 0 /* No CID */ + \
+    RETRY_TAG_LEN)
+
+ssize_t
+lsquic_tg_generate_retry (struct token_generator *,
+        unsigned char *buf, size_t bufsz,
+        const unsigned char *scid_buf, size_t scid_len,
+        const struct sockaddr *sa_peer, const struct lsquic_cid *odcid);
+
+ssize_t
+lsquic_tg_generate_resume (struct token_generator *,
+        unsigned char *buf, size_t bufsz,
+        const struct sockaddr *sa_peer);
+
+int
+lsquic_tg_validate_token (struct token_generator *,
+        const struct lsquic_packet_in *, const struct sockaddr *,
+        struct lsquic_cid *);
+
+size_t
+lsquic_tg_token_size (const struct token_generator *tokgen,
+                enum token_type token_type, const struct sockaddr *sa_peer);
+
 #endif

src/liblsquic/lsquic_trans_params.c

@@ -19,6 +19,7 @@
 #include "Ws2tcpip.h"
 #endif
 
+#include "lsquic.h"
 #include "lsquic_byteswap.h"
 #include "lsquic_int_types.h"
 #include "lsquic_types.h"
@@ -54,6 +55,7 @@ tpi_val_2_enum (uint64_t tpi_val)
     case 14:        return TPI_ACTIVE_CONNECTION_ID_LIMIT;
     case 15:        return TPI_INITIAL_SOURCE_CID;
     case 16:        return TPI_RETRY_SOURCE_CID;
+    case 17:        return TPI_VERSION_INFORMATION;
     case 0x20:      return TPI_MAX_DATAGRAM_FRAME_SIZE;
 #if LSQUIC_TEST_QUANTUM_READINESS
     case 0xC37:     return TPI_QUANTUM_READINESS;
@@ -87,6 +89,7 @@ static const unsigned enum_2_tpi_val[LAST_TPI + 1] =
     [TPI_ACTIVE_CONNECTION_ID_LIMIT]        =  0xE,
     [TPI_INITIAL_SOURCE_CID]                =  0xF,
     [TPI_RETRY_SOURCE_CID]                  =  0x10,
+    [TPI_VERSION_INFORMATION]               =  0x11,
     [TPI_MAX_DATAGRAM_FRAME_SIZE]           =  0x20,
 #if LSQUIC_TEST_QUANTUM_READINESS
     [TPI_QUANTUM_READINESS]                 =  0xC37,
@@ -267,6 +270,8 @@ lsquic_tp_encode (const struct transport_params *params, int is_server,
 #if LSQUIC_TEST_QUANTUM_READINESS
     const size_t quantum_sz = lsquic_tp_get_quantum_sz();
 #endif
+    lsquic_ver_tag_t tag;
+    int i;
 
     need = 0;
     set = params->tp_set;   /* Will turn bits off for default values */
@@ -357,6 +362,16 @@ lsquic_tp_encode (const struct transport_params *params, int is_server,
             need += (1 << bits[tpi][0]) + 1 /* Zero length byte */;
         }
 
+    if (set & (1 << TPI_VERSION_INFORMATION))
+    {
+        tpi = TPI_VERSION_INFORMATION;
+        bits[tpi][0] = vint_val2bits(enum_2_tpi_val[tpi]);
+        bits[tpi][1] = vint_val2bits(params->tp_version_cnt << 2);
+        need += (1 << bits[tpi][0])
+                +  (1 << bits[tpi][1])
+                +  (params->tp_version_cnt << 2);
+    }
+
     if (need > bufsz || need > UINT16_MAX)
     {
         errno = ENOBUFS;
@@ -458,6 +473,17 @@ lsquic_tp_encode (const struct transport_params *params, int is_server,
                 p += quantum_sz;
                 break;
 #endif
+            case TPI_VERSION_INFORMATION:
+                //FIXME: generate supported version info.
+                vint_write(p, params->tp_version_cnt << 2,
+                                            bits[tpi][1], 1 << bits[tpi][1]);
+                p += 1 << bits[tpi][1];
+                for(i = 0; i < params->tp_version_cnt; ++i)
+                {
+                    tag = lsquic_ver2tag(params->tp_version_info[i]);
+                    WRITE_TO_P(&tag, 4);
+                }
+                break;
             }
         }
 
@@ -480,6 +506,7 @@ lsquic_tp_decode (const unsigned char *const buf, size_t bufsz,
     enum transport_param_id tpi;
     unsigned set_of_ids;
     int s;
+    lsquic_ver_tag_t tag;
 
     p = buf;
     end = buf + bufsz;
@@ -643,6 +670,31 @@ lsquic_tp_decode (const unsigned char *const buf, size_t bufsz,
             if (q != p + len)
                 return -1;
             break;
+        case TPI_VERSION_INFORMATION:
+            if (len & 0x3)
+                return -1;
+            q = p;
+            while(q < p + len)
+            {
+                memmove(&tag, q, 4);
+                if (tag == 0)
+                    return -1;
+                int ver = lsquic_tag2ver(tag);
+                if (ver != -1)
+                {
+                    if (params->tp_version_cnt > 0)
+                        params->tp_versions |= 1 << ver;
+                    if (params->tp_version_cnt < sizeof(params->tp_version_info))
+                        params->tp_version_info[params->tp_version_cnt++] = ver;
+                }
+                else if (params->tp_version_cnt == 0)
+                    return -1;
+                q += 4;
+            }
+            if (!is_server && (params->tp_versions
+                            & (1 << params->tp_chosen_version)) == 0)
+                return -1;
+            break;
         default:
             /* Do nothing: skip this transport parameter */
             break;
@@ -693,6 +745,7 @@ lsquic_tp_to_str (const struct transport_params *params, char *buf, size_t sz)
     enum transport_param_id tpi;
     char tok_str[sizeof(params->tp_stateless_reset_token) * 2 + 1];
     char addr_str[INET6_ADDRSTRLEN];
+    int i;
 
     for (tpi = 0; tpi <= MAX_NUMERIC_TPI; ++tpi)
         if (params->tp_set & (1 << tpi))
@@ -766,6 +819,23 @@ lsquic_tp_to_str (const struct transport_params *params, char *buf, size_t sz)
                 return;
         }
     }
+    if (params->tp_set & (1 << TPI_VERSION_INFORMATION))
+    {
+        nw = snprintf(buf, end - buf, "; version information: chosen: %s, available:",
+                      lsquic_ver2str[params->tp_chosen_version]);
+        buf += nw;
+        if (buf >= end)
+            return;
+        for(i = 1; i < params->tp_version_cnt; ++i)
+        {
+            nw = snprintf(buf, end - buf, " %s",
+                        lsquic_ver2str[params->tp_version_info[i]]);
+           buf += nw;
+            if (buf >= end)
+                return;
+         }
+
+    }
 }
 
 
@@ -988,6 +1058,8 @@ lsquic_tp_encode_27 (const struct transport_params *params, int is_server,
                 p += quantum_sz;
                 break;
 #endif
+            case TPI_VERSION_INFORMATION:
+                break;
             }
         }
 

src/liblsquic/lsquic_trans_params.h

@@ -64,6 +64,7 @@ enum transport_param_id
 #if LSQUIC_TEST_QUANTUM_READINESS
     TPI_QUANTUM_READINESS,
 #endif
+    TPI_VERSION_INFORMATION,
     TPI_STATELESS_RESET_TOKEN,              LAST_TPI = TPI_STATELESS_RESET_TOKEN
 };
 
@@ -106,6 +107,10 @@ struct transport_params
 #define tp_original_dest_cid tp_cids[TP_CID_IDX(TPI_ORIGINAL_DEST_CID)]
 #define tp_initial_source_cid tp_cids[TP_CID_IDX(TPI_INITIAL_SOURCE_CID)]
 #define tp_retry_source_cid tp_cids[TP_CID_IDX(TPI_RETRY_SOURCE_CID)]
+    unsigned   tp_versions;
+    uint8_t    tp_version_cnt;
+    uint8_t    tp_version_info[7];
+#define tp_chosen_version tp_version_info[0]
 };
 
 #define MAX_TP_STR_SZ ((LAST_TPI + 1) *                                     \

src/liblsquic/lsquic_version.c

@@ -18,7 +18,8 @@ static const unsigned char version_tags[N_LSQVER][4] =
     [LSQVER_ID27] = { 0xFF, 0, 0, 27, },
     [LSQVER_ID29] = { 0xFF, 0, 0, 29, },
     [LSQVER_I001] = {    0, 0, 0, 1, },
-    [LSQVER_VERNEG] = { 0xFA, 0xFA, 0xFA, 0xFA, },
+    [LSQVER_I002] = { 0x6B, 0x33, 0x43, 0xCF },
+    [LSQVER_RESVED] = { 0xFA, 0xFA, 0xFA, 0xFA, },
 };
 
 
@@ -47,6 +48,44 @@ lsquic_tag2ver (uint32_t ver_tag)
 }
 
 
+enum lsquic_version
+lsquic_tag2ver_fast (const unsigned char *tag)
+{
+    unsigned char ch;
+    ch = *tag;
+    if (ch == 'Q' && *(tag + 1) == '0')
+    {
+        if (*(tag + 2) == '5' && *(tag + 3) == '0')
+            return LSQVER_050;
+        else if (*(tag + 2) == '4' && *(tag + 3) == '6')
+            return LSQVER_046;
+    }
+    else if (ch == 0x6b && *(tag + 1) == 0x33
+             && *(tag + 2) == 0x43 && *(tag + 3) == 0xcf)
+    {
+        return LSQVER_I002;
+    }
+    else if (ch == '\0' && *(tag + 1) == 0 && *(tag + 2) == 0)
+    {
+        if (*(tag + 3) == 0x01)
+            return LSQVER_I001;
+        else if (*(tag + 3) == 0x00)
+            return LSQVER_VERNEG;
+    }
+    else if (ch == 0xff && *(tag + 1) == 0 && *(tag + 2) == 0)
+    {
+        if (*(tag + 3) == 0x1D)
+            return LSQVER_ID29;
+        else if (*(tag + 3) == 0x1B)
+            return LSQVER_ID27;
+    }
+    else if ((ch & 0xf) == 0xa && (*(tag + 1) & 0xf) == 0xa
+            && (*(tag + 2) & 0xf) == 0xa && (*(tag + 3) & 0xf) == 0xa)
+        return LSQVER_RESVED;
+    return N_LSQVER;
+}
+
+
 const char *const lsquic_ver2str[N_LSQVER] = {
     [LSQVER_043] = "Q043",
     [LSQVER_046] = "Q046",
@@ -54,7 +93,8 @@ const char *const lsquic_ver2str[N_LSQVER] = {
     [LSQVER_ID27] = "FF00001B",
     [LSQVER_ID29] = "FF00001D",
     [LSQVER_I001] = "00000001",
-    [LSQVER_VERNEG] = "FAFAFAFA",
+    [LSQVER_I002] = "6B3343CF",
+    [LSQVER_RESVED] = "FAFAFAFA",
 };
 
 
@@ -65,7 +105,8 @@ const char *const lsquic_ver2altstr[N_LSQVER] = {
     [LSQVER_ID27] = "h3-27",
     [LSQVER_ID29] = "h3-29",
     [LSQVER_I001] = "h3",
-    [LSQVER_VERNEG] = "VERNEG",
+    [LSQVER_I002] = "h3-v2",
+    [LSQVER_RESVED] = "RESERVED",
 };
 
 

src/liblsquic/lsquic_version.h

@@ -14,6 +14,9 @@ lsquic_ver2tag (unsigned version);
 enum lsquic_version
 lsquic_tag2ver (uint32_t ver_tag);
 
+enum lsquic_version
+lsquic_tag2ver_fast (const unsigned char * tag);
+
 extern const char *const lsquic_ver2str[];
 
 int

src/lshpack

@@ -1 +1 @@
-Subproject commit a84a6b9f6d1e2dd5605c399fc2edcfeea8291258
+Subproject commit 32e96f10593c7cb8553cd8c9c12721100ae9e924

tests/test_ack.c

@@ -179,7 +179,7 @@ run_test (const struct test *test)
     size_t sz;
     unsigned char buf[0x1000];
 
-    pf = select_pf_by_ver(LSQVER_ID27);
+    pf = select_pf_by_ver(LSQVER_I001);
     if (!test->skip_gen)
     {
         rechist.acki = &test->acki;

tests/test_ackparse_ietf.c

@@ -20,8 +20,8 @@
 
 static struct lsquic_conn lconn = LSCONN_INITIALIZER_CIDLEN(lconn, 0);
 
-//static const struct parse_funcs *const pf = select_pf_by_ver(LSQVER_ID27); // will not work on MSVC
-#define pf ((const struct parse_funcs *const)select_pf_by_ver(LSQVER_ID27))
+//static const struct parse_funcs *const pf = select_pf_by_ver(LSQVER_I001); // will not work on MSVC
+#define pf ((const struct parse_funcs *const)select_pf_by_ver(LSQVER_I001))
 
 
 static void

tests/test_crypto_gen.c

@@ -103,7 +103,7 @@ main (void)
     const struct test tests[] = {
 
         {   .lineno     = __LINE__,
-            .pf         = select_pf_by_ver(LSQVER_ID27),
+            .pf         = select_pf_by_ver(LSQVER_I001),
             .offset     = 0,
             .data_sz    = 10,
             .data       = "0123456789",
@@ -119,7 +119,7 @@ main (void)
         },
 
         {   .lineno     = __LINE__,
-            .pf         = select_pf_by_ver(LSQVER_ID27),
+            .pf         = select_pf_by_ver(LSQVER_I001),
             .offset     = 500,
             .data_sz    = 10,
             .data       = "0123456789",

tests/test_frame_reader.c

@@ -1134,8 +1134,9 @@ test_one_frt (const struct frame_reader_test *frt)
     stream.conn_pub = &conn_pub;
     conn_pub.lconn = &lconn;
 
-  top:
     lsquic_mm_init(&mm);
+
+  top:
     lshpack_dec_init(&hdec);
     memset(&input, 0, sizeof(input));
     memcpy(input.in_buf, frt->frt_buf, frt->frt_bufsz);

tests/test_h3_framing.c

@@ -63,7 +63,7 @@
 #define MIN(a, b) ((a) < (b) ? (a) : (b))
 #define MAX(a, b) ((a) > (b) ? (a) : (b))
 
-static const struct parse_funcs *g_pf = select_pf_by_ver(LSQVER_ID27);
+static const struct parse_funcs *g_pf = select_pf_by_ver(LSQVER_I001);
 
 struct test_ctl_settings
 {
@@ -343,7 +343,7 @@ init_test_objs (struct test_objs *tobjs, unsigned initial_conn_window,
     memset(tobjs, 0, sizeof(*tobjs));
     LSCONN_INITIALIZE(&tobjs->lconn);
     tobjs->lconn.cn_pf = g_pf;
-    tobjs->lconn.cn_version = LSQVER_ID27;
+    tobjs->lconn.cn_version = LSQVER_I001;
     tobjs->lconn.cn_esf_c = &lsquic_enc_session_common_ietf_v1;
     network_path.np_pack_size = packet_sz;
     tobjs->lconn.cn_if = &our_conn_if;
@@ -376,7 +376,7 @@ init_test_objs (struct test_objs *tobjs, unsigned initial_conn_window,
     tobjs->stream_if = &stream_if;
     tobjs->stream_if_ctx = &test_ctx;
     tobjs->ctor_flags = stream_ctor_flags;
-    if ((1 << tobjs->lconn.cn_version) & LSQUIC_IETF_VERSIONS)
+    //if ((1 << tobjs->lconn.cn_version) & LSQUIC_IETF_VERSIONS)
     {
         lsquic_qeh_init(&tobjs->qeh, &tobjs->lconn);
         s = lsquic_qeh_settings(&tobjs->qeh, 0, 0, 0, 0);
@@ -395,7 +395,7 @@ deinit_test_objs (struct test_objs *tobjs)
     lsquic_send_ctl_cleanup(&tobjs->send_ctl);
     lsquic_malo_destroy(tobjs->conn_pub.packet_out_malo);
     lsquic_mm_cleanup(&tobjs->eng_pub.enp_mm);
-    if ((1 << tobjs->lconn.cn_version) & LSQUIC_IETF_VERSIONS)
+    //if ((1 << tobjs->lconn.cn_version) & LSQUIC_IETF_VERSIONS)
         lsquic_qeh_cleanup(&tobjs->qeh);
 }
 

tests/test_packet_resize.c

@@ -214,7 +214,7 @@ new_packet (struct test_ctx *ctx)
                                      */
 
     packet_out = lsquic_packet_out_new(&ctx->enpub.enp_mm, ctx->enpub.enp_mm.malo.packet_out, 1,
-                         &ctx->lconn, PACKNO_BITS_0, 0, NULL, &ctx->path, HETY_NOT_SET);
+                         &ctx->lconn, PACKNO_BITS_0, 0, NULL, &ctx->path, HETY_SHORT);
     if (packet_out)
         packet_out->po_packno = packno++;
 

tests/test_send_headers.c

@@ -185,8 +185,8 @@ init_test_objs (struct test_objs *tobjs, unsigned initial_conn_window,
     int s;
     memset(tobjs, 0, sizeof(*tobjs));
     LSCONN_INITIALIZE(&tobjs->lconn);
-    tobjs->lconn.cn_pf = select_pf_by_ver(LSQVER_ID27);
-    tobjs->lconn.cn_version = LSQVER_ID27;
+    tobjs->lconn.cn_pf = select_pf_by_ver(LSQVER_I001);
+    tobjs->lconn.cn_version = LSQVER_I001;
     tobjs->lconn.cn_esf_c = &lsquic_enc_session_common_ietf_v1;
     network_path.np_pack_size = IQUIC_MAX_IPv4_PACKET_SZ;
     tobjs->lconn.cn_if = &our_conn_if;

tests/test_stream.c

@@ -363,7 +363,7 @@ init_test_objs (struct test_objs *tobjs, unsigned initial_conn_window,
     LSCONN_INITIALIZE(&tobjs->lconn);
     tobjs->lconn.cn_pf = pf ? pf : g_pf;
     tobjs->lconn.cn_version = tobjs->lconn.cn_pf == &lsquic_parse_funcs_ietf_v1 ?
-        LSQVER_ID27 : LSQVER_043;
+        LSQVER_I001 : LSQVER_043;
     tobjs->lconn.cn_esf_c = &lsquic_enc_session_common_gquic_1;
     network_path.np_pack_size = 1370;
     tobjs->lconn.cn_if = &our_conn_if;
@@ -1620,7 +1620,7 @@ test_termination (void)
         {
             init_test_ctl_settings(&g_ctl_settings);
             g_ctl_settings.tcs_schedule_stream_packets_immediately = 1;
-            init_test_objs(&tobjs, 0x4000, 0x4000, select_pf_by_ver(LSQVER_ID27));
+            init_test_objs(&tobjs, 0x4000, 0x4000, select_pf_by_ver(LSQVER_I001));
             tf->func(&tobjs);
             deinit_test_objs(&tobjs);
         }
@@ -2619,7 +2619,7 @@ test_changing_pack_size (void)
     enum lsquic_version versions_to_test[3] =
     {
         LSQVER_046,
-        LSQVER_ID27,
+        LSQVER_I001,
     };
 
     for (i = 0; i < 3; i++)
@@ -3005,7 +3005,7 @@ test_resize_buffered (void)
     ssize_t nw;
     struct test_objs tobjs;
     struct lsquic_stream *streams[1];
-    const struct parse_funcs *const pf = select_pf_by_ver(LSQVER_ID27);
+    const struct parse_funcs *const pf = select_pf_by_ver(LSQVER_I001);
     char buf[0x10000];
     unsigned char buf_out[0x10000];
     int s, fin;
@@ -3068,7 +3068,7 @@ test_resize_scheduled (void)
     ssize_t nw;
     struct test_objs tobjs;
     struct lsquic_stream *streams[1];
-    const struct parse_funcs *const pf = select_pf_by_ver(LSQVER_ID27);
+    const struct parse_funcs *const pf = select_pf_by_ver(LSQVER_I001);
     char buf[0x10000];
     unsigned char buf_out[0x10000];
     int s, fin;
@@ -3260,8 +3260,8 @@ test_packetization (int schedule_stream_packets_immediately, int dispatch_once,
 
     if (g_use_crypto_ctor)
     {
-        stream_ids[0] = ENC_LEV_CLEAR;
-        stream_ids[1] = ENC_LEV_INIT;
+        stream_ids[0] = ENC_LEV_INIT;
+        stream_ids[1] = ENC_LEV_HSK;
     }
     else
     {
@@ -3754,7 +3754,7 @@ main (int argc, char **argv)
 
     /* Redo some tests using crypto streams and frames */
     g_use_crypto_ctor = 1;
-    g_pf = select_pf_by_ver(LSQVER_ID27);
+    g_pf = select_pf_by_ver(LSQVER_I001);
     main_test_packetization();
 
     return 0;

tests/test_streamgen.c

@@ -371,7 +371,7 @@ main (void)
          */
 
         {   .lineno     = __LINE__,
-            .pf         = select_pf_by_ver(LSQVER_ID27),
+            .pf         = select_pf_by_ver(LSQVER_I001),
             .fin        = { 0, 1, },
             .offset     = 0x0807060504030201UL,
             .stream_id  = 0x210,
@@ -391,7 +391,7 @@ main (void)
         },
 
         {   .lineno     = __LINE__,
-            .pf         = select_pf_by_ver(LSQVER_ID27),
+            .pf         = select_pf_by_ver(LSQVER_I001),
             .fin        = { 0, 0, },
             .offset     = 0,
             .stream_id  = 0x210,
@@ -410,7 +410,7 @@ main (void)
         },
 
         {   .lineno     = __LINE__,
-            .pf         = select_pf_by_ver(LSQVER_ID27),
+            .pf         = select_pf_by_ver(LSQVER_I001),
             .fin        = { 0, 0, },
             .offset     = 0,
             .stream_id  = 0x21,
@@ -428,7 +428,7 @@ main (void)
         },
 
         {   .lineno     = __LINE__,
-            .pf         = select_pf_by_ver(LSQVER_ID27),
+            .pf         = select_pf_by_ver(LSQVER_I001),
             .fin        = { 0, 0, },
             .offset     = 0x0807060504030201UL,
             .stream_id  = 0x210,
@@ -448,7 +448,7 @@ main (void)
         },
 
         {   .lineno     = __LINE__,
-            .pf         = select_pf_by_ver(LSQVER_ID27),
+            .pf         = select_pf_by_ver(LSQVER_I001),
             .fin        = { 1, 0, },
             .offset     = 0x0807060504030201UL,
             .stream_id  = 0x210,
@@ -466,7 +466,7 @@ main (void)
         },
 
         {   .lineno     = __LINE__,
-            .pf         = select_pf_by_ver(LSQVER_ID27),
+            .pf         = select_pf_by_ver(LSQVER_I001),
             .fin        = { 1, 0, },
             .offset     = 0x0807060504030201UL,
             .stream_id  = 0x210,

tests/test_streamparse.c

@@ -274,7 +274,7 @@ main (void)
 
         {   "Balls to the wall: every possible bit is set",
             __LINE__,
-            select_pf_by_ver(LSQVER_ID27),
+            select_pf_by_ver(LSQVER_I001),
           /*  TYPE   OFF    DLEN   FIN   */
             { 0x10 | 1<<2 | 1<<1 | 1<<0,
               0x41, 0x23,                                       /* Stream ID */
@@ -293,7 +293,7 @@ main (void)
 
         {   "Balls to the wall #2: every possible bit is set except FIN",
             __LINE__,
-            select_pf_by_ver(LSQVER_ID27),
+            select_pf_by_ver(LSQVER_I001),
           /*  TYPE   OFF    DLEN   FIN   */
             { 0x10 | 1<<2 | 1<<1 | 0<<0,
               0x81, 0x23, 0x00, 0xE4,                           /* Stream ID */
@@ -312,7 +312,7 @@ main (void)
 
         {   "Data length is zero",
             __LINE__,
-            select_pf_by_ver(LSQVER_ID27),
+            select_pf_by_ver(LSQVER_I001),
           /*  TYPE   OFF    DLEN   FIN   */
             { 0x10 | 1<<2 | 0<<1 | 0<<0,
               0x81, 0x23, 0x00, 0xE4,                           /* Stream ID */
@@ -330,7 +330,7 @@ main (void)
 
         {   "Sanity check: what happens when data length is zero #1",
             __LINE__,
-            select_pf_by_ver(LSQVER_ID27),
+            select_pf_by_ver(LSQVER_I001),
           /*  TYPE   OFF    DLEN   FIN   */
             { 0x10 | 1<<2 | 1<<1 | 0<<0,
               0x81, 0x23, 0x00, 0xE4,                           /* Stream ID */
@@ -349,7 +349,7 @@ main (void)
 
         {   "Sanity check: what happens when data length is zero #2",
             __LINE__,
-            select_pf_by_ver(LSQVER_ID27),
+            select_pf_by_ver(LSQVER_I001),
           /*  TYPE   OFF    DLEN   FIN   */
             { 0x10 | 1<<2 | 1<<1 | 0<<0,
               0x81, 0x23, 0x00, 0xE4,                           /* Stream ID */
@@ -368,7 +368,7 @@ main (void)
 
         {   "Sanity check: what happens when data length is zero #3",
             __LINE__,
-            select_pf_by_ver(LSQVER_ID27),
+            select_pf_by_ver(LSQVER_I001),
           /*  TYPE   OFF    DLEN   FIN   */
             { 0x10 | 0<<2 | 1<<1 | 0<<0,
               0x81, 0x23, 0x00, 0xE4,                           /* Stream ID */
@@ -386,7 +386,7 @@ main (void)
 
         {   "Sanity check: what happens when data length is zero #3",
             __LINE__,
-            select_pf_by_ver(LSQVER_ID27),
+            select_pf_by_ver(LSQVER_I001),
           /*  TYPE   OFF    DLEN   FIN   */
             { 0x10 | 1<<2 | 1<<1 | 1<<0,
               0x81, 0x23, 0x00, 0xE4,                           /* Stream ID */
@@ -405,7 +405,7 @@ main (void)
 
         {   "Check data bounds #1",
             __LINE__,
-            select_pf_by_ver(LSQVER_ID27),
+            select_pf_by_ver(LSQVER_I001),
           /*  TYPE   OFF    DLEN   FIN   */
             { 0x10 | 1<<2 | 1<<1 | 1<<0,
               0x81, 0x23, 0x00, 0xE4,                           /* Stream ID */
@@ -424,7 +424,7 @@ main (void)
 
         {   "Check data bounds #2",
             __LINE__,
-            select_pf_by_ver(LSQVER_ID27),
+            select_pf_by_ver(LSQVER_I001),
           /*  TYPE   OFF    DLEN   FIN   */
             { 0x10 | 1<<2 | 1<<1 | 1<<0,
               0x81, 0x23, 0x00, 0xE4,                           /* Stream ID */

tests/test_tokgen.c

@@ -35,6 +35,7 @@ main (int argc, char **argv)
     printf("\n");
 
     lsquic_tg_destroy(tg);
+    lsquic_stock_shared_hash_destroy(enpub.enp_shi_ctx);
 
     return 0;
 }

tests/test_trapa.c

@@ -72,7 +72,7 @@ static const struct trapa_test tests[] =
             .tp_max_idle_timeout = 10 * 1000,
             .tp_max_ack_delay = TP_DEF_MAX_ACK_DELAY,
             .tp_active_connection_id_limit = 7,
-            .tp_initial_source_cid = { .len = 8, .u_cid.id = 0x0807060504030201ull, },
+            .tp_initial_source_cid = { .len = 8, .buf = "\x01\x02\x03\x04\x05\x06\x07\x08", },
         },
         .is_server = 0,
         .enc_len = 36,