Release 2.8.9

- [BUGFIX] Use ls-qpack 0.11.1
- [OPTIMIZATION] Generate random bytes in batches.
- Change loss_bits transport parameter ID to 0x1057 following latest
  draft.
- Randomize period with which PINGs are sent to elicit ACKs.
- Some refactoring and code cleanup.

CHANGELOG

@@ -1,3 +1,12 @@
+2020-01-16
+    - 2.8.9
+    - [BUGFIX] Use ls-qpack 0.11.1
+    - [OPTIMIZATION] Generate random bytes in batches.
+    - Change loss_bits transport parameter ID to 0x1057 following
+      latest draft.
+    - Randomize period with which PINGs are sent to elicit ACKs.
+    - Some refactoring and code cleanup.
+
 2020-01-14
     - 2.8.8
     - [BUGFIX] Invalid read when parsing IETF transport parameters

include/lsquic.h

@@ -25,7 +25,7 @@ extern "C" {
 
 #define LSQUIC_MAJOR_VERSION 2
 #define LSQUIC_MINOR_VERSION 8
-#define LSQUIC_PATCH_VERSION 8
+#define LSQUIC_PATCH_VERSION 9
 
 /**
  * Engine flags:

src/liblsquic/CMakeLists.txt

@@ -9,6 +9,7 @@ SET(lsquic_STAT_SRCS
     lsquic_cfcw.c
     lsquic_chsk_stream.c
     lsquic_conn.c
+    lsquic_crand.c
     lsquic_crt_compress.c
     lsquic_crypto.c
     lsquic_cubic.c

src/liblsquic/Makefile.am

@@ -14,6 +14,7 @@ liblsquic_a_SOURCES =  ls-qpack/lsqpack.c \
     lsquic_cfcw.c \
     lsquic_chsk_stream.c \
     lsquic_conn.c \
+    lsquic_crand.c \
     lsquic_crt_compress.c \
     lsquic_crypto.c \
     lsquic_cubic.c \

src/liblsquic/lsquic_bbr.c

@@ -29,9 +29,12 @@
 #include "lsquic_conn_public.h"
 #include "lsquic_util.h"
 #include "lsquic_malo.h"
+#include "lsquic_crand.h"
+#include "lsquic_mm.h"
+#include "lsquic_engine_public.h"
 
 #define LSQUIC_LOGGER_MODULE LSQLM_BBR
-#define LSQUIC_LOG_CONN_ID lsquic_conn_log_cid(bbr->bbr_conn)
+#define LSQUIC_LOG_CONN_ID lsquic_conn_log_cid(bbr->bbr_conn_pub->lconn)
 #include "lsquic_logger.h"
 
 #define MIN(a, b) ((a) < (b) ? (a) : (b))
@@ -165,8 +168,8 @@ lsquic_bbr_init (void *cong_ctl, const struct lsquic_conn_public *conn_pub,
                                                 enum quic_ft_bit retx_frames)
 {
     struct lsquic_bbr *const bbr = cong_ctl;
-    bbr->bbr_conn = conn_pub->lconn;
-    lsquic_bw_sampler_init(&bbr->bbr_bw_sampler, bbr->bbr_conn, retx_frames);
+    bbr->bbr_conn_pub = conn_pub;
+    lsquic_bw_sampler_init(&bbr->bbr_bw_sampler, conn_pub->lconn, retx_frames);
 
     bbr->bbr_rtt_stats = &conn_pub->rtt_stats;
     bbr->bbr_mode = BBR_MODE_STARTUP;
@@ -665,13 +668,16 @@ on_exit_startup (struct lsquic_bbr *bbr, lsquic_time_t now)
 static void
 enter_probe_bw_mode (struct lsquic_bbr *bbr, lsquic_time_t now)
 {
+    uint8_t rand;
+
     set_mode(bbr, BBR_MODE_PROBE_BW);
     bbr->bbr_cwnd_gain = kCwndGain;
 
     // Pick a random offset for the gain cycle out of {0, 2..7} range. 1 is
     // excluded because in that case increased gain and decreased gain would not
     // follow each other.
-    bbr->bbr_cycle_current_offset = rand() % (kGainCycleLength - 1);
+    rand = lsquic_crand_get_byte(bbr->bbr_conn_pub->enpub->enp_crand);
+    bbr->bbr_cycle_current_offset = rand % (kGainCycleLength - 1);
     if (bbr->bbr_cycle_current_offset >= 1)
         ++bbr->bbr_cycle_current_offset;
 

src/liblsquic/lsquic_bbr.h

@@ -33,7 +33,7 @@
 
 struct lsquic_bbr
 {
-    struct lsquic_conn   *bbr_conn;
+    const struct lsquic_conn_public  *bbr_conn_pub;
 
     enum bbr_mode
     {

src/liblsquic/lsquic_crand.c

@@ -0,0 +1,47 @@
+/* Copyright (c) 2017 - 2020 LiteSpeed Technologies Inc.  See LICENSE. */
+#include <openssl/rand.h>
+#include <stdint.h>
+
+#include "lsquic_crand.h"
+
+
+#define OFF_MASK(crand_) ((sizeof((crand_)->rand_buf) * 2) - 1)
+
+
+uint8_t
+lsquic_crand_get_nybble (struct crand *crand)
+{
+    uint8_t byte;
+
+    if (crand->nybble_off == 0)
+        RAND_bytes(crand->rand_buf, sizeof(crand->rand_buf));
+
+    byte = crand->rand_buf[crand->nybble_off / 2];
+    if (crand->nybble_off & 1)
+        byte >>= 4;
+    else
+        byte &= 0xF;
+    crand->nybble_off += 1;
+    crand->nybble_off &= OFF_MASK(crand);
+    return byte;
+}
+
+
+uint8_t
+lsquic_crand_get_byte (struct crand *crand)
+{
+    uint8_t byte;
+
+    if (crand->nybble_off & 1)
+        return (lsquic_crand_get_nybble(crand) << 4)
+             |  lsquic_crand_get_nybble(crand);
+    else
+    {
+        if (crand->nybble_off == 0)
+            RAND_bytes(crand->rand_buf, sizeof(crand->rand_buf));
+        byte = crand->rand_buf[crand->nybble_off / 2];
+        crand->nybble_off += 2;
+        crand->nybble_off &= OFF_MASK(crand);
+        return byte;
+    }
+}

src/liblsquic/lsquic_crand.h

@@ -0,0 +1,23 @@
+/* Copyright (c) 2017 - 2020 LiteSpeed Technologies Inc.  See LICENSE. */
+/*
+ * lsquic_crand.h -- cached random bytes
+ *
+ * The idea is to reduce number of calls to RAND_bytes()
+ */
+
+#ifndef LSQUIC_CRAND_H
+#define LSQUIC_CRAND_H 1
+
+struct crand
+{
+    unsigned        nybble_off;     /* Increments 2 per byte */
+    uint8_t         rand_buf[256];  /* Must be power of two */
+};
+
+uint8_t
+lsquic_crand_get_nybble (struct crand *);
+
+uint8_t
+lsquic_crand_get_byte (struct crand *);
+
+#endif

src/liblsquic/lsquic_crypto.c

@@ -30,12 +30,6 @@ static const char s_hs_signature[] = "QUIC CHLO and server config signature";
 static int crypto_inited = 0;
 
 
-void rand_bytes(void *data, int len)
-{
-    RAND_bytes(data, len);
-}
-
-
 uint64_t fnv1a_64(const uint8_t * data, int len)
 {
     uint64_t hash = UINT64_C(14695981039346656037);
@@ -408,7 +402,7 @@ void gen_nonce_c(unsigned char *buf, uint64_t orbit)
     p += 4;
     memcpy(p, &orbit, 8);
     p += 8;
-    rand_bytes(p, 20);
+    RAND_bytes(p, 20);
     p += 20;
 }
 

src/liblsquic/lsquic_crypto.h

@@ -30,9 +30,6 @@ struct x509_st;
 
 void crypto_init(void);
 
-/* XXX: why have a wrapper around RAND_bytes? */
-void rand_bytes(void *data, int len);
-
 
 int export_key_material_simple(unsigned char *ikm, uint32_t ikm_len,
                         unsigned char *salt, int salt_len,

src/liblsquic/lsquic_engine.c

@@ -76,6 +76,7 @@
 #include "lsquic_http1x_if.h"
 #include "lsquic_parse_common.h"
 #include "lsquic_handshake.h"
+#include "lsquic_crand.h"
 
 #define LSQUIC_LOGGER_MODULE LSQLM_ENGINE
 #include "lsquic_logger.h"
@@ -257,6 +258,7 @@ struct lsquic_engine
     unsigned                           last_logged_ae_why;
     int                                last_tick_diff;
 #endif
+    struct crand                       crand;
 };
 
 
@@ -556,6 +558,7 @@ lsquic_engine_new (unsigned flags,
     engine->pub.enp_tokgen = lsquic_tg_new(&engine->pub);
     if (!engine->pub.enp_tokgen)
         return NULL;
+    engine->pub.enp_crand = &engine->crand;
     if (flags & ENG_SERVER)
     {
         engine->pr_queue = prq_create(

src/liblsquic/lsquic_engine_public.h

@@ -16,6 +16,7 @@ struct stack_st_X509;
 struct lsquic_hash;
 struct lsquic_stream_if;
 struct ssl_ctx_st;
+struct crand;
 
 enum warning_type
 {
@@ -60,6 +61,7 @@ struct lsquic_engine_public {
     }                               enp_flags;
     unsigned char                   enp_ver_tags_buf[ sizeof(lsquic_ver_tag_t) * N_LSQVER ];
     unsigned                        enp_ver_tags_len;
+    struct crand                   *enp_crand;
 };
 
 /* Put connection onto the Tickable Queue if it is not already on it.  If

src/liblsquic/lsquic_full_conn_ietf.c

@@ -66,6 +66,7 @@
 #include "lsquic_ietf.h"
 #include "lsquic_push_promise.h"
 #include "lsquic_headers.h"
+#include "lsquic_crand.h"
 
 #define LSQUIC_LOGGER_MODULE LSQLM_CONN
 #define LSQUIC_LOG_CONN_ID ietf_full_conn_ci_get_log_cid(&conn->ifc_conn)
@@ -358,6 +359,7 @@ struct ietf_full_conn
     unsigned char               ifc_active_cids_limit;
     unsigned char               ifc_active_cids_count;
     unsigned char               ifc_first_active_cid_seqno;
+    unsigned char               ifc_ping_unretx_thresh;
     unsigned                    ifc_last_retire_prior_to;
     lsquic_time_t               ifc_last_live_update;
     struct conn_path            ifc_paths[N_PATHS];
@@ -942,6 +944,7 @@ ietf_full_conn_init (struct ietf_full_conn *conn,
     conn->ifc_max_req_id = VINT_MAX_VALUE + 1;
     conn->ifc_idle_to = enpub->enp_settings.es_idle_timeout * 1000 * 1000;
     conn->ifc_ping_period = enpub->enp_settings.es_ping_period * 1000 * 1000;
+    conn->ifc_ping_unretx_thresh = 20;
     return 0;
 }
 
@@ -1394,13 +1397,15 @@ generate_ack_frame_for_pns (struct ietf_full_conn *conn,
     else
         conn->ifc_flags &= ~IFC_ACK_HAD_MISS;
     LSQ_DEBUG("Put %d bytes of ACK frame into packet on outgoing queue", w);
-    /* TODO: randomize the 20 to be 20 +/- 8 */
-    if (conn->ifc_n_cons_unretx >= 20 &&
+    if (conn->ifc_n_cons_unretx >= conn->ifc_ping_unretx_thresh &&
                 !lsquic_send_ctl_have_outgoing_retx_frames(&conn->ifc_send_ctl))
     {
         LSQ_DEBUG("schedule PING frame after %u non-retx "
                                     "packets sent", conn->ifc_n_cons_unretx);
         conn->ifc_send_flags |= SF_SEND_PING;
+        /* This gives a range [12, 27]: */
+        conn->ifc_ping_unretx_thresh = 12
+                    + lsquic_crand_get_nybble(conn->ifc_enpub->enp_crand);
     }
 
     conn->ifc_n_slack_akbl[pns] = 0;
@@ -5717,17 +5722,6 @@ process_regular_packet (struct ietf_full_conn *conn,
         if (0 == (conn->ifc_flags & (IFC_ACK_QUED_INIT << pns)))
         {
             frame_types = packet_in->pi_frame_types;
-#if 0   /* TODO */
-            if ((conn->fc_flags & FC_GOING_AWAY)
-                && lsquic_hash_count(conn->fc_pub.all_streams) < 3)
-            {
-                /* Ignore PING frames if we are going away and there are no
-                 * active streams.  (HANDSHAKE and HEADERS streams are the
-                 * two streams that are always in the all_streams hash).
-                 */
-                frame_types &= ~(1 << QUIC_FRAME_PING);
-            }
-#endif
             if (frame_types & IQUIC_FRAME_ACKABLE_MASK)
             {
                 was_missing = packet_in->pi_packno !=

src/liblsquic/lsquic_handshake.c

@@ -1571,7 +1571,7 @@ lsquic_enc_session_gen_chlo (enc_session_t *enc_session_p,
                                             ++n_tags;           /* PUBS */
             MSG_LEN_ADD(msg_len, sizeof(enc_session->hs_ctx.nonc));
                                             ++n_tags;           /* NONC */
-            rand_bytes(enc_session->priv_key, 32);
+            RAND_bytes(enc_session->priv_key, 32);
             c255_get_pub_key(enc_session->priv_key, pub_key);
             gen_nonce_c(enc_session->hs_ctx.nonc, enc_session->info->orbt);
         }
@@ -1832,9 +1832,9 @@ get_valid_scfg (const struct lsquic_enc_session *enc_session,
         return NULL;
 
     temp_scfg = &s_server_config.lsc_scfg->info;
-    rand_bytes(temp_scfg->skt_key, sizeof(temp_scfg->skt_key));
-    rand_bytes(temp_scfg->sscid, sizeof(temp_scfg->sscid));
-    rand_bytes(temp_scfg->priv_key, sizeof(temp_scfg->priv_key));
+    RAND_bytes(temp_scfg->skt_key, sizeof(temp_scfg->skt_key));
+    RAND_bytes(temp_scfg->sscid, sizeof(temp_scfg->sscid));
+    RAND_bytes(temp_scfg->priv_key, sizeof(temp_scfg->priv_key));
     c255_get_pub_key(temp_scfg->priv_key, spubs + 3);
     temp_scfg->aead = settings->es_aead;
     temp_scfg->kexs = settings->es_kexs;
@@ -2038,7 +2038,7 @@ gen_rej1_data (struct lsquic_enc_session *enc_session, uint8_t *data,
         lsquic_str_prealloc(&enc_session->ssno, SNO_LENGTH);
         lsquic_str_setlen(&enc_session->ssno, SNO_LENGTH);
     }
-    rand_bytes(lsquic_str_buf(&enc_session->ssno), SNO_LENGTH);
+    RAND_bytes((uint8_t *) lsquic_str_buf(&enc_session->ssno), SNO_LENGTH);
 
     MW_BEGIN(&mw, QTAG_REJ, 7, data);
     MW_WRITE_LS_STR(&mw, QTAG_STK, &enc_session->sstk);
@@ -2088,8 +2088,8 @@ gen_shlo_data (uint8_t *buf, size_t buf_len, struct lsquic_enc_session *enc_sess
     if (MSG_LEN_VAL(msg_len) > buf_len)
         return -1;
 
-    rand_bytes(nonce, 32);
-    rand_bytes(enc_session->priv_key, 32);
+    RAND_bytes(nonce, 32);
+    RAND_bytes(enc_session->priv_key, 32);
     c255_get_pub_key(enc_session->priv_key, (unsigned char *)pub_key);
     if (lsquic_str_len(&enc_session->sstk) != STK_LENGTH)
     {
@@ -2105,7 +2105,7 @@ gen_shlo_data (uint8_t *buf, size_t buf_len, struct lsquic_enc_session *enc_sess
         lsquic_str_prealloc(&enc_session->ssno, SNO_LENGTH);
         lsquic_str_setlen(&enc_session->ssno, SNO_LENGTH);
     }
-    rand_bytes(lsquic_str_buf(&enc_session->ssno), SNO_LENGTH);
+    RAND_bytes((uint8_t *) lsquic_str_buf(&enc_session->ssno), SNO_LENGTH);
 
     MW_BEGIN(&mw, QTAG_SHLO, 9 + include_reset_token, buf);
     MW_WRITE_LS_STR(&mw, QTAG_STK, &enc_session->sstk);
@@ -2833,8 +2833,8 @@ gen_stk (lsquic_server_config_t *server_config, const struct sockaddr *ip_addr,
     else
         memcpy(stk, &((struct sockaddr_in6 *)ip_addr)->sin6_addr, 16);
     memcpy(stk + 16, &tm, 8);
-    rand_bytes(stk + 24, STK_LENGTH - 24 - 12);
-    rand_bytes(stk_out + STK_LENGTH - 12, 12);
+    RAND_bytes(stk + 24, STK_LENGTH - 24 - 12);
+    RAND_bytes(stk_out + STK_LENGTH - 12, 12);
     aes_aead_enc(&server_config->lsc_stk_ctx, NULL, 0, stk_out + STK_LENGTH - 12, 12, stk,
                  STK_LENGTH - 12 - 12, stk_out, &out_len);
 }

src/liblsquic/lsquic_pr_queue.c

@@ -35,6 +35,7 @@
 #include "lsquic_sizes.h"
 #include "lsquic_handshake.h"
 #include "lsquic_xxhash.h"
+#include "lsquic_crand.h"
 
 #define LSQUIC_LOGGER_MODULE LSQLM_PRQ
 #include "lsquic_logger.h"
@@ -106,19 +107,9 @@ struct pr_queue
     unsigned char               prq_pubres_g_buf[GQUIC_RESET_SZ];
     unsigned char               prq_verneg_g_buf[1 + GQUIC_CID_LEN
                                                                 + N_LSQVER * 4];
-    /* We generate random nybbles in batches */
-#define NYBBLE_COUNT_BITS 4
-#define NYBBLE_COUNT (1 << NYBBLE_COUNT_BITS)
-#define NYBBLE_MASK (NYBBLE_COUNT - 1)
-    unsigned                    prq_rand_nybble_off;
-    uint8_t                     prq_rand_nybble_buf[NYBBLE_COUNT * 2];
 };
 
 
-static uint8_t
-get_rand_byte (struct pr_queue *);
-
-
 static int
 comp_reqs (const void *s1, const void *s2, size_t n)
 {
@@ -209,7 +200,6 @@ prq_create (unsigned max_elems, unsigned max_conns,
     prq->prq_verneg_g_sz = verneg_g_sz;
     prq->prq_pubres_g_sz = (unsigned) prst_g_sz;
     prq->prq_enpub       = enpub;
-    prq->prq_rand_nybble_off = 0;
 
     LSQ_INFO("initialized queue of size %d", max_elems);
 
@@ -290,7 +280,7 @@ lsquic_prq_new_req (struct pr_queue *prq, enum packet_req_type type,
         max = MIN(IQUIC_MAX_SRST_SIZE, data_sz - 1u);
         if (max > IQUIC_MIN_SRST_SIZE)
         {
-            rand = get_rand_byte(prq);
+            rand = lsquic_crand_get_byte(prq->prq_enpub->enp_crand);
             size = IQUIC_MIN_SRST_SIZE + rand % (max - IQUIC_MIN_SRST_SIZE);
         }
         else
@@ -430,31 +420,6 @@ get_evconn (struct pr_queue *prq)
 }
 
 
-static uint8_t
-get_rand_nybble (struct pr_queue *prq)
-{
-    uint8_t byte;
-
-    if (prq->prq_rand_nybble_off == 0)
-        RAND_bytes(prq->prq_rand_nybble_buf, sizeof(prq->prq_rand_nybble_buf));
-
-    byte = prq->prq_rand_nybble_buf[prq->prq_rand_nybble_off / 2];
-    if (prq->prq_rand_nybble_off & 1)
-        byte >>= 4;
-    else
-        byte &= 0xF;
-    prq->prq_rand_nybble_off = (prq->prq_rand_nybble_off + 1) & NYBBLE_MASK;
-    return byte;
-}
-
-
-static uint8_t
-get_rand_byte (struct pr_queue *prq)
-{
-    return (get_rand_nybble(prq) << 4) | get_rand_nybble(prq);
-}
-
-
 struct lsquic_conn *
 prq_next_conn (struct pr_queue *prq)
 {
@@ -509,7 +474,7 @@ prq_next_conn (struct pr_queue *prq)
         len = gen_verneg(packet_out->po_data, max_bufsz(prq),
                     /* Flip SCID/DCID here: */ &req->pr_dcid, &req->pr_scid,
                     prq->prq_enpub->enp_settings.es_versions,
-                    get_rand_byte(prq));
+                    lsquic_crand_get_byte(prq->prq_enpub->enp_crand));
         if (len > 0)
             packet_out->po_data_sz = len;
         else

src/liblsquic/lsquic_qdec_hdl.c

@@ -520,6 +520,7 @@ qdh_supply_hset_to_stream (struct qpack_dec_hdl *qdh,
 }
 
 
+/* Releases qlist */
 static int
 qdh_process_qlist (struct qpack_dec_hdl *qdh,
             struct lsquic_stream *stream, struct lsqpack_header_list *qlist)
@@ -562,7 +563,8 @@ qdh_header_read_results (struct qpack_dec_hdl *qdh,
         }
         else
         {
-            assert(0);  /* XXX TODO What do we do here? */
+            LSQ_WARN("read header status is DONE but header list is not set");
+            assert(0);
             return LQRHS_ERROR;
         }
     }

src/liblsquic/lsquic_send_ctl.c

@@ -49,6 +49,7 @@
 #include "lsqpack.h"
 #include "lsquic_frab_list.h"
 #include "lsquic_qdec_hdl.h"
+#include "lsquic_crand.h"
 
 #define LSQUIC_LOGGER_MODULE LSQLM_SENDCTL
 #define LSQUIC_LOG_CONN_ID lsquic_conn_log_cid(ctl->sc_conn_pub->lconn)
@@ -2966,8 +2967,8 @@ lsquic_send_ctl_cidlen_change (struct lsquic_send_ctl *ctl,
 void
 lsquic_send_ctl_begin_optack_detection (struct lsquic_send_ctl *ctl)
 {
-    unsigned char buf[1];
+    uint8_t rand;
 
-    RAND_bytes(buf, sizeof(buf));
-    ctl->sc_gap = ctl->sc_cur_packno + 1 + buf[0];
+    rand = lsquic_crand_get_byte(ctl->sc_enpub->enp_crand);
+    ctl->sc_gap = ctl->sc_cur_packno + 1 + rand;
 }

src/liblsquic/lsquic_trans_params.h

@@ -57,7 +57,7 @@ enum trapa_flags
 #define TPI_QUANTUM_READINESS 3127
     TRAPA_QUANTUM_READY = 1 << 5,   /* Include "Quantum Readiness" TP */
 #endif
-#define TPI_QL_BITS 0x1055     /* 1055 is 133t for "loss" */
+#define TPI_QL_BITS 0x1057     /* 1057 is 133t for "lost" */
     TRAPA_QL_BITS       = 1 << 6,   /* tp_loss_bits contains valid value */
     TRAPA_QL_BITS_OLD   = 1 << 7,   /* Send old-school boolean loss_bits TP.
                                      * Not set on decoded transport parameters.

test/prog.c

@@ -488,6 +488,7 @@ prog_usr2_handler (int fd, short what, void *arg)
     LSQ_NOTICE("Got SIGUSR2, cool down engine");
     prog->prog_flags |= PROG_FLAG_COOLDOWN;
     lsquic_engine_cooldown(prog->prog_engine);
+    prog_process_conns(prog);
 }