diff --git a/bin/prog.c b/bin/prog.c index b779f92..82bc7ca 100644 --- a/bin/prog.c +++ b/bin/prog.c @@ -171,6 +171,8 @@ prog_print_common_options (const struct prog *prog, FILE *out) " sndbuf=12345 # Sets SO_SNDBUF\n" " rcvbuf=12345 # Sets SO_RCVBUF\n" " -W Use stock PMI (malloc & free)\n" +" -O BURST Use UDP GSO (if available). BURST factor is the max packets\n" +" that can be aggregated in single sendmsg.\n" ); #if HAVE_SENDMMSG @@ -226,6 +228,34 @@ prog_print_common_options (const struct prog *prog, FILE *out) ); } +#if HAVE_GSO +/* Test at runtime if the GSO support is available. setsockopt(UDP_SEGMENT) + * should be successful if the GSO is supported. + * Returns non-zero if GSO supported. */ +int supports_gso(void) +{ + int fd = socket(AF_INET, SOCK_DGRAM, 0); + int gso_size = 1400; // just for test + + if(fd < 0) { + LSQ_ERROR("weird! socket failed"); + return 0; + } + if (setsockopt(fd, SOL_UDP, UDP_SEGMENT, &gso_size, sizeof(gso_size))) { + LSQ_INFO("gso setsockopt failed. GSO not supp"); + close(fd); + return 0; + } + LSQ_INFO("GSO is supported"); + close(fd); + return 1; +} +#else +int supports_gso(void) +{ + return 0; +} +#endif // HAVE_GSO int prog_set_opt (struct prog *prog, int opt, const char *arg) @@ -237,6 +267,11 @@ prog_set_opt (struct prog *prog, int opt, const char *arg) switch (opt) { + case 'O': + if(supports_gso()) { + prog->prog_gso_burst = (unsigned)atoi(arg); + } + return 0; #if LSQUIC_DONTFRAG_SUPPORTED case 'D': { diff --git a/bin/prog.h b/bin/prog.h index 1638d23..7125f3d 100644 --- a/bin/prog.h +++ b/bin/prog.h @@ -25,6 +25,7 @@ struct prog unsigned short prog_max_packet_size; int prog_version_cleared; unsigned long prog_read_count; + unsigned prog_gso_burst; #if HAVE_SENDMMSG int prog_use_sendmmsg; #endif @@ -74,7 +75,7 @@ prog_init (struct prog *, unsigned lsquic_engine_flags, struct sport_head *, # define IP_DONTFRAG_FLAG "" #endif -#define PROG_OPTS "i:km:c:y:L:l:o:H:s:S:Y:z:G:W" RECVMMSG_FLAG SENDMMSG_FLAG \ +#define PROG_OPTS "O:i:km:c:y:L:l:o:H:s:S:Y:z:G:W" RECVMMSG_FLAG SENDMMSG_FLAG \ IP_DONTFRAG_FLAG /* Returns: diff --git a/bin/test_common.c b/bin/test_common.c index a7c0ec7..368a8c3 100644 --- a/bin/test_common.c +++ b/bin/test_common.c @@ -14,6 +14,7 @@ #ifndef WIN32 #include #include +#include #include #include #include @@ -1304,6 +1305,7 @@ enum ctl_what #if ECN_SUPPORTED CW_ECN = 1 << 1, #endif + CW_PKTLEN = 1 << 2, }; static void @@ -1313,7 +1315,7 @@ setup_control_msg ( #else WSAMSG #endif - *msg, enum ctl_what cw, + *msg, enum ctl_what cw, uint16_t pktlen, const struct lsquic_out_spec *spec, unsigned char *buf, size_t bufsz) { struct cmsghdr *cmsg; @@ -1417,6 +1419,17 @@ setup_control_msg ( } cw &= ~CW_ECN; } +#endif +#if HAVE_GSO + else if (cw & CW_PKTLEN) + { + cmsg->cmsg_level = SOL_UDP; + cmsg->cmsg_type = UDP_SEGMENT; + cmsg->cmsg_len = CMSG_LEN(sizeof(uint16_t)); + ctl_len += CMSG_SPACE(sizeof(uint16_t)); + *(uint16_t *)CMSG_DATA(cmsg) = pktlen; + cw &= ~CW_PKTLEN; + } #endif else assert(0); @@ -1429,6 +1442,18 @@ setup_control_msg ( #endif } +#ifndef NDEBUG +void check_if_single_peer(const struct lsquic_out_spec *specs, + unsigned count) +{ + void *ctx; + unsigned i; + for (i = 1, ctx = specs[i].peer_ctx; + i < count; + ctx = specs[i].peer_ctx, ++i) + assert(ctx == specs[i - 1].peer_ctx); +} +#endif #if HAVE_SENDMMSG static int @@ -1436,18 +1461,11 @@ send_packets_using_sendmmsg (const struct lsquic_out_spec *specs, unsigned count) { #ifndef NDEBUG - { - /* This only works for a single port! If the specs contain more - * than one socket, this function does *NOT* work. We check it - * here just in case: - */ - void *ctx; - unsigned i; - for (i = 1, ctx = specs[i].peer_ctx; - i < count; - ctx = specs[i].peer_ctx, ++i) - assert(ctx == specs[i - 1].peer_ctx); - } + /* This only works for a single port! If the specs contain more + * than one socket, this function does *NOT* work. We check it + * here just in case: + */ + check_if_single_peer(specs, count); #endif const struct service_port *const sport = specs[0].peer_ctx; @@ -1518,7 +1536,7 @@ send_packets_using_sendmmsg (const struct lsquic_out_spec *specs, else if (cw) { prev_ancil_key = ancil_key; - setup_control_msg(&mmsgs[i].msg_hdr, cw, &specs[i], ancil[i].buf, + setup_control_msg(&mmsgs[i].msg_hdr, cw, 0, &specs[i], ancil[i].buf, sizeof(ancil[i].buf)); } else @@ -1584,9 +1602,13 @@ find_sport (struct prog *prog, const struct sockaddr *local_sa) #endif - +/* + * Non-zero pktlen indicates use of UDP GSO. pktlen is used to create gso_size + * CMSG needed for UDP GSO. + */ static int -send_packets_one_by_one (const struct lsquic_out_spec *specs, unsigned count) +send_packets_one_by_one (const struct lsquic_out_spec *specs, unsigned count, + uint16_t pktlen) { const struct service_port *sport; enum ctl_what cw; @@ -1611,6 +1633,7 @@ send_packets_one_by_one (const struct lsquic_out_spec *specs, unsigned count) #if ECN_SUPPORTED + CMSG_SPACE(sizeof(int)) #endif + + CMSG_SPACE(sizeof(uint16_t)) ]; struct cmsghdr cmsg; } ancil; @@ -1699,6 +1722,10 @@ send_packets_one_by_one (const struct lsquic_out_spec *specs, unsigned count) ancil_key |= specs[n].ecn; } #endif + if (pktlen) + { + cw |= CW_PKTLEN; + } if (cw && prev_ancil_key == ancil_key) { /* Reuse previous ancillary message */ @@ -1707,7 +1734,7 @@ send_packets_one_by_one (const struct lsquic_out_spec *specs, unsigned count) else if (cw) { prev_ancil_key = ancil_key; - setup_control_msg(&msg, cw, &specs[n], ancil.buf, sizeof(ancil.buf)); + setup_control_msg(&msg, cw, pktlen, &specs[n], ancil.buf, sizeof(ancil.buf)); } else { @@ -1764,18 +1791,125 @@ send_packets_one_by_one (const struct lsquic_out_spec *specs, unsigned count) } } +#if HAVE_GSO +/* UDP GSO + * Refs: + * https://lwn.net/Articles/752956/ + * http://vger.kernel.org/lpc_net2018_talks/willemdebruijn-lpc2018-udpgso-paper-DRAFT-1.pdf + */ + +static int send_iovecs_gso(struct lsquic_out_spec *spec, + struct iovec *vecs, unsigned vcnt) +{ + spec->iov = vecs; + spec->iovlen = vcnt; + LSQ_DEBUG("GSO with burst:%d", vcnt); + return send_packets_one_by_one (spec, 1, vcnt > 1? vecs[0].iov_len: 0); +} + +/* return true if specs match */ +int match_spec(const struct lsquic_out_spec *s1, + const struct lsquic_out_spec *s2) +{ + if (s1->local_sa != s2->local_sa || + s1->dest_sa != s2->dest_sa || + s1->peer_ctx != s2->peer_ctx || + s1->ecn != s2->ecn) + return 0; + return 1; +} + +/* + * To use GSO the flow here is: + * a. Check all the equal length iovs and batch them in single iovec array and + * pass it in spec. + * b. Batching inter-spec iovs can happen only if all the spec parameters + * (peer_ctx, sa, ecn) match. + * c. The last packet in the iovec array can be of smaller length then all the + * earlier packets. + */ +static int +send_packets_using_gso (const unsigned burst, + const struct lsquic_out_spec *specs, + unsigned count) +{ + struct iovec vecs[burst]; + struct iovec *inv; + struct lsquic_out_spec newspec; + unsigned i, j, vcnt = 0; + int ret; + + if (count == 0 || specs[0].iovlen == 0) { + LSQ_ERROR("Sanity failed. count=%d, specs iovlen=%zu", + count, specs[0].iovlen); + return 0; + } + + /* Coalesce packets with same lengths, except that the last packet can be + * of smaller length. Colasece max gso_burst packets. */ + for (i = 0; i < count; ++i) + { + if (vcnt == 0) { + memcpy(&newspec, &specs[i], sizeof(struct lsquic_out_spec)); + } else if(!match_spec(&newspec, &specs[i])) { + // new specs dont match prev ones, so send the previous iovec batch + ret = send_iovecs_gso (&newspec, vecs, vcnt); + if(ret == 0) { + LSQ_ERROR("Partial send1"); + return i-1; + } + vcnt = 0; + continue; + } + for (j = 0; j < specs[i].iovlen; ++j) + { + inv = &specs[i].iov[j]; + if (vcnt == 0) { + vecs[vcnt++] = *inv; + continue; + } + if (inv->iov_len > vecs[0].iov_len) { + ret = send_iovecs_gso (&newspec, vecs, vcnt); + vcnt = 0; + } else if ((inv->iov_len != vecs[0].iov_len) || (vcnt >= (burst-1))) { + vecs[vcnt++] = *inv; + ret = send_iovecs_gso (&newspec, vecs, vcnt); + vcnt = 0; + } else { + vecs[vcnt++] = *inv; + } + if(vcnt == 0 && ret == 0) { + LSQ_ERROR("Partial send2"); + return i-1; + } + } + } + if (vcnt) { + ret = send_iovecs_gso (&newspec, vecs, vcnt); + if(ret == 0) { + LSQ_ERROR("Partial send3"); + return count-1; + } + } + return count; +} +#endif // HAVE_GSO int sport_packets_out (void *ctx, const struct lsquic_out_spec *specs, unsigned count) { -#if HAVE_SENDMMSG const struct prog *prog = ctx; - if (prog->prog_use_sendmmsg) - return send_packets_using_sendmmsg(specs, count); +#if HAVE_GSO + if (prog->prog_gso_burst > 0) + return send_packets_using_gso(prog->prog_gso_burst, specs, count); else #endif - return send_packets_one_by_one(specs, count); +#if HAVE_SENDMMSG + if (prog->prog_use_sendmmsg) + return send_packets_using_sendmmsg(specs, count); +#endif + return send_packets_one_by_one(specs, count, 0); } diff --git a/bin/test_common.h b/bin/test_common.h index 4ad7472..11683a3 100644 --- a/bin/test_common.h +++ b/bin/test_common.h @@ -143,6 +143,21 @@ destroy_lsquic_reader_ctx (struct reader_ctx *ctx); #define LITESPEED_ID "lsquic" "/" TOSTRING(LSQUIC_MAJOR_VERSION) "." \ TOSTRING(LSQUIC_MINOR_VERSION) "." TOSTRING(LSQUIC_PATCH_VERSION) +#ifndef WIN32 +#ifndef UDP_SEGMENT +#define UDP_SEGMENT 103 +#endif +#endif + +#if __linux__ +#ifndef HAVE_GSO +#if defined(SOL_UDP) && defined(UDP_SEGMENT) +#include +#define HAVE_GSO 1 +#endif +#endif +#endif + struct header_buf { unsigned off;