1.13.0: [FEATURE, API Change] HTTP header bypass

Add ability to create custom header set objects via callbacks.
This avoids reading and re-parsing headers from the stream.

See test/http_client.c for example implementation.  (Use -B flag
to turn it on).

CHANGELOG

@@ -1,3 +1,10 @@
+2018-08-27
+
+    - 1.13.0
+    - [FEATURE, API Change] Add ability to create custom header set
+      objects via callbacks.  This avoids reading and re-parsing
+      headers from the stream.
+
 2018-08-27
 
     - 1.12.4

include/lsquic.h

@@ -24,8 +24,8 @@ extern "C" {
 #endif
 
 #define LSQUIC_MAJOR_VERSION 1
-#define LSQUIC_MINOR_VERSION 12
-#define LSQUIC_PATCH_VERSION 4
+#define LSQUIC_MINOR_VERSION 13
+#define LSQUIC_PATCH_VERSION 0
 
 /**
  * Engine flags:
@@ -498,6 +498,72 @@ struct lsquic_packout_mem_if
 
 struct stack_st_X509;
 
+/**
+ * When headers are processed, various errors may occur.  They are listed
+ * in this enum.
+ */
+enum lsquic_header_status
+{
+    LSQUIC_HDR_OK,
+    /** Duplicate pseudo-header */
+    LSQUIC_HDR_ERR_DUPLICATE_PSDO_HDR,
+    /** Not all request pseudo-headers are present */
+    LSQUIC_HDR_ERR_INCOMPL_REQ_PSDO_HDR,
+    /** Unnecessary request pseudo-header present in the response */
+    LSQUIC_HDR_ERR_UNNEC_REQ_PSDO_HDR,
+    /** Not all response pseudo-headers are present */
+    LSQUIC_HDR_ERR_INCOMPL_RESP_PSDO_HDR,
+    /** Unnecessary response pseudo-header present in the response. */
+    LSQUIC_HDR_ERR_UNNEC_RESP_PSDO_HDR,
+    /** Unknown pseudo-header */
+    LSQUIC_HDR_ERR_UNKNOWN_PSDO_HDR,
+    /** Uppercase letter in header */
+    LSQUIC_HDR_ERR_UPPERCASE_HEADER,
+    /** Misplaced pseudo-header */
+    LSQUIC_HDR_ERR_MISPLACED_PSDO_HDR,
+    /** Missing pseudo-header */
+    LSQUIC_HDR_ERR_MISSING_PSDO_HDR,
+    /** Header or headers are too large */
+    LSQUIC_HDR_ERR_HEADERS_TOO_LARGE,
+    /** Cannot allocate any more memory. */
+    LSQUIC_HDR_ERR_NOMEM,
+};
+
+struct lsquic_hset_if
+{
+    /**
+     * Create a new header set.  This object is (and must be) fetched from a
+     * stream by calling @ref lsquic_stream_get_hset() before the stream can
+     * be read.
+     */
+    void *              (*hsi_create_header_set)(void *hsi_ctx,
+                                                        int is_push_promise);
+    /**
+     * Process new header.  Return 0 on success, -1 if there is a problem with
+     * the header.  -1 is treated as a stream error: the associated stream is
+     * reset.
+     *
+     * `hdr_set' is the header set object returned by
+     * @ref hsi_create_header_set().
+     *
+     * `name_idx' is set to the index in the HPACK static table whose entry's
+     * name element matches `name'.  If there is no such match, `name_idx' is
+     * set to zero.
+     *
+     * If `name' is NULL, this means that no more header are going to be
+     * added to the set.
+     */
+    enum lsquic_header_status (*hsi_process_header)(void *hdr_set,
+                                    unsigned name_idx,
+                                    const char *name, unsigned name_len,
+                                    const char *value, unsigned value_len);
+    /**
+     * Discard header set.  This is called for unclaimed header sets and
+     * header sets that had an error.
+     */
+    void                (*hsi_discard_header_set)(void *hdr_set);
+};
+
 /* TODO: describe this important data structure */
 typedef struct lsquic_engine_api
 {
@@ -525,6 +591,14 @@ typedef struct lsquic_engine_api
     int                                (*ea_verify_cert)(void *verify_ctx,
                                                 struct stack_st_X509 *chain);
     void                                *ea_verify_ctx;
+
+    /**
+     * Optional header set interface.  If not specified, the incoming headers
+     * are converted to HTTP/1.x format and are read from stream and have to
+     * be parsed again.
+     */
+    const struct lsquic_hset_if         *ea_hsi_if;
+    void                                *ea_hsi_ctx;
 } lsquic_engine_api_t;
 
 /**
@@ -694,6 +768,20 @@ struct lsquic_http_headers
 int lsquic_stream_send_headers(lsquic_stream_t *s,
                                const lsquic_http_headers_t *h, int eos);
 
+/**
+ * Get header set associated with the stream.  The header set is created by
+ * @ref hsi_create_header_set() callback.  After this call, the ownership of
+ * the header set is trasnferred to the caller.
+ *
+ * This call must precede calls to @ref lsquic_stream_read() and
+ * @ref lsquic_stream_readv().
+ *
+ * If the optional header set interface (@ref ea_hsi_if) is not specified,
+ * this function returns NULL.
+ */
+void *
+lsquic_stream_get_hset (lsquic_stream_t *);
+
 int lsquic_conn_is_push_enabled(lsquic_conn_t *c);
 
 /** Possible values for how are 0, 1, and 2.  See shutdown(2). */
@@ -743,16 +831,15 @@ lsquic_stream_refuse_push (lsquic_stream_t *s);
  *
  * @param ref_stream_id   Stream ID in response to which push promise was
  *                            sent.
- * @param headers         Uncompressed request headers.
- * @param headers_sz      Size of uncompressed request headers, not counting
- *                          the NUL byte.
+ * @param hdr_set         Header set.  This object was passed to or generated
+ *                            by @ref lsquic_conn_push_stream().
  *
  * @retval   0  Success.
  * @retval  -1  This is not a pushed stream.
  */
 int
 lsquic_stream_push_info (const lsquic_stream_t *, uint32_t *ref_stream_id,
-                         const char **headers, size_t *headers_sz);
+                         void **hdr_set);
 
 /** Return current priority of the stream */
 unsigned lsquic_stream_priority (const lsquic_stream_t *s);

src/liblsquic/CMakeLists.txt

@@ -53,6 +53,7 @@ SET(lsquic_STAT_SRCS
     lsquic_min_heap.c
     lshpack.c
     lsquic_parse_Q044.c
+    lsquic_http1x_if.c
     )
 
 

src/liblsquic/lshpack.c

@@ -5412,10 +5412,7 @@ lshpack_enc_cleanup (struct lshpack_enc *enc)
 
 
 //not find return 0, otherwise return the index
-#if !LS_HPACK_EMIT_TEST_CODE
-static
-#endif
-       unsigned
+unsigned
 lshpack_enc_get_stx_tab_id (const char *name, lshpack_strlen_t name_len,
                 const char *val, lshpack_strlen_t val_len, int *val_matched)
 {
@@ -6197,6 +6194,7 @@ struct dec_table_entry
 {
     uint16_t    dte_name_len;
     uint16_t    dte_val_len;
+    uint8_t     dte_name_idx;
     char        dte_buf[0];     /* Contains both name and value */
 };
 
@@ -6430,7 +6428,7 @@ hdec_get_table_entry (struct lshpack_dec *dec, uint32_t index)
 static
 #endif
        int
-lshpack_dec_push_entry (struct lshpack_dec *dec, const char *name,
+lshpack_dec_push_entry (struct lshpack_dec *dec, uint8_t name_idx, const char *name,
                         uint16_t name_len, const char *val, uint16_t val_len)
 {
     struct dec_table_entry *entry;
@@ -6450,6 +6448,7 @@ lshpack_dec_push_entry (struct lshpack_dec *dec, const char *name,
     dec->hpd_cur_capacity += DYNAMIC_ENTRY_OVERHEAD + name_len + val_len;
     entry->dte_name_len = name_len;
     entry->dte_val_len = val_len;
+    entry->dte_name_idx = name_idx;
     memcpy(DTE_NAME(entry), name, name_len);
     memcpy(DTE_VALUE(entry), val, val_len);
     return 0;
@@ -6459,7 +6458,8 @@ lshpack_dec_push_entry (struct lshpack_dec *dec, const char *name,
 int
 lshpack_dec_decode (struct lshpack_dec *dec,
     const unsigned char **src, const unsigned char *src_end,
-    char *dst, char *const dst_end, uint16_t *name_len, uint16_t *val_len)
+    char *dst, char *const dst_end, uint16_t *name_len, uint16_t *val_len,
+    uint32_t *name_idx)
 {
     struct dec_table_entry *entry;
     uint32_t index, new_capacity;
@@ -6532,6 +6532,7 @@ lshpack_dec_decode (struct lshpack_dec *dec,
 
         indexed_type = 1;
     }
+    *name_idx = index;
 
     char *const name = dst;
     if (index > 0)
@@ -6562,6 +6563,8 @@ lshpack_dec_decode (struct lshpack_dec *dec,
 
             *name_len = entry->dte_name_len;
             memcpy(name, DTE_NAME(entry), *name_len);
+            if (entry->dte_name_idx)
+                *name_idx = entry->dte_name_idx;
             if (indexed_type == 3)
             {
                 if (entry->dte_name_len + entry->dte_val_len > dst_end - dst)
@@ -6592,7 +6595,9 @@ lshpack_dec_decode (struct lshpack_dec *dec,
 
     if (indexed_type == 0)
     {
-        if (0 != lshpack_dec_push_entry(dec, name, *name_len,
+        if (index > HPACK_STATIC_TABLE_SIZE)
+            index = 0;
+        if (0 != lshpack_dec_push_entry(dec, index, name, *name_len,
                                             name + *name_len, *val_len))
             return -1;  //error
     }

src/liblsquic/lshpack.h

@@ -101,7 +101,7 @@ int
 lshpack_dec_decode (struct lshpack_dec *dec,
     const unsigned char **src, const unsigned char *src_end,
     char *dst, char *const dst_end, lshpack_strlen_t *name_len,
-    lshpack_strlen_t *val_len);
+    lshpack_strlen_t *val_len, uint32_t *name_idx);
 
 void
 lshpack_dec_set_max_capacity (struct lshpack_dec *, unsigned);
@@ -156,6 +156,10 @@ struct lshpack_dec
     struct lshpack_arr hpd_dyn_table;
 };
 
+unsigned
+lshpack_enc_get_stx_tab_id (const char *name, lshpack_strlen_t name_len,
+                const char *val, lshpack_strlen_t val_len, int *val_matched);
+
 #ifdef __cplusplus
 }
 #endif

src/liblsquic/lsquic_engine.c

@@ -55,6 +55,7 @@
 #include "lsquic_hash.h"
 #include "lsquic_attq.h"
 #include "lsquic_min_heap.h"
+#include "lsquic_http1x_if.h"
 
 #define LSQUIC_LOGGER_MODULE LSQLM_ENGINE
 #include "lsquic_logger.h"
@@ -325,6 +326,16 @@ lsquic_engine_new (unsigned flags,
     engine->stream_if_ctx   = api->ea_stream_if_ctx;
     engine->packets_out     = api->ea_packets_out;
     engine->packets_out_ctx = api->ea_packets_out_ctx;
+    if (api->ea_hsi_if)
+    {
+        engine->pub.enp_hsi_if  = api->ea_hsi_if;
+        engine->pub.enp_hsi_ctx = api->ea_hsi_ctx;
+    }
+    else
+    {
+        engine->pub.enp_hsi_if  = lsquic_http1x_if;
+        engine->pub.enp_hsi_ctx = NULL;
+    }
     if (api->ea_pmi)
     {
         engine->pub.enp_pmi      = api->ea_pmi;

src/liblsquic/lsquic_engine_public.h

@@ -14,6 +14,8 @@ struct stack_st_X509;
 struct lsquic_engine_public {
     struct lsquic_mm                enp_mm;
     struct lsquic_engine_settings   enp_settings;
+    const struct lsquic_hset_if    *enp_hsi_if;
+    void                           *enp_hsi_ctx;
     int                           (*enp_verify_cert)(void *verify_ctx,
                                             struct stack_st_X509 *chain);
     void                           *enp_verify_ctx;

src/liblsquic/lsquic_ev_log.c

@@ -18,7 +18,7 @@
 #include "lsquic_packet_out.h"
 #include "lsquic_parse.h"
 #include "lsquic_frame_common.h"
-#include "lsquic_frame_reader.h"
+#include "lsquic_headers.h"
 #include "lsquic_str.h"
 #include "lsquic_handshake.h"
 #include "lsquic_ev_log.h"
@@ -208,6 +208,7 @@ void
 lsquic_ev_log_http_headers_in (lsquic_cid_t cid, int is_server,
                                         const struct uncompressed_headers *uh)
 {
+    const struct http1x_headers *h1h;
     const char *cr, *p;
 
     if (uh->uh_flags & UH_PP)
@@ -220,13 +221,17 @@ lsquic_ev_log_http_headers_in (lsquic_cid_t cid, int is_server,
             uh->uh_stream_id, uh->uh_oth_stream_id, uh->uh_weight,
             (int) uh->uh_exclusive, !!(uh->uh_flags & UH_FIN));
 
-    for (p = uh->uh_headers; p < uh->uh_headers + uh->uh_size; p = cr + 2)
+    if (uh->uh_flags & UH_H1H)
     {
-        cr = strchr(p, '\r');
-        if (cr && cr > p)
-            LCID("  %.*s", (int) (cr - p), p);
-        else
-            break;
+        h1h = uh->uh_hset;
+        for (p = h1h->h1h_buf; p < h1h->h1h_buf + h1h->h1h_size; p = cr + 2)
+        {
+            cr = strchr(p, '\r');
+            if (cr && cr > p)
+                LCID("  %.*s", (int) (cr - p), p);
+            else
+                break;
+        }
     }
 }