CentOS7安裝QAT 1.7加密卡

1. 準備材料

1.1 機器

物理PC一臺（記作A），安裝QAT加密卡硬件和CentOS 7系統。

1.2 Tools

登錄A，執行：

# yum install -ykernel kernel-devel openssl-devel pciutils zlib-devel gcc libudev-devel boost-devel

如果內核有更新，重啓。

1.3 Code

1.3.1 QAT相關

https://01.org/sites/default/files/downloads/intelr-quickassist-technology/qat1.7.upstream.l.1.0.3-42.tar.gz

https://github.com/intel/QAT_Engine/archive/v0.5.34.tar.gz

1.3.2 OpenSSL

https://github.com/openssl/openssl/archive/OpenSSL_1_1_0g.tar.gz

1.3.3 應用相關

http://nginx.org/download/nginx-1.12.2.tar.gz

http://www.haproxy.org/download/1.8/src/haproxy-1.8.4.tar.gz

只能用瀏覽器下載。

2. Make && Make install

2.1 QAT driver

登錄A，執行：

# mkdir /tmp/QAT

# tar zxvf qat1.7.upstream.l.*.tar.gz-C /tmp/QAT

# cd /tmp/QAT

# chmod -R o-rwx *

# ./configure

# make

# make install

2.2 OpenSSL

登錄A，執行：

# tar xzf OpenSSL_1_1_0g.tar.gz

# cd openssl-OpenSSL_1_1_0g

# ./config--prefix=/usr/local/ssl

# make depend (ifrecommended by the OpenSSL* build system)

# make

# make install

# exportOPENSSL_ENGINES=/usr/local/ssl/lib/engines-1.1

# echo/usr/local/ssl/lib > /etc/ld.so.conf.d/qat.conf

# ldconfig

2.3 QAT Engine

2.3.1 qat_contig_mem

# tar zxfQAT_Engine-0.5.34.tar.gz

# cd QAT_Engine-0.5.34/qat_contig_mem

# make

# make load

# make test

2.3.2 QAT_Engine

# cd QAT_Engine-0.5.34

# ./configure --with-qat_dir=/tmp/QAT\

--with-openssl_dir=/path/to/openssl-OpenSSL_1_1_0g\

--with-openssl_install_dir=/usr/local/ssl\

--enable-upstream_driver \

--enable-usdm

# make

# make install

2.3.3 QAT config files

# cp /path/to/QAT_Engine-0.5.34/qat/config/c6xx/multi_process_event-driven_optimized/*/etc

# service qat_servicestop

# service qat_servicestart

2.4 Nginx

nginx-qat.patch:

diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c
index a1baefb..e783b5f 100644
--- a/src/core/ngx_connection.c
+++ b/src/core/ngx_connection.c
@@ -957,6 +957,18 @@ ngx_configure_listening_sockets(ngx_cycle_t *cycle)
     return;
 }

+#if (NGX_SSL)
+void
+ngx_asynch_del_conn(ngx_connection_t *c)
+{
+    if (ngx_del_async_conn) {
+        if (c->num_async_fds) {
+            ngx_del_async_conn(c, NGX_DISABLE_EVENT);
+            c->num_async_fds--;
+        }
+    }
+}
+#endif

 void
 ngx_close_listening_sockets(ngx_cycle_t *cycle)
@@ -986,7 +998,11 @@ ngx_close_listening_sockets(ngx_cycle_t *cycle)
                      * for closed shared listening sockets unless
                      * the events was explicitly deleted
                      */
-
+#if (NGX_SSL)
+                    if (c->asynch) {
+                        ngx_asynch_del_conn(c);
+                    }
+#endif
                     ngx_del_event(c->read, NGX_READ_EVENT, 0);

                 } else {
@@ -1035,6 +1051,9 @@ ngx_get_connection(ngx_socket_t s, ngx_log_t *log)
 {
     ngx_uint_t         instance;
     ngx_event_t       *rev, *wev;
+#if (NGX_SSL)
+    ngx_event_t       *aev;
+#endif
     ngx_connection_t  *c;

     /* disable warning: Win32 SOCKET is u_int while UNIX socket is int */
@@ -1071,11 +1090,17 @@ ngx_get_connection(ngx_socket_t s, ngx_log_t *log)

     rev = c->read;
     wev = c->write;
+#if (NGX_SSL)
+    aev = c->async;
+#endif

     ngx_memzero(c, sizeof(ngx_connection_t));

     c->read = rev;
     c->write = wev;
+#if (NGX_SSL)
+    c->async = aev;
+#endif
     c->fd = s;
     c->log = log;

@@ -1083,17 +1108,32 @@ ngx_get_connection(ngx_socket_t s, ngx_log_t *log)

     ngx_memzero(rev, sizeof(ngx_event_t));
     ngx_memzero(wev, sizeof(ngx_event_t));
+#if (NGX_SSL)
+    ngx_memzero(aev, sizeof(ngx_event_t));
+#endif

     rev->instance = !instance;
     wev->instance = !instance;
+#if (NGX_SSL)
+    aev->instance = !instance;
+#endif

     rev->index = NGX_INVALID_INDEX;
     wev->index = NGX_INVALID_INDEX;
+#if (NGX_SSL)
+    aev->index = NGX_INVALID_INDEX;
+#endif

     rev->data = c;
     wev->data = c;
+#if (NGX_SSL)
+    aev->data = c;
+#endif

     wev->write = 1;
+#if (NGX_SSL)
+    aev->async = 1;
+#endif

     return c;
 }
@@ -1132,11 +1172,26 @@ ngx_close_connection(ngx_connection_t *c)
         ngx_del_timer(c->write);
     }

+#if (NGX_SSL)
+    if (c->async->timer_set) {
+        ngx_del_timer(c->async);
+    }
+
+    if (c->asynch) {
+        ngx_asynch_del_conn(c);
+    }
+#endif
     if (!c->shared) {
         if (ngx_del_conn) {
             ngx_del_conn(c, NGX_CLOSE_EVENT);

         } else {
+#if (NGX_SSL)
+            if (c->asynch) {
+                ngx_asynch_del_conn(c);
+            }
+#endif
+
             if (c->read->active || c->read->disabled) {
                 ngx_del_event(c->read, NGX_READ_EVENT, NGX_CLOSE_EVENT);
             }
@@ -1155,8 +1210,17 @@ ngx_close_connection(ngx_connection_t *c)
         ngx_delete_posted_event(c->write);
     }

+#if (NGX_SSL)
+    if (c->async->posted) {
+        ngx_delete_posted_event(c->async);
+    }
+#endif
+
     c->read->closed = 1;
     c->write->closed = 1;
+#if (NGX_SSL)
+    c->async->closed = 1;
+#endif

     ngx_reusable_connection(c, 0);

diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h
index 1d3e3a3..3a481d9 100644
--- a/src/core/ngx_connection.h
+++ b/src/core/ngx_connection.h
@@ -122,8 +122,14 @@ struct ngx_connection_s {
     void               *data;
     ngx_event_t        *read;
     ngx_event_t        *write;
+#if (NGX_SSL)
+    ngx_event_t        *async;
+#endif

     ngx_socket_t        fd;
+#if (NGX_SSL)
+    ngx_socket_t        async_fd;
+#endif

     ngx_recv_pt         recv;
     ngx_send_pt         send;
@@ -149,6 +155,7 @@ struct ngx_connection_s {

 #if (NGX_SSL || NGX_COMPAT)
     ngx_ssl_connection_t  *ssl;
+    ngx_flag_t          asynch;
 #endif

     struct sockaddr    *local_sockaddr;
@@ -181,6 +188,9 @@ struct ngx_connection_s {
     unsigned            tcp_nopush:2;    /* ngx_connection_tcp_nopush_e */

     unsigned            need_last_buf:1;
+#if (NGX_SSL)
+    unsigned char       num_async_fds;
+#endif

 #if (NGX_HAVE_AIO_SENDFILE || NGX_COMPAT)
     unsigned            busy_count:2;
@@ -205,6 +215,7 @@ struct ngx_connection_s {

 ngx_listening_t *ngx_create_listening(ngx_conf_t *cf, struct sockaddr *sockaddr,
     socklen_t socklen);
+void ngx_asynch_del_conn(ngx_connection_t *c);
 ngx_int_t ngx_clone_listening(ngx_conf_t *cf, ngx_listening_t *ls);
 ngx_int_t ngx_set_inherited_sockets(ngx_cycle_t *cycle);
 ngx_int_t ngx_open_listening_sockets(ngx_cycle_t *cycle);
diff --git a/src/core/ngx_cycle.h b/src/core/ngx_cycle.h
index 2b48ccb..f79da12 100644
--- a/src/core/ngx_cycle.h
+++ b/src/core/ngx_cycle.h
@@ -71,7 +71,9 @@ struct ngx_cycle_s {
     ngx_connection_t         *connections;
     ngx_event_t              *read_events;
     ngx_event_t              *write_events;
-
+#if (NGX_SSL)
+    ngx_event_t              *async_events;
+#endif
     ngx_cycle_t              *old_cycle;

     ngx_str_t                 conf_file;
diff --git a/src/event/modules/ngx_epoll_module.c b/src/event/modules/ngx_epoll_module.c
index 76aee08..b45202e 100644
--- a/src/event/modules/ngx_epoll_module.c
+++ b/src/event/modules/ngx_epoll_module.c
@@ -122,6 +122,11 @@ static ngx_int_t ngx_epoll_notify(ngx_event_handler_pt handler);
 #endif
 static ngx_int_t ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer,
     ngx_uint_t flags);
+#if (NGX_SSL)
+static ngx_int_t ngx_epoll_add_async_connection(ngx_connection_t *c);
+static ngx_int_t ngx_epoll_del_async_connection(ngx_connection_t *c,
+        ngx_uint_t flags);
+#endif

 #if (NGX_HAVE_FILE_AIO)
 static void ngx_epoll_eventfd_handler(ngx_event_t *ev);
@@ -196,6 +201,13 @@ static ngx_event_module_t  ngx_epoll_module_ctx = {
         ngx_epoll_process_events,        /* process the events */
         ngx_epoll_init,                  /* init the events */
         ngx_epoll_done,                  /* done the events */
+#if (NGX_SSL)
+        ngx_epoll_add_async_connection,  /* add an async conn */
+        ngx_epoll_del_async_connection   /* del an async conn */
+#else
+        NULL,
+        NULL
+#endif
     }
 };

@@ -758,6 +770,54 @@ ngx_epoll_del_connection(ngx_connection_t *c, ngx_uint_t flags)
     return NGX_OK;
 }

+#if (NGX_SSL)
+static ngx_int_t
+ngx_epoll_add_async_connection(ngx_connection_t *c)
+{
+    struct epoll_event  ee;
+
+    ee.events = EPOLLIN|EPOLLOUT|EPOLLET|EPOLLRDHUP;
+    ee.data.ptr = (void *) ((uintptr_t) c | (c->async->async << 1) | c->async->instance);
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "epoll add async connection: fd:%d ev:%08XD", c->async_fd, ee.events);
+    if (epoll_ctl(ep, EPOLL_CTL_ADD, c->async_fd, &ee) == -1) {
+        ngx_log_error(NGX_LOG_ALERT, c->log, ngx_errno,
+                      "async add conn epoll_ctl(EPOLL_CTL_ADD, %d) failed", c->async_fd);
+        return NGX_ERROR;
+    }
+
+    c->async->active = 1;
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_epoll_del_async_connection(ngx_connection_t *c, ngx_uint_t flags)
+{
+    int                 op;
+    struct epoll_event  ee;
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "epoll del async connection: fd:%d", c->async_fd);
+
+    op = EPOLL_CTL_DEL;
+    ee.events = 0;
+    ee.data.ptr = NULL;
+    if (epoll_ctl(ep, op, c->async_fd, &ee) == -1) {
+        ngx_log_error(NGX_LOG_ALERT, c->log, ngx_errno,
+                      "async del conn epoll_ctl(%d, %d) failed", op, c->async_fd);
+        c->async_fd = -1;
+        return NGX_ERROR;
+    }
+    c->async_fd = -1;
+    c->async->active = 0;
+
+    return NGX_OK;
+}
+#endif
+

 #if (NGX_HAVE_EVENTFD)

@@ -791,6 +851,10 @@ ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags)
     ngx_event_t       *rev, *wev;
     ngx_queue_t       *queue;
     ngx_connection_t  *c;
+#if (NGX_SSL)
+    ngx_int_t          async;
+    ngx_event_t       *aev;
+#endif

     /* NGX_TIMER_INFINITE == INFTIM */

@@ -837,7 +901,12 @@ ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags)
         c = event_list[i].data.ptr;

         instance = (uintptr_t) c & 1;
+#if (NGX_SSL)
+        async = ((uintptr_t) c & 2) >> 1;
+        c = (ngx_connection_t *) ((uintptr_t) c & (uintptr_t) ~3);
+#else
         c = (ngx_connection_t *) ((uintptr_t) c & (uintptr_t) ~1);
+#endif

         rev = c->read;

@@ -880,7 +949,11 @@ ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags)
         }
 #endif

+#if (NGX_SSL)
+        if ((revents & EPOLLIN) && rev->active && !async) {
+#else
         if ((revents & EPOLLIN) && rev->active) {
+#endif

 #if (NGX_HAVE_EPOLLRDHUP)
             if (revents & EPOLLRDHUP) {
@@ -905,7 +978,11 @@ ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags)

         wev = c->write;

+#if (NGX_SSL)
+        if ((revents & EPOLLOUT) && wev->active && !async) {
+#else
         if ((revents & EPOLLOUT) && wev->active) {
+#endif

             if (c->fd == -1 || wev->instance != instance) {

@@ -931,6 +1008,32 @@ ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags)
                 wev->handler(wev);
             }
         }
+#if (NGX_SSL)
+        aev = c->async;
+
+        if ((revents & EPOLLIN) && aev->active && async) {
+
+            if (c->async_fd == -1 || aev->instance!= instance) {
+                /*
+                 * the stale event from a file descriptor
+                 * that was just closed in this iteration
+                 */
+
+                ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                               "epoll: stale event %p", c);
+                continue;
+            }
+
+            aev->ready = 1;
+
+            if (flags & NGX_POST_EVENTS) {
+                ngx_post_event(aev, &ngx_posted_events);
+
+            } else {
+                aev->handler(aev);
+            }
+        }
+#endif
     }

     return NGX_OK;
diff --git a/src/event/ngx_event.c b/src/event/ngx_event.c
index 57af813..ed973d2 100644
--- a/src/event/ngx_event.c
+++ b/src/event/ngx_event.c
@@ -170,7 +170,7 @@ static ngx_event_module_t  ngx_event_core_module_ctx = {
     ngx_event_core_create_conf,            /* create configuration */
     ngx_event_core_init_conf,              /* init configuration */

-    { NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
+    { NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}
 };


@@ -569,6 +569,9 @@ ngx_event_process_init(ngx_cycle_t *cycle)
 {
     ngx_uint_t           m, i;
     ngx_event_t         *rev, *wev;
+#if (NGX_SSL)
+    ngx_event_t         *aev;
+#endif
     ngx_listening_t     *ls;
     ngx_connection_t    *c, *next, *old;
     ngx_core_conf_t     *ccf;
@@ -711,6 +714,19 @@ ngx_event_process_init(ngx_cycle_t *cycle)
         wev[i].closed = 1;
     }

+#if (NGX_SSL)
+    cycle->async_events = ngx_alloc(sizeof(ngx_event_t) * cycle->connection_n,
+            cycle->log);
+    if (cycle->async_events == NULL) {
+        return NGX_ERROR;
+    }
+
+    aev = cycle->async_events;
+    for (i = 0; i < cycle->connection_n; i++) {
+        aev[i].closed = 1;
+        aev[i].instance = 1;
+    }
+#endif
     i = cycle->connection_n;
     next = NULL;

@@ -721,6 +737,10 @@ ngx_event_process_init(ngx_cycle_t *cycle)
         c[i].read = &cycle->read_events[i];
         c[i].write = &cycle->write_events[i];
         c[i].fd = (ngx_socket_t) -1;
+#if (NGX_SSL)
+        c[i].async = &cycle->async_events[i];
+        c[i].async_fd = (ngx_socket_t) -1;
+#endif

         next = &c[i];
     } while (i);
diff --git a/src/event/ngx_event.h b/src/event/ngx_event.h
index 053bd16..6a9ded8 100644
--- a/src/event/ngx_event.h
+++ b/src/event/ngx_event.h
@@ -32,6 +32,10 @@ struct ngx_event_s {

     unsigned         write:1;

+#if (NGX_SSL)
+    unsigned         async:1;
+#endif
+
     unsigned         accept:1;

     /* used to detect the stale events in kqueue and epoll */
@@ -108,6 +112,9 @@ struct ngx_event_s {
 #endif

     ngx_event_handler_pt  handler;
+#if (NGX_SSL)
+    ngx_event_handler_pt  saved_handler;
+#endif


 #if (NGX_HAVE_IOCP)
@@ -191,6 +198,8 @@ typedef struct {

     ngx_int_t  (*init)(ngx_cycle_t *cycle, ngx_msec_t timer);
     void       (*done)(ngx_cycle_t *cycle);
+    ngx_int_t  (*add_async_conn)(ngx_connection_t *c);
+    ngx_int_t  (*del_async_conn)(ngx_connection_t *c, ngx_uint_t flags);
 } ngx_event_actions_t;


@@ -415,6 +424,8 @@ extern ngx_uint_t            ngx_use_epoll_rdhup;
 #define ngx_del_event        ngx_event_actions.del
 #define ngx_add_conn         ngx_event_actions.add_conn
 #define ngx_del_conn         ngx_event_actions.del_conn
+#define ngx_add_async_conn   ngx_event_actions.add_async_conn
+#define ngx_del_async_conn   ngx_event_actions.del_async_conn

 #define ngx_notify           ngx_event_actions.notify

diff --git a/src/event/ngx_event_accept.c b/src/event/ngx_event_accept.c
index 7756370..f8cec14 100644
--- a/src/event/ngx_event_accept.c
+++ b/src/event/ngx_event_accept.c
@@ -249,6 +249,9 @@ ngx_event_accept(ngx_event_t *ev)

         rev->log = log;
         wev->log = log;
+#if (NGX_SSL)
+        c->async->log = log;
+#endif

         /*
          * TODO: MT: - ngx_atomic_fetch_add()
@@ -735,6 +738,11 @@ ngx_disable_accept_events(ngx_cycle_t *cycle, ngx_uint_t all)

 #endif

+#if (NGX_SSL)
+        if (c->asynch) {
+            ngx_asynch_del_conn(c);
+        }
+#endif
         if (ngx_del_event(c->read, NGX_READ_EVENT, NGX_DISABLE_EVENT)
             == NGX_ERROR)
         {
diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c
index 8c7c677..a581967 100644
--- a/src/event/ngx_event_openssl.c
+++ b/src/event/ngx_event_openssl.c
@@ -115,6 +115,13 @@ int  ngx_ssl_next_certificate_index;
 int  ngx_ssl_certificate_name_index;
 int  ngx_ssl_stapling_index;

+static void
+ngx_ssl_empty_handler(ngx_event_t *aev)
+{
+    ngx_log_debug0(NGX_LOG_DEBUG_HTTP, aev->log, 0, "ssl empty handler");
+
+    return;
+}

 ngx_int_t
 ngx_ssl_init(ngx_log_t *log)
@@ -336,6 +343,10 @@ ngx_ssl_create(ngx_ssl_t *ssl, ngx_uint_t protocols, void *data)
     SSL_CTX_set_mode(ssl->ctx, SSL_MODE_NO_AUTO_CHAIN);
 #endif

+    if (ssl->asynch) {
+        SSL_CTX_set_mode(ssl->ctx, SSL_MODE_ASYNC);
+    }
+
     SSL_CTX_set_read_ahead(ssl->ctx, 1);

     SSL_CTX_set_info_callback(ssl->ctx, ngx_ssl_info_callback);
@@ -366,6 +377,26 @@ ngx_ssl_certificates(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_array_t *certs,
     return NGX_OK;
 }

+static ENGINE *
+ngx_ssl_setup_engine(const char *engine)
+{
+    ENGINE *e = NULL;
+
+#ifndef OPENSSL_NO_ENGINE
+    if (engine) {
+        if ((e = ENGINE_by_id(engine)) == NULL) {
+            return NULL;
+        }
+
+        ENGINE_ctrl_cmd(e, "SET_USER_INTERFACE", 0, NULL, 0, 1);
+        if (!ENGINE_set_default(e, ENGINE_METHOD_ALL)) {
+            ENGINE_free(e);
+            return NULL;
+        }
+    }
+#endif
+    return e;
+}

 ngx_int_t
 ngx_ssl_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *cert,
@@ -375,6 +406,7 @@ ngx_ssl_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *cert,
     X509        *x509;
     u_long       n;
     ngx_str_t   *pwd;
+    char        *qat_engine = "qat";
     ngx_uint_t   tries;

     if (ngx_conf_full_name(cf->cycle, cert, 1) != NGX_OK) {
@@ -494,6 +526,12 @@ ngx_ssl_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *cert,

     BIO_free(bio);

+    ssl->engine = ngx_ssl_setup_engine(qat_engine);
+    if (!ssl->engine) {
+        ngx_ssl_error(NGX_LOG_NOTICE, ssl->log, 0,
+                "Load ENGINE(\"%s\") failed", qat_engine);
+    }
+
     if (ngx_strncmp(key->data, "engine:", sizeof("engine:") - 1) == 0) {

 #ifndef OPENSSL_NO_ENGINE
@@ -1193,6 +1231,7 @@ ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c, ngx_uint_t flags)
     }

     c->ssl = sc;
+    c->asynch = ssl->asynch;

     return NGX_OK;
 }
@@ -1211,6 +1250,100 @@ ngx_ssl_set_session(ngx_connection_t *c, ngx_ssl_session_t *session)
     return NGX_OK;
 }

+ngx_int_t
+ngx_ssl_async_process_fds(ngx_connection_t *c)
+{
+    OSSL_ASYNC_FD *add_fds = NULL;
+    OSSL_ASYNC_FD *del_fds = NULL;
+    size_t        num_add_fds = 0;
+    size_t        num_del_fds = 0;
+    unsigned      loop = 0;
+
+    ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "ngx_ssl_async_process_fds called");
+
+    if (!ngx_del_async_conn || !ngx_add_async_conn) {
+        ngx_ssl_error(NGX_LOG_ALERT, c->log, 0,
+                      "Async notifications not supported");
+        return 0;
+    }
+
+    SSL_get_changed_async_fds(c->ssl->connection, NULL, &num_add_fds,
+                              NULL, &num_del_fds);
+
+    if (num_add_fds) {
+        add_fds = ngx_alloc(num_add_fds * sizeof(OSSL_ASYNC_FD), c->log);
+        if (add_fds == NULL) {
+            ngx_ssl_error(NGX_LOG_ALERT, c->log, 0,
+                          "Memory Allocation Error");
+            return 0;
+        }
+    }
+
+    if (num_del_fds) {
+        del_fds = ngx_alloc(num_del_fds * sizeof(OSSL_ASYNC_FD), c->log);
+        if (del_fds == NULL) {
+            ngx_ssl_error(NGX_LOG_ALERT, c->log, 0,
+                          "Memory Allocation Error");
+            if (add_fds)
+                ngx_free(add_fds);
+            return 0;
+        }
+    }
+
+    SSL_get_changed_async_fds(c->ssl->connection, add_fds, &num_add_fds,
+                              del_fds, &num_del_fds);
+
+    if (num_del_fds) {
+        for (loop = 0; loop < num_del_fds; loop++) {
+            c->async_fd = del_fds[loop];
+            if (c->num_async_fds) {
+                ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, "%s: deleting fd = %d", __func__, c->async_fd);
+                ngx_del_async_conn(c, NGX_DISABLE_EVENT);
+                c->num_async_fds--;
+            }
+        }
+    }
+    if (num_add_fds) {
+        for (loop = 0; loop < num_add_fds; loop++) {
+            if (c->num_async_fds == 0) {
+                c->num_async_fds++;
+                c->async_fd = add_fds[loop];
+                ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, "%s: adding fd = %d", __func__, c->async_fd);
+                ngx_add_async_conn(c);
+            }
+        }
+    }
+
+    if (add_fds)
+        ngx_free(add_fds);
+    if (del_fds)
+        ngx_free(del_fds);
+
+    return 1;
+}
+
+static void
+ngx_ssl_handshake_async_handler(ngx_event_t *aev)
+{
+    ngx_connection_t  *c;
+
+    c = aev->data;
+
+    ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "SSL handshake async handler");
+
+    aev->ready = 0;
+    aev->handler = ngx_ssl_empty_handler;
+    c->read->handler = c->read->saved_handler;
+
+    if (ngx_ssl_handshake(c) == NGX_AGAIN) {
+        return;
+    }
+
+    c->ssl->handler(c);
+}
+

 ngx_int_t
 ngx_ssl_handshake(ngx_connection_t *c)
@@ -1226,6 +1359,10 @@ ngx_ssl_handshake(ngx_connection_t *c)

     if (n == 1) {

+        if (c->asynch && ngx_ssl_async_process_fds(c) == 0) {
+            return NGX_ERROR;
+        }
+
         if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
             return NGX_ERROR;
         }
@@ -1307,6 +1444,10 @@ ngx_ssl_handshake(ngx_connection_t *c)
     ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "SSL_get_error: %d", sslerr);

     if (sslerr == SSL_ERROR_WANT_READ) {
+        if (c->asynch && ngx_ssl_async_process_fds(c) == 0) {
+            return NGX_ERROR;
+        }
+
         c->read->ready = 0;
         c->read->handler = ngx_ssl_handshake_handler;
         c->write->handler = ngx_ssl_handshake_handler;
@@ -1323,6 +1464,10 @@ ngx_ssl_handshake(ngx_connection_t *c)
     }

     if (sslerr == SSL_ERROR_WANT_WRITE) {
+        if (c->asynch && ngx_ssl_async_process_fds(c) == 0) {
+            return NGX_ERROR;
+        }
+
         c->write->ready = 0;
         c->read->handler = ngx_ssl_handshake_handler;
         c->write->handler = ngx_ssl_handshake_handler;
@@ -1338,6 +1483,21 @@ ngx_ssl_handshake(ngx_connection_t *c)
         return NGX_AGAIN;
     }

+    if (c->asynch && sslerr == SSL_ERROR_WANT_ASYNC) {
+        c->async->handler = ngx_ssl_handshake_async_handler;
+        c->read->saved_handler = c->read->handler;
+        c->read->handler = ngx_ssl_empty_handler;
+
+        ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                "SSL ASYNC WANT recieved: \"%s\"", __func__);
+
+        if (ngx_ssl_async_process_fds(c) == 0) {
+            return NGX_ERROR;
+        }
+
+        return NGX_AGAIN;
+    }
+
     err = (sslerr == SSL_ERROR_SYSCALL) ? ngx_errno : 0;

     c->ssl->no_wait_shutdown = 1;
@@ -1378,6 +1538,11 @@ ngx_ssl_handshake_handler(ngx_event_t *ev)
         return;
     }

+    /*
+     * empty the handler of async event to avoid
+     * going back to previous ssl shutdown state
+     */
+    c->async->handler = ngx_ssl_empty_handler;
     c->ssl->handler(c);
 }

@@ -1518,6 +1683,22 @@ ngx_ssl_recv(ngx_connection_t *c, u_char *buf, size_t size)
     }
 }

+static void
+ngx_ssl_read_async_handler(ngx_event_t *aev)
+{
+    ngx_connection_t  *c;
+
+    c = aev->data;
+
+    ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "SSL read async handler");
+
+    aev->ready = 0;
+    aev->handler = ngx_ssl_empty_handler;
+    c->read->handler = c->read->saved_handler;
+
+    c->read->handler(c->read);
+}

 static ngx_int_t
 ngx_ssl_handle_recv(ngx_connection_t *c, int n)
@@ -1549,6 +1730,10 @@ ngx_ssl_handle_recv(ngx_connection_t *c, int n)

     if (n > 0) {

+        if (c->asynch && ngx_ssl_async_process_fds(c) == 0) {
+            return NGX_ERROR;
+        }
+
         if (c->ssl->saved_write_handler) {

             c->write->handler = c->ssl->saved_write_handler;
@@ -1572,6 +1757,10 @@ ngx_ssl_handle_recv(ngx_connection_t *c, int n)
     ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "SSL_get_error: %d", sslerr);

     if (sslerr == SSL_ERROR_WANT_READ) {
+        if (c->asynch && ngx_ssl_async_process_fds(c) == 0) {
+            return NGX_ERROR;
+        }
+
         c->read->ready = 0;
         return NGX_AGAIN;
     }
@@ -1581,6 +1770,10 @@ ngx_ssl_handle_recv(ngx_connection_t *c, int n)
         ngx_log_error(NGX_LOG_INFO, c->log, 0,
                       "peer started SSL renegotiation");

+        if (c->asynch && ngx_ssl_async_process_fds(c) == 0) {
+            return NGX_ERROR;
+        }
+
         c->write->ready = 0;

         if (ngx_handle_write_event(c->write, 0) != NGX_OK) {
@@ -1599,6 +1792,21 @@ ngx_ssl_handle_recv(ngx_connection_t *c, int n)
         return NGX_AGAIN;
     }

+    if (c->asynch && sslerr == SSL_ERROR_WANT_ASYNC) {
+        c->async->handler = ngx_ssl_read_async_handler;
+        c->read->saved_handler = c->read->handler;
+        c->read->handler = ngx_ssl_empty_handler;
+
+        ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                "SSL ASYNC WANT recieved: \"%s\"", __func__);
+
+        if (ngx_ssl_async_process_fds(c) == 0) {
+            return NGX_ERROR;
+        }
+
+        return NGX_AGAIN;
+    }
+
     c->ssl->no_wait_shutdown = 1;
     c->ssl->no_send_shutdown = 1;

@@ -1787,6 +1995,22 @@ ngx_ssl_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
     return in;
 }

+static void
+ngx_ssl_write_async_handler(ngx_event_t *aev)
+{
+    ngx_connection_t  *c;
+
+    c = aev->data;
+
+    ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "SSL write async handler");
+
+    aev->ready = 0;
+    aev->handler = ngx_ssl_empty_handler;
+    c->read->handler = c->read->saved_handler;
+
+    c->write->handler(c->write);
+}

 ssize_t
 ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size)
@@ -1804,6 +2028,10 @@ ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size)

     if (n > 0) {

+        if (c->asynch && ngx_ssl_async_process_fds(c) == 0) {
+            return NGX_ERROR;
+        }
+
         if (c->ssl->saved_read_handler) {

             c->read->handler = c->ssl->saved_read_handler;
@@ -1829,6 +2057,9 @@ ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size)
     ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "SSL_get_error: %d", sslerr);

     if (sslerr == SSL_ERROR_WANT_WRITE) {
+        if (c->asynch && ngx_ssl_async_process_fds(c) == 0) {
+            return NGX_ERROR;
+        }
         c->write->ready = 0;
         return NGX_AGAIN;
     }
@@ -1838,6 +2069,9 @@ ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size)
         ngx_log_error(NGX_LOG_INFO, c->log, 0,
                       "peer started SSL renegotiation");

+        if (c->asynch && ngx_ssl_async_process_fds(c) == 0) {
+            return NGX_ERROR;
+        }
         c->read->ready = 0;

         if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
@@ -1857,6 +2091,21 @@ ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size)
         return NGX_AGAIN;
     }

+    if (c->asynch && sslerr == SSL_ERROR_WANT_ASYNC) {
+        c->async->handler = ngx_ssl_write_async_handler;
+        c->read->saved_handler = c->read->handler;
+        c->read->handler = ngx_ssl_empty_handler;
+
+        ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                "SSL ASYNC WANT recieved: \"%s\"", __func__);
+
+        if (ngx_ssl_async_process_fds(c) == 0) {
+            return NGX_ERROR;
+        }
+
+        return NGX_AGAIN;
+    }
+
     c->ssl->no_wait_shutdown = 1;
     c->ssl->no_send_shutdown = 1;
     c->write->error = 1;
@@ -1888,6 +2137,40 @@ ngx_ssl_free_buffer(ngx_connection_t *c)
     }
 }

+static void
+ngs_ssl_shutdown_async_del_conn(ngx_connection_t *c)
+{
+    ngx_ssl_async_process_fds(c);
+    ngx_asynch_del_conn(c);
+    ngx_del_conn(c, NGX_DISABLE_EVENT);
+}
+
+static void
+ngx_ssl_shutdown_async_handler(ngx_event_t *aev)
+{
+    ngx_connection_t           *c;
+    ngx_connection_handler_pt   handler;
+
+    c = aev->data;
+    handler = c->ssl->handler;
+
+    if (aev->timedout) {
+        c->timedout = 1;
+    }
+
+    ngx_log_debug0(NGX_LOG_DEBUG_EVENT, aev->log, 0,
+                   "SSL shutdown async handler");
+
+    aev->ready = 0;
+    aev->handler = ngx_ssl_empty_handler;
+    c->read->handler = c->read->saved_handler;
+
+    if (ngx_ssl_shutdown(c) == NGX_AGAIN) {
+        return;
+    }
+
+    handler(c);
+}

 ngx_int_t
 ngx_ssl_shutdown(ngx_connection_t *c)
@@ -1902,6 +2185,21 @@ ngx_ssl_shutdown(ngx_connection_t *c)
          * Avoid calling SSL_shutdown() if handshake wasn't completed.
          */

+        if (c->asynch) {
+            /* Check if there is inflight request */
+            if (SSL_want_async(c->ssl->connection) && !c->timedout) {
+                c->async->handler = ngx_ssl_shutdown_async_handler;
+                ngx_ssl_async_process_fds(c);
+                ngx_add_timer(c->async, 300);
+                return NGX_AGAIN;
+            }
+
+            /* Ignore errors from ngx_ssl_async_process_fds as
+             * we want to carry on and close the SSL connection
+             * anyway. */
+            ngs_ssl_shutdown_async_del_conn(c);
+        }
+
         SSL_free(c->ssl->connection);
         c->ssl = NULL;

@@ -1941,13 +2239,26 @@ ngx_ssl_shutdown(ngx_connection_t *c)
     /* before 0.9.8m SSL_shutdown() returned 0 instead of -1 on errors */

     if (n != 1 && ERR_peek_error()) {
+        if (c->asynch && ngx_ssl_async_process_fds(c) == 0) {
+            return NGX_ERROR;
+        }
         sslerr = SSL_get_error(c->ssl->connection, n);
-
         ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
                        "SSL_get_error: %d", sslerr);
     }
+    else if (c->asynch && n == -1) {
+        sslerr = SSL_get_error(c->ssl->connection, n);
+        ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                       "SSL_get_error async: %d", sslerr);
+    }

     if (n == 1 || sslerr == 0 || sslerr == SSL_ERROR_ZERO_RETURN) {
+        if (c->asynch) {
+            /* Ignore errors from ngx_ssl_async_process_fds as
+             * we want to carry on and close the SSL connection
+             * anyway. */
+            ngs_ssl_shutdown_async_del_conn(c);
+        }
         SSL_free(c->ssl->connection);
         c->ssl = NULL;

@@ -1955,6 +2266,9 @@ ngx_ssl_shutdown(ngx_connection_t *c)
     }

     if (sslerr == SSL_ERROR_WANT_READ || sslerr == SSL_ERROR_WANT_WRITE) {
+        if (c->asynch && ngx_ssl_async_process_fds(c) == 0) {
+            return NGX_ERROR;
+        }
         c->read->handler = ngx_ssl_shutdown_handler;
         c->write->handler = ngx_ssl_shutdown_handler;

@@ -1973,6 +2287,28 @@ ngx_ssl_shutdown(ngx_connection_t *c)
         return NGX_AGAIN;
     }

+    if (c->asynch) {
+        if (sslerr == SSL_ERROR_WANT_ASYNC) {
+            c->async->handler = ngx_ssl_shutdown_async_handler;
+            c->read->saved_handler = ngx_ssl_shutdown_handler;
+            c->read->handler = ngx_ssl_empty_handler;
+            c->write->handler = ngx_ssl_shutdown_handler;
+
+            ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                    "SSL ASYNC WANT recieved: \"%s\"", __func__);
+
+            /* Ignore errors from ngx_ssl_async_process_fds as
+             * we want to carry on anyway. */
+            ngx_ssl_async_process_fds(c);
+            return NGX_AGAIN;
+        }
+
+        /* Ignore errors from ngx_ssl_async_process_fds as
+         * we want to carry on and close the SSL connection
+         * anyway. */
+        ngs_ssl_shutdown_async_del_conn(c);
+    }
+
     err = (sslerr == SSL_ERROR_SYSCALL) ? ngx_errno : 0;

     ngx_ssl_connection_error(c, sslerr, err, "SSL_shutdown() failed");
@@ -2003,6 +2339,11 @@ ngx_ssl_shutdown_handler(ngx_event_t *ev)
         return;
     }

+    /*
+     * empty the handler of async event to avoid
+     * going back to previous ssl shutdown state
+     */
+    c->async->handler = ngx_ssl_empty_handler;
     handler(c);
 }

@@ -3145,6 +3486,9 @@ ngx_ssl_cleanup_ctx(void *data)
     }

     SSL_CTX_free(ssl->ctx);
+    if (ssl->engine) {
+        ENGINE_free(ssl->engine);
+    }
 }


diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h
index e093e10..3629d07 100644
--- a/src/event/ngx_event_openssl.h
+++ b/src/event/ngx_event_openssl.h
@@ -57,7 +57,9 @@
 struct ngx_ssl_s {
     SSL_CTX                    *ctx;
     ngx_log_t                  *log;
+    ENGINE                     *engine;
     size_t                      buffer_size;
+    ngx_flag_t                  asynch;
 };


@@ -187,6 +189,7 @@ ngx_int_t ngx_ssl_set_session(ngx_connection_t *c, ngx_ssl_session_t *session);

 ngx_int_t ngx_ssl_check_host(ngx_connection_t *c, ngx_str_t *name);

+#define ngx_ssl_waiting_for_async(c) SSL_waiting_for_async(c->ssl->connection)

 ngx_int_t ngx_ssl_get_protocol(ngx_connection_t *c, ngx_pool_t *pool,
     ngx_str_t *s);
@@ -239,7 +242,7 @@ ngx_int_t ngx_ssl_shutdown(ngx_connection_t *c);
 void ngx_cdecl ngx_ssl_error(ngx_uint_t level, ngx_log_t *log, ngx_err_t err,
     char *fmt, ...);
 void ngx_ssl_cleanup_ctx(void *data);
-
+ngx_int_t ngx_ssl_async_process_fds(ngx_connection_t *c);

 extern int  ngx_ssl_connection_index;
 extern int  ngx_ssl_server_conf_index;
diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c
index 2771ac1..d893ee7 100644
--- a/src/http/modules/ngx_http_ssl_module.c
+++ b/src/http/modules/ngx_http_ssl_module.c
@@ -43,6 +43,8 @@ static char *ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf,

 static char *ngx_http_ssl_enable(ngx_conf_t *cf, ngx_command_t *cmd,
     void *conf);
+static char *ngx_http_ssl_enable_asynch(ngx_conf_t *cf, ngx_command_t *cmd,
+    void *conf);
 static char *ngx_http_ssl_password_file(ngx_conf_t *cf, ngx_command_t *cmd,
     void *conf);
 static char *ngx_http_ssl_session_cache(ngx_conf_t *cf, ngx_command_t *cmd,
@@ -79,6 +81,13 @@ static ngx_command_t  ngx_http_ssl_commands[] = {
       offsetof(ngx_http_ssl_srv_conf_t, enable),
       NULL },

+    { ngx_string("ssl_asynch"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG,
+      ngx_http_ssl_enable_asynch,
+      NGX_HTTP_SRV_CONF_OFFSET,
+      offsetof(ngx_http_ssl_srv_conf_t, enable_asynch),
+      NULL },
+
     { ngx_string("ssl_certificate"),
       NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1,
       ngx_conf_set_str_array_slot,
@@ -541,6 +550,7 @@ ngx_http_ssl_create_srv_conf(ngx_conf_t *cf)
      */

     sscf->enable = NGX_CONF_UNSET;
+    sscf->enable_asynch = NGX_CONF_UNSET;
     sscf->prefer_server_ciphers = NGX_CONF_UNSET;
     sscf->buffer_size = NGX_CONF_UNSET_SIZE;
     sscf->verify = NGX_CONF_UNSET_UINT;
@@ -578,6 +588,17 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child)
         }
     }

+    if (conf->enable_asynch == NGX_CONF_UNSET) {
+        if (prev->enable_asynch == NGX_CONF_UNSET) {
+            conf->enable_asynch = 0;
+
+        } else {
+            conf->enable_asynch = prev->enable_asynch;
+            conf->file = prev->file;
+            conf->line = prev->line;
+        }
+    }
+
     ngx_conf_merge_value(conf->session_timeout,
                          prev->session_timeout, 300);

@@ -650,6 +671,7 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child)
             return NGX_CONF_ERROR;
         }

+        conf->ssl.asynch = conf->enable_asynch;
     } else {

         if (conf->certificates == NULL) {
@@ -822,6 +844,36 @@ ngx_http_ssl_enable(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
     return NGX_CONF_OK;
 }

+static char *
+ngx_http_ssl_enable_asynch(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
+{
+    ngx_http_ssl_srv_conf_t *sscf = conf;
+
+    char  *rv;
+
+    ngx_flag_t       *pssl, *pssl_asynch;
+
+    rv = ngx_conf_set_flag_slot(cf, cmd, conf);
+
+    if (rv != NGX_CONF_OK) {
+        return rv;
+    }
+
+    /* If ssl_asynch on is configured, then ssl on is configured by default
+     * This will align 'ssl_asynch on;' and 'listen port ssl' diretives
+     * */
+    pssl = (ngx_flag_t *) ((char *)conf + offsetof(ngx_http_ssl_srv_conf_t, enable));
+    pssl_asynch = (ngx_flag_t *) ((char *)conf + cmd->offset);
+
+    if(*pssl_asynch && *pssl != 1) {
+        *pssl = *pssl_asynch;
+    }
+
+    sscf->file = cf->conf_file->file.name.data;
+    sscf->line = cf->conf_file->line;
+
+    return NGX_CONF_OK;
+}

 static char *
 ngx_http_ssl_password_file(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
diff --git a/src/http/modules/ngx_http_ssl_module.h b/src/http/modules/ngx_http_ssl_module.h
index 57f5941..b97417a 100644
--- a/src/http/modules/ngx_http_ssl_module.h
+++ b/src/http/modules/ngx_http_ssl_module.h
@@ -17,6 +17,8 @@
 typedef struct {
     ngx_flag_t                      enable;

+    ngx_flag_t                      enable_asynch;
+
     ngx_ssl_t                       ssl;

     ngx_flag_t                      prefer_server_ciphers;
diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c
index 476f039..693c4f0 100644
--- a/src/http/ngx_http_request.c
+++ b/src/http/ngx_http_request.c
@@ -447,9 +447,21 @@ ngx_http_wait_request_handler(ngx_event_t *rev)
          * We are trying to not hold c->buffer's memory for an idle connection.
          */

-        if (ngx_pfree(c->pool, b->start) == NGX_OK) {
-            b->start = NULL;
+        /* For the Async implementation we need the same buffer to be used
+         * again on any async calls that have not completed.
+         * As such we need to turn off this optimisation if an async request
+         * is still in progress.
+         */
+
+#if (NGX_HTTP_SSL)
+        if ((c->asynch && !ngx_ssl_waiting_for_async(c)) || !c->asynch) {
+#endif
+            if (ngx_pfree(c->pool, b->start) == NGX_OK) {
+                b->start = NULL;
+            }
+#if (NGX_HTTP_SSL)
         }
+#endif

         return;
     }
@@ -1385,12 +1397,22 @@ ngx_http_read_request_header(ngx_http_request_t *r)
         return n;
     }

-    if (rev->ready) {
+#if (NGX_HTTP_SSL)
+    if (c->asynch) {
         n = c->recv(c, r->header_in->last,
+                r->header_in->end - r->header_in->last);
+    }
+    else {
+#endif
+        if (rev->ready) {
+            n = c->recv(c, r->header_in->last,
                     r->header_in->end - r->header_in->last);
-    } else {
-        n = NGX_AGAIN;
+        } else {
+            n = NGX_AGAIN;
+        }
+#if (NGX_HTTP_SSL)
     }
+#endif

     if (n == NGX_AGAIN) {
         if (!rev->timer_set) {
@@ -2954,52 +2976,62 @@ ngx_http_set_keepalive(ngx_http_request_t *r)
      * c->pool and are freed too.
      */

-    b = c->buffer;
-
-    if (ngx_pfree(c->pool, b->start) == NGX_OK) {
+    /* For the Async implementation we need the same buffer to be used
+     * again on any async calls that have not completed.
+     * As such we need to turn off this optimisation if an async request
+     * is still in progress.
+     */

-        /*
-         * the special note for ngx_http_keepalive_handler() that
-         * c->buffer's memory was freed
-         */
+#if (NGX_HTTP_SSL)
+    if ((c->asynch && !ngx_ssl_waiting_for_async(c)) || !c->asynch) {
+#endif
+        b = c->buffer;

-        b->pos = NULL;
+        if (ngx_pfree(c->pool, b->start) == NGX_OK) {

-    } else {
-        b->pos = b->start;
-        b->last = b->start;
-    }
+            /*
+             * the special note for ngx_http_keepalive_handler() that
+             * c->buffer's memory was freed
+             */

-    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, "hc free: %p",
-                   hc->free);
+            b->pos = NULL;

-    if (hc->free) {
-        for (cl = hc->free; cl; /* void */) {
-            ln = cl;
-            cl = cl->next;
-            ngx_pfree(c->pool, ln->buf->start);
-            ngx_free_chain(c->pool, ln);
+        } else {
+            b->pos = b->start;
+            b->last = b->start;
         }

-        hc->free = NULL;
-    }
+        ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, "hc free: %p",
+                hc->free);

-    ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0, "hc busy: %p %i",
-                   hc->busy, hc->nbusy);
+        if (hc->free) {
+            for (cl = hc->free; cl; /* void */) {
+                ln = cl;
+                cl = cl->next;
+                ngx_pfree(c->pool, ln->buf->start);
+                ngx_free_chain(c->pool, ln);
+            }

-    if (hc->busy) {
-        for (cl = hc->busy; cl; /* void */) {
-            ln = cl;
-            cl = cl->next;
-            ngx_pfree(c->pool, ln->buf->start);
-            ngx_free_chain(c->pool, ln);
+            hc->free = NULL;
         }

-        hc->busy = NULL;
-        hc->nbusy = 0;
-    }
+        ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0, "hc busy: %p %i",
+                hc->busy, hc->nbusy);

+        if (hc->busy) {
+            for (cl = hc->busy; cl; /* void */) {
+                ln = cl;
+                cl = cl->next;
+                ngx_pfree(c->pool, ln->buf->start);
+                ngx_free_chain(c->pool, ln);
+            }
+
+            hc->busy = NULL;
+            hc->nbusy = 0;
+        }
 #if (NGX_HTTP_SSL)
+    }
+
     if (c->ssl) {
         ngx_ssl_free_buffer(c);
     }
@@ -3008,6 +3040,11 @@ ngx_http_set_keepalive(ngx_http_request_t *r)
     rev->handler = ngx_http_keepalive_handler;

     if (wev->active && (ngx_event_flags & NGX_USE_LEVEL_EVENT)) {
+#if (NGX_HTTP_SSL)
+        if (c->asynch) {
+            ngx_asynch_del_conn(c);
+        }
+#endif
         if (ngx_del_event(wev, NGX_WRITE_EVENT, 0) != NGX_OK) {
             ngx_http_close_connection(c);
             return;
@@ -3152,15 +3189,27 @@ ngx_http_keepalive_handler(ngx_event_t *rev)
          * Like ngx_http_set_keepalive() we are trying to not hold
          * c->buffer's memory for a keepalive connection.
          */
+
+        /* For the Async implementation we need the same buffer to be used
+         * again on any async calls that have not completed.
+         * As such we need to turn off this optimisation if an async request
+         * is still in progress.
+         */

-        if (ngx_pfree(c->pool, b->start) == NGX_OK) {
+#if (NGX_HTTP_SSL)
+        if ((c->asynch && !ngx_ssl_waiting_for_async(c)) || !c->asynch) {
+#endif
+            if (ngx_pfree(c->pool, b->start) == NGX_OK) {

-            /*
-             * the special note that c->buffer's memory was freed
-             */
+                /*
+                 * the special note that c->buffer's memory was freed
+                 */

-            b->pos = NULL;
+                b->pos = NULL;
+            }
+#if (NGX_HTTP_SSL)
         }
+#endif

         return;
     }
@@ -3229,6 +3278,11 @@ ngx_http_set_lingering_close(ngx_http_request_t *r)
     wev->handler = ngx_http_empty_handler;

     if (wev->active && (ngx_event_flags & NGX_USE_LEVEL_EVENT)) {
+#if (NGX_HTTP_SSL)
+        if (c->asynch) {
+            ngx_asynch_del_conn(c);
+        }
+#endif
         if (ngx_del_event(wev, NGX_WRITE_EVENT, 0) != NGX_OK) {
             ngx_http_close_request(r, 0);
             return;
diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c
index 3695286..79bc8e9 100644
--- a/src/http/ngx_http_upstream.c
+++ b/src/http/ngx_http_upstream.c
@@ -1285,7 +1285,11 @@ ngx_http_upstream_check_broken_connection(ngx_http_request_t *r,
         if ((ngx_event_flags & NGX_USE_LEVEL_EVENT) && ev->active) {

             event = ev->write ? NGX_WRITE_EVENT : NGX_READ_EVENT;
-
+#if (NGX_HTTP_SSL)
+            if (c->asynch) {
+                ngx_asynch_del_conn(c);
+            }
+#endif
             if (ngx_del_event(ev, event, 0) != NGX_OK) {
                 ngx_http_upstream_finalize_request(r, u,
                                                NGX_HTTP_INTERNAL_SERVER_ERROR);
@@ -1412,7 +1416,11 @@ ngx_http_upstream_check_broken_connection(ngx_http_request_t *r,
     if ((ngx_event_flags & NGX_USE_LEVEL_EVENT) && ev->active) {

         event = ev->write ? NGX_WRITE_EVENT : NGX_READ_EVENT;
-
+#if (NGX_HTTP_SSL)
+        if (c->asynch) {
+            ngx_asynch_del_conn(c);
+        }
+#endif
         if (ngx_del_event(ev, event, 0) != NGX_OK) {
             ngx_http_upstream_finalize_request(r, u,
                                                NGX_HTTP_INTERNAL_SERVER_ERROR);
diff --git a/src/os/unix/ngx_process_cycle.c b/src/os/unix/ngx_process_cycle.c
index 1710ea8..7228198 100644
--- a/src/os/unix/ngx_process_cycle.c
+++ b/src/os/unix/ngx_process_cycle.c
@@ -1027,6 +1027,11 @@ ngx_channel_handler(ngx_event_t *ev)
         if (n == NGX_ERROR) {

             if (ngx_event_flags & NGX_USE_EPOLL_EVENT) {
+#if (NGX_HTTP_SSL)
+                if (c->asynch) {
+                    ngx_asynch_del_conn(c);
+                }
+#endif
                 ngx_del_conn(c, 0);
             }

# tar xzf nginx-1.12.2.tar.gz

# cd nginx-1.12.2/

# patch –Np1 –i /path/to/nginx-qat.patch

# OPENSSL_LIB=/usr/local/ssl

# ./configure --with-http_ssl_module \

   --conf-path=/etc/nginx/nginx.conf \

   --http-log-path=/var/log/nginx/access.log \

   --error-log-path=/var/log/nginx/error.log \

    --with-debug \

   --with-cc-opt="-I$OPENSSL_LIB/include" \

   --with-ld-opt="-Wl,-rpath=$OPENSSL_LIB/lib -L$OPENSSL_LIB/lib"

# make

# make install

2.5 HAProxy

# tar xzf haproxy-1.8.4.tar.gz

# cd haproxy-1.8.4

#make TARGET=linux26 USE_OPENSSL=1SSL_INC=/usr/local/ssl/include SSL_LIB=/usr/local/ssl/lib ADDLIB=-ldl

3.測試性能

3.1 OpenSSL

3.1.1 QAT Engine加載

# cd /path/to/openssl-OpenSSL_1_1_0g/apps

# ./openssl engine -t -c-vvvv qat

(qat) Reference implementation of QAT crypto engine
 [RSA, DSA, DH, AES-128-CBC-HMAC-SHA1, AES-128-CBC-HMAC-SHA256, AES-256-CBC-HMAC-SHA1, AES-256-CBC-HMAC-SHA256, TLS1-PRF]
     [ available ]
     ENABLE_EXTERNAL_POLLING: Enables the external polling interface to the engine.
          (input flags): NO_INPUT
     POLL: Polls the engine for any completed requests
          (input flags): NO_INPUT
     SET_INSTANCE_FOR_THREAD: Set instance to be used by this thread
          (input flags): NUMERIC
     GET_NUM_OP_RETRIES: Get number of retries
          (input flags): NO_INPUT
     SET_MAX_RETRY_COUNT: Set maximum retry count
          (input flags): NUMERIC
     SET_INTERNAL_POLL_INTERVAL: Set internal polling interval
          (input flags): NUMERIC
     GET_EXTERNAL_POLLING_FD: Returns non blocking fd for crypto engine
          (input flags): NO_INPUT
     ENABLE_EVENT_DRIVEN_POLLING_MODE: Set event driven polling mode
          (input flags): NO_INPUT
     GET_NUM_CRYPTO_INSTANCES: Get the number of crypto instances
          (input flags): NO_INPUT
     DISABLE_EVENT_DRIVEN_POLLING_MODE: Unset event driven polling mode
          (input flags): NO_INPUT
     SET_EPOLL_TIMEOUT: Set epoll_wait timeout
          (input flags): NUMERIC
     SET_CRYPTO_SMALL_PACKET_OFFLOAD_THRESHOLD: Set QAT small packet threshold
          (input flags): STRING
     ENABLE_INLINE_POLLING: Enables the inline polling mode.
          (input flags): NO_INPUT

3.1.2 RSA 2048

軟件同步：

# ./openssl speed-elapsed rsa2048

硬件同步：

# ./openssl speed-engine qat -elapsed rsa2048

硬件異步：

# ./openssl speed-engine qat -elapsed -async_jobs 72 rsa2048

3.1.3 ECDH

軟件同步：

# ./openssl speed-elapsed ecdh

硬件同步：

# ./openssl speed-engine qat -elapsed ecdh

硬件異步：

# ./openssl speed-engine qat -elapsed -async_jobs 36 ecdh

3.1.4 ECDSA

軟件同步：

# ./openssl speed-elapsed ecdsa

硬件同步：

# ./openssl speed-engine qat -elapsed ecdsa

硬件異步：

# ./openssl speed-engine qat -elapsed -async_jobs 36 ecdsa

3.1.5 aes-128-cbc-hmac-sha256

軟件同步：

# ./openssl speed-elapsed -multi 2 -evp aes-128-cbc-hmac-sha256

硬件同步：

# ./openssl speed-engine qat -elapsed -multi 2 -evp aes-128-cbc-hmac-sha256

硬件異步：

# ./openssl speed-engine qat -elapsed -async_jobs 128 -multi 2 -evp aes-128-cbc-hmac-sha256

3.2 Nginx

3.2.1配置文件

/etc/nginx/nginx.conf:

user root;
worker_processes auto;
pid /run/nginx.pid;

events {
        worker_connections 768;
}

http {

        sendfile on;
        tcp_nopush on;
        tcp_nodelay on;
        keepalive_timeout 65;
        types_hash_max_size 2048;

        include /etc/nginx/mime.types;
        default_type application/octet-stream;

        ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
        ssl_prefer_server_ciphers on;

        access_log /var/log/nginx/access.log;
        error_log /var/log/nginx/error.log;

        gzip on;
        gzip_disable "msie6";

        include /etc/nginx/conf.d/*.conf;
}

/etc/nginx/conf.d/v.conf:

upstream domain1 {
    server 181.1.1.1;
}

server {
    listen       447 ssl;

    ssl_asynch          on;
    ssl_protocols       TLSv1 TLSv1.1 TLSv1.2;
    ssl_certificate     /root/cert/ecdsa.pem;
    ssl_certificate_key /root/cert/ecdsa.key;
        location / {
                proxy_pass http://domain1;
    }
}

3.2.2 運行

# /usr/local/nginx/sbin/nginx

3.3 HAproxy

3.3.1 配置文件

ssl.cfg:

global
    maxconn 100
    ssl-mode-async
    ssl-engine qat

defaults
        mode http
        timeout connect 5s
        timeout client 5s
        timeout server 5s

frontend myfrontend
        bind :447 ssl crt /root/hap-cert/cert.pem crt /root/hap-cert/
        default_backend mybackend

backend mybackend
        server s3 181.1.1.1:80

3.3.2 運行

# cd haproxy-1.8.4

# ./haproxy –f ssl.cfg

3.4 測試

# cat /sys/kernel/debug/qat_c6xx_04\:00.0/fw_counters

跑流量。完畢後再次執行：

# cat /sys/kernel/debug/qat_c6xx_04\:00.0/fw_counters

如果數據發送變化，說明QAT生效。