database/sql query 超時設置

參考文章:Query

問題描述

接上文，mysql的同步訪問問題解決後，繼續壓測，發現訪問mysql的耗時逐漸增大，影響模塊整體處理能力。我設置了readTimeout，沒有任何反應，請求耗時還是會超過readTimeout所設置的值。因爲：

• readTimeout只能限制連接數據讀取時間，如果程序發生在獲取連接前等待時間過長，無法通過這個參數設置，且readTimeout*3纔是真正的讀取超時時間，因爲會重試3次。
• 當設置的MaxOpenConn消耗殆盡時，再次進入的請求會夯死在database/sql內部，直到有空閒連接可操作。

顯然Query方法是無法滿足我們的需求。

解決方法

翻源碼，發現了一個新方法QueryContext,通過傳入context，該context有超時時間，那麼就達到了該查詢請求有了超時設置，nice。

// conn returns a newly-opened or cached *driverConn.
func (db *DB) conn(ctx context.Context, strategy connReuseStrategy) (*driverConn, error) {
    db.mu.Lock()
    if db.closed {
        db.mu.Unlock()
        return nil, errDBClosed
    }
    // Check if the context is expired.
    select {
    default:
    case <-ctx.Done():
        db.mu.Unlock()
        return nil, ctx.Err()
    }
    lifetime := db.maxLifetime

    // Prefer a free connection, if possible.
    numFree := len(db.freeConn)
    if strategy == cachedOrNewConn && numFree > 0 {
        conn := db.freeConn[0]
        copy(db.freeConn, db.freeConn[1:])
        db.freeConn = db.freeConn[:numFree-1]
        conn.inUse = true
        db.mu.Unlock()
        if conn.expired(lifetime) {
            conn.Close()
            return nil, driver.ErrBadConn
        }
        // Lock around reading lastErr to ensure the session resetter finished.
        conn.Lock()
        err := conn.lastErr
        conn.Unlock()
        if err == driver.ErrBadConn {
            conn.Close()
            return nil, driver.ErrBadConn
        }
        return conn, nil
    }

    // Out of free connections or we were asked not to use one. If we're not
    // allowed to open any more connections, make a request and wait.
    if db.maxOpen > 0 && db.numOpen >= db.maxOpen {
        // Make the connRequest channel. It's buffered so that the
        // connectionOpener doesn't block while waiting for the req to be read.
        req := make(chan connRequest, 1)
        reqKey := db.nextRequestKeyLocked()
        db.connRequests[reqKey] = req
        db.waitCount++
        db.mu.Unlock()

        waitStart := time.Now()

        // Timeout the connection request with the context.
        select {
        case <-ctx.Done():
            // Remove the connection request and ensure no value has been sent
            // on it after removing.
            db.mu.Lock()
            delete(db.connRequests, reqKey)
            db.mu.Unlock()

            atomic.AddInt64(&db.waitDuration, int64(time.Since(waitStart)))

            select {
            default:
            case ret, ok := <-req:
                if ok && ret.conn != nil {
                    db.putConn(ret.conn, ret.err, false)
                }
            }
            return nil, ctx.Err()
        case ret, ok := <-req:
            atomic.AddInt64(&db.waitDuration, int64(time.Since(waitStart)))

            if !ok {
                return nil, errDBClosed
            }
            if ret.err == nil && ret.conn.expired(lifetime) {
                ret.conn.Close()
                return nil, driver.ErrBadConn
            }
            if ret.conn == nil {
                return nil, ret.err
            }
            // Lock around reading lastErr to ensure the session resetter finished.
            ret.conn.Lock()
            err := ret.conn.lastErr
            ret.conn.Unlock()
            if err == driver.ErrBadConn {
                ret.conn.Close()
                return nil, driver.ErrBadConn
            }
            return ret.conn, ret.err
        }
    }

    db.numOpen++ // optimistically
    db.mu.Unlock()
    ci, err := db.connector.Connect(ctx)
    if err != nil {
        db.mu.Lock()
        db.numOpen-- // correct for earlier optimism
        db.maybeOpenNewConnections()
        db.mu.Unlock()
        return nil, err
    }
    db.mu.Lock()
    dc := &driverConn{
        db:        db,
        createdAt: nowFunc(),
        ci:        ci,
        inUse:     true,
    }
    db.addDepLocked(dc, dc)
    db.mu.Unlock()
    return dc, nil
}

使用該訪問進行query查詢

sqlCtx, _ := context.WithTimeout(context.Background(), time.Duration(p.queryTimeout)*time.Millisecond)
     //defer cancel()
 rows, err := p.orderClient.QueryContext(sqlCtx, sqlStr)

壓測數據

時間單位都是ms

thread_num	mysql_query_timeout	max_conn_num	max_idle_conn	qps	mysql_err_num	mysql_err_max_time	mysql_succ_num	mysql_succ_max_time
1000	1000	0	50	4000	10	1500	6280	317
1000	1000	50	50	3500	135	1020	5370	833
1000	2000	50	50	3300	39	2020	5040	1172
400	2000	50	50	2800	3	2015	4620	640
400	1000	50	50	2800	9	1020	4646	484

數據顯示SetMaxOpenConns爲0的情況下，設置context是沒有效果的。但是>0後，該timeout設置有效，原因也很好理解，因爲maxOpenConn=0時，請求不會等待空閒連接。所以也不會有ctx.Done()邏輯存在。