Redis源碼分析：過期key刪除與設置key的過期時間

Redis中設置key過期時間與過期key的處理流程

在Redis中，可以再設置值的時候就設置該Key的過期時間，也可以通過在expire命令來設置某個key值的過期時間，並且在瞭解完設置過期時間之後，來查看一下Redis的過期key的處理流程。本文先來了解一下Redis的過期鍵的相關流程。

Redis設置key的過期時間

通過set設置過期時間

/* SET key value [NX] [XX] [EX <seconds>] [PX <milliseconds>] */
void setCommand(client *c) {
    int j;
    robj *expire = NULL;
    int unit = UNIT_SECONDS;
    int flags = OBJ_SET_NO_FLAGS;

    for (j = 3; j < c->argc; j++) {                                     // 判斷set過程中輸入的幾個參數
        char *a = c->argv[j]->ptr;
        robj *next = (j == c->argc-1) ? NULL : c->argv[j+1];            // 獲取下一個輸入參數

        if ((a[0] == 'n' || a[0] == 'N') &&
            (a[1] == 'x' || a[1] == 'X') && a[2] == '\0' &&
            !(flags & OBJ_SET_XX))
        {
            flags |= OBJ_SET_NX;                                        // 判斷是否是NX標誌，只有在鍵不存在的情況下操作
        } else if ((a[0] == 'x' || a[0] == 'X') &&
                   (a[1] == 'x' || a[1] == 'X') && a[2] == '\0' &&
                   !(flags & OBJ_SET_NX))
        {
            flags |= OBJ_SET_XX;                                        // 判斷是否是XX標誌，只有在鍵存在的情況下才操作
        } else if ((a[0] == 'e' || a[0] == 'E') &&
                   (a[1] == 'x' || a[1] == 'X') && a[2] == '\0' &&
                   !(flags & OBJ_SET_PX) && next)
        {
            flags |= OBJ_SET_EX;                                        // 是否設置的過期時間爲秒
            unit = UNIT_SECONDS;
            expire = next;                                                      
            j++;
        } else if ((a[0] == 'p' || a[0] == 'P') &&
                   (a[1] == 'x' || a[1] == 'X') && a[2] == '\0' &&
                   !(flags & OBJ_SET_EX) && next)
        {
            flags |= OBJ_SET_PX;                                        // 是否設置的過期時間爲毫秒
            unit = UNIT_MILLISECONDS;
            expire = next;
            j++;
        } else {
            addReply(c,shared.syntaxerr);                               // 參數解析失敗則返回語法錯誤
            return;
        }
    }

    c->argv[2] = tryObjectEncoding(c->argv[2]);                         // 對傳入值進行編碼
    setGenericCommand(c,flags,c->argv[1],c->argv[2],expire,unit,NULL,NULL);   // 調用設置命令
}

從整個流程來看，set在檢查完輸入的相關標誌之後，就直接調用了setGenericCommand方法來進行設置，該方法就是最終對key進行過期設置的函數，

void setGenericCommand(client *c, int flags, robj *key, robj *val, robj *expire, int unit, robj *ok_reply, robj *abort_reply) {
    long long milliseconds = 0; /* initialized to avoid any harmness warning */

    if (expire) {
        if (getLongLongFromObjectOrReply(c, expire, &milliseconds, NULL) != C_OK)   // 將過期時間進行轉換，如果失敗則返回
            return;
        if (milliseconds <= 0) {                                                    // 如果轉換成微秒時間小於0則傳入的時間有誤
            addReplyErrorFormat(c,"invalid expire time in %s",c->cmd->name);
            return;
        }
        if (unit == UNIT_SECONDS) milliseconds *= 1000;                             // 轉換成毫秒
    }

    if ((flags & OBJ_SET_NX && lookupKeyWrite(c->db,key) != NULL) ||        
        (flags & OBJ_SET_XX && lookupKeyWrite(c->db,key) == NULL))                  // 檢查是否需要建存在的情況 如果是NX表示鍵不存在則操作 如果XX鍵存在才能操作
    {
        addReply(c, abort_reply ? abort_reply : shared.nullbulk);                   // 如果不滿足條件則返回中止命令
        return;
    }
    setKey(c->db,key,val);                                                          // 往對應的數據庫中設置該key
    server.dirty++;
    if (expire) setExpire(c,c->db,key,mstime()+milliseconds);                       // 如果有過期時間，則調用setExpire來進行過期時間的設置
    notifyKeyspaceEvent(NOTIFY_STRING,"set",key,c->db->id);                         // 通知事件
    if (expire) notifyKeyspaceEvent(NOTIFY_GENERIC, 
        "expire",key,c->db->id); 
    addReply(c, ok_reply ? ok_reply : shared.ok);                                   // 返回設置成功
}

通過流程的概述可知，在最後需要調用setExpire來真正的將對應的過期時間設置進去。

void setExpire(client *c, redisDb *db, robj *key, long long when) {
    dictEntry *kde, *de;

    /* Reuse the sds from the main dict in the expire dict */
    kde = dictFind(db->dict,key->ptr);                          // 從db中查找到對應的key的entry
    serverAssertWithInfo(NULL,key,kde != NULL);                 // 判斷是否未找到
    de = dictAddOrFind(db->expires,dictGetKey(kde));            // 在expires中查找對應的entry的key值
    dictSetSignedIntegerVal(de,when);                           // 設置過期時間

    int writable_slave = server.masterhost && server.repl_slave_ro == 0;
    if (c && writable_slave && !(c->flags & CLIENT_MASTER))   
        rememberSlaveKeyWithExpire(db,key);
}

主要通過dictSetSignedIntegerVal來設置到已經查找到的dictEntry中的v對應的s64字段，從而保存查找的時間，至此我們知道了設置的過期的key會被添加到expires這個字典中單獨去保存。

通過expire來設置過期時間

/* EXPIRE key seconds */
void expireCommand(client *c) {
    expireGenericCommand(c,mstime(),UNIT_SECONDS);  // 通過秒來過期
}

/* This is the generic command implementation for EXPIRE, PEXPIRE, EXPIREAT
 * and PEXPIREAT. Because the commad second argument may be relative or absolute
 * the "basetime" argument is used to signal what the base time is (either 0
 * for *AT variants of the command, or the current time for relative expires).
 *
 * unit is either UNIT_SECONDS or UNIT_MILLISECONDS, and is only used for
 * the argv[2] parameter. The basetime is always specified in milliseconds. */
void expireGenericCommand(client *c, long long basetime, int unit) {
    robj *key = c->argv[1], *param = c->argv[2];                                    // 獲取參數
    long long when; /* unix time in milliseconds when the key will expire. */

    if (getLongLongFromObjectOrReply(c, param, &when, NULL) != C_OK)                // 轉換過期時間
        return;

    if (unit == UNIT_SECONDS) when *= 1000;                                         
    when += basetime;                                                               // 獲取將來過期的時間

    /* No key, return zero. */
    if (lookupKeyWrite(c->db,key) == NULL) {                                        // 先查找該Key 如果沒有查找到則返回空
        addReply(c,shared.czero);
        return;
    }

    /* EXPIRE with negative TTL, or EXPIREAT with a timestamp into the past
     * should never be executed as a DEL when load the AOF or in the context
     * of a slave instance.
     *
     * Instead we take the other branch of the IF statement setting an expire
     * (possibly in the past) and wait for an explicit DEL from the master. */
    if (when <= mstime() && !server.loading && !server.masterhost) {                // 如果過期的時間小於當前獲取的時間 如果server不是loading狀態並且不是Master模式
        robj *aux;

        int deleted = server.lazyfree_lazy_expire ? dbAsyncDelete(c->db,key) :
                                                    dbSyncDelete(c->db,key);        // 是否是惰性刪除
        serverAssertWithInfo(c,key,deleted);                                        // 檢查該key是否刪除
        server.dirty++;

        /* Replicate/AOF this as an explicit DEL or UNLINK. */
        aux = server.lazyfree_lazy_expire ? shared.unlink : shared.del;             
        rewriteClientCommandVector(c,2,aux,key);
        signalModifiedKey(c->db,key);                                               // 是否是在事務中如果事務中下一個則報錯
        notifyKeyspaceEvent(NOTIFY_GENERIC,"del",key,c->db->id);
        addReply(c, shared.cone);
        return;
    } else {
        setExpire(c,c->db,key,when);                                                // 設置過期刪除的時間
        addReply(c,shared.cone);                                                    // 返回成功
        signalModifiedKey(c->db,key);                                       
        notifyKeyspaceEvent(NOTIFY_GENERIC,"expire",key,c->db->id);
        server.dirty++;
        return;
    }
}

通過該epxire命令的執行可知，如果該鍵還未過期則調用setExpire來設置該鍵的過期時間，該函數的執行流程已經分析過。

過期key刪除

通過get等命令獲取key時

檢查鍵的是否過程主要是通過在獲取到key時，然後獲取key時，來判斷該key是否已經過期。在getCommand中，會調用到如下流程

void getCommand(client *c) {
    getGenericCommand(c);
}

int getGenericCommand(client *c) {
    robj *o;

    if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.nullbulk)) == NULL)  // 先查找該key
        return C_OK;

    if (o->type != OBJ_STRING) {
        addReply(c,shared.wrongtypeerr);
        return C_ERR;
    } else {
        addReplyBulk(c,o);
        return C_OK;
    }
}

robj *lookupKeyReadOrReply(client *c, robj *key, robj *reply) {
    robj *o = lookupKeyRead(c->db, key);    // 查找該key
    if (!o) addReply(c,reply);
    return o;
}

robj *lookupKeyRead(redisDb *db, robj *key) {
    return lookupKeyReadWithFlags(db,key,LOOKUP_NONE);
}

在lookupKeyReadWithFlags函數中，有該expireIfNeeded來判斷是否過期刪除；

int expireIfNeeded(redisDb *db, robj *key) {
    if (!keyIsExpired(db,key)) return 0;                                        // 檢查是否是過期了 如果沒有過期則返回0

    /* If we are running in the context of a slave, instead of
     * evicting the expired key from the database, we return ASAP:
     * the slave key expiration is controlled by the master that will
     * send us synthesized DEL operations for expired keys.
     *
     * Still we try to return the right information to the caller,
     * that is, 0 if we think the key should be still valid, 1 if
     * we think the key is expired at this time. */
    if (server.masterhost != NULL) return 1;

    /* Delete the key */
    server.stat_expiredkeys++;
    propagateExpire(db,key,server.lazyfree_lazy_expire);            
    notifyKeyspaceEvent(NOTIFY_EXPIRED,
        "expired",key,db->id);
    return server.lazyfree_lazy_expire ? dbAsyncDelete(db,key) :
                                         dbSyncDelete(db,key);              // 判斷是通過惰性刪除還是同步刪除
}

這就是在獲取的過程中來刪除已經過期的key，刪除的過程中既要在數據庫中保存的數據刪除還需要在過期字典中刪除。

定時任務刪除過期key

從以上流程可知，設置key的過期都是設置了過期時間之後就執行完成，此時，還需要知道什麼時候會去檢查哪些key過期了呢，什麼時候去檢查呢，答案就是在Redis服務器啓動的時候，初始化了一個serverCron函數，定時執行；在該函數中有如下過程；

databasesCron  -> activeExpireCycle

activeExpireCycle函數就是刪除對應過期key的執行函數；

void activeExpireCycle(int type) {
    /* This function has some global state in order to continue the work
     * incrementally across calls. */
    static unsigned int current_db = 0; /* Last DB tested. */
    static int timelimit_exit = 0;      /* Time limit hit in previous call? */
    static long long last_fast_cycle = 0; /* When last fast cycle ran. */

    int j, iteration = 0;
    int dbs_per_call = CRON_DBS_PER_CALL;
    long long start = ustime(), timelimit, elapsed;

    /* When clients are paused the dataset should be static not just from the
     * POV of clients not being able to write, but also from the POV of
     * expires and evictions of keys not being performed. */
    if (clientsArePaused()) return;

    if (type == ACTIVE_EXPIRE_CYCLE_FAST) {                                     // 是否是快速的過期
        /* Don't start a fast cycle if the previous cycle did not exit
         * for time limit. Also don't repeat a fast cycle for the same period
         * as the fast cycle total duration itself. */
        if (!timelimit_exit) return;                                                // 如果過期時間不存在則返回
        if (start < last_fast_cycle + ACTIVE_EXPIRE_CYCLE_FAST_DURATION*2) return;
        last_fast_cycle = start;                                                    // 開始時間設置成最後一次時間
    }

    /* We usually should test CRON_DBS_PER_CALL per iteration, with
     * two exceptions:
     *
     * 1) Don't test more DBs than we have.
     * 2) If last time we hit the time limit, we want to scan all DBs
     * in this iteration, as there is work to do in some DB and we don't want
     * expired keys to use memory for too much time. */
    if (dbs_per_call > server.dbnum || timelimit_exit)
        dbs_per_call = server.dbnum;                                                // 限制訪問的數據庫數量

    /* We can use at max ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC percentage of CPU time
     * per iteration. Since this function gets called with a frequency of
     * server.hz times per second, the following is the max amount of
     * microseconds we can spend in this function. */
    timelimit = 1000000*ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC/server.hz/100;           // 獲取過期的時間限制值
    timelimit_exit = 0;
    if (timelimit <= 0) timelimit = 1;

    if (type == ACTIVE_EXPIRE_CYCLE_FAST)
        timelimit = ACTIVE_EXPIRE_CYCLE_FAST_DURATION; /* in microseconds. */       // 轉換成微秒

    /* Accumulate some global stats as we expire keys, to have some idea
     * about the number of keys that are already logically expired, but still
     * existing inside the database. */
    long total_sampled = 0;
    long total_expired = 0;

    for (j = 0; j < dbs_per_call && timelimit_exit == 0; j++) {                     // 遍歷對應的數據庫 檢查是否需要停止
        int expired;
        redisDb *db = server.db+(current_db % server.dbnum);                        // 獲取對應的數據庫

        /* Increment the DB now so we are sure if we run out of time
         * in the current DB we'll restart from the next. This allows to
         * distribute the time evenly across DBs. */
        current_db++;

        /* Continue to expire if at the end of the cycle more than 25%
         * of the keys were expired. */
        do {
            unsigned long num, slots;
            long long now, ttl_sum;
            int ttl_samples;
            iteration++;

            /* If there is nothing to expire try next DB ASAP. */
            if ((num = dictSize(db->expires)) == 0) {                               // 獲取過期字典列表 如果該數據庫沒有過期字典則停止
                db->avg_ttl = 0;
                break;
            }
            slots = dictSlots(db->expires);
            now = mstime();

            /* When there are less than 1% filled slots getting random
             * keys is expensive, so stop here waiting for better times...
             * The dictionary will be resized asap. */
            if (num && slots > DICT_HT_INITIAL_SIZE &&
                (num*100/slots < 1)) break;

            /* The main collection cycle. Sample random keys among keys
             * with an expire set, checking for expired ones. */
            expired = 0;
            ttl_sum = 0;
            ttl_samples = 0;

            if (num > ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP)
                num = ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP;             // 獲取過期key的次數

            while (num--) {
                dictEntry *de;
                long long ttl;

                if ((de = dictGetRandomKey(db->expires)) == NULL) break;        // 隨機獲取key
                ttl = dictGetSignedIntegerVal(de)-now;                          
                if (activeExpireCycleTryExpire(db,de,now)) expired++;           // 如果超時了則過期刪除掉
                if (ttl > 0) {
                    /* We want the average TTL of keys yet not expired. */
                    ttl_sum += ttl;                                             // 如果沒有過期則獲取過期時間
                    ttl_samples++;
                }
                total_sampled++;
            }
            total_expired += expired;                                           // 加上已經刪除的key的數量

            /* Update the average TTL stats for this database. */
            if (ttl_samples) {
                long long avg_ttl = ttl_sum/ttl_samples;

                /* Do a simple running average with a few samples.
                 * We just use the current estimate with a weight of 2%
                 * and the previous estimate with a weight of 98%. */
                if (db->avg_ttl == 0) db->avg_ttl = avg_ttl;
                db->avg_ttl = (db->avg_ttl/50)*49 + (avg_ttl/50);               // 更新平均過期的比率
            }

            /* We can't block forever here even if there are many keys to
             * expire. So after a given amount of milliseconds return to the
             * caller waiting for the other active expire cycle. */
            if ((iteration & 0xf) == 0) { /* check once every 16 iterations. */
                elapsed = ustime()-start;
                if (elapsed > timelimit) {
                    timelimit_exit = 1;
                    server.stat_expired_time_cap_reached_count++;
                    break;
                }
            }
            /* We don't repeat the cycle if there are less than 25% of keys
             * found expired in the current DB. */
        } while (expired > ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP/4);             // 如果找到的值過期的比率不足25%則停止
    }

    elapsed = ustime()-start;
    latencyAddSampleIfNeeded("expire-cycle",elapsed/1000);                      // 設置執行消耗的時間

    /* Update our estimate of keys existing but yet to be expired.
     * Running average with this sample accounting for 5%. */
    double current_perc;
    if (total_sampled) {
        current_perc = (double)total_expired/total_sampled;
    } else
        current_perc = 0;
    server.stat_expired_stale_perc = (current_perc*0.05)+
                                     (server.stat_expired_stale_perc*0.95);
}

該函數主要分兩種情況下的刪除；

1. 快速刪除，即該函數將嘗試不超過一定時間內不重複執行的策略，即不超過兩個1000微秒
2. 慢刪除，即通過時間限制爲REDIS_HZ週期的百分比，來執行刪除的過期key

通過隨機的抽取百分之二十五的key來達到刪除過期key的目的，從而保證了一些過期的key在很久不訪問時佔用內存資源。

總結

本文主要是大致瞭解了一下，在Redis中key的過期設置的主要的流程，Redis是如何通過訪問key時檢測與定時任務來清理過期的key從而節省空間。由於本人才疏學淺，如有錯誤請批評指正。