利用feather快速處理大數據

Feather是一個快速、輕量級的存儲框架，可以在應用在pandas的Dataframe數據結構中。

讀寫數據

import feather
import pandas as pd

def read_csv_feature(file_in):
    # 讀
    f = open(file_in, encoding='utf-8')
    reader = pd.read_csv(f, sep=',',iterator=True)
    loop = True
    chunkSize= 10000
    chunks = []
    while loop:
        try:
            chunk = reader.get_chunk(chunkSize)
            chunks.apped(chunk)
        except StopIteration:
            loop = False
            print('Iteration is stopped')
        df = pd.concat(chunks, ignore_index=True)

    return df


def write_csv_feature(file_in, file_out):
    # 寫
    df = read_csv_feature(file_in)
    print(df.count())
    feather = feather.write_dataframe(df, file_out)