加上令人頭疼的公司的事情, 導致我很忙... 哎...
爲了保持博客的新鮮度, 就貼一點充數的資料吧..呵呵...
在我前段時間搞的一個開源的mapreduce小框架中, 對於map, reduce數據流的讀取, 我抽象出一個迭代器, 進行數據的讀取.
這樣我的數據無論來自local Disk, 來自memory, 來時NFS都一視同仁...
在寫一個module之前,我先想象其接口及用法:
%%% usage:
%%% Iter = make_iter(prev_fun, next_fun, Ｓtate),
%%% {Next , Iter2} = next(Iter),
%%% {Next2, Iter3} = next(Iter2),
%%% {Next, Iter2} = prev(Iter3).我們首先通過調用make_iter/3 構建一個迭代器, 第一個參數是一個函數用來獲取前一條記錄, 第二個參數用來獲取下一條
記錄. State是一個內部狀態信息, 用來保存當前的狀態.
構造好iter後, 我們可以通過調用prev/1, next/1 來遍歷這個Iter.
這個實現,有點OO的感覺在裏面, Iter可以理解爲一個對象,其State爲內部數據, prev_fun, next_fun爲對應的方法.
爲了讓我們的iter模塊更豐富,我們仿照lists module, 加入了foldl/3 和 foldr/3 兩個函數.
當然還可以讓這個iter功能更強大,比如是否可以循環迭代?是否可以雙向或單向迭代?等等.有興趣朋友可以實現哦.
好了,貼上代碼:
-module(heng_iter).
-author('[email protected]').
-vsn('0.1').
-include("heng_master.hrl").
-export([make_iter/3]).
-export([next/1, prev/1]).
-export([foldl/3, foldr/3]).
-record(iter,
{
%type = normal :: 'normal' | 'cyclic', %
next = nil :: 'nil' | fun(),
prev = nil :: 'nil' | fun(),
state
}).
%% @doc get the next item
next(#iter{next = nil}) ->
{error, edirection};
next(Iter = #iter{next = Next, state = State}) ->
case Next(State) of
nil -> % to the end
{nil, Iter};
{V, State2} ->
{{value, V}, Iter#iter{state = State2}}
end.
%% @doc get the item in the backward
prev(#iter{prev = nil}) ->
{error, edirection};
prev(Iter = #iter{prev = Prev, state = State}) ->
case Prev(State) of
nil -> % to the end
{nil, Iter};
{V, State2} ->
{{value, V}, Iter#iter{state = State2}}
end.
%-spec foldl(Fun :: fun(), Acc :: term(), Iter :: iter()) ->
% term().
foldl(Fun, Acc, Iter) when is_function(Fun, 2) ->
{V, Iter2} = next(Iter),
foldl0(Fun, Acc, V, Iter2).
foldl0(_Fun, Acc, nil, _Iter) ->
Acc;
foldl0(Fun, Acc, {value, V}, Iter) ->
Acc2 = Fun(V, Acc),
{V2, Iter2} = next(Iter),
foldl0(Fun, Acc2, V2, Iter2).
foldr(Fun, Acc, Iter) when is_function(Fun, 2) ->
{V, Iter2} = prev(Iter),
foldr0(Fun, Acc, V, Iter2).
foldr0(_Fun, Acc, nil, _Iter) ->
Acc;
foldr0(Fun, Acc, {value, V}, Iter) ->
Acc2 = Fun(V, Acc),
{V2, Iter2} = prev(Iter),
foldr0(Fun, Acc2, V2, Iter2).
%% @doc 'fward' equal 'forward' , 'bid' equal 'bidirectional'
make_iter(Next, Prev, State)
when is_function(Next) andalso is_function(Prev)
%andalso ((Type =:= 'normal') or (Type =:= 'cyclic'))
->
#iter{%type = Type,
next = Next,
prev = Prev,
state = State
}.最後是一個小例子, 我們定義一個迭代器,用來訪問list, 當然這個做法似乎有點多此一舉..
(這段代碼摘自 iter 模塊中的測試代碼 )
%% @doc some test
basic_test_() ->
Data = ["begin", "one", "two", "there", "four", "end"],
ExpR1 = "begin one two there four end",
ExpR2 = "end four three two one begin",
%% 定義一個迭代器, 其調用listnext/1 獲取下一條記錄, 調用listprev/1 獲取上一條記錄, State爲一個tuple
%% 分別保存:數據的begin, last, length 和具體的數據(也就是list)
Iter = make_iter(fun listnext/1, fun listprev/1, {0, 0, length(Data), Data}),
R =
?MODULE:foldl(fun(Str, Acc) ->
[Str, $ | Acc]
end,
[],
Iter),
Str = lists:flatten(lists:reverse(R)), % Str 爲 ExpR1
Iter2 = make_iter(fun listnext/1, fun listprev/1, {length(Data)+1, 1, length(Data), Data}),
R2 =
?MODULE:foldr(fun(E, Acc) ->
[E, $ | Acc]
end,
[],
Iter2),
Str2 = lists:flatten(lists:reverse(R2)), % Str 爲 ExpR2
[
?_assert(Str = ExpR1),
?_assert(Str2 = ExpR2)
].
%% 用來獲取下一條記錄
listnext({Last, _First, Last, _Data}) ->
nil;
listnext({Pos, First, Last, Data}) ->
Pos2 = Pos + 1,
V = lists:nth(Pos2, Data),
{V, {Pos2, First, Last, Data}}.
%% 用來獲取前一條記錄
listprev({0, _First, _Last, _Data}) ->
nil;
listprev({First, First, _Last, _Data}) ->
nil;
listprev({Pos, First, Last, Data}) ->
Pos2 = Pos - 1,
V = lists:nth(Pos2, Data),
{V, {Pos2, First, Last, Data}}. 好了就是這些, 有了這個module, 在mapreduce框架中,用戶可以自定義各種的iter 實現, map, reduce會依據用戶的iter
進行具體的數據讀取,然後進行處理..
詳細的代碼, 請參考此處:
[url]http://code.google.com/p/heng/source/browse/trunk/master/heng_iter.erl[/url]