From e8385d7d86f5597c5640532e0cffc5438a3e82d7 Mon Sep 17 00:00:00 2001 From: lsowen Date: Fri, 2 Nov 2012 22:50:17 -0400 Subject: [PATCH 1/3] move forward on the transition to ets over mnesia --- src/ezic.erl | 10 +++++----- src/ezic_db_ets.erl | 30 ++++++++++++++++++++++++++++-- src/ezic_db_ets_admin.erl | 18 ++++++++---------- src/ezic_flatten.erl | 34 +++++++++++++++------------------- src/ezic_zone.erl | 3 ++- 5 files changed, 58 insertions(+), 37 deletions(-) diff --git a/src/ezic.erl b/src/ezic.erl index b2390f6..6a4f12c 100644 --- a/src/ezic.erl +++ b/src/ezic.erl @@ -36,7 +36,7 @@ localtime(TzName) -> utc_to_local(UTCDatetime, TzName) -> NormalDatetime= ezic_date:normalize(UTCDatetime, u), - #flatzone{offset=Offset, dstoffset=DSTOffset}= ezic_db:flatzone(NormalDatetime, TzName), + #flatzone{offset=Offset, dstoffset=DSTOffset}= ezic_db_ets:flatzone(NormalDatetime, TzName), ezic_date:add_offset( ezic_date:add_offset( @@ -47,7 +47,7 @@ utc_to_local(UTCDatetime, TzName) -> local_to_utc(LocalDatetime, TzName) -> NormalDatetime= ezic_date:normalize(LocalDatetime, w), - #flatzone{offset=Offset, dstoffset=DSTOffset}= ezic_db:flatzone(NormalDatetime, TzName), + #flatzone{offset=Offset, dstoffset=DSTOffset}= ezic_db_ets:flatzone(NormalDatetime, TzName), ezic_date:add_offset( ezic_date:add_offset( @@ -94,8 +94,8 @@ zf() -> reflatten() -> - ezic_db:wipe(flatzone), - ezic_flatten:flatten(). + ezic_db_ets:wipe(flatzone), + ezic_db_ets:flatten(). @@ -116,7 +116,7 @@ test() -> current_as_of_utc(UTCDatetime, TzName) -> CZone= ezic_zone:current_as_of_utc(UTCDatetime, TzName), RuleName= CZone#zone.rule, - Rules= ezic_db:rules(RuleName), + Rules= ezic_db_ets:rules(RuleName), CRule= ezic_rule:current_as_of_utc(UTCDatetime, Rules), {ok, CZone, CRule}. diff --git a/src/ezic_db_ets.erl b/src/ezic_db_ets.erl index af9d85c..742dc04 100644 --- a/src/ezic_db_ets.erl +++ b/src/ezic_db_ets.erl @@ -10,6 +10,9 @@ , rules/1 , flatzone/2 , all/1 + %, insert_all/1 + , wipe/1 + , flatten/0 ]). @@ -37,18 +40,29 @@ rules(TzName) -> gen_server:call(?MODULE, {rules, TzName}). - flatzone(Date, TzName) -> gen_server:call(?MODULE, {flatzone, Date, TzName}). - all(Tab) -> gen_server:call(?MODULE, {all, Tab}). +%insert_all(Records) -> +% gen_server:call(?MODULE, {insert_all, Records}). +%wipe() -> +% gen_server:call(?MODULE, {wipe}). + + +wipe(Tab) -> + gen_server:call(?MODULE, {wipe, Tab}). + + +flatten() -> + gen_server:call(?MODULE, {flatten}). + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% GEN_SERVER @@ -78,6 +92,18 @@ handle_call({all, Table}, _, Ets) -> handle_call({flatzone, Date, Name}, _, Ets) -> Matches= ets:select(Ets, ezic_flatten:ms(Date, Name)), {reply, Matches, Ets}; +%handle_call({insert_all, Records}, _, Ets) -> +% Result= ets:insert(Ets, Records), +% {reply, Result, Ets}; +%handle_call({wipe}, _, Ets) -> +% Result= ets:delete(Ets), +% {reply, Result, Ets}; +handle_call({wipe, Tab}, _, Ets) -> + Result= ets:delete(Ets, Tab), + {reply, Result, Ets}; +handle_call({flatten}, _, Ets) -> + Result= ezic_flatten:flatten(Ets), + {reply, Result, Ets}; handle_call(_, _, Ets) -> {noreply, Ets}. diff --git a/src/ezic_db_ets_admin.erl b/src/ezic_db_ets_admin.erl index 219f800..aecda26 100644 --- a/src/ezic_db_ets_admin.erl +++ b/src/ezic_db_ets_admin.erl @@ -37,11 +37,11 @@ init() -> {ok, DbDir}= application:get_env(db_dir), Filename= filename:join(DbDir, ?DB_FILENAME), - Tabfile= case db_sane(Filename) of + Ets= case db_sane(Filename) of true -> load_tabfile(Filename); false -> create_tables(Filename) end, - {ok, Tabfile}. + {ok, Ets}. % erases all data @@ -61,7 +61,7 @@ insert_all(Record) -> % erases date for one table wipe(Tab) -> - not_done. + ezic_db_ets:wipe(Tab). @@ -84,13 +84,14 @@ db_sane(Filename) -> %% creates the dets table and populates it. create_tables(Filename) -> - Tab= ets:new(ezic_ets_db, [duplicate_bag]), - ets:insert(Tab, ezic_loader:load()), + Ets= ets:new(ezic_ets_db, [duplicate_bag]), + ets:insert(Ets, ezic_loader:load()), + ezic_flatten:flatten(Ets), % save to disk - ets:tab2file(Tab, Filename), + ets:tab2file(Ets, Filename), - Tab. + Ets. @@ -105,6 +106,3 @@ load_tabfile(Filename) -> ets_from_dets(Dets) -> Ets= ets:new(tzdb, ?ETS_OPTS), dets:to_ets(Dets, Ets). - - - diff --git a/src/ezic_flatten.erl b/src/ezic_flatten.erl index 4ef4cc0..112c7e7 100644 --- a/src/ezic_flatten.erl +++ b/src/ezic_flatten.erl @@ -1,5 +1,6 @@ -module(ezic_flatten). -include("include/ezic.hrl"). +-define(NODEBUG, true). -include_lib("eunit/include/eunit.hrl"). @@ -14,20 +15,15 @@ -export([ - flatten/0 + flatten/1 , contains_date/2 , ms/2 ]). -%% debug --export([flatten_all_zones/1]). - - - -flatten() -> - AllZones= ezic_db:get_all(zone), - flatten_all_zones(AllZones), +flatten(Ets) -> + AllZones= ets:lookup(Ets, zone), + flatten_all_zones(Ets, AllZones), done. @@ -79,16 +75,16 @@ ms(Date, Name) -> % recursively processes sets of similar zones until they're all done, % passing zone sets to flatten_zone_set/1 -flatten_all_zones([]) -> +flatten_all_zones(Ets, []) -> done; -flatten_all_zones([Z1|_]= AllZones) -> +flatten_all_zones(Ets, [Z1|_]= AllZones) -> io:format("Flattening zones: ~s~n", [Z1#zone.name]), {CurrentZones, RestZones}= ezic_zone:split_by_name(Z1, AllZones), - Flats= flatten_zone_set(CurrentZones), - ezic_db:insert_all(Flats), + Flats= flatten_zone_set(Ets, CurrentZones), + ets:insert(Ets, Flats), - flatten_all_zones(RestZones). + flatten_all_zones(Ets, RestZones). @@ -113,8 +109,8 @@ contains_date2(FlatZone=#flatzone{wall_from=From, wall_to=To}, {Dt,#tztime{time= %% gathers relevant rules and creates flat periods of the same gmt %% offset (#flatzone). This is a recursive solution, eliminating Zones %% from the list until it's been exhausted -flatten_zone_set(Zones) -> - flatten_zone_set(?MINFLAT, Zones, [], none). +flatten_zone_set(Ets, Zones) -> + flatten_zone_set(Ets, ?MINFLAT, Zones, [], none). % flatten_zone_set(FromTime, Zones, Flats) -> [#flatzone{}] @@ -123,7 +119,7 @@ flatten_zone_set(Zones) -> % Flats= [#flatzone{}] % % assumes a new zone every time it is called -flatten_zone_set(FromTimeStub=#flatzone{utc_from=UTCFrom, dstoffset=DSTOffset} +flatten_zone_set(Ets, FromTimeStub=#flatzone{utc_from=UTCFrom, dstoffset=DSTOffset} , Zones %[Z1=#zone{rule=RuleName, until=UntilTime, gmtoff=Offset} | _RestZones], , Flats , CurrentRule) -> @@ -145,7 +141,7 @@ flatten_zone_set(FromTimeStub=#flatzone{utc_from=UTCFrom, dstoffset=DSTOffset} %% we gather all rules that _may_ apply - Rules= ezic_db:rules(RuleName), + Rules= ets:select(Ets, [{#rule{name=RuleName, _='_'}, [], ['$_']}]), ?debugVal(FromTime), @@ -169,7 +165,7 @@ flatten_zone_set(FromTimeStub=#flatzone{utc_from=UTCFrom, dstoffset=DSTOffset} ?debugMsg("maxyear reached from flatten_zone_set"), FinalFlats; false -> - flatten_zone_set(NextFlat, RestZones, FinalFlats, EndingRule) + flatten_zone_set(Ets, NextFlat, RestZones, FinalFlats, EndingRule) catch exit:Reason -> ?debugMsg("bad year for nextflat:"), diff --git a/src/ezic_zone.erl b/src/ezic_zone.erl index b2a93ce..df61374 100644 --- a/src/ezic_zone.erl +++ b/src/ezic_zone.erl @@ -1,5 +1,6 @@ -module(ezic_zone). -include("include/ezic.hrl"). +-define(NODEBUG, true). -include_lib("eunit/include/eunit.hrl"). -export([ @@ -41,7 +42,7 @@ current(TzName) -> current_as_of_utc(erlang:universaltime(), TzName). current_as_of_utc(UTCDatetime, TzName) -> - Zones= ezic_db:zones(TzName), + Zones= ezic_db_ets:zones(TzName), get_zone_utc(UTCDatetime, Zones). From fe6db38bec2d28b67818cbeafb57c8b52cf29a3d Mon Sep 17 00:00:00 2001 From: lsowen Date: Fri, 2 Nov 2012 23:06:34 -0400 Subject: [PATCH 2/3] flatzone call was returning a list. Update return value convention in ezic_db_ets:flatzone() to match ezic_db:flatzone(). --- src/ezic_db_ets.erl | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/ezic_db_ets.erl b/src/ezic_db_ets.erl index 742dc04..f3c570f 100644 --- a/src/ezic_db_ets.erl +++ b/src/ezic_db_ets.erl @@ -90,8 +90,18 @@ handle_call({all, Table}, _, Ets) -> Matches= ets:lookup(Ets, Table), {reply, Matches, Ets}; handle_call({flatzone, Date, Name}, _, Ets) -> - Matches= ets:select(Ets, ezic_flatten:ms(Date, Name)), - {reply, Matches, Ets}; + FlatZones= ets:select(Ets, ezic_flatten:ms(Date, Name)), + Result= case length(FlatZones) of + 1 -> + hd(FlatZones); + 2 -> + erlang:error(ambiguous_zone, FlatZones); + 0 -> + erlang:error(no_zone); + _ -> + erlang:error(should_not_happen, {FlatZones, Date, Name}) + end, + {reply, Result, Ets}; %handle_call({insert_all, Records}, _, Ets) -> % Result= ets:insert(Ets, Records), % {reply, Result, Ets}; From 9d0df77b2950e76b5dec7c4a7865f95d2e2557c6 Mon Sep 17 00:00:00 2001 From: lsowen Date: Wed, 7 Nov 2012 14:06:27 -0500 Subject: [PATCH 3/3] Now using named ets tables. More closely mirrors the design of the mnesia backed version. --- src/ezic_db_ets.erl | 40 +++++++++++++++++----------------- src/ezic_db_ets_admin.erl | 45 ++++++++++++++++++++++++++++++++------- src/ezic_flatten.erl | 28 ++++++++++++------------ 3 files changed, 71 insertions(+), 42 deletions(-) diff --git a/src/ezic_db_ets.erl b/src/ezic_db_ets.erl index f3c570f..dd1d32c 100644 --- a/src/ezic_db_ets.erl +++ b/src/ezic_db_ets.erl @@ -80,17 +80,17 @@ init(_) -> %%~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -handle_call({zones, Name}, _, Ets) -> - Matches= ets:select(Ets, [{#zone{name=Name, _='_'}, [], ['$_']}]), - {reply, Matches, Ets}; -handle_call({rules, Name}, _, Ets) -> - Matches= ets:select(Ets, [{#rule{name=Name, _='_'}, [], ['$_']}]), - {reply, Matches, Ets}; -handle_call({all, Table}, _, Ets) -> - Matches= ets:lookup(Ets, Table), - {reply, Matches, Ets}; -handle_call({flatzone, Date, Name}, _, Ets) -> - FlatZones= ets:select(Ets, ezic_flatten:ms(Date, Name)), +handle_call({zones, Name}, _, State) -> + Matches= ets:select(zone, [{#zone{name=Name, _='_'}, [], ['$_']}]), + {reply, Matches, State}; +handle_call({rules, Name}, _, State) -> + Matches= ets:select(rule, [{#rule{name=Name, _='_'}, [], ['$_']}]), + {reply, Matches, State}; +handle_call({all, Tab}, _, State) -> + Matches= ets:lookup(Tab, Tab), + {reply, Matches, State}; +handle_call({flatzone, Date, Name}, _, State) -> + FlatZones= ets:select(flatzone, ezic_flatten:ms(Date, Name)), Result= case length(FlatZones) of 1 -> hd(FlatZones); @@ -101,21 +101,21 @@ handle_call({flatzone, Date, Name}, _, Ets) -> _ -> erlang:error(should_not_happen, {FlatZones, Date, Name}) end, - {reply, Result, Ets}; + {reply, Result, State}; %handle_call({insert_all, Records}, _, Ets) -> % Result= ets:insert(Ets, Records), % {reply, Result, Ets}; %handle_call({wipe}, _, Ets) -> % Result= ets:delete(Ets), % {reply, Result, Ets}; -handle_call({wipe, Tab}, _, Ets) -> - Result= ets:delete(Ets, Tab), - {reply, Result, Ets}; -handle_call({flatten}, _, Ets) -> - Result= ezic_flatten:flatten(Ets), - {reply, Result, Ets}; -handle_call(_, _, Ets) -> - {noreply, Ets}. +handle_call({wipe, Tab}, _, State) -> + Result= ets:delete(Tab, Tab), + {reply, Result, State}; +handle_call({flatten}, _, State) -> + Result= ezic_flatten:flatten(), + {reply, Result, State}; +handle_call(_, _, State) -> + {noreply, State}. %%~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/src/ezic_db_ets_admin.erl b/src/ezic_db_ets_admin.erl index aecda26..163ef2c 100644 --- a/src/ezic_db_ets_admin.erl +++ b/src/ezic_db_ets_admin.erl @@ -37,11 +37,11 @@ init() -> {ok, DbDir}= application:get_env(db_dir), Filename= filename:join(DbDir, ?DB_FILENAME), - Ets= case db_sane(Filename) of + case db_sane(Filename) of true -> load_tabfile(Filename); false -> create_tables(Filename) end, - {ok, Ets}. + {ok, []}. % erases all data @@ -84,21 +84,50 @@ db_sane(Filename) -> %% creates the dets table and populates it. create_tables(Filename) -> - Ets= ets:new(ezic_ets_db, [duplicate_bag]), - ets:insert(Ets, ezic_loader:load()), - ezic_flatten:flatten(Ets), + {ok, Zones, Rules, _, _} = ezic_record:separate(ezic_loader:load()), + + ets:new(zone, [duplicate_bag, named_table]), + ets:insert(zone, Zones), + + ets:new(rule, [duplicate_bag, named_table]), + ets:insert(rule, Rules), + + ets:new(flatzone, [duplicate_bag, named_table]), + ezic_flatten:flatten(), + + % combine into one ets + Ets= ets:new(ezic_db_ets, [duplicate_bag]), + ets:insert(Ets, ets:lookup(zone, zone)), + ets:insert(Ets, ets:lookup(rule, rule)), + ets:insert(Ets, ets:lookup(flatzone, flatzone)), % save to disk ets:tab2file(Ets, Filename), - Ets. + ets:delete(Ets), + + ok. %% loads or creates the dets table. load_tabfile(Filename) -> - {ok, Name}= ets:file2tab(Filename), - Name. + {ok, Ets}= ets:file2tab(Filename), + + Zones= ets:lookup(Ets, zone), + ets:new(zone, [duplicate_bag, named_table]), + ets:insert(zone, Zones), + + Rules= ets:lookup(Ets, rule), + ets:new(rule, [duplicate_bag, named_table]), + ets:insert(rule, Rules), + + FlatZones= ets:lookup(Ets, flatzone), + ets:new(flatzone, [duplicate_bag, named_table]), + ets:insert(flatzone, FlatZones), + + ets:delete(Ets), + ok. diff --git a/src/ezic_flatten.erl b/src/ezic_flatten.erl index 112c7e7..c94cb38 100644 --- a/src/ezic_flatten.erl +++ b/src/ezic_flatten.erl @@ -15,15 +15,15 @@ -export([ - flatten/1 + flatten/0 , contains_date/2 , ms/2 ]). -flatten(Ets) -> - AllZones= ets:lookup(Ets, zone), - flatten_all_zones(Ets, AllZones), +flatten() -> + AllZones= ets:lookup(zone, zone), + flatten_all_zones(AllZones), done. @@ -75,16 +75,16 @@ ms(Date, Name) -> % recursively processes sets of similar zones until they're all done, % passing zone sets to flatten_zone_set/1 -flatten_all_zones(Ets, []) -> +flatten_all_zones([]) -> done; -flatten_all_zones(Ets, [Z1|_]= AllZones) -> +flatten_all_zones([Z1|_]= AllZones) -> io:format("Flattening zones: ~s~n", [Z1#zone.name]), {CurrentZones, RestZones}= ezic_zone:split_by_name(Z1, AllZones), - Flats= flatten_zone_set(Ets, CurrentZones), - ets:insert(Ets, Flats), + Flats= flatten_zone_set(CurrentZones), + ets:insert(flatzone, Flats), - flatten_all_zones(Ets, RestZones). + flatten_all_zones(RestZones). @@ -109,8 +109,8 @@ contains_date2(FlatZone=#flatzone{wall_from=From, wall_to=To}, {Dt,#tztime{time= %% gathers relevant rules and creates flat periods of the same gmt %% offset (#flatzone). This is a recursive solution, eliminating Zones %% from the list until it's been exhausted -flatten_zone_set(Ets, Zones) -> - flatten_zone_set(Ets, ?MINFLAT, Zones, [], none). +flatten_zone_set(Zones) -> + flatten_zone_set(?MINFLAT, Zones, [], none). % flatten_zone_set(FromTime, Zones, Flats) -> [#flatzone{}] @@ -119,7 +119,7 @@ flatten_zone_set(Ets, Zones) -> % Flats= [#flatzone{}] % % assumes a new zone every time it is called -flatten_zone_set(Ets, FromTimeStub=#flatzone{utc_from=UTCFrom, dstoffset=DSTOffset} +flatten_zone_set(FromTimeStub=#flatzone{utc_from=UTCFrom, dstoffset=DSTOffset} , Zones %[Z1=#zone{rule=RuleName, until=UntilTime, gmtoff=Offset} | _RestZones], , Flats , CurrentRule) -> @@ -141,7 +141,7 @@ flatten_zone_set(Ets, FromTimeStub=#flatzone{utc_from=UTCFrom, dstoffset=DSTOffs %% we gather all rules that _may_ apply - Rules= ets:select(Ets, [{#rule{name=RuleName, _='_'}, [], ['$_']}]), + Rules= ets:select(rule, [{#rule{name=RuleName, _='_'}, [], ['$_']}]), ?debugVal(FromTime), @@ -165,7 +165,7 @@ flatten_zone_set(Ets, FromTimeStub=#flatzone{utc_from=UTCFrom, dstoffset=DSTOffs ?debugMsg("maxyear reached from flatten_zone_set"), FinalFlats; false -> - flatten_zone_set(Ets, NextFlat, RestZones, FinalFlats, EndingRule) + flatten_zone_set(NextFlat, RestZones, FinalFlats, EndingRule) catch exit:Reason -> ?debugMsg("bad year for nextflat:"),