From 9ad8f3c8266ff85a19e72b7803db875bb0f28944 Mon Sep 17 00:00:00 2001 From: aj Date: Sun, 7 Nov 2010 18:32:37 -0800 Subject: [PATCH] lots of work towards flattening the records. stalled at ezic_rule:next_event/4 --- include/ezic.hrl | 2 +- notes/flattening.svg | 122 ++++++++++++++++++++++++++++++++ src/ezic.erl | 10 +-- src/ezic_date.erl | 53 ++++++++++++++ src/ezic_db.erl | 112 ++++++++++++++++++------------ src/ezic_flatten.erl | 145 +++++++++++++++++++++++++++++++++++++++ src/ezic_loader.erl | 33 ++++++--- src/ezic_record.erl | 1 + src/ezic_rule.erl | 67 ++++++------------ src/ezic_zone.erl | 32 ++++++--- test/ezic_zone_tests.erl | 14 ++++ test/test.erl | 3 +- 12 files changed, 479 insertions(+), 115 deletions(-) create mode 100644 notes/flattening.svg create mode 100644 src/ezic_flatten.erl create mode 100644 test/ezic_zone_tests.erl diff --git a/include/ezic.hrl b/include/ezic.hrl index 87405d5..800def2 100644 --- a/include/ezic.hrl +++ b/include/ezic.hrl @@ -17,7 +17,7 @@ % a flattened zone record, giving specific information for a specific time range. --record(flatzone, {tzname, tzrule, from, to, gmtoff}). +-record(flatzone, {tzname, wall_from, wall_to, std_from, std_to, utc_from, utc_to, offset}). diff --git a/notes/flattening.svg b/notes/flattening.svg new file mode 100644 index 0000000..8e0c65b --- /dev/null +++ b/notes/flattening.svg @@ -0,0 +1,122 @@ + + + + + + + + + + + + image/svg+xml + + + + + + + + DST point changes + Zone point changes + + + + + + + + + + diff --git a/src/ezic.erl b/src/ezic.erl index 518127c..385f95c 100644 --- a/src/ezic.erl +++ b/src/ezic.erl @@ -49,7 +49,8 @@ utc_to_local(UTCDatetime, TzName) -> next_timechange(TzName) -> next_timechange(erlang:universaltime(), TzName). -next_timechange(UtcDatetime, TzName) -> +next_timechange(UTCDatetime, TzName) -> + UTCDatetime, TzName, not_done. @@ -63,7 +64,8 @@ next_timechange(UtcDatetime, TzName) -> load(Folder) -> - ezic_loader:load(Folder). + ezic_loader:load(Folder), + ezic_flatten:flatten(). dev_start() -> @@ -106,6 +108,6 @@ time_offset(Zone, Rule) -> date_add(Datetime, SecDiff) -> ?gs2dt(?dt2gs(Datetime) + SecDiff). -date_subtract(Datetime, SecDiff) -> - ?gs2dt(?dt2gs(Datetime) - SecDiff). +%% date_subtract(Datetime, SecDiff) -> +%% ?gs2dt(?dt2gs(Datetime) - SecDiff). diff --git a/src/ezic_date.erl b/src/ezic_date.erl index 750c803..34730e0 100644 --- a/src/ezic_date.erl +++ b/src/ezic_date.erl @@ -8,10 +8,14 @@ -endif. + + -export([for/2]). -export([month_to_num/1, day_to_num/1]). -export([compare_times/2, compare_datetimes/2, normalize/1, overlap/2, date_between/2]). +-export([add_seconds/2, all_times/3]). + @@ -218,3 +222,52 @@ date_between_normal(D1, {D2s, D2e}) -> andalso compare_datetimes_normal(D1, D2e). + + +add_seconds(Datetime, Seconds) -> + calendar:gregorian_seconds_to_datetime( + calendar:date_to_gregorian_seconds(Datetime) + Seconds + ). + + + + + +% returns {WallTime, StdTime, UtcTime} +% where each is a datetime tuple: {{Y,M,D}{HH,MM,SS}} + +% universal time given +all_times(Datetime=#tztime{time=UTCTime, flag=Flag}, Offset, DSTOffset) + when Flag=:=u; Flag=:=g; Flag=:=z -> + OSec= calendar:time_to_seconds(Offset), + DSTSec= calendar:time_to_seconds(DSTOffset), + + STDTime= add_seconds(UTCTime, OSec), + WallTime= add_seconds(STDTime, DSTSec), + + {WallTime, STDTime, UTCTime}; + + +% standard time given +all_times(Datetime=#tztime{time=STDTime, flag=s}, Offset, DSTOffset) -> + OSec= calendar:time_to_seconds(Offset), + DSTSec= calendar:time_to_seconds(DSTOffset), + + UTCTime= add_seconds(STDTime, -1*OSec), + WallTime= add_seconds(STDTime, DSTSec), + + {WallTime, STDTime, UTCTime}; + + +% wall time given +all_times(Datetime=#tztime{time=WallTime, flag=Flag}, Offset, DSTOffset) + when Flag=:=w, Flag=:=undefined -> + OSec= calendar:time_to_seconds(Offset), + DSTSec= calendar:time_to_seconds(DSTOffset), + + STDTime= add_seconds(WallTime, -1*DSTSec), + UTCTime= add_seconds(STDTime, -1*OSec), + + {WallTime, STDTime, UTCTime}. + + diff --git a/src/ezic_db.erl b/src/ezic_db.erl index 17250af..3db93e4 100644 --- a/src/ezic_db.erl +++ b/src/ezic_db.erl @@ -1,13 +1,28 @@ - -module(ezic_db). -include("include/ezic.hrl"). -include_lib("stdlib/include/qlc.hrl"). -export([zones/1, rules/1]). --export([wipe/0, init/0, insert_all/1, getall/1]). +-export([wipe/0, init/0, insert_all/1, get_all/1]). +-define(create(Record), + {atomic, ok} = mnesia:create_table(Record, + [{type, bag} + , {disc_copies, [node()]} + , {attributes, record_info(fields, Record)} + ])). + + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% READ - db reading methods +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + + +% retrieve all zones by name zones(Name) -> F = fun()-> Q = qlc:q([Z || Z=#zone{name=N} <- mnesia:table(zone), N=:=Name]), @@ -16,6 +31,7 @@ zones(Name) -> {atomic, Zones}= mnesia:transaction(F), Zones. +% retrieve all rules by name rules(Name) -> F = fun()-> Q = qlc:q([R || R=#rule{name=N} <- mnesia:table(rule), N=:=Name]), @@ -24,46 +40,22 @@ rules(Name) -> {atomic, Rules}= mnesia:transaction(F), Rules. +% get all records from table +get_all(Tab) when is_atom(Tab) -> + F = fun() -> + Q = qlc:q([R || R<- mnesia:table(Tab)]), + qlc:e(Q) + end, + {atomic, Ret}= mnesia:transaction(F), + Ret. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% WRITE - insertion/edit methods +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - --define(create(Record), - {atomic, ok} = mnesia:create_table(Record, [{type, bag}, {disc_copies, [node()]},{attributes, record_info(fields, Record)}])). - - -init() -> - create_tabs(mnesia:create_schema([node()])), - mnesia:wait_for_tables([rule, zone, link, leap], 3000). - -wipe() -> - mnesia:stop(), - mnesia:delete_schema([node()]). - - -create_tabs(ok) -> - mnesia:start(), - - ?create(rule), - ?create(zone), - ?create(link), - ?create(leap), - - %% {atomic, ok}= mnesia:create_table(rule, [{disc_copies, [node()]},{attributes, record_info(fields, rule)}]), - %% {atomic, ok}= mnesia:create_table(zone, [{disc_copies, [node()]},{attributes, record_info(fields, zone)}]), - %% {atomic, ok}= mnesia:create_table(leap, [{disc_copies, [node()]},{attributes, record_info(fields, leap)}]), - %% {atomic, ok}= mnesia:create_table(link, [{disc_copies, [node()]},{attributes, record_info(fields, link)}]); - - ok; -create_tabs({error, {_, {already_exists,_}}}) -> - mnesia:start(), - ok; -create_tabs(E) -> - ?debug(E). - - insert_all(Records) -> mnesia:transaction( fun() -> @@ -74,10 +66,40 @@ insert_all(Records) -> -getall(Tab) when is_atom(Tab) -> - F = fun() -> - Q = qlc:q([R || R<- mnesia:table(Tab)]), - qlc:e(Q) - end, - mnesia:transaction(F). - + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% ADMIN - initialization and administration methods +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + +% initialize the db +init() -> + create_tabs(mnesia:create_schema([node()])), + mnesia:wait_for_tables([rule, zone, link, leap], 3000). + + +% WARNING: deletes all db files +wipe() -> + mnesia:stop(), + mnesia:delete_schema([node()]). + + +create_tabs(ok) -> + mnesia:start(), + + ?create(rule), + ?create(zone), + ?create(link), + ?create(leap), + ?create(flatzone), + + ok; +create_tabs({error, {_, {already_exists,_}}}) -> + mnesia:start(), + ok; +create_tabs(E) -> + ?debug(E). + + + + diff --git a/src/ezic_flatten.erl b/src/ezic_flatten.erl new file mode 100644 index 0000000..de519ba --- /dev/null +++ b/src/ezic_flatten.erl @@ -0,0 +1,145 @@ +-module(ezic_flatten). +-include("include/ezic.hrl"). + +-define(FLAT(W,S,U), #flatzone{wall_from=W, std_from=S, utc_from=U}). +-define(ENDFLAT(F,W,S,U), F#flatzone{wall_to=W, std_to=S, utc_to=U}). +-define(MINFLAT, ?FLAT(minimum,minimum,minimum)). + + + + +-export([flatten/0]). + + + + +flatten() -> + AllZones= ezic_db:get_all(zone), + flatten_zones(AllZones), + not_done. + + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% PRIVATE +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + + +make_next_flat(#flatzone{wall_to=W, std_to=S, utc_to=U, offset=O}) -> + make_next_flat({W,S,U}, O); +make_next_flat(EndTimes) -> + make_next_flat(EndTimes, undefined). + + +make_next_flat({W,S,U}, O) -> + FW= ezic_date:add_seconds(W,1), + FS= ezic_date:add_seconds(S,1), + FU= ezic_date:add_seconds(U,1), + Flat= ?FLAT(FW, FS, FU), + Flat#flatzone{offset=O}. + + + +end_flat(Flat=#flatzone{}, {W,S,U}) -> + ?ENDFLAT(Flat,W,S,U). + + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% INTERNAL +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + + + +% recursively processes sets of similar zones until they're all done, passing zone sets to flatten_zone_set/1 +flatten_zones(Zones) -> + flatten_zones(Zones, []). + +flatten_zones([], Flats) -> + Flats; +flatten_zones([Z1|_]= AllZones, Flats) -> + {CurrentZones, RestZones}= ezic_zone:split_by_name(Z1, AllZones), + + SortedZones= lists:sort(fun ezic_zone:sort_ascending/2, CurrentZones), + NewFlats= lists:merge(Flats, flatten_zone_set(SortedZones)), + + flatten_zones(RestZones, NewFlats). + + + + + + + + + + + + +% takes zones one-by-one, gathering relevant rules and create flat periods of the same gmt offset (#flatzone) +flatten_zone_set(Zones) -> + flatten_zone_set(?MINFLAT, Zones, []). + + +% flatten_zone_set(FromTime, Zones, Flats) -> [#flatzone{}] +% FromTime= #flatzone{*_from =/= undefined} +% Zones= [#zone{}] +% Flats= [#flatzone{}] +% +% assumes a new zone every time it is called +flatten_zone_set(_, [], Flats) -> + Flats; +flatten_zone_set(FromTimeStub, [Z1=#zone{rule=RuleName, until=UntilTime, gmtoff=Offset} | RestZones] = Zones, Flats) -> + + %% we have a flatzone with start times, we populate the base offset + FromTime= FromTimeStub#flatzone{offset=Offset}, + + + + %% we gather all rules that _may_ apply (same year) + Rules= ezic_db:rules(RuleName), + RelevantRules= ezic_rule:filter(FromTime, UntilTime, Rules), + SortedRules= lists:sort(fun ezic_rule:sort_ascending/2, RelevantRules), + + %% Then we plot points, bouncing from rule to rule, ending when the zone ends + {ok, NextFromTimeStub, NewFlats}= flatten_zone(FromTime, Z1, SortedRules), + + + not_done. + + + + + + + + + + +% flattens a single zone into multiple flatzones. +% FromTime has its *_from fields and offset field populated. (it should anyway) +% @todo invalid input checking +% Rules are sorted +flatten_zone(BaseFlat, Zone, Rules) -> + flatten_zone(BaseFlat, Zone, Rules, []). + +flatten_zone(BaseFlat, Zone=#zone{until=Until, gmtoff=Offset}, Rules, Flats) -> + case ezic_rule:next_event(BaseFlat, Until, Offset, Rules) of + % is this enough information to compare all times corectly? + {end_until, EndTimes} -> + NewFlats= [end_flat(BaseFlat, EndTimes) | Flats], + NewFrom= make_next_flat(EndTimes), + {ok, NewFrom, NewFlats}; % base case + + {end_rule, EndTimes, NextOffset} -> + NewFlats= [end_flat(BaseFlat, EndTimes) | Flats], + NewFrom= make_next_flat(EndTimes, NextOffset), + flatten_zone(NewFrom, Zone, Rules, NewFlats) % tail recursion + + end. + + + + diff --git a/src/ezic_loader.erl b/src/ezic_loader.erl index 6488fb4..d895381 100644 --- a/src/ezic_loader.erl +++ b/src/ezic_loader.erl @@ -15,8 +15,6 @@ load(File) -> end, {ok, Zones, Rules, Leaps, Links} = ezic_compile:separate(Records), -% ezic_compile:flatten(Zones, Rules). - ezic_db:wipe(), ezic_db:init(), @@ -30,6 +28,7 @@ load(File) -> + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % INTERNAL %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -38,7 +37,6 @@ load(File) -> % returns a list of tzdata records for all files in folder % note: assumes every file in the folder is a tzdata file -% this method does not recurse. parse_dir(Folder) -> {ok, Files} = file:list_dir(Folder), lists:flatten([ parse_file(filename:join(Folder,File)) || File <- Files ]). @@ -52,24 +50,33 @@ parse_file(File) -> + parse_lines(eof, _, Records) -> Records; parse_lines({ok, Line}, File, Records) -> StrLine= clean_line(Line), - case length(StrLine) > 0 of - false -> parse_lines(file:read_line(File), File, Records); - true -> parse_lines_2(StrLine, File, Records) - end. + NewRecords= case length(StrLine) > 0 of + false -> Records; + true -> [parse_to_record(StrLine, File, Records) | Records] + end, + parse_lines(file:read_line(File), File, NewRecords). -parse_lines_2(Line, File, Records) -> - %?debug("Line: ~p", [Line]), + +% the Line has data, so we parse it, build a record, and return it +parse_to_record(Line, File, Records) -> [Type | Data] = string:tokens(Line, " \t"), {ok, PrevType, PrevName} = prev_rec_type(Records), {ok, Record} = build_record(Type, Data, {PrevType, PrevName}), - parse_lines(file:read_line(File), File, [Record|Records]). + +% parse_lines(file:read_line(File), File, [Record|Records]). + + Record. + + +% retrieves the previous record type, in case Zone continuations occur (multiline) prev_rec_type([]) -> {ok, null, null}; prev_rec_type([#zone{name=Name}|_]) -> @@ -77,6 +84,9 @@ prev_rec_type([#zone{name=Name}|_]) -> prev_rec_type(List) when is_list(List) -> {ok, void, void}. + + + clean_line(Line) -> Line1= string:strip(Line), Line2= string:strip(Line1, both, $\n), @@ -105,3 +115,6 @@ build_record("Leap", Data,_) -> build_record(Type, Data, PT) -> {error, {badLine, Type, Data, PT}}. + + + diff --git a/src/ezic_record.erl b/src/ezic_record.erl index c36bd16..0189ed2 100644 --- a/src/ezic_record.erl +++ b/src/ezic_record.erl @@ -136,6 +136,7 @@ parse_save(X) -> parse_save_rev(XRev). + parse_time("-") -> #tztime{time={0,0,0}}; parse_time(TS) -> diff --git a/src/ezic_rule.erl b/src/ezic_rule.erl index a9f66eb..dc66fa0 100644 --- a/src/ezic_rule.erl +++ b/src/ezic_rule.erl @@ -2,62 +2,39 @@ -include("include/ezic.hrl"). --export([current/2, current_set/2, sort/2]). --export([from_time/1, dst_sec/1]). - - - -current(_, []) -> - none; -current(Now, Rules) -> - Current_SetRules= lists:filter(fun(R)-> current_set(Now, R) end, Rules), -% ?debug("Current_SetRules: ~p", [Current_SetRules]), - SRules= lists:sort(fun sort/2, Current_SetRules), - CRule= choose_rule(SRules), - CRule. +-export([next_event/4]). -% newest first -sort(R1, R2) -> - T1= from_time(R1), - T2= from_time(R2), - ezic_date:compare_datetimes(T2, T1). +% determines the next zone or dst change event +% returns the latest possible times the previous period is in effect (`Until -1sec` in most cases) +% Rules are sorted +next_event(BaseFlat, ZoneEnd, Offset, []) -> + {end_until, make_times(ZoneEnd, Offset)}; +% BaseFlat has offset defined +% @todo ensure offset defined on BaseFlat +% Rules are sorted in ascending order (oldest first) +next_event(BaseFlat=#flatzone{utc_from=#tztime{time={{Year,_,_},_}}}, + ZoneEnd, ZoneOffset, [Rule|RestRules]) -> - -current_set(Now, Rule=#rule{from=F, to=T}) -> - {{Y,_,_},_} = Now, - case ezic_date:date_between(Y, {F,T}) of - false -> false; - true -> current_set2(Now, Rule) - end. - - -current_set2(Now, #rule{in=Month, on=Day, at=Time}) -> - {{Y,_,_},_} = Now, - RTime= {{Y, Month, Day}, Time}, - ezic_date:compare_datetimes(RTime, Now). - - - -from_time(#rule{from=F, in=M, on=O, at=A}) -> - {{F,M,O},A}. + not_done. -choose_rule([]) -> - none; -choose_rule([H|_]) -> - H. +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% PRIVATE +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +% create endtimes for a #flatzone, with no dst offset +make_times(Until, Offset) -> + make_times(Until, Offset, 0). + +make_times(Until, Offset, DSTOffset) -> + ezic_date:all_times(Until, Offset, DSTOffset). -dst_sec(none) -> - 0; -dst_sec(Rule) -> - calendar:time_to_seconds((Rule#rule.save)#tztime.time). - diff --git a/src/ezic_zone.erl b/src/ezic_zone.erl index ea07cb7..54abd78 100644 --- a/src/ezic_zone.erl +++ b/src/ezic_zone.erl @@ -1,11 +1,13 @@ -module(ezic_zone). -include("include/ezic.hrl"). +-include_lib("eunit/include/eunit.hrl"). -export([ current/1 , current_as_of_utc/2 - , sort/2 - , revsort/2 + , split_by_name/2 + , sort_ascending/2 + , sort_descending/2 , offset_sec/1 ]). @@ -28,8 +30,8 @@ current_as_of_utc(UTCDatetime, TzName) -> get_zone_utc(_, []) -> erlang:error(no_current); -get_zone_utc(UTCDatetime, Zones) -> - SortedZones= lists:sort(fun revsort/2, Zones), +get_zone_utc(_UTCDatetime, Zones) -> + _SortedZones= lists:sort(fun sort_descending/2, Zones), %% [CZone|_]= lists:dropwhile(fun(SZ)-> older_zone_utc(SZ, UTCDatetime) end, SortedZones), %% CZone. @@ -43,17 +45,29 @@ get_zone_utc(UTCDatetime, Zones) -> -sort(#zone{until=U1}, #zone{until=U2}) -> + +% returns {[SimilarZone], [DifferentZone]} +% where Similar Zones are those with the same name as Zone +% and DifferentZones are all the rest +split_by_name(#zone{name=N}, Zones) -> + lists:partition( + fun(#zone{name=NI}) -> + N =:= NI + end + , Zones). + + +sort_descending(#zone{until=U1}, #zone{until=U2}) -> ezic_date:compare_datetimes(U2, U1). -revsort(Z1=#zone{}, Z2=#zone{}) -> - not sort(Z1, Z2). +sort_ascending(Z1=#zone{}, Z2=#zone{}) -> + not sort_descending(Z1, Z2). % @bug doesn't normalize times. Times are self-relative (to standard time), and Now may come from any timezone unless we're careful about that. -older_zone(#zone{until=Until}, UTCDatetime) -> - ezic_date:compare_datetimes(Until, UTCDatetime). +%% older_zone(#zone{until=Until}, UTCDatetime) -> +%% ezic_date:compare_datetimes(Until, UTCDatetime). diff --git a/test/ezic_zone_tests.erl b/test/ezic_zone_tests.erl new file mode 100644 index 0000000..5db7fff --- /dev/null +++ b/test/ezic_zone_tests.erl @@ -0,0 +1,14 @@ +-module(ezic_zone_tests). +-include("include/ezic.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +split_by_name_test_() -> + AZones= [A=#zone{name=a}, #zone{name=a}, #zone{name=a}], + BZones= [B=#zone{name=b}, #zone{name=b}, #zone{name=b}], + AllZones= lists:merge(AZones, BZones), + + [ + ?_assertEqual({AZones, BZones}, ezic_zone:split_by_name(A, AllZones)) + , ?_assertEqual({BZones, AZones}, ezic_zone:split_by_name(B, AllZones)) + , ?_assertEqual({[], AllZones}, ezic_zone:split_by_name(#zone{name=c}, AllZones)) + ]. diff --git a/test/test.erl b/test/test.erl index f82ef40..9fdce40 100644 --- a/test/test.erl +++ b/test/test.erl @@ -4,4 +4,5 @@ all() -> eunit:test(ezic_record), - eunit:test(ezic_date). + eunit:test(ezic_date), + eunit:test(ezic_zone).