RSS

Erlang

15 Jan

Bulk loading data into Mnesia

Mnesia is a DBMS which comes with the erlang distribution.Which is appropriate for
telecommunication applications which are realtime and continues
Mnesia provides read,write operations which can be used to load and search data.

Here the problem is bulk loading of data into mnesia table
say 1 million records.[records which has about 12 attributes]
tables which are disc_copies and disc_only_copies

create tables,both disc_copies and disc_only_copies
In disc_copies table kept in RAM as well as on disc.Where as in disc_only_copies table kept on disc_only

mnesia_create_tables(NodeList) ->
case mnesia:start() of
ok ->
error_logger:error_report([“Mnesia start ok”]),

mnesia:create_table(person,
[{disc_only_copies, NodeList},{type, set},
{attributes, record_info(fields, person)}]),

mnesia:create_table(subscriber,
[{disc_copies, NodeList},{type, set},
{attributes, record_info(fields, subscriber)}]);

{error, Reason} ->
error_logger:error_report([“Mnesia start error: “, Reason]),
{error, Reason}
end.

For bulk writing we can use two functions. one is mnesia:dirty_write and other one is
mnesia transaction mechanism.
Dirty operations are performance wise better than transaction mechanism.Its because dirty operations are bypassing much of processing
In transaction mechanism action is either performed or aborted.

Use dirty method for loading data.

dirty_populate_disc_copy(Rec) ->
dirty_populate_disc_copy(Rec,5).

dirty_populate_disc_copy(N,NumProc) ->
From =N,
To= N-trunc(N/5),
dirty_populate_disc_copy(N,NumProc,From,To).

dirty_populate_disc_copy(N,NumProc,From,To)->
if
NumProc > 0 ->
io:format(“From=~w To=~w Pocess=~w~n”,[From,To,NumProc]),
spawn(bench_db,populate_subscriber,[From,To,dirty]),
dirty_populate_disc_copy(N,NumProc-1,To,To-trunc(N/5));
NumProc ==0 ->
no_spawn
end,
ok.

Used 5 spawns to load data into mnesia tables.
specify integer value for Rec .It will load that number of records into mnesia table.
%% ——————————————————————-
%% Populate the subscriber table :disc_copies
%% ——————————————————————-

populate_subscriber(From,To,Option) ->
random:seed(),
Stime =erlang:localtime(),
statistics(runtime),
statistics(wall_clock),
do_populate_subscriber(From, To,Option),
{_, RT} = statistics(runtime),
{_, WC} = statistics(wall_clock),
Etime=erlang:localtime(),
io:format(“Start=~p End=~p~n~n”, [Stime, Etime]),
io:format(“Run Time=~p Wall Clock Time=(~p) milliseconds~n”, [RT, WC]).

do_populate_subscriber(FId,TId,Option) when FId > TId ->
Name = “abcddddd”,
GroupId = random:uniform(100) – 1,
Subscr = #subscriber{subscriber_number = FId,
subscriber_name = Name,
group_id = GroupId,
variable_id = GroupId,
variable_name = “sports”,
marketing_value = “12.4454555”,
length_variable = false,
location = 9064949,
active_sessions = 450000,
changed_by = <>,
changed_time = <>,
suffix = “222222”
},
if
Option == dirty ->
save_record_dirty(subscriber,Subscr);
Option == trans ->
save_record_trans(subscriber,Subscr)
end,
do_populate_subscriber(FId-1,TId,Option);

do_populate_subscriber(_,_,_) ->
io:format(” totally ~p bytes~n”,
[mnesia:table_info(subscriber, memory) * 4]),
ok.

save_record_dirty(DB,Record) ->
mnesia:dirty_write(DB,Record).

Find the complete module here


Advertisements
 
Leave a comment

Posted by on January 15, 2009 in technology

 

Tags: , , , , ,

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out / Change )

Twitter picture

You are commenting using your Twitter account. Log Out / Change )

Facebook photo

You are commenting using your Facebook account. Log Out / Change )

Google+ photo

You are commenting using your Google+ account. Log Out / Change )

Connecting to %s

 
%d bloggers like this: