/[cvs]/nfo/perl/libs/Data/Transfer/Sync/Core.pm
ViewVC logotype

Contents of /nfo/perl/libs/Data/Transfer/Sync/Core.pm

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.4 - (show annotations)
Sun Feb 9 05:01:10 2003 UTC (21 years, 5 months ago) by joko
Branch: MAIN
Changes since 1.3: +55 -410 lines
+ major structure changes
- refactored code to sister modules

1 ## $Id: Core.pm,v 1.3 2003/01/20 17:01:14 joko Exp $
2 ##
3 ## Copyright (c) 2002 Andreas Motl <andreas.motl@ilo.de>
4 ##
5 ## See COPYRIGHT section in pod text below for usage and distribution rights.
6 ##
7 ## ----------------------------------------------------------------------------------------
8 ## $Log: Core.pm,v $
9 ## Revision 1.3 2003/01/20 17:01:14 joko
10 ## + cosmetics and debugging
11 ## + probably refactored code to new plugin-modules 'Metadata.pm' and/or 'StorageInterface.pm' (guess it was the last one...)
12 ##
13 ## Revision 1.2 2003/01/19 02:05:42 joko
14 ## - removed pod-documentation: now in Data/Transfer/Sync.pod
15 ##
16 ## Revision 1.1 2003/01/19 01:23:04 joko
17 ## + new from Data/Transfer/Sync.pm
18 ##
19 ## Revision 1.11 2002/12/23 07:10:59 joko
20 ## + using MD5 for checksum generation again - the 32-bit integer hash from DBI seems to be too lazy
21 ##
22 ## Revision 1.10 2002/12/19 01:07:16 joko
23 ## + fixed output done via $logger
24 ##
25 ## Revision 1.9 2002/12/16 07:02:34 jonen
26 ## + added comment
27 ##
28 ## Revision 1.8 2002/12/15 02:03:09 joko
29 ## + fixed logging-messages
30 ## + additional metadata-checks
31 ##
32 ## Revision 1.7 2002/12/13 21:49:34 joko
33 ## + sub configure
34 ## + sub checkOptions
35 ##
36 ## Revision 1.6 2002/12/06 04:49:10 jonen
37 ## + disabled output-puffer here
38 ##
39 ## Revision 1.5 2002/12/05 08:06:05 joko
40 ## + bugfix with determining empty fields (Null) with DBD::CSV
41 ## + debugging
42 ## + updated comments
43 ##
44 ## Revision 1.4 2002/12/03 15:54:07 joko
45 ## + {import}-flag is now {prepare}-flag
46 ##
47 ## Revision 1.3 2002/12/01 22:26:59 joko
48 ## + minor cosmetics for logging
49 ##
50 ## Revision 1.2 2002/12/01 04:43:25 joko
51 ## + mapping deatil entries may now be either an ARRAY or a HASH
52 ## + erase flag is used now (for export-operations)
53 ## + expressions to refer to values inside deep nested structures
54 ## - removed old mappingV2-code
55 ## + cosmetics
56 ## + sub _erase_all
57 ##
58 ## Revision 1.1 2002/11/29 04:45:50 joko
59 ## + initial check in
60 ##
61 ## Revision 1.1 2002/10/10 03:44:21 cvsjoko
62 ## + new
63 ## ----------------------------------------------------------------------------------------
64
65
66 package Data::Transfer::Sync::Core;
67
68 use strict;
69 use warnings;
70
71 use mixin::with qw( Data::Transfer::Sync );
72
73
74 # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - main
75
76 use Data::Dumper;
77
78 use misc::HashExt;
79 use Data::Compare::Struct qw( getDifference isEmpty );
80 use Data::Storage::Container;
81 use DesignPattern::Object;
82
83 # get logger instance
84 my $logger = Log::Dispatch::Config->instance;
85
86 $| = 1;
87
88 =pod
89 sub new {
90 my $invocant = shift;
91 my $class = ref($invocant) || $invocant;
92 my $self = {};
93 $logger->debug( __PACKAGE__ . "->new(@_)" );
94 bless $self, $class;
95 $self->configure(@_);
96 return $self;
97 }
98 =cut
99
100
101 sub _init {
102 my $self = shift;
103
104 # build new container if necessary
105 $self->{container} = Data::Storage::Container->new() if !$self->{container};
106
107 # add storages to container (optional)
108 foreach (keys %{$self->{storages}}) {
109 $self->{container}->addStorage($_, $self->{storages}->{$_});
110 }
111
112 return 1;
113
114 }
115
116 sub _initV1 {
117 my $self = shift;
118 $logger->debug( __PACKAGE__ . "->_initV1" );
119 die("this should not be reached!");
120 # tag storages with id-authority and checksum-provider information
121 # TODO: better store tag inside metadata to hold bits together!
122 map { $self->{container}->{storage}->{$_}->{isIdentAuthority} = 1 } @{$self->{id_authorities}};
123 map { $self->{container}->{storage}->{$_}->{isChecksumAuthority} = 1; } @{$self->{checksum_authorities}};
124 map { $self->{container}->{storage}->{$_}->{isWriteProtected} = 1; } @{$self->{write_protected}};
125 }
126
127
128 # TODO: abstract the hardwired use of "source" and "target" in here somehow - hmmmm....... /(="ยง/%???
129 sub _run {
130
131 my $self = shift;
132
133 #print "verbose: ", $self->{verbose}, "\n";
134 $self->{verbose} = 1;
135
136 $logger->debug( __PACKAGE__ . "->_run" );
137
138 # for statistics
139 my $tc = OneLineDumpHash->new( {} );
140 my $results;
141
142 # set of objects is already in $self->{args}
143 # TODO: make independent of the terminology "object..."
144 $results = $self->{args}->{objectSet} if $self->{args}->{objectSet};
145
146 # apply filter
147 if (my $filter = $self->{meta}->{source}->{filter}) {
148 #print Dumper($filter);
149 #exit;
150 $results ||= $self->_getNodeList('source', $filter);
151 }
152
153 # get reference to node list from convenient method provided by CORE-HANDLE
154 $results ||= $self->_getNodeList('source');
155
156 # checkpoint: do we actually have a list to iterate through?
157 if (!$results || !@{$results}) {
158 $logger->notice( __PACKAGE__ . "->_run: No nodes to synchronize." );
159 return;
160 }
161
162
163
164 # check if we actually *have* a synchronization method
165 if (!$self->{options}->{metadata}->{syncMethod}) {
166 $logger->critical( __PACKAGE__ . "->_run: No synchronization method (checksum|timestamp) specified" );
167 return;
168 }
169
170
171 # dereference
172 my @results = @{$results};
173
174 # iterate through set
175 foreach my $source_node_real (@results) {
176
177 $tc->{total}++;
178
179 #print "======================== iter", "\n";
180
181 # clone object (in case we have to modify it here)
182 # TODO:
183 # - is a "deep_copy" needed here if occouring modifications take place?
184 # - puuhhhh, i guess a deep_copy would destroy tangram mechanisms?
185 # - after all, just take care for now that this object doesn't get updated!
186 my $source_node = $source_node_real;
187
188 # modify entry - handle new style callbacks (the readers)
189 #print Dumper($source_node);
190 #exit;
191
192 my $descent = 'source';
193
194 # handle callbacks right now while scanning them (asymmetric to the writers)
195 my $map_callbacks = {};
196 if (my $callbacks = $self->{meta}->{$descent}->{Callback}) {
197
198 my $error = 0;
199
200 foreach my $node (keys %{$callbacks->{read}}) {
201
202 my $object = $source_node;
203 my $value; # = $source_node->{$node};
204
205 # ------------ half-redundant: make $self->callCallback($object, $value, $opts)
206 #my $perl_callback = $self->{meta}->{$descent}->{node} . '::' . $node . '_read';
207 my $perl_callback = $self->{meta}->{$descent}->{nodeType} . '::' . $node . '_read';
208 my $evalstring = 'return ' . $perl_callback . '( { object => $object, property => $node, value => $value, storage => $self->{meta}->{$descent}->{storage} } );';
209 #print $evalstring, "\n"; exit;
210 my $cb_result = eval($evalstring);
211 if ($@) {
212 die $@;
213 $error = 1;
214 print $@, "\n";
215 }
216 # ------------ half-redundant: make $self->callCallback($object, $value, $opts)
217
218 $source_node->{$node} = $cb_result;
219
220 }
221
222 }
223
224 #print Dumper($source_node);
225
226 # exclude defined fields (simply delete from object)
227 map { delete $source_node->{$_} } @{$self->{meta}->{source}->{subnodes_exclude}};
228
229 # here we accumulate information about the status of the current node (payload/row/object/item/entry)
230 $self->{node} = {};
231 $self->{node}->{source}->{payload} = $source_node;
232
233 #print "res - ident", "\n";
234
235 # determine ident of entry
236 my $identOK = $self->_resolveNodeIdent('source');
237 #if (!$identOK && lc $self->{args}->{direction} ne 'import') {
238 if (!$identOK) {
239 #print Dumper($self->{meta}->{source});
240 $logger->critical( __PACKAGE__ . "->_run: No ident found in source node ( nodeName='$self->{meta}->{source}->{nodeName}', nodeType='$self->{meta}->{source}->{nodeType}') try to \"prepare\" this node first?" );
241 return;
242 }
243
244 #print "l" if $self->{verbose};
245 my $statOK = $self->_statloadNode('target', $self->{node}->{source}->{ident});
246
247 # mark node as new either if there's no ident or if stat/load failed
248 if (!$statOK) {
249 $self->{node}->{status}->{new} = 1;
250 print "n" if $self->{verbose};
251 }
252
253 #print "checksum", "\n";
254
255 print ":" if $self->{verbose};
256
257 #print Dumper($self);
258
259 # determine status of entry by synchronization method
260 if ( lc $self->{options}->{metadata}->{syncMethod} eq 'checksum' ) {
261 #if ( $statOK && (lc $self->{args}->{method} eq 'checksum') ) {
262 #if ( !$self->{node}->{status}->{new} && (lc $self->{args}->{method} eq 'checksum') ) {
263
264 # calculate checksum of source node
265 #$self->_calcChecksum('source');
266 if (!$self->_readChecksum('source')) {
267 $logger->warning( __PACKAGE__ . "->_run: Could not find \"source\" entry with ident=\"$self->{node}->{source}->{ident}\"" );
268 $tc->{skip}++;
269 print "s" if $self->{verbose};
270 next;
271 }
272
273 # get checksum from synchronization target
274 $self->_readChecksum('target');
275 #if (!$self->_readChecksum('target')) {
276 # $logger->critical( __PACKAGE__ . "->_readChecksum: Could not find \"target\" entry with ident=\"$self->{node}->{source}->{ident}\"" );
277 # next;
278 #}
279
280 # pre flight check: do we actually have a checksum provided?
281 #if (!$self->{node}->{source}->{checksum}) {
282 # print "Source checksum for entry with ident \"$self->{node}->{source}->{ident}\" could not be calculated, maybe it's missing?.", "\n";
283 # return;
284 #}
285
286 # determine if entry is "new" or "dirty"
287 # after all, this seems to be the point where the hammer falls.....
288 print "c" if $self->{verbose};
289 #print Dumper($self->{node});
290 #exit;
291 $self->{node}->{status}->{new} = !$self->{node}->{target}->{checksum};
292 if (!$self->{node}->{status}->{new}) {
293 $self->{node}->{status}->{dirty} =
294 $self->{node}->{status}->{new} ||
295 (!$self->{node}->{source}->{checksum} || !$self->{node}->{target}->{checksum}) ||
296 ($self->{node}->{source}->{checksum} ne $self->{node}->{target}->{checksum}) ||
297 $self->{args}->{force};
298 }
299
300 } else {
301 $logger->warning( __PACKAGE__ . "->_run: Synchronization method '$self->{options}->{metadata}->{syncMethod}' is not implemented" );
302 $tc->{skip}++;
303 print "s" if $self->{verbose};
304 next;
305
306 }
307
308 # first reaction on entry-status: continue with next entry if the current is already "in sync"
309 if (!$self->{node}->{status}->{new} && !$self->{node}->{status}->{dirty}) {
310 $tc->{in_sync}++;
311 next;
312 }
313
314 # build map to actually transfer the data from source to target
315 $self->buildAttributeMap();
316
317
318 #print Dumper($self->{node}); exit;
319
320 #print "attempt", "\n";
321
322 # additional (new) checks for feature "write-protection"
323 if ($self->{meta}->{target}->{storage}->{isWriteProtected}) {
324 $tc->{attempt_transfer}++;
325 print "\n" if $self->{verbose};
326 $logger->notice( __PACKAGE__ . "->_run: Target is write-protected. Will not insert or modify node. " .
327 "(Ident: $self->{node}->{source}->{ident} " . "Dump:\n" . Dumper($self->{node}->{source}->{payload}) . ")" );
328 print "\n" if $self->{verbose};
329 $tc->{skip}++;
330 next;
331 }
332
333 #print Dumper($self);
334 #exit;
335
336 # transfer contents of map to target
337 if ($self->{node}->{status}->{new}) {
338 $tc->{attempt_new}++;
339 $self->_doTransferToTarget('insert');
340 # asymmetry: refetch node from target to re-calculate new ident and checksum (TODO: is IdentAuthority of relevance here?)
341 #print Dumper($self->{node});
342 $self->_statloadNode('target', $self->{node}->{target}->{ident}, 1);
343 $self->_readChecksum('target');
344
345 } elsif ($self->{node}->{status}->{dirty}) {
346 $tc->{attempt_modify}++;
347 # asymmetry: get ident before updating (TODO: is IdentAuthority of relevance here?)
348 $self->{node}->{target}->{ident} = $self->{node}->{map}->{$self->{meta}->{target}->{IdentProvider}->{arg}};
349 $self->_doTransferToTarget('update');
350 $self->_readChecksum('target');
351 }
352
353 if ($self->{node}->{status}->{ok}) {
354 $tc->{ok}++;
355 print "t" if $self->{verbose};
356 }
357
358 if ($self->{node}->{status}->{error}) {
359 $tc->{error}++;
360 push( @{$tc->{error_per_row}}, $self->{node}->{status}->{error} );
361 print "e" if $self->{verbose};
362 }
363
364 # trace
365 #print Dumper($self);
366 #exit;
367
368 # change ident in source (take from target), if transfer was ok and target is an IdentAuthority
369 # this is (for now) called a "retransmit" indicated by a "r"-character when verbosing
370 if ($self->{node}->{status}->{ok} && $self->{options}->{target}->{storage}->{idAuthority}) {
371 print "r" if $self->{verbose};
372 #print Dumper($self->{meta});
373 #print Dumper($self->{node});
374 #exit;
375 $self->_doModifySource_IdentChecksum($self->{node}->{target}->{ident});
376 }
377
378 }
379
380 print "\n" if $self->{verbose};
381
382 # build user-message from some stats
383 my $msg = "statistics: $tc";
384
385 if ($tc->{error_per_row}) {
386 $msg .= "\n";
387 $msg .= "errors from \"error_per_row\":" . "\n";
388 $msg .= Dumper($tc->{error_per_row});
389 }
390
391 # todo!!!
392 #sysevent( { usermsg => $msg, level => $level }, $taskEvent );
393 $logger->info( __PACKAGE__ . "->_run: $msg" );
394
395 return $tc;
396
397 }
398
399
400 # refactor this as some core-function to do a generic dump resolving data-encapsulations of e.g. Set::Object
401 sub _dumpCompact {
402 my $self = shift;
403
404 #my $vars = \@_;
405 my @data = ();
406
407 my $count = 0;
408 foreach (@_) {
409 my $item = {};
410 foreach my $key (keys %$_) {
411 my $val = $_->{$key};
412
413 #print Dumper($val);
414
415 if (ref $val eq 'Set::Object') {
416 #print "========================= SET", "\n";
417 #print Dumper($val);
418 #print Dumper($val->members());
419 #$val = $val->members();
420 #$vars->[$count]->{$key} = $val->members() if $val->can("members");
421 #$item->{$key} = $val->members() if $val->can("members");
422 $item->{$key} = $val->members();
423 #print Dumper($vars->[$count]->{$key});
424
425 } else {
426 $item->{$key} = $val;
427 }
428
429 }
430 push @data, $item;
431 $count++;
432 }
433
434 #print "Dump:", Dumper(@data), "\n";
435
436 $Data::Dumper::Indent = 0;
437 my $result = Dumper(@data);
438 $Data::Dumper::Indent = 2;
439 return $result;
440
441 }
442
443
444
445 sub _doTransferToTarget {
446 my $self = shift;
447 my $action = shift;
448 #print Dumper($self->{node});
449 #exit;
450 $self->_modifyNode('target', $action, $self->{node}->{map});
451 }
452
453
454 sub _doModifySource_IdentChecksum {
455 my $self = shift;
456 my $ident_new = shift;
457 # this changes an old node to a new one including ident and checksum
458 # TODO:
459 # - eventually introduce an external resource to store this data to
460 # - we won't have to "re"-modify the source node here
461 my $map = {
462 $self->{meta}->{source}->{IdentProvider}->{arg} => $ident_new,
463 cs => $self->{node}->{target}->{checksum},
464 };
465
466 #print Dumper($map);
467 #print Dumper($self->{node});
468 #exit;
469
470 $self->_modifyNode('source', 'update', $map);
471 }
472
473
474
475 sub _prepareNode_MetaProperties {
476 my $self = shift;
477 my $descent = shift;
478
479 $logger->info( __PACKAGE__ . "->_prepareNode_MetaProperties( descent $descent )" );
480
481 # TODO: this should (better) be: "my $firstnode = $self->_getFirstNode($descent);"
482 my $list = $self->_getNodeList($descent);
483
484 # get first node
485 my $firstnode = $list->[0];
486
487 # check if node contains meta properties/nodes
488 # TODO: "cs" is hardcoded here!
489 my @required = ( $self->{meta}->{$descent}->{IdentProvider}->{arg}, 'cs' );
490 my @found = keys %$firstnode;
491 #my @diff = getDifference(\@found, \@required);
492 my $diff = getDifference(\@required, \@found);
493 #print Dumper(@found);
494 #print Dumper(@required);
495 #print Dumper(@diff);
496 #if (!$#diff || $#diff == -1) {
497 if (isEmpty($diff)) {
498 $logger->warning( __PACKAGE__ . "->_prepareNode_MetaProperties: node is lacking meta properties - will try to alter..." );
499 foreach (@required) {
500 my $sql = "ALTER TABLE $self->{meta}->{$descent}->{node} ADD COLUMN $_";
501 #print "sql: $sql", "\n";
502 my $res = $self->{meta}->{$descent}->{storage}->sendCommand($sql);
503 #print Dumper($res->getStatus());
504 }
505 }
506
507 }
508
509 sub _prepareNode_DummyIdent {
510 my $self = shift;
511 my $descent = shift;
512
513 $logger->info( __PACKAGE__ . "->_prepareNode_DummyIdent( descent $descent )" );
514
515 my $list = $self->_getNodeList($descent);
516 #print Dumper($list);
517 my $i = 0;
518 my $ident_base = 5678983;
519 my $ident_appendix = '0001';
520 foreach my $node (@$list) {
521 my $ident_dummy = $i + $ident_base;
522 $ident_dummy .= $ident_appendix;
523 my $map = {
524 $self->{meta}->{$descent}->{IdentProvider}->{arg} => $ident_dummy,
525 cs => undef,
526 };
527
528 # diff lists and ...
529 my $diff = getDifference([keys %$node], [keys %$map]);
530 next if $#{$diff} == -1;
531
532 # ... build criteria including all columns
533 my @crits;
534 foreach my $property (@$diff) {
535 next if !$property;
536 my $value = $node->{$property};
537 next if !$value;
538 push @crits, "$property='" . quotesql($value) . "'";
539 }
540 my $crit = join ' AND ', @crits;
541 print "p" if $self->{verbose};
542
543 #print Dumper($map);
544 #print Dumper($crit);
545
546 $self->_modifyNode($descent, 'update', $map, $crit);
547 $i++;
548 }
549
550 print "\n" if $self->{verbose};
551
552 if (!$i) {
553 $logger->warning( __PACKAGE__ . "->_prepareNode_DummyIdent: no nodes touched" );
554 }
555
556 }
557
558 # TODO: handle this in an abstract way (wipe out use of 'source' and/or 'target' inside core)
559 sub _otherSide {
560 my $self = shift;
561 my $descent = shift;
562 return 'source' if $descent eq 'target';
563 return 'target' if $descent eq 'source';
564 return '';
565 }
566
567 sub _erase_all {
568 my $self = shift;
569 my $descent = shift;
570 #my $node = shift;
571 my $node = $self->{meta}->{$descent}->{node};
572 $self->{meta}->{$descent}->{storage}->eraseAll($node);
573 }
574
575 1;

MailToCvsAdmin">MailToCvsAdmin
ViewVC Help
Powered by ViewVC 1.1.26 RSS 2.0 feed