/[cvs]/nfo/perl/libs/Data/Transfer/Sync/Core.pm
ViewVC logotype

Contents of /nfo/perl/libs/Data/Transfer/Sync/Core.pm

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.5 - (show annotations)
Tue Feb 11 05:30:47 2003 UTC (21 years, 5 months ago) by joko
Branch: MAIN
Changes since 1.4: +47 -17 lines
+ minor fixes and some debugging mud

1 ## $Id: Core.pm,v 1.4 2003/02/09 05:01:10 joko Exp $
2 ##
3 ## Copyright (c) 2002 Andreas Motl <andreas.motl@ilo.de>
4 ##
5 ## See COPYRIGHT section in pod text below for usage and distribution rights.
6 ##
7 ## ----------------------------------------------------------------------------------------
8 ## $Log: Core.pm,v $
9 ## Revision 1.4 2003/02/09 05:01:10 joko
10 ## + major structure changes
11 ## - refactored code to sister modules
12 ##
13 ## Revision 1.3 2003/01/20 17:01:14 joko
14 ## + cosmetics and debugging
15 ## + probably refactored code to new plugin-modules 'Metadata.pm' and/or 'StorageInterface.pm' (guess it was the last one...)
16 ##
17 ## Revision 1.2 2003/01/19 02:05:42 joko
18 ## - removed pod-documentation: now in Data/Transfer/Sync.pod
19 ##
20 ## Revision 1.1 2003/01/19 01:23:04 joko
21 ## + new from Data/Transfer/Sync.pm
22 ##
23 ## Revision 1.11 2002/12/23 07:10:59 joko
24 ## + using MD5 for checksum generation again - the 32-bit integer hash from DBI seems to be too lazy
25 ##
26 ## Revision 1.10 2002/12/19 01:07:16 joko
27 ## + fixed output done via $logger
28 ##
29 ## Revision 1.9 2002/12/16 07:02:34 jonen
30 ## + added comment
31 ##
32 ## Revision 1.8 2002/12/15 02:03:09 joko
33 ## + fixed logging-messages
34 ## + additional metadata-checks
35 ##
36 ## Revision 1.7 2002/12/13 21:49:34 joko
37 ## + sub configure
38 ## + sub checkOptions
39 ##
40 ## Revision 1.6 2002/12/06 04:49:10 jonen
41 ## + disabled output-puffer here
42 ##
43 ## Revision 1.5 2002/12/05 08:06:05 joko
44 ## + bugfix with determining empty fields (Null) with DBD::CSV
45 ## + debugging
46 ## + updated comments
47 ##
48 ## Revision 1.4 2002/12/03 15:54:07 joko
49 ## + {import}-flag is now {prepare}-flag
50 ##
51 ## Revision 1.3 2002/12/01 22:26:59 joko
52 ## + minor cosmetics for logging
53 ##
54 ## Revision 1.2 2002/12/01 04:43:25 joko
55 ## + mapping deatil entries may now be either an ARRAY or a HASH
56 ## + erase flag is used now (for export-operations)
57 ## + expressions to refer to values inside deep nested structures
58 ## - removed old mappingV2-code
59 ## + cosmetics
60 ## + sub _erase_all
61 ##
62 ## Revision 1.1 2002/11/29 04:45:50 joko
63 ## + initial check in
64 ##
65 ## Revision 1.1 2002/10/10 03:44:21 cvsjoko
66 ## + new
67 ## ----------------------------------------------------------------------------------------
68
69
70 package Data::Transfer::Sync::Core;
71
72 use strict;
73 use warnings;
74
75 use mixin::with qw( Data::Transfer::Sync );
76
77
78 # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - main
79
80 use Data::Dumper;
81
82 use misc::HashExt;
83 use Data::Compare::Struct qw( getDifference isEmpty );
84 use Data::Storage::Container;
85 use DesignPattern::Object;
86 use libdb qw( quotesql );
87
88 # get logger instance
89 my $logger = Log::Dispatch::Config->instance;
90
91 $| = 1;
92
93 =pod
94 sub new {
95 my $invocant = shift;
96 my $class = ref($invocant) || $invocant;
97 my $self = {};
98 $logger->debug( __PACKAGE__ . "->new(@_)" );
99 bless $self, $class;
100 $self->configure(@_);
101 return $self;
102 }
103 =cut
104
105
106 sub _init {
107 my $self = shift;
108
109 # build new container if necessary
110 $self->{container} = Data::Storage::Container->new() if !$self->{container};
111
112 # add storages to container (optional)
113 foreach (keys %{$self->{storages}}) {
114 $self->{container}->addStorage($_, $self->{storages}->{$_});
115 }
116
117 # trace
118 #print Dumper($self);
119 #exit;
120
121 return 1;
122
123 }
124
125 sub _initV1 {
126 my $self = shift;
127 $logger->debug( __PACKAGE__ . "->_initV1" );
128 die("this should not be reached!");
129 # tag storages with id-authority and checksum-provider information
130 # TODO: better store tag inside metadata to hold bits together!
131 map { $self->{container}->{storage}->{$_}->{isIdentAuthority} = 1 } @{$self->{id_authorities}};
132 map { $self->{container}->{storage}->{$_}->{isChecksumAuthority} = 1; } @{$self->{checksum_authorities}};
133 map { $self->{container}->{storage}->{$_}->{isWriteProtected} = 1; } @{$self->{write_protected}};
134 }
135
136
137 # TODO: abstract the hardwired use of "source" and "target" in here somehow - hmmmm....... /(="ยง/%???
138 sub _run {
139
140 my $self = shift;
141
142 #print "verbose: ", $self->{verbose}, "\n";
143 $self->{verbose} = 1;
144
145 $logger->debug( __PACKAGE__ . "->_run" );
146
147 # for statistics
148 my $tc = OneLineDumpHash->new( {} );
149 my $results;
150
151 # set of objects is already in $self->{args}
152 # TODO: make independent of the terminology "object..."
153 $results = $self->{args}->{objectSet} if $self->{args}->{objectSet};
154
155 # apply filter
156 if (my $filter = $self->{meta}->{source}->{filter}) {
157 #print Dumper($filter);
158 #exit;
159 $results ||= $self->_getNodeList('source', $filter);
160 }
161
162 # get reference to node list from convenient method provided by CORE-HANDLE
163 $results ||= $self->_getNodeList('source');
164
165 # checkpoint: do we actually have a list to iterate through?
166 if (!$results || !@{$results}) {
167 $logger->notice( __PACKAGE__ . "->_run: No nodes to synchronize." );
168 return;
169 }
170
171
172
173 # check if we actually *have* a synchronization method
174 if (!$self->{options}->{metadata}->{syncMethod}) {
175 $logger->critical( __PACKAGE__ . "->_run: No synchronization method (checksum|timestamp) specified" );
176 return;
177 }
178
179
180 # dereference
181 my @results = @{$results};
182
183 # iterate through set
184 foreach my $source_node_real (@results) {
185
186 print ":" if $self->{verbose};
187
188 $tc->{total}++;
189
190 #print "======================== iter", "\n";
191
192 # clone object (in case we have to modify it here)
193 # TODO:
194 # - is a "deep_copy" needed here if occouring modifications take place?
195 # - puuhhhh, i guess a deep_copy would destroy tangram mechanisms?
196 # - after all, just take care for now that this object doesn't get updated!
197 # - so, just use its reference for now - if some cloning is needed in future, do this here!
198 my $source_node = $source_node_real;
199
200 # modify entry - handle new style callbacks (the readers)
201 #print Dumper($source_node);
202 #exit;
203
204 my $descent = 'source';
205
206 # handle callbacks right now while scanning them (asymmetric to the writers)
207 my $map_callbacks = {};
208 if (my $callbacks = $self->{meta}->{$descent}->{Callback}) {
209
210 # trace
211 #print Dumper($callbacks);
212 #exit;
213
214 my $error = 0;
215
216 foreach my $node (keys %{$callbacks->{read}}) {
217
218 my $object = $source_node;
219 my $value; # = $source_node->{$node};
220
221 #print Dumper($self->{options});
222
223 # ------------ half-redundant: make $self->callCallback($object, $value, $opts)
224 #my $perl_callback = $self->{meta}->{$descent}->{node} . '::' . $node . '_read';
225 my $perl_callback = $self->{meta}->{$descent}->{nodeType} . '::' . $node . '_read';
226 my $evalstring = 'return ' . $perl_callback . '( { object => $object, property => $node, value => $value, storage => $self->{meta}->{$descent}->{storage} } );';
227 #print $evalstring, "\n"; exit;
228 my $cb_result = eval($evalstring);
229 if ($@) {
230 #die $@;
231 $logger->error( __PACKAGE__ . "->_run: $@" );
232 $error = 1;
233 #print $@, "\n";
234 return;
235 }
236 # ------------ half-redundant: make $self->callCallback($object, $value, $opts)
237
238 $source_node->{$node} = $cb_result;
239
240 }
241
242 }
243
244 # trace
245 #print Dumper($source_node);
246
247 # exclude defined fields (simply delete from object)
248 map { delete $source_node->{$_} } @{$self->{meta}->{source}->{subnodes_exclude}};
249
250 # here we accumulate information about the status of the current node (payload/row/object/item/entry)
251 $self->{node} = {};
252 $self->{node}->{source}->{payload} = $source_node;
253
254 # trace
255 #print Dumper($self->{node});
256 #exit;
257
258 # determine ident of entry
259 my $identOK = $self->_resolveNodeIdent('source');
260 #if (!$identOK && lc $self->{args}->{direction} ne 'import') {
261 if (!$identOK) {
262 #print Dumper($self->{meta}->{source});
263 $logger->critical( __PACKAGE__ . "->_run: No ident found in source node ( nodeName='$self->{meta}->{source}->{nodeName}', nodeType='$self->{meta}->{source}->{nodeType}') try to \"prepare\" this node first?" );
264 return;
265 }
266
267 #print "l" if $self->{verbose};
268 my $statOK = $self->_statloadNode('target', $self->{node}->{source}->{ident});
269
270 # mark node as new either if there's no ident or if stat/load failed
271 if (!$statOK) {
272 $self->{node}->{status}->{new} = 1;
273 print "n" if $self->{verbose};
274 }
275
276 #print "checksum", "\n";
277
278 #print Dumper($self);
279
280 # determine status of entry by synchronization method
281 if ( lc $self->{options}->{metadata}->{syncMethod} eq 'checksum' ) {
282 #if ( $statOK && (lc $self->{args}->{method} eq 'checksum') ) {
283 #if ( !$self->{node}->{status}->{new} && (lc $self->{args}->{method} eq 'checksum') ) {
284
285 # calculate checksum of source node
286 #$self->_calcChecksum('source');
287 if (!$self->_readChecksum('source')) {
288 $logger->warning( __PACKAGE__ . "->_run: Could not find \"source\" entry with ident=\"$self->{node}->{source}->{ident}\"" );
289 $tc->{skip}++;
290 print "s" if $self->{verbose};
291 next;
292 }
293
294 # get checksum from synchronization target
295 $self->_readChecksum('target');
296 #if (!$self->_readChecksum('target')) {
297 # $logger->critical( __PACKAGE__ . "->_readChecksum: Could not find \"target\" entry with ident=\"$self->{node}->{source}->{ident}\"" );
298 # next;
299 #}
300
301 # pre flight check: do we actually have a checksum provided?
302 #if (!$self->{node}->{source}->{checksum}) {
303 # print "Source checksum for entry with ident \"$self->{node}->{source}->{ident}\" could not be calculated, maybe it's missing?.", "\n";
304 # return;
305 #}
306
307 # determine if entry is "new" or "dirty"
308 # after all, this seems to be the point where the hammer falls.....
309 print "c" if $self->{verbose};
310
311 # trace
312 #print Dumper($self->{node});
313 #exit;
314
315 $self->{node}->{status}->{new} = !$self->{node}->{target}->{checksum};
316 if (!$self->{node}->{status}->{new}) {
317 $self->{node}->{status}->{dirty} =
318 $self->{node}->{status}->{new} ||
319 (!$self->{node}->{source}->{checksum} || !$self->{node}->{target}->{checksum}) ||
320 ($self->{node}->{source}->{checksum} ne $self->{node}->{target}->{checksum}) ||
321 $self->{args}->{force};
322 }
323
324 } else {
325 $logger->warning( __PACKAGE__ . "->_run: Synchronization method '$self->{options}->{metadata}->{syncMethod}' is not implemented" );
326 $tc->{skip}++;
327 print "s" if $self->{verbose};
328 next;
329
330 }
331
332 # first reaction on entry-status: continue with next entry if the current is already "in sync"
333 if (!$self->{node}->{status}->{new} && !$self->{node}->{status}->{dirty}) {
334 $tc->{in_sync}++;
335 next;
336 }
337
338 # build map to actually transfer the data from source to target
339 $self->buildAttributeMap();
340
341
342 # trace
343 #print Dumper($self->{node}); exit;
344 #print "attempt", "\n";
345
346 # additional (new) checks for feature "write-protection"
347 if ($self->{meta}->{target}->{storage}->{isWriteProtected}) {
348 $tc->{attempt_transfer}++;
349 print "\n" if $self->{verbose};
350 $logger->notice( __PACKAGE__ . "->_run: Target is write-protected. Will not insert or modify node. " .
351 "(Ident: $self->{node}->{source}->{ident} " . "Dump:\n" . Dumper($self->{node}->{source}->{payload}) . ")" );
352 print "\n" if $self->{verbose};
353 $tc->{skip}++;
354 next;
355 }
356
357 # trace
358 #print Dumper($self);
359 #exit;
360
361 # transfer contents of map to target
362 if ($self->{node}->{status}->{new}) {
363 $tc->{attempt_new}++;
364 $self->_doTransferToTarget('insert');
365 # asymmetry: refetch node from target to re-calculate new ident and checksum (TODO: is IdentAuthority of relevance here?)
366 #print Dumper($self->{node});
367 $self->_statloadNode('target', $self->{node}->{target}->{ident}, 1);
368 $self->_readChecksum('target');
369
370 } elsif ($self->{node}->{status}->{dirty}) {
371 $tc->{attempt_modify}++;
372 # asymmetry: get ident before updating (TODO: is IdentAuthority of relevance here?)
373 $self->{node}->{target}->{ident} = $self->{node}->{map}->{$self->{meta}->{target}->{IdentProvider}->{arg}};
374 $self->_doTransferToTarget('update');
375 $self->_readChecksum('target');
376 }
377
378 if ($self->{node}->{status}->{ok}) {
379 $tc->{ok}++;
380 print "t" if $self->{verbose};
381 }
382
383 if ($self->{node}->{status}->{error}) {
384 $tc->{error}++;
385 push( @{$tc->{error_per_row}}, $self->{node}->{status}->{error} );
386 print "e" if $self->{verbose};
387 }
388
389 # trace
390 #print Dumper($self);
391 #exit;
392
393 # change ident in source (take from target), if transfer was ok and target is an IdentAuthority
394 # this is (for now) called a "retransmit" indicated by a "r"-character when verbosing
395 #if ($self->{node}->{status}->{ok} && $self->{options}->{target}->{storage}->{idAuthority}) {
396 if ($self->{node}->{status}->{ok} && $self->{meta}->{target}->{isIdentAuthority}) {
397 print "r" if $self->{verbose};
398 #print Dumper($self->{meta});
399 #print Dumper($self->{node});
400 #exit;
401 $self->_doModifySource_IdentChecksum($self->{node}->{target}->{ident});
402 }
403
404 }
405
406 print "\n" if $self->{verbose};
407
408 # build user-message from some stats
409 my $msg = "statistics: $tc";
410
411 if ($tc->{error_per_row}) {
412 $msg .= "\n";
413 $msg .= "errors from \"error_per_row\":" . "\n";
414 $msg .= Dumper($tc->{error_per_row});
415 }
416
417 # todo!!!
418 #sysevent( { usermsg => $msg, level => $level }, $taskEvent );
419 $logger->info( __PACKAGE__ . "->_run: $msg" );
420
421 return $tc;
422
423 }
424
425
426 # refactor this as some core-function to do a generic dump resolving data-encapsulations of e.g. Set::Object
427 sub _dumpCompact {
428 my $self = shift;
429
430 #my $vars = \@_;
431 my @data = ();
432
433 my $count = 0;
434 foreach (@_) {
435 my $item = {};
436 foreach my $key (keys %$_) {
437 my $val = $_->{$key};
438
439 #print Dumper($val);
440
441 if (ref $val eq 'Set::Object') {
442 #print "========================= SET", "\n";
443 #print Dumper($val);
444 #print Dumper($val->members());
445 #$val = $val->members();
446 #$vars->[$count]->{$key} = $val->members() if $val->can("members");
447 #$item->{$key} = $val->members() if $val->can("members");
448 $item->{$key} = $val->members();
449 #print Dumper($vars->[$count]->{$key});
450
451 } else {
452 $item->{$key} = $val;
453 }
454
455 }
456 push @data, $item;
457 $count++;
458 }
459
460 #print "Dump:", Dumper(@data), "\n";
461
462 $Data::Dumper::Indent = 0;
463 my $result = Dumper(@data);
464 $Data::Dumper::Indent = 2;
465 return $result;
466
467 }
468
469
470
471 sub _doTransferToTarget {
472 my $self = shift;
473 my $action = shift;
474
475 # trace
476 #print Dumper($self->{meta});
477 #print Dumper($self->{node});
478 #exit;
479
480 $self->_modifyNode('target', $action, $self->{node}->{map});
481 }
482
483
484 sub _doModifySource_IdentChecksum {
485 my $self = shift;
486 my $ident_new = shift;
487 # this changes an old node to a new one including ident and checksum
488 # TODO:
489 # - eventually introduce an external resource to store this data to
490 # - we won't have to "re"-modify the source node here
491 my $map = {
492 $self->{meta}->{source}->{IdentProvider}->{arg} => $ident_new,
493 cs => $self->{node}->{target}->{checksum},
494 };
495
496 #print Dumper($map);
497 #print Dumper($self->{node});
498 #exit;
499
500 $self->_modifyNode('source', 'update', $map);
501 }
502
503
504
505 sub _prepareNode_MetaProperties {
506 my $self = shift;
507 my $descent = shift;
508
509 $logger->info( __PACKAGE__ . "->_prepareNode_MetaProperties( descent $descent )" );
510
511 # TODO: this should (better) be: "my $firstnode = $self->_getFirstNode($descent);"
512 my $list = $self->_getNodeList($descent);
513
514 # get first node
515 my $firstnode = $list->[0];
516
517 # check if node contains meta properties/nodes
518 # TODO: "cs" is hardcoded here!
519 my @required = ( $self->{meta}->{$descent}->{IdentProvider}->{arg}, 'cs' );
520 my @found = keys %$firstnode;
521 #my @diff = getDifference(\@found, \@required);
522 my $diff = getDifference(\@required, \@found);
523 #print Dumper(@found);
524 #print Dumper(@required);
525 #print Dumper(@diff);
526 #if (!$#diff || $#diff == -1) {
527 if (isEmpty($diff)) {
528 $logger->warning( __PACKAGE__ . "->_prepareNode_MetaProperties: node is lacking meta properties - will try to alter..." );
529 foreach (@required) {
530 my $sql = "ALTER TABLE $self->{meta}->{$descent}->{node} ADD COLUMN $_";
531 #print "sql: $sql", "\n";
532 my $res = $self->{meta}->{$descent}->{storage}->sendCommand($sql);
533 #print Dumper($res->getStatus());
534 }
535 }
536
537 }
538
539 sub _prepareNode_DummyIdent {
540 my $self = shift;
541 my $descent = shift;
542
543 $logger->info( __PACKAGE__ . "->_prepareNode_DummyIdent( descent $descent )" );
544
545 my $list = $self->_getNodeList($descent);
546 #print Dumper($list);
547 my $i = 0;
548 my $ident_base = 5678983;
549 my $ident_appendix = '0001';
550 foreach my $node (@$list) {
551 my $ident_dummy = $i + $ident_base;
552 $ident_dummy .= $ident_appendix;
553 my $map = {
554 $self->{meta}->{$descent}->{IdentProvider}->{arg} => $ident_dummy,
555 cs => undef,
556 };
557
558 # diff lists and ...
559 my $diff = getDifference([keys %$node], [keys %$map]);
560 next if $#{$diff} == -1;
561
562 # ... build criteria including all columns
563 my @crits;
564 foreach my $property (@$diff) {
565 next if !$property;
566 my $value = $node->{$property};
567 next if !$value;
568 push @crits, "$property='" . quotesql($value) . "'";
569 }
570 my $crit = join ' AND ', @crits;
571 print "p" if $self->{verbose};
572
573 #print Dumper($map);
574 #print Dumper($crit);
575
576 $self->_modifyNode($descent, 'update', $map, $crit);
577 $i++;
578 }
579
580 print "\n" if $self->{verbose};
581
582 if (!$i) {
583 $logger->warning( __PACKAGE__ . "->_prepareNode_DummyIdent: no nodes touched" );
584 }
585
586 }
587
588 # TODO: handle this in an abstract way (wipe out use of 'source' and/or 'target' inside core)
589 sub _otherSide {
590 my $self = shift;
591 my $descent = shift;
592 return 'source' if $descent eq 'target';
593 return 'target' if $descent eq 'source';
594 return '';
595 }
596
597 sub _erase_all {
598 my $self = shift;
599 my $descent = shift;
600 #my $node = shift;
601 my $node = $self->{meta}->{$descent}->{node};
602 $self->{meta}->{$descent}->{storage}->eraseAll($node);
603 }
604
605 1;

MailToCvsAdmin">MailToCvsAdmin
ViewVC Help
Powered by ViewVC 1.1.26 RSS 2.0 feed