/[cvs]/nfo/perl/libs/Data/Transfer/Sync/Core.pm
ViewVC logotype

Contents of /nfo/perl/libs/Data/Transfer/Sync/Core.pm

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.7 - (show annotations)
Fri Feb 21 08:01:11 2003 UTC (21 years, 4 months ago) by joko
Branch: MAIN
Changes since 1.6: +11 -5 lines
debugging, logging
renamed module

1 ## $Id: Core.pm,v 1.6 2003/02/14 14:03:49 joko Exp $
2 ##
3 ## Copyright (c) 2002 Andreas Motl <andreas.motl@ilo.de>
4 ##
5 ## See COPYRIGHT section in pod text below for usage and distribution rights.
6 ##
7 ## ----------------------------------------------------------------------------------------
8 ## $Log: Core.pm,v $
9 ## Revision 1.6 2003/02/14 14:03:49 joko
10 ## + logging, debugging
11 ## - refactored code to sister module
12 ##
13 ## Revision 1.5 2003/02/11 05:30:47 joko
14 ## + minor fixes and some debugging mud
15 ##
16 ## Revision 1.4 2003/02/09 05:01:10 joko
17 ## + major structure changes
18 ## - refactored code to sister modules
19 ##
20 ## Revision 1.3 2003/01/20 17:01:14 joko
21 ## + cosmetics and debugging
22 ## + probably refactored code to new plugin-modules 'Metadata.pm' and/or 'StorageInterface.pm' (guess it was the last one...)
23 ##
24 ## Revision 1.2 2003/01/19 02:05:42 joko
25 ## - removed pod-documentation: now in Data/Transfer/Sync.pod
26 ##
27 ## Revision 1.1 2003/01/19 01:23:04 joko
28 ## + new from Data/Transfer/Sync.pm
29 ##
30 ## Revision 1.11 2002/12/23 07:10:59 joko
31 ## + using MD5 for checksum generation again - the 32-bit integer hash from DBI seems to be too lazy
32 ##
33 ## Revision 1.10 2002/12/19 01:07:16 joko
34 ## + fixed output done via $logger
35 ##
36 ## Revision 1.9 2002/12/16 07:02:34 jonen
37 ## + added comment
38 ##
39 ## Revision 1.8 2002/12/15 02:03:09 joko
40 ## + fixed logging-messages
41 ## + additional metadata-checks
42 ##
43 ## Revision 1.7 2002/12/13 21:49:34 joko
44 ## + sub configure
45 ## + sub checkOptions
46 ##
47 ## Revision 1.6 2002/12/06 04:49:10 jonen
48 ## + disabled output-puffer here
49 ##
50 ## Revision 1.5 2002/12/05 08:06:05 joko
51 ## + bugfix with determining empty fields (Null) with DBD::CSV
52 ## + debugging
53 ## + updated comments
54 ##
55 ## Revision 1.4 2002/12/03 15:54:07 joko
56 ## + {import}-flag is now {prepare}-flag
57 ##
58 ## Revision 1.3 2002/12/01 22:26:59 joko
59 ## + minor cosmetics for logging
60 ##
61 ## Revision 1.2 2002/12/01 04:43:25 joko
62 ## + mapping deatil entries may now be either an ARRAY or a HASH
63 ## + erase flag is used now (for export-operations)
64 ## + expressions to refer to values inside deep nested structures
65 ## - removed old mappingV2-code
66 ## + cosmetics
67 ## + sub _erase_all
68 ##
69 ## Revision 1.1 2002/11/29 04:45:50 joko
70 ## + initial check in
71 ##
72 ## Revision 1.1 2002/10/10 03:44:21 cvsjoko
73 ## + new
74 ## ----------------------------------------------------------------------------------------
75
76
77 package Data::Transfer::Sync::Core;
78
79 use strict;
80 use warnings;
81
82 use mixin::with qw( Data::Transfer::Sync );
83
84
85 # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - main
86
87 use Data::Dumper;
88
89 #use misc::HashExt;
90 use Hash::Serializer;
91 use Data::Compare::Struct qw( getDifference isEmpty );
92 use Data::Storage::Container;
93 use DesignPattern::Object;
94 use libdb qw( quotesql );
95
96 # get logger instance
97 my $logger = Log::Dispatch::Config->instance;
98
99 $| = 1;
100
101 =pod
102 sub new {
103 my $invocant = shift;
104 my $class = ref($invocant) || $invocant;
105 my $self = {};
106 $logger->debug( __PACKAGE__ . "->new(@_)" );
107 bless $self, $class;
108 $self->configure(@_);
109 return $self;
110 }
111 =cut
112
113
114 sub _init {
115 my $self = shift;
116
117 # build new container if necessary
118 $self->{container} = Data::Storage::Container->new() if !$self->{container};
119
120 # add storages to container (optional)
121 foreach (keys %{$self->{storages}}) {
122 $self->{container}->addStorage($_, $self->{storages}->{$_});
123 }
124
125 # trace
126 #print Dumper($self);
127 #exit;
128
129 return 1;
130
131 }
132
133 sub _initV1 {
134 my $self = shift;
135 $logger->debug( __PACKAGE__ . "->_initV1" );
136 die("this should not be reached!");
137 # tag storages with id-authority and checksum-provider information
138 # TODO: better store tag inside metadata to hold bits together!
139 map { $self->{container}->{storage}->{$_}->{isIdentAuthority} = 1 } @{$self->{id_authorities}};
140 map { $self->{container}->{storage}->{$_}->{isChecksumAuthority} = 1; } @{$self->{checksum_authorities}};
141 map { $self->{container}->{storage}->{$_}->{isWriteProtected} = 1; } @{$self->{write_protected}};
142 }
143
144
145 # TODO: abstract the hardwired use of "source" and "target" in here somehow - hmmmm....... /(="ยง/%???
146 sub _run {
147
148 my $self = shift;
149
150 #print "verbose: ", $self->{verbose}, "\n";
151 $self->{verbose} = 1;
152
153 $logger->debug( __PACKAGE__ . "->_run" );
154
155 # for statistics
156 my $tc = Hash::Serializer->new( {} );
157 my $results;
158
159 # set of objects is already in $self->{args}
160 # TODO: make independent of the terminology "object..."
161 $results = $self->{args}->{objectSet} if $self->{args}->{objectSet};
162
163 # apply filter
164 if (my $filter = $self->{meta}->{source}->{filter}) {
165 #print Dumper($filter);
166 #exit;
167 $results ||= $self->_getNodeList('source', $filter);
168 }
169
170 # get reference to node list from convenient method provided by CORE-HANDLE
171 $results ||= $self->_getNodeList('source');
172
173 # checkpoint: do we actually have a list to iterate through?
174 if (!$results || !@{$results}) {
175 $logger->notice( __PACKAGE__ . "->_run: No nodes to synchronize." );
176 return;
177 }
178
179
180
181 # check if we actually *have* a synchronization method
182 if (!$self->{options}->{metadata}->{syncMethod}) {
183 $logger->critical( __PACKAGE__ . "->_run: No synchronization method (checksum|timestamp) specified" );
184 return;
185 }
186
187
188 # dereference
189 my @results = @{$results};
190
191 # iterate through set
192 foreach my $source_node_real (@results) {
193
194 print ":" if $self->{verbose};
195
196 $tc->{total}++;
197
198 #print "======================== iter", "\n";
199
200 # clone object (in case we have to modify it here)
201 # TODO:
202 # - is a "deep_copy" needed here if occouring modifications take place?
203 # - puuhhhh, i guess a deep_copy would destroy tangram mechanisms?
204 # - after all, just take care for now that this object doesn't get updated!
205 # - so, just use its reference for now - if some cloning is needed in future, do this here!
206 my $source_node = $source_node_real;
207
208 # modify entry - handle new style callbacks (the readers)
209
210 # trace
211 #print Dumper($source_node);
212 #exit;
213
214 my $descent = 'source';
215
216 # handle callbacks right now while scanning them (asymmetric to the writers)
217 my $map_callbacks = {};
218 if (my $callbacks = $self->{meta}->{$descent}->{Callback}) {
219
220 # trace
221 #print Dumper($callbacks);
222 #exit;
223
224 my $error = 0;
225
226 foreach my $node (keys %{$callbacks->{read}}) {
227
228 my $object = $source_node;
229 my $value; # = $source_node->{$node};
230
231 # trace
232 #print Dumper($self->{options});
233
234 # ------------ half-redundant: make $self->callCallback($object, $value, $opts)
235 #my $perl_callback = $self->{meta}->{$descent}->{node} . '::' . $node . '_read';
236 my $perl_callback = $self->{meta}->{$descent}->{nodeType} . '::' . $node . '_read';
237 my $evalstring = 'return ' . $perl_callback . '( { object => $object, property => $node, value => $value, storage => $self->{meta}->{$descent}->{storage} } );';
238 #print $evalstring, "\n"; exit;
239 my $cb_result = eval($evalstring);
240 if ($@) {
241 $error = 1;
242 $logger->error( __PACKAGE__ . "->_run: $@" );
243 next;
244 }
245 # ------------ half-redundant: make $self->callCallback($object, $value, $opts)
246
247 $source_node->{$node} = $cb_result;
248
249 }
250
251 }
252
253 # trace
254 #print Dumper($source_node);
255
256 # exclude defined fields (simply delete from object)
257 map { delete $source_node->{$_} } @{$self->{meta}->{source}->{subnodes_exclude}};
258
259 # here we accumulate information about the status of the current node (payload/row/object/item/entry)
260 $self->{node} = {};
261 $self->{node}->{source}->{payload} = $source_node;
262
263 # trace
264 #print Dumper($self->{node});
265 #exit;
266
267 # determine ident of entry
268 my $identOK = $self->_resolveNodeIdent('source');
269 #if (!$identOK && lc $self->{args}->{direction} ne 'import') {
270 if (!$identOK) {
271 #print Dumper($self->{meta}->{source});
272 $logger->critical( __PACKAGE__ . "->_run: No ident found in source node ( nodeName='$self->{meta}->{source}->{nodeName}', nodeType='$self->{meta}->{source}->{nodeType}') try to \"prepare\" this node first?" );
273 return;
274 }
275
276 #print "l" if $self->{verbose};
277 my $statOK = $self->_statloadNode('target', $self->{node}->{source}->{ident});
278
279 # mark node as new either if there's no ident or if stat/load failed
280 if (!$statOK) {
281 $self->{node}->{status}->{new} = 1;
282 print "n" if $self->{verbose};
283 }
284
285 #print "checksum", "\n";
286
287 #print Dumper($self);
288
289 # determine status of entry by synchronization method
290 if ( lc $self->{options}->{metadata}->{syncMethod} eq 'checksum' ) {
291 #if ( $statOK && (lc $self->{args}->{method} eq 'checksum') ) {
292 #if ( !$self->{node}->{status}->{new} && (lc $self->{args}->{method} eq 'checksum') ) {
293
294 # calculate checksum of source node
295 #$self->_calcChecksum('source');
296 if (!$self->_readChecksum('source')) {
297 $logger->warning( __PACKAGE__ . "->_run: Could not find \"source\" entry with ident=\"$self->{node}->{source}->{ident}\"" );
298 $tc->{skip}++;
299 print "s" if $self->{verbose};
300 next;
301 }
302
303 # get checksum from synchronization target
304 $self->_readChecksum('target');
305 #if (!$self->_readChecksum('target')) {
306 # $logger->critical( __PACKAGE__ . "->_readChecksum: Could not find \"target\" entry with ident=\"$self->{node}->{source}->{ident}\"" );
307 # next;
308 #}
309
310 # pre flight check: do we actually have a checksum provided?
311 #if (!$self->{node}->{source}->{checksum}) {
312 # print "Source checksum for entry with ident \"$self->{node}->{source}->{ident}\" could not be calculated, maybe it's missing?.", "\n";
313 # return;
314 #}
315
316 # determine if entry is "new" or "dirty"
317 # after all, this seems to be the point where the hammer falls.....
318 print "c" if $self->{verbose};
319
320 # trace
321 #print Dumper($self->{node});
322 #exit;
323
324 $self->{node}->{status}->{new} = !$self->{node}->{target}->{checksum};
325 if (!$self->{node}->{status}->{new}) {
326 $self->{node}->{status}->{dirty} =
327 $self->{node}->{status}->{new} ||
328 (!$self->{node}->{source}->{checksum} || !$self->{node}->{target}->{checksum}) ||
329 ($self->{node}->{source}->{checksum} ne $self->{node}->{target}->{checksum}) ||
330 $self->{args}->{force};
331 }
332
333 } else {
334 $logger->warning( __PACKAGE__ . "->_run: Synchronization method '$self->{options}->{metadata}->{syncMethod}' is not implemented" );
335 $tc->{skip}++;
336 print "s" if $self->{verbose};
337 next;
338
339 }
340
341 # first reaction on entry-status: continue with next entry if the current is already "in sync"
342 if (!$self->{node}->{status}->{new} && !$self->{node}->{status}->{dirty}) {
343 $tc->{in_sync}++;
344 next;
345 }
346
347 # build map to actually transfer the data from source to target
348 $self->buildAttributeMap();
349
350
351 # trace
352 #print Dumper($self->{node}); exit;
353 #print "attempt", "\n";
354
355 # additional (new) checks for feature "write-protection"
356 if ($self->{meta}->{target}->{storage}->{isWriteProtected}) {
357 $tc->{attempt_transfer}++;
358 print "\n" if $self->{verbose};
359 $logger->notice( __PACKAGE__ . "->_run: Target is write-protected. Will not insert or modify node. " .
360 "(Ident: $self->{node}->{source}->{ident} " . "Dump:\n" . Dumper($self->{node}->{source}->{payload}) . ")" );
361 print "\n" if $self->{verbose};
362 $tc->{skip}++;
363 next;
364 }
365
366 # trace
367 #print Dumper($self);
368 #exit;
369
370 # transfer contents of map to target
371 if ($self->{node}->{status}->{new}) {
372 $tc->{attempt_new}++;
373 $self->_doTransferToTarget('insert');
374 # asymmetry: refetch node from target to re-calculate new ident and checksum (TODO: is IdentAuthority of relevance here?)
375 #print Dumper($self->{node});
376 $self->_statloadNode('target', $self->{node}->{target}->{ident}, 1);
377 $self->_readChecksum('target');
378
379 } elsif ($self->{node}->{status}->{dirty}) {
380 $tc->{attempt_modify}++;
381 # asymmetry: get ident before updating (TODO: is IdentAuthority of relevance here?)
382 $self->{node}->{target}->{ident} = $self->{node}->{map}->{$self->{meta}->{target}->{IdentProvider}->{arg}};
383 $self->_doTransferToTarget('update');
384 $self->_readChecksum('target');
385 }
386
387 if ($self->{node}->{status}->{ok}) {
388 $tc->{ok}++;
389 print "t" if $self->{verbose};
390 }
391
392 if ($self->{node}->{status}->{error}) {
393 $tc->{error}++;
394 push( @{$tc->{error_per_row}}, $self->{node}->{status}->{error} );
395 print "e" if $self->{verbose};
396 }
397
398 # trace
399 #print Dumper($self);
400 #exit;
401
402 # change ident in source (take from target), if transfer was ok and target is an IdentAuthority
403 # this is (for now) called a "retransmit" indicated by a "r"-character when verbosing
404 #if ($self->{node}->{status}->{ok} && $self->{options}->{target}->{storage}->{idAuthority}) {
405 if ($self->{node}->{status}->{ok} && $self->{meta}->{target}->{isIdentAuthority}) {
406 print "r" if $self->{verbose};
407 #print Dumper($self->{meta});
408 #print Dumper($self->{node});
409 #exit;
410 $self->_doModifySource_IdentChecksum($self->{node}->{target}->{ident});
411 }
412
413 }
414
415 print "\n" if $self->{verbose};
416
417 # build user-message from some stats
418 my $msg = "statistics: $tc";
419
420 if ($tc->{error_per_row}) {
421 $msg .= "\n";
422 $msg .= "errors from \"error_per_row\":" . "\n";
423 $msg .= Dumper($tc->{error_per_row});
424 }
425
426 # todo!!!
427 #sysevent( { usermsg => $msg, level => $level }, $taskEvent );
428 #$logger->info( __PACKAGE__ . "->_run: $msg" );
429 $logger->info($msg . "\n");
430
431 return $tc;
432
433 }
434
435
436 # refactor this as some core-function to do a generic dump resolving data-encapsulations of e.g. Set::Object
437 sub _dumpCompact {
438 my $self = shift;
439
440 #my $vars = \@_;
441 my @data = ();
442
443 my $count = 0;
444 foreach (@_) {
445 my $item = {};
446 foreach my $key (keys %$_) {
447 my $val = $_->{$key};
448
449 #print Dumper($val);
450
451 if (ref $val eq 'Set::Object') {
452 #print "========================= SET", "\n";
453 #print Dumper($val);
454 #print Dumper($val->members());
455 #$val = $val->members();
456 #$vars->[$count]->{$key} = $val->members() if $val->can("members");
457 #$item->{$key} = $val->members() if $val->can("members");
458 $item->{$key} = $val->members();
459 #print Dumper($vars->[$count]->{$key});
460
461 } else {
462 $item->{$key} = $val;
463 }
464
465 }
466 push @data, $item;
467 $count++;
468 }
469
470 #print "Dump:", Dumper(@data), "\n";
471
472 $Data::Dumper::Indent = 0;
473 my $result = Dumper(@data);
474 $Data::Dumper::Indent = 2;
475 return $result;
476
477 }
478
479
480
481 sub _doTransferToTarget {
482 my $self = shift;
483 my $action = shift;
484
485 # trace
486 #print Dumper($self->{meta});
487 #print Dumper($self->{node});
488 #exit;
489
490 $self->_modifyNode('target', $action, $self->{node}->{map});
491 }
492
493
494 sub _doModifySource_IdentChecksum {
495 my $self = shift;
496 my $ident_new = shift;
497 # this changes an old node to a new one including ident and checksum
498 # TODO:
499 # - eventually introduce an external resource to store this data to
500 # - we won't have to "re"-modify the source node here
501 my $map = {
502 $self->{meta}->{source}->{IdentProvider}->{arg} => $ident_new,
503 cs => $self->{node}->{target}->{checksum},
504 };
505
506 #print Dumper($map);
507 #print Dumper($self->{node});
508 #exit;
509
510 $self->_modifyNode('source', 'update', $map);
511 }
512
513
514
515 sub _prepareNode_MetaProperties {
516 my $self = shift;
517 my $descent = shift;
518
519 $logger->info( __PACKAGE__ . "->_prepareNode_MetaProperties( descent $descent )" );
520
521 # TODO: this should (better) be: "my $firstnode = $self->_getFirstNode($descent);"
522 my $list = $self->_getNodeList($descent);
523
524 # get first node
525 my $firstnode = $list->[0];
526
527 # check if node contains meta properties/nodes
528 # TODO: "cs" is hardcoded here!
529 my @required = ( $self->{meta}->{$descent}->{IdentProvider}->{arg}, 'cs' );
530 my @found = keys %$firstnode;
531 #my @diff = getDifference(\@found, \@required);
532 my $diff = getDifference(\@required, \@found);
533 #print Dumper(@found);
534 #print Dumper(@required);
535 #print Dumper(@diff);
536 #if (!$#diff || $#diff == -1) {
537 if (isEmpty($diff)) {
538 $logger->warning( __PACKAGE__ . "->_prepareNode_MetaProperties: node is lacking meta properties - will try to alter..." );
539 foreach (@required) {
540 my $sql = "ALTER TABLE $self->{meta}->{$descent}->{node} ADD COLUMN $_";
541 #print "sql: $sql", "\n";
542 my $res = $self->{meta}->{$descent}->{storage}->sendCommand($sql);
543 #print Dumper($res->getStatus());
544 }
545 }
546
547 }
548
549 sub _prepareNode_DummyIdent {
550 my $self = shift;
551 my $descent = shift;
552
553 $logger->info( __PACKAGE__ . "->_prepareNode_DummyIdent( descent $descent )" );
554
555 my $list = $self->_getNodeList($descent);
556 #print Dumper($list);
557 my $i = 0;
558 my $ident_base = 5678983;
559 my $ident_appendix = '0001';
560 foreach my $node (@$list) {
561 my $ident_dummy = $i + $ident_base;
562 $ident_dummy .= $ident_appendix;
563 my $map = {
564 $self->{meta}->{$descent}->{IdentProvider}->{arg} => $ident_dummy,
565 cs => undef,
566 };
567
568 # diff lists and ...
569 my $diff = getDifference([keys %$node], [keys %$map]);
570 next if $#{$diff} == -1;
571
572 # ... build criteria including all columns
573 my @crits;
574 foreach my $property (@$diff) {
575 next if !$property;
576 my $value = $node->{$property};
577 next if !$value;
578 push @crits, "$property='" . quotesql($value) . "'";
579 }
580 my $crit = join ' AND ', @crits;
581 print "p" if $self->{verbose};
582
583 #print Dumper($map);
584 #print Dumper($crit);
585
586 $self->_modifyNode($descent, 'update', $map, $crit);
587 $i++;
588 }
589
590 #print "\n" if $self->{verbose};
591
592 if (!$i) {
593 $logger->warning( __PACKAGE__ . "->_prepareNode_DummyIdent: no nodes touched" );
594 }
595
596 }
597
598 # TODO: handle this in an abstract way (wipe out use of 'source' and/or 'target' inside core)
599 sub _otherSide {
600 my $self = shift;
601 my $descent = shift;
602 return 'source' if $descent eq 'target';
603 return 'target' if $descent eq 'source';
604 return '';
605 }
606
607
608 1;
609 __END__

MailToCvsAdmin">MailToCvsAdmin
ViewVC Help
Powered by ViewVC 1.1.26 RSS 2.0 feed