/[cvs]/nfo/perl/libs/Data/Transfer/Sync/Core.pm
ViewVC logotype

Diff of /nfo/perl/libs/Data/Transfer/Sync/Core.pm

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.4 by joko, Sun Feb 9 05:01:10 2003 UTC revision 1.13 by joko, Sat Jun 19 16:53:38 2004 UTC
# Line 1  Line 1 
1    ## -------------------------------------------------------------------------
2    ##
3  ##    $Id$  ##    $Id$
4  ##  ##
5  ##    Copyright (c) 2002  Andreas Motl <andreas.motl@ilo.de>  ##    Copyright (c) 2002  Andreas Motl <andreas.motl@ilo.de>
6  ##  ##
7  ##    See COPYRIGHT section in pod text below for usage and distribution rights.  ##    See COPYRIGHT section in pod text below for usage and distribution rights.
8  ##  ##
9  ##    ----------------------------------------------------------------------------------------  ## -------------------------------------------------------------------------
10  ##    $Log$  ##    $Log$
11    ##    Revision 1.13  2004/06/19 16:53:38  joko
12    ##    fix: handle local checksum only if database is configured with "hasLocalChecksum"
13    ##
14    ##    Revision 1.12  2004/06/19 01:45:08  joko
15    ##    introduced "local checksum"-mechanism
16    ##    moved _dumpCompact to ::Compare::Checksum
17    ##
18    ##    Revision 1.11  2004/05/11 20:03:48  jonen
19    ##    bugfix[joko] related to Attribute Map
20    ##
21    ##    Revision 1.10  2003/06/25 23:03:57  joko
22    ##    no debugging
23    ##
24    ##    Revision 1.9  2003/05/13 08:17:52  joko
25    ##    buildAttributeMap now propagates error
26    ##
27    ##    Revision 1.8  2003/03/27 15:31:15  joko
28    ##    fixes to modules regarding new namespace(s) below Data::Mungle::*
29    ##
30    ##    Revision 1.7  2003/02/21 08:01:11  joko
31    ##    debugging, logging
32    ##    renamed module
33    ##
34    ##    Revision 1.6  2003/02/14 14:03:49  joko
35    ##    + logging, debugging
36    ##    - refactored code to sister module
37    ##
38    ##    Revision 1.5  2003/02/11 05:30:47  joko
39    ##    + minor fixes and some debugging mud
40    ##
41  ##    Revision 1.4  2003/02/09 05:01:10  joko  ##    Revision 1.4  2003/02/09 05:01:10  joko
42  ##    + major structure changes  ##    + major structure changes
43  ##    - refactored code to sister modules  ##    - refactored code to sister modules
# Line 52  Line 84 
84  ##    + minor cosmetics for logging  ##    + minor cosmetics for logging
85  ##  ##
86  ##    Revision 1.2  2002/12/01 04:43:25  joko  ##    Revision 1.2  2002/12/01 04:43:25  joko
87  ##    + mapping deatil entries may now be either an ARRAY or a HASH  ##    + mapping detail entries may now be either an ARRAY or a HASH
88  ##    + erase flag is used now (for export-operations)  ##    + erase flag is used now (for export-operations)
89  ##    + expressions to refer to values inside deep nested structures  ##    + expressions to refer to values inside deep nested structures
90  ##    - removed old mappingV2-code  ##    - removed old mappingV2-code
# Line 64  Line 96 
96  ##  ##
97  ##    Revision 1.1  2002/10/10 03:44:21  cvsjoko  ##    Revision 1.1  2002/10/10 03:44:21  cvsjoko
98  ##    + new  ##    + new
99  ##    ----------------------------------------------------------------------------------------  ## -------------------------------------------------------------------------
100    
101    
102  package Data::Transfer::Sync::Core;  package Data::Transfer::Sync::Core;
# Line 79  use mixin::with qw( Data::Transfer::Sync Line 111  use mixin::with qw( Data::Transfer::Sync
111    
112  use Data::Dumper;  use Data::Dumper;
113    
114  use misc::HashExt;  #use misc::HashExt;
115  use Data::Compare::Struct qw( getDifference isEmpty );  use Hash::Serializer;
116    use Data::Mungle::Compare::Struct qw( getDifference isEmpty );
117    use Data::Mungle::Transform::Deep qw( deep_copy expand );
118  use Data::Storage::Container;  use Data::Storage::Container;
119  use DesignPattern::Object;  use DesignPattern::Object;
120    use shortcuts::database qw( quotesql );
121    
122  # get logger instance  # get logger instance
123  my $logger = Log::Dispatch::Config->instance;  my $logger = Log::Dispatch::Config->instance;
# Line 113  sub _init { Line 148  sub _init {
148      $self->{container}->addStorage($_, $self->{storages}->{$_});      $self->{container}->addStorage($_, $self->{storages}->{$_});
149    }    }
150        
151      # trace
152        #print Dumper($self);
153        #exit;
154    
155    return 1;    return 1;
156        
157  }  }
# Line 140  sub _run { Line 179  sub _run {
179    $logger->debug( __PACKAGE__ . "->_run" );    $logger->debug( __PACKAGE__ . "->_run" );
180    
181    # for statistics    # for statistics
182    my $tc = OneLineDumpHash->new( {} );    my $tc = Hash::Serializer->new( {} );
183    my $results;    my $results;
184        
185    # set of objects is already in $self->{args}    # set of objects is already in $self->{args}
# Line 163  sub _run { Line 202  sub _run {
202      return;      return;
203    }    }
204    
205      #print Dumper(@$results);
206      #exit;
207    
208    # check if we actually *have* a synchronization method    # check if we actually *have* a synchronization method
209    if (!$self->{options}->{metadata}->{syncMethod}) {    if (!$self->{options}->{metadata}->{syncMethod}) {
# Line 174  sub _run { Line 214  sub _run {
214        
215    # dereference    # dereference
216    my @results = @{$results};    my @results = @{$results};
217      #print Dumper(@results);
218    
219    # iterate through set    # iterate through set
220    foreach my $source_node_real (@results) {    foreach my $source_node_real (@results) {
221    
222        print ":" if $self->{verbose};
223    
224        #print Dumper($source_node_real);
225    
226      $tc->{total}++;      $tc->{total}++;
227    
228  #print "========================  iter", "\n";      #print "=" x 80, "\n";
229    
230      # clone object (in case we have to modify it here)      # clone object (in case we have to modify it here)
231      # TODO:      # TODO:
232      #   - is a "deep_copy" needed here if occouring modifications take place?      #   - is a "deep_copy" needed here if occouring modifications take place?
233      #   - puuhhhh, i guess a deep_copy would destroy tangram mechanisms?      #   - puuhhhh, i guess a deep_copy would destroy tangram mechanisms?
234      #   - after all, just take care for now that this object doesn't get updated!      #   - after all, just take care for now that this object doesn't get updated!
235        #   - so, just use its reference for now - if some cloning is needed in future, do this here!
236      my $source_node = $source_node_real;      my $source_node = $source_node_real;
237        #my $source_node = expand($source_node_real);
238    
239      # modify entry - handle new style callbacks (the readers)      # modify entry - handle new style callbacks (the readers)
240  #print Dumper($source_node);  
241  #exit;      # trace
242        #print Dumper($source_node);
243        #exit;
244    
245      my $descent = 'source';      my $descent = 'source';
246    
# Line 199  sub _run { Line 248  sub _run {
248      my $map_callbacks = {};      my $map_callbacks = {};
249      if (my $callbacks = $self->{meta}->{$descent}->{Callback}) {      if (my $callbacks = $self->{meta}->{$descent}->{Callback}) {
250    
251          # trace
252            #print Dumper($callbacks);
253            #exit;
254    
255        my $error = 0;        my $error = 0;
256    
257        foreach my $node (keys %{$callbacks->{read}}) {        foreach my $node (keys %{$callbacks->{read}}) {
258                    
259            #print "cb_node: $node", "\n";
260            
261          my $object = $source_node;          my $object = $source_node;
262          my $value; # = $source_node->{$node};          my $value; # = $source_node->{$node};
263    
264          # trace
265            #print Dumper($self->{options});
266    
267          # ------------  half-redundant: make $self->callCallback($object, $value, $opts)          # ------------  half-redundant: make $self->callCallback($object, $value, $opts)
268          #my $perl_callback = $self->{meta}->{$descent}->{node} . '::' . $node . '_read';          #my $perl_callback = $self->{meta}->{$descent}->{node} . '::' . $node . '_read';
269          my $perl_callback = $self->{meta}->{$descent}->{nodeType} . '::' . $node . '_read';          my $perl_callback = $self->{meta}->{$descent}->{nodeType} . '::' . $node . '_read';
# Line 213  sub _run { Line 271  sub _run {
271          #print $evalstring, "\n"; exit;          #print $evalstring, "\n"; exit;
272          my $cb_result = eval($evalstring);          my $cb_result = eval($evalstring);
273          if ($@) {          if ($@) {
           die $@;  
274            $error = 1;            $error = 1;
275            print $@, "\n";            $logger->error( __PACKAGE__ . "->_run: $@" );
276              next;
277          }          }
278          # ------------  half-redundant: make $self->callCallback($object, $value, $opts)          # ------------  half-redundant: make $self->callCallback($object, $value, $opts)
279                    
# Line 225  sub _run { Line 283  sub _run {
283    
284      }      }
285    
286  #print Dumper($source_node);      # trace
287          #print Dumper($source_node);
288    
289      # exclude defined fields  (simply delete from object)      # exclude defined fields  (simply delete from object)
290      map { delete $source_node->{$_} } @{$self->{meta}->{source}->{subnodes_exclude}};      map { delete $source_node->{$_} } @{$self->{meta}->{source}->{subnodes_exclude}};
# Line 234  sub _run { Line 293  sub _run {
293      $self->{node} = {};      $self->{node} = {};
294      $self->{node}->{source}->{payload} = $source_node;      $self->{node}->{source}->{payload} = $source_node;
295    
296  #print "res - ident", "\n";      # trace
297          #print Dumper($self->{node});
298          #exit;
299    
300      # determine ident of entry      # determine ident of entry
301      my $identOK = $self->_resolveNodeIdent('source');      my $identOK = $self->_resolveNodeIdent('source');
# Line 254  sub _run { Line 315  sub _run {
315        print "n" if $self->{verbose};        print "n" if $self->{verbose};
316      }      }
317    
 #print "checksum", "\n";  
       
     print ":" if $self->{verbose};  
   
     #print Dumper($self);  
       
318      # determine status of entry by synchronization method      # determine status of entry by synchronization method
319      if ( lc $self->{options}->{metadata}->{syncMethod} eq 'checksum' ) {      if ( lc $self->{options}->{metadata}->{syncMethod} eq 'checksum' ) {
320      #if ( $statOK && (lc $self->{args}->{method} eq 'checksum') ) {      #if ( $statOK && (lc $self->{args}->{method} eq 'checksum') ) {
321      #if ( !$self->{node}->{status}->{new} && (lc $self->{args}->{method} eq 'checksum') ) {      #if ( !$self->{node}->{status}->{new} && (lc $self->{args}->{method} eq 'checksum') ) {
322                
323        # new 2004-06-17: calculate local checksum of source node
324        # new 2004-06-19: ... only if requested
325        if ($self->{options}->{source}->{storage}->{handle}->{locator}->{hasLocalChecksum}) {
326          $self->handleLocalChecksum('source');
327        }
328        
329      # calculate checksum of source node      # calculate checksum of source node
330        #$self->_calcChecksum('source');        #$self->_calcChecksum('source');
331        if (!$self->_readChecksum('source')) {        if (!$self->readChecksum('source')) {
332          $logger->warning( __PACKAGE__ . "->_run: Could not find \"source\" entry with ident=\"$self->{node}->{source}->{ident}\"" );          $logger->warning( __PACKAGE__ . "->_run: Could not find \"source\" entry with ident=\"$self->{node}->{source}->{ident}\"" );
333          $tc->{skip}++;          $tc->{skip}++;
334          print "s" if $self->{verbose};          print "s" if $self->{verbose};
# Line 275  sub _run { Line 336  sub _run {
336        }        }
337                
338        # get checksum from synchronization target        # get checksum from synchronization target
339        $self->_readChecksum('target');        $self->readChecksum('target');
340        #if (!$self->_readChecksum('target')) {        #if (!$self->readChecksum('target')) {
341        #  $logger->critical( __PACKAGE__ . "->_readChecksum: Could not find \"target\" entry with ident=\"$self->{node}->{source}->{ident}\"" );        #  $logger->critical( __PACKAGE__ . "->readChecksum: Could not find \"target\" entry with ident=\"$self->{node}->{source}->{ident}\"" );
342        #  next;        #  next;
343        #}        #}
344        
# Line 290  sub _run { Line 351  sub _run {
351        # determine if entry is "new" or "dirty"        # determine if entry is "new" or "dirty"
352        # after all, this seems to be the point where the hammer falls.....        # after all, this seems to be the point where the hammer falls.....
353        print "c" if $self->{verbose};        print "c" if $self->{verbose};
354  #print Dumper($self->{node});  
355  #exit;        # trace
356            #print Dumper($self->{node});
357            #exit;
358            #print "LOCAL: ", $self->{node}->{source}->{checksum_local_storage}, " <-> ", $self->{node}->{source}->{checksum_local_calculated}, "\n";
359            #print "REMOTE: ", $self->{node}->{source}->{checksum}, " <-> ", $self->{node}->{target}->{checksum}, "\n";
360    
361          # calculate new/dirty status
362        $self->{node}->{status}->{new} = !$self->{node}->{target}->{checksum};        $self->{node}->{status}->{new} = !$self->{node}->{target}->{checksum};
363        if (!$self->{node}->{status}->{new}) {        if (!$self->{node}->{status}->{new}) {
364          $self->{node}->{status}->{dirty} =          $self->{node}->{status}->{dirty} =
# Line 301  sub _run { Line 368  sub _run {
368            $self->{args}->{force};            $self->{args}->{force};
369        }        }
370    
371          # new 2004-06-17: also check if local checksum is inconsistent
372          # new 2004-06-19: ... only if requested
373          if ($self->{options}->{source}->{storage}->{handle}->{locator}->{hasLocalChecksum} and
374            ($self->{node}->{source}->{checksum_local_storage} ne $self->{node}->{source}->{checksum_local_calculated}) ) {
375            $self->{node}->{status}->{dirty_local} = 1;
376            $self->{node}->{status}->{dirty} = 1;
377          }
378    
379      } else {      } else {
380        $logger->warning( __PACKAGE__ . "->_run: Synchronization method '$self->{options}->{metadata}->{syncMethod}' is not implemented" );        $logger->warning( __PACKAGE__ . "->_run: Synchronization method '$self->{options}->{metadata}->{syncMethod}' is not implemented" );
381        $tc->{skip}++;        $tc->{skip}++;
382        print "s" if $self->{verbose};        print "s" if $self->{verbose};
383        next;        next;
384              }
385    
386        # new 2004-06-17: also update local checksum
387        if ($self->{node}->{status}->{dirty_local}) {
388          $tc->{locally_modified}++;
389          print "[lm]" if $self->{verbose};
390          $self->_doModify_LocalChecksum('source');
391      }      }
392    
393      # first reaction on entry-status: continue with next entry if the current is already "in sync"      # first reaction on entry-status: continue with next entry if the current is already "in sync"
# Line 315  sub _run { Line 396  sub _run {
396        next;        next;
397      }      }
398    
399      # build map to actually transfer the data from source to target      #print Dumper($self->{node}->{source});
     $self->buildAttributeMap();  
   
400    
401  #print Dumper($self->{node}); exit;      # build map to actually transfer the data from source to target
402        if (!$self->buildAttributeMap()) {
403          #$logger->warning( __PACKAGE__ . "->_run: Attribute Map could not be created. Will not insert or modify node.");
404          push( @{$tc->{error_per_row}}, "Attribute Map could not be created. Will not insert or modify node $self->{node}->{source}->{ident}.");
405          #push( @{$tc->{error_per_row}}, "Attribute Map could not be created. Will not insert or modify node " . Dumper($self->{node}->{source}) . ".");
406          $tc->{error}++;
407          print "e" if $self->{verbose};
408          next;
409        }
410    
411  #print "attempt", "\n";      # trace
412          #print Dumper($self->{node}); exit;
413          #print "attempt", "\n";
414    
415      # additional (new) checks for feature "write-protection"      # additional (new) checks for feature "write-protection"
416      if ($self->{meta}->{target}->{storage}->{isWriteProtected}) {      if ($self->{meta}->{target}->{storage}->{isWriteProtected}) {
# Line 334  sub _run { Line 423  sub _run {
423        next;        next;
424      }      }
425    
426  #print Dumper($self);      # trace
427  #exit;        #print Dumper($self);
428          #exit;
429    
430      # transfer contents of map to target      # transfer contents of map to target
431      if ($self->{node}->{status}->{new}) {      if ($self->{node}->{status}->{new}) {
# Line 344  sub _run { Line 434  sub _run {
434        # asymmetry: refetch node from target to re-calculate new ident and checksum (TODO: is IdentAuthority of relevance here?)        # asymmetry: refetch node from target to re-calculate new ident and checksum (TODO: is IdentAuthority of relevance here?)
435        #print Dumper($self->{node});        #print Dumper($self->{node});
436        $self->_statloadNode('target', $self->{node}->{target}->{ident}, 1);        $self->_statloadNode('target', $self->{node}->{target}->{ident}, 1);
437        $self->_readChecksum('target');        $self->readChecksum('target');
438    
439      } elsif ($self->{node}->{status}->{dirty}) {      } elsif ($self->{node}->{status}->{dirty}) {
440        $tc->{attempt_modify}++;        $tc->{attempt_modify}++;
441        # asymmetry: get ident before updating (TODO: is IdentAuthority of relevance here?)        # asymmetry: get ident before updating (TODO: is IdentAuthority of relevance here?)
442        $self->{node}->{target}->{ident} = $self->{node}->{map}->{$self->{meta}->{target}->{IdentProvider}->{arg}};        $self->{node}->{target}->{ident} = $self->{node}->{map}->{$self->{meta}->{target}->{IdentProvider}->{arg}};
443        $self->_doTransferToTarget('update');        $self->_doTransferToTarget('update');
444        $self->_readChecksum('target');        $self->readChecksum('target');
445      }      }
446    
447      if ($self->{node}->{status}->{ok}) {      if ($self->{node}->{status}->{ok}) {
# Line 371  sub _run { Line 461  sub _run {
461            
462      # change ident in source (take from target), if transfer was ok and target is an IdentAuthority      # change ident in source (take from target), if transfer was ok and target is an IdentAuthority
463      # this is (for now) called a "retransmit" indicated by a "r"-character when verbosing      # this is (for now) called a "retransmit" indicated by a "r"-character when verbosing
464      if ($self->{node}->{status}->{ok} && $self->{options}->{target}->{storage}->{idAuthority}) {      #if ($self->{node}->{status}->{ok} && $self->{options}->{target}->{storage}->{idAuthority}) {
465        if ($self->{node}->{status}->{ok} && $self->{meta}->{target}->{isIdentAuthority}) {
466        print "r" if $self->{verbose};        print "r" if $self->{verbose};
467        #print Dumper($self->{meta});        #print Dumper($self->{meta});
468        #print Dumper($self->{node});        #print Dumper($self->{node});
469        #exit;        #exit;
470        $self->_doModifySource_IdentChecksum($self->{node}->{target}->{ident});        $self->_doModifySource_IdentChecksum($self->{node}->{target}->{ident});
471      }      }
472        
473        #print "UNLOAD", "\n";
474        #$self->{meta}->{source}->{storage}->unload( $self->{node}->{source}->{payload} );
475    
476    }    }
477    
# Line 394  sub _run { Line 488  sub _run {
488            
489      # todo!!!      # todo!!!
490      #sysevent( { usermsg => $msg, level => $level }, $taskEvent );      #sysevent( { usermsg => $msg, level => $level }, $taskEvent );
491      $logger->info( __PACKAGE__ . "->_run: $msg" );      #$logger->info( __PACKAGE__ . "->_run: $msg" );
492        $logger->info($msg . "\n");
493    
494    return $tc;    return $tc;
495    
496  }  }
497    
498    
 # refactor this as some core-function to do a generic dump resolving data-encapsulations of e.g. Set::Object  
 sub _dumpCompact {  
   my $self = shift;  
   
   #my $vars = \@_;  
   my @data = ();  
   
   my $count = 0;  
   foreach (@_) {  
     my $item = {};  
     foreach my $key (keys %$_) {  
       my $val = $_->{$key};  
   
 #print Dumper($val);  
   
       if (ref $val eq 'Set::Object') {  
         #print "========================= SET", "\n";  
 #print Dumper($val);  
         #print Dumper($val->members());  
         #$val = $val->members();  
         #$vars->[$count]->{$key} = $val->members() if $val->can("members");  
         #$item->{$key} = $val->members() if $val->can("members");  
         $item->{$key} = $val->members();  
         #print Dumper($vars->[$count]->{$key});  
   
       } else {  
         $item->{$key} = $val;  
       }  
   
     }  
     push @data, $item;  
     $count++;  
   }  
   
 #print "Dump:", Dumper(@data), "\n";  
   
   $Data::Dumper::Indent = 0;  
   my $result = Dumper(@data);  
   $Data::Dumper::Indent = 2;  
   return $result;  
     
 }  
   
   
   
499  sub _doTransferToTarget {  sub _doTransferToTarget {
500    my $self = shift;    my $self = shift;
501    my $action = shift;    my $action = shift;
502    #print Dumper($self->{node});      
503    #exit;    # trace
504        #print Dumper($self->{meta});
505        #print Dumper($self->{node});
506        #exit;
507        
508    $self->_modifyNode('target', $action, $self->{node}->{map});    $self->_modifyNode('target', $action, $self->{node}->{map});
509  }  }
510    
# Line 474  sub _doModifySource_IdentChecksum { Line 528  sub _doModifySource_IdentChecksum {
528    $self->_modifyNode('source', 'update', $map);    $self->_modifyNode('source', 'update', $map);
529  }  }
530    
531    sub _doModify_LocalChecksum {
532      my $self = shift;
533      my $descent = shift;
534      my $map = {
535        cs_local => $self->{node}->{$descent}->{checksum_local_calculated},
536      };
537      $self->_modifyNode($descent, 'update', $map);
538    }
539    
540  sub _prepareNode_MetaProperties {  sub _prepareNode_MetaProperties {
541    my $self = shift;    my $self = shift;
# Line 510  sub _prepareNode_MetaProperties { Line 571  sub _prepareNode_MetaProperties {
571    
572  }  }
573    
574    # TODO: load column-metadata from reversed mapping-metadata
575  sub _prepareNode_DummyIdent {  sub _prepareNode_DummyIdent {
576    my $self = shift;    my $self = shift;
577    my $descent = shift;    my $descent = shift;
578    
579      #print Dumper($self->{options});
580    $logger->info( __PACKAGE__ . "->_prepareNode_DummyIdent( descent $descent )" );    $logger->info( __PACKAGE__ . "->_prepareNode_DummyIdent( descent $descent )" );
581    
582    my $list = $self->_getNodeList($descent);    my $list = $self->_getNodeList($descent);
# Line 527  sub _prepareNode_DummyIdent { Line 590  sub _prepareNode_DummyIdent {
590      my $map = {      my $map = {
591        $self->{meta}->{$descent}->{IdentProvider}->{arg} => $ident_dummy,        $self->{meta}->{$descent}->{IdentProvider}->{arg} => $ident_dummy,
592        cs => undef,        cs => undef,
593          cs_local => undef,
594      };      };
595            
596      # diff lists and ...      # diff lists and ...
# Line 568  sub _otherSide { Line 632  sub _otherSide {
632    return '';    return '';
633  }  }
634    
 sub _erase_all {  
   my $self = shift;  
   my $descent = shift;  
   #my $node = shift;  
   my $node = $self->{meta}->{$descent}->{node};  
   $self->{meta}->{$descent}->{storage}->eraseAll($node);  
 }  
635    
636  1;  1;
637    __END__
Legend:
Removed from v.1.4  
changed lines
  Added in v.1.13
MailToCvsAdmin">MailToCvsAdmin
 ViewVC Help
Powered by ViewVC 1.1.26 RSS 2.0 feed