/[cvs]/nfo/perl/libs/Data/Transfer/Sync/Core.pm
ViewVC logotype

Diff of /nfo/perl/libs/Data/Transfer/Sync/Core.pm

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.6 by joko, Fri Feb 14 14:03:49 2003 UTC revision 1.12 by joko, Sat Jun 19 01:45:08 2004 UTC
# Line 1  Line 1 
1    ## -------------------------------------------------------------------------
2    ##
3  ##    $Id$  ##    $Id$
4  ##  ##
5  ##    Copyright (c) 2002  Andreas Motl <andreas.motl@ilo.de>  ##    Copyright (c) 2002  Andreas Motl <andreas.motl@ilo.de>
6  ##  ##
7  ##    See COPYRIGHT section in pod text below for usage and distribution rights.  ##    See COPYRIGHT section in pod text below for usage and distribution rights.
8  ##  ##
9  ##    ----------------------------------------------------------------------------------------  ## -------------------------------------------------------------------------
10  ##    $Log$  ##    $Log$
11    ##    Revision 1.12  2004/06/19 01:45:08  joko
12    ##    introduced "local checksum"-mechanism
13    ##    moved _dumpCompact to ::Compare::Checksum
14    ##
15    ##    Revision 1.11  2004/05/11 20:03:48  jonen
16    ##    bugfix[joko] related to Attribute Map
17    ##
18    ##    Revision 1.10  2003/06/25 23:03:57  joko
19    ##    no debugging
20    ##
21    ##    Revision 1.9  2003/05/13 08:17:52  joko
22    ##    buildAttributeMap now propagates error
23    ##
24    ##    Revision 1.8  2003/03/27 15:31:15  joko
25    ##    fixes to modules regarding new namespace(s) below Data::Mungle::*
26    ##
27    ##    Revision 1.7  2003/02/21 08:01:11  joko
28    ##    debugging, logging
29    ##    renamed module
30    ##
31  ##    Revision 1.6  2003/02/14 14:03:49  joko  ##    Revision 1.6  2003/02/14 14:03:49  joko
32  ##    + logging, debugging  ##    + logging, debugging
33  ##    - refactored code to sister module  ##    - refactored code to sister module
# Line 59  Line 81 
81  ##    + minor cosmetics for logging  ##    + minor cosmetics for logging
82  ##  ##
83  ##    Revision 1.2  2002/12/01 04:43:25  joko  ##    Revision 1.2  2002/12/01 04:43:25  joko
84  ##    + mapping deatil entries may now be either an ARRAY or a HASH  ##    + mapping detail entries may now be either an ARRAY or a HASH
85  ##    + erase flag is used now (for export-operations)  ##    + erase flag is used now (for export-operations)
86  ##    + expressions to refer to values inside deep nested structures  ##    + expressions to refer to values inside deep nested structures
87  ##    - removed old mappingV2-code  ##    - removed old mappingV2-code
# Line 71  Line 93 
93  ##  ##
94  ##    Revision 1.1  2002/10/10 03:44:21  cvsjoko  ##    Revision 1.1  2002/10/10 03:44:21  cvsjoko
95  ##    + new  ##    + new
96  ##    ----------------------------------------------------------------------------------------  ## -------------------------------------------------------------------------
97    
98    
99  package Data::Transfer::Sync::Core;  package Data::Transfer::Sync::Core;
# Line 86  use mixin::with qw( Data::Transfer::Sync Line 108  use mixin::with qw( Data::Transfer::Sync
108    
109  use Data::Dumper;  use Data::Dumper;
110    
111  use misc::HashExt;  #use misc::HashExt;
112  use Data::Compare::Struct qw( getDifference isEmpty );  use Hash::Serializer;
113    use Data::Mungle::Compare::Struct qw( getDifference isEmpty );
114    use Data::Mungle::Transform::Deep qw( deep_copy expand );
115  use Data::Storage::Container;  use Data::Storage::Container;
116  use DesignPattern::Object;  use DesignPattern::Object;
117  use libdb qw( quotesql );  use shortcuts::database qw( quotesql );
118    
119  # get logger instance  # get logger instance
120  my $logger = Log::Dispatch::Config->instance;  my $logger = Log::Dispatch::Config->instance;
# Line 152  sub _run { Line 176  sub _run {
176    $logger->debug( __PACKAGE__ . "->_run" );    $logger->debug( __PACKAGE__ . "->_run" );
177    
178    # for statistics    # for statistics
179    my $tc = OneLineDumpHash->new( {} );    my $tc = Hash::Serializer->new( {} );
180    my $results;    my $results;
181        
182    # set of objects is already in $self->{args}    # set of objects is already in $self->{args}
# Line 175  sub _run { Line 199  sub _run {
199      return;      return;
200    }    }
201    
202      #print Dumper(@$results);
203      #exit;
204    
205    # check if we actually *have* a synchronization method    # check if we actually *have* a synchronization method
206    if (!$self->{options}->{metadata}->{syncMethod}) {    if (!$self->{options}->{metadata}->{syncMethod}) {
# Line 186  sub _run { Line 211  sub _run {
211        
212    # dereference    # dereference
213    my @results = @{$results};    my @results = @{$results};
214      #print Dumper(@results);
215    
216    # iterate through set    # iterate through set
217    foreach my $source_node_real (@results) {    foreach my $source_node_real (@results) {
218    
219      print ":" if $self->{verbose};      print ":" if $self->{verbose};
220    
221        #print Dumper($source_node_real);
222    
223      $tc->{total}++;      $tc->{total}++;
224    
225  #print "========================  iter", "\n";      #print "=" x 80, "\n";
226    
227      # clone object (in case we have to modify it here)      # clone object (in case we have to modify it here)
228      # TODO:      # TODO:
# Line 203  sub _run { Line 231  sub _run {
231      #   - after all, just take care for now that this object doesn't get updated!      #   - after all, just take care for now that this object doesn't get updated!
232      #   - so, just use its reference for now - if some cloning is needed in future, do this here!      #   - so, just use its reference for now - if some cloning is needed in future, do this here!
233      my $source_node = $source_node_real;      my $source_node = $source_node_real;
234        #my $source_node = expand($source_node_real);
235    
236      # modify entry - handle new style callbacks (the readers)      # modify entry - handle new style callbacks (the readers)
237    
238      # trace      # trace
239        #print Dumper($source_node);      #print Dumper($source_node);
240        #exit;      #exit;
241    
242      my $descent = 'source';      my $descent = 'source';
243    
# Line 224  sub _run { Line 253  sub _run {
253    
254        foreach my $node (keys %{$callbacks->{read}}) {        foreach my $node (keys %{$callbacks->{read}}) {
255                    
256            #print "cb_node: $node", "\n";
257            
258          my $object = $source_node;          my $object = $source_node;
259          my $value; # = $source_node->{$node};          my $value; # = $source_node->{$node};
260    
# Line 281  sub _run { Line 312  sub _run {
312        print "n" if $self->{verbose};        print "n" if $self->{verbose};
313      }      }
314    
 #print "checksum", "\n";  
       
     #print Dumper($self);  
       
315      # determine status of entry by synchronization method      # determine status of entry by synchronization method
316      if ( lc $self->{options}->{metadata}->{syncMethod} eq 'checksum' ) {      if ( lc $self->{options}->{metadata}->{syncMethod} eq 'checksum' ) {
317      #if ( $statOK && (lc $self->{args}->{method} eq 'checksum') ) {      #if ( $statOK && (lc $self->{args}->{method} eq 'checksum') ) {
318      #if ( !$self->{node}->{status}->{new} && (lc $self->{args}->{method} eq 'checksum') ) {      #if ( !$self->{node}->{status}->{new} && (lc $self->{args}->{method} eq 'checksum') ) {
319                
320        # calculate local checksum of source node
321        $self->handleLocalChecksum('source');
322        
323      # calculate checksum of source node      # calculate checksum of source node
324        #$self->_calcChecksum('source');        #$self->_calcChecksum('source');
325        if (!$self->_readChecksum('source')) {        if (!$self->readChecksum('source')) {
326          $logger->warning( __PACKAGE__ . "->_run: Could not find \"source\" entry with ident=\"$self->{node}->{source}->{ident}\"" );          $logger->warning( __PACKAGE__ . "->_run: Could not find \"source\" entry with ident=\"$self->{node}->{source}->{ident}\"" );
327          $tc->{skip}++;          $tc->{skip}++;
328          print "s" if $self->{verbose};          print "s" if $self->{verbose};
# Line 300  sub _run { Line 330  sub _run {
330        }        }
331                
332        # get checksum from synchronization target        # get checksum from synchronization target
333        $self->_readChecksum('target');        $self->readChecksum('target');
334        #if (!$self->_readChecksum('target')) {        #if (!$self->readChecksum('target')) {
335        #  $logger->critical( __PACKAGE__ . "->_readChecksum: Could not find \"target\" entry with ident=\"$self->{node}->{source}->{ident}\"" );        #  $logger->critical( __PACKAGE__ . "->readChecksum: Could not find \"target\" entry with ident=\"$self->{node}->{source}->{ident}\"" );
336        #  next;        #  next;
337        #}        #}
338        
# Line 319  sub _run { Line 349  sub _run {
349        # trace        # trace
350          #print Dumper($self->{node});          #print Dumper($self->{node});
351          #exit;          #exit;
352            #print "LOCAL: ", $self->{node}->{source}->{checksum_local_storage}, " <-> ", $self->{node}->{source}->{checksum_local_calculated}, "\n";
353            #print "REMOTE: ", $self->{node}->{source}->{checksum}, " <-> ", $self->{node}->{target}->{checksum}, "\n";
354    
355          # calculate new/dirty status
356        $self->{node}->{status}->{new} = !$self->{node}->{target}->{checksum};        $self->{node}->{status}->{new} = !$self->{node}->{target}->{checksum};
357        if (!$self->{node}->{status}->{new}) {        if (!$self->{node}->{status}->{new}) {
358          $self->{node}->{status}->{dirty} =          $self->{node}->{status}->{dirty} =
# Line 329  sub _run { Line 362  sub _run {
362            $self->{args}->{force};            $self->{args}->{force};
363        }        }
364    
365          # new 2004-06-17: also check if local checksum is inconsistent
366          if ($self->{node}->{source}->{checksum_local_storage} ne $self->{node}->{source}->{checksum_local_calculated}) {
367            $self->{node}->{status}->{dirty_local} = 1;
368            $self->{node}->{status}->{dirty} = 1;
369          }
370    
371      } else {      } else {
372        $logger->warning( __PACKAGE__ . "->_run: Synchronization method '$self->{options}->{metadata}->{syncMethod}' is not implemented" );        $logger->warning( __PACKAGE__ . "->_run: Synchronization method '$self->{options}->{metadata}->{syncMethod}' is not implemented" );
373        $tc->{skip}++;        $tc->{skip}++;
374        print "s" if $self->{verbose};        print "s" if $self->{verbose};
375        next;        next;
376              }
377    
378        # new 2004-06-17: also update local checksum
379        if ($self->{node}->{status}->{dirty_local}) {
380          $tc->{locally_modified}++;
381          print "[lm]" if $self->{verbose};
382          $self->_doModify_LocalChecksum('source');
383      }      }
384    
385      # first reaction on entry-status: continue with next entry if the current is already "in sync"      # first reaction on entry-status: continue with next entry if the current is already "in sync"
# Line 343  sub _run { Line 388  sub _run {
388        next;        next;
389      }      }
390    
391      # build map to actually transfer the data from source to target      #print Dumper($self->{node}->{source});
     $self->buildAttributeMap();  
392    
393        # build map to actually transfer the data from source to target
394        if (!$self->buildAttributeMap()) {
395          #$logger->warning( __PACKAGE__ . "->_run: Attribute Map could not be created. Will not insert or modify node.");
396          push( @{$tc->{error_per_row}}, "Attribute Map could not be created. Will not insert or modify node $self->{node}->{source}->{ident}.");
397          #push( @{$tc->{error_per_row}}, "Attribute Map could not be created. Will not insert or modify node " . Dumper($self->{node}->{source}) . ".");
398          $tc->{error}++;
399          print "e" if $self->{verbose};
400          next;
401        }
402    
403      # trace      # trace
404        #print Dumper($self->{node}); exit;        #print Dumper($self->{node}); exit;
# Line 373  sub _run { Line 426  sub _run {
426        # asymmetry: refetch node from target to re-calculate new ident and checksum (TODO: is IdentAuthority of relevance here?)        # asymmetry: refetch node from target to re-calculate new ident and checksum (TODO: is IdentAuthority of relevance here?)
427        #print Dumper($self->{node});        #print Dumper($self->{node});
428        $self->_statloadNode('target', $self->{node}->{target}->{ident}, 1);        $self->_statloadNode('target', $self->{node}->{target}->{ident}, 1);
429        $self->_readChecksum('target');        $self->readChecksum('target');
430    
431      } elsif ($self->{node}->{status}->{dirty}) {      } elsif ($self->{node}->{status}->{dirty}) {
432        $tc->{attempt_modify}++;        $tc->{attempt_modify}++;
433        # asymmetry: get ident before updating (TODO: is IdentAuthority of relevance here?)        # asymmetry: get ident before updating (TODO: is IdentAuthority of relevance here?)
434        $self->{node}->{target}->{ident} = $self->{node}->{map}->{$self->{meta}->{target}->{IdentProvider}->{arg}};        $self->{node}->{target}->{ident} = $self->{node}->{map}->{$self->{meta}->{target}->{IdentProvider}->{arg}};
435        $self->_doTransferToTarget('update');        $self->_doTransferToTarget('update');
436        $self->_readChecksum('target');        $self->readChecksum('target');
437      }      }
438    
439      if ($self->{node}->{status}->{ok}) {      if ($self->{node}->{status}->{ok}) {
# Line 408  sub _run { Line 461  sub _run {
461        #exit;        #exit;
462        $self->_doModifySource_IdentChecksum($self->{node}->{target}->{ident});        $self->_doModifySource_IdentChecksum($self->{node}->{target}->{ident});
463      }      }
464        
465        #print "UNLOAD", "\n";
466        #$self->{meta}->{source}->{storage}->unload( $self->{node}->{source}->{payload} );
467    
468    }    }
469    
# Line 424  sub _run { Line 480  sub _run {
480            
481      # todo!!!      # todo!!!
482      #sysevent( { usermsg => $msg, level => $level }, $taskEvent );      #sysevent( { usermsg => $msg, level => $level }, $taskEvent );
483      $logger->info( __PACKAGE__ . "->_run: $msg" );      #$logger->info( __PACKAGE__ . "->_run: $msg" );
484        $logger->info($msg . "\n");
485    
486    return $tc;    return $tc;
487    
488  }  }
489    
490    
 # refactor this as some core-function to do a generic dump resolving data-encapsulations of e.g. Set::Object  
 sub _dumpCompact {  
   my $self = shift;  
   
   #my $vars = \@_;  
   my @data = ();  
   
   my $count = 0;  
   foreach (@_) {  
     my $item = {};  
     foreach my $key (keys %$_) {  
       my $val = $_->{$key};  
   
 #print Dumper($val);  
   
       if (ref $val eq 'Set::Object') {  
         #print "========================= SET", "\n";  
 #print Dumper($val);  
         #print Dumper($val->members());  
         #$val = $val->members();  
         #$vars->[$count]->{$key} = $val->members() if $val->can("members");  
         #$item->{$key} = $val->members() if $val->can("members");  
         $item->{$key} = $val->members();  
         #print Dumper($vars->[$count]->{$key});  
   
       } else {  
         $item->{$key} = $val;  
       }  
   
     }  
     push @data, $item;  
     $count++;  
   }  
   
 #print "Dump:", Dumper(@data), "\n";  
   
   $Data::Dumper::Indent = 0;  
   my $result = Dumper(@data);  
   $Data::Dumper::Indent = 2;  
   return $result;  
     
 }  
   
   
   
491  sub _doTransferToTarget {  sub _doTransferToTarget {
492    my $self = shift;    my $self = shift;
493    my $action = shift;    my $action = shift;
# Line 508  sub _doModifySource_IdentChecksum { Line 520  sub _doModifySource_IdentChecksum {
520    $self->_modifyNode('source', 'update', $map);    $self->_modifyNode('source', 'update', $map);
521  }  }
522    
523    sub _doModify_LocalChecksum {
524      my $self = shift;
525      my $descent = shift;
526      my $map = {
527        cs_local => $self->{node}->{$descent}->{checksum_local_calculated},
528      };
529      $self->_modifyNode($descent, 'update', $map);
530    }
531    
532  sub _prepareNode_MetaProperties {  sub _prepareNode_MetaProperties {
533    my $self = shift;    my $self = shift;
# Line 544  sub _prepareNode_MetaProperties { Line 563  sub _prepareNode_MetaProperties {
563    
564  }  }
565    
566    # TODO: load column-metadata from reversed mapping-metadata
567  sub _prepareNode_DummyIdent {  sub _prepareNode_DummyIdent {
568    my $self = shift;    my $self = shift;
569    my $descent = shift;    my $descent = shift;
570    
571      #print Dumper($self->{options});
572    $logger->info( __PACKAGE__ . "->_prepareNode_DummyIdent( descent $descent )" );    $logger->info( __PACKAGE__ . "->_prepareNode_DummyIdent( descent $descent )" );
573    
574    my $list = $self->_getNodeList($descent);    my $list = $self->_getNodeList($descent);
# Line 561  sub _prepareNode_DummyIdent { Line 582  sub _prepareNode_DummyIdent {
582      my $map = {      my $map = {
583        $self->{meta}->{$descent}->{IdentProvider}->{arg} => $ident_dummy,        $self->{meta}->{$descent}->{IdentProvider}->{arg} => $ident_dummy,
584        cs => undef,        cs => undef,
585          cs_local => undef,
586      };      };
587            
588      # diff lists and ...      # diff lists and ...
Legend:
Removed from v.1.6  
changed lines
  Added in v.1.12
MailToCvsAdmin">MailToCvsAdmin
 ViewVC Help
Powered by ViewVC 1.1.26 RSS 2.0 feed