/[cvs]/nfo/perl/libs/Data/Transfer/Sync.pm
ViewVC logotype

Diff of /nfo/perl/libs/Data/Transfer/Sync.pm

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.5 by joko, Thu Dec 5 08:06:05 2002 UTC revision 1.11 by joko, Mon Dec 23 07:10:59 2002 UTC
# Line 6  Line 6 
6  ##  ##
7  ##    ----------------------------------------------------------------------------------------  ##    ----------------------------------------------------------------------------------------
8  ##    $Log$  ##    $Log$
9    ##    Revision 1.11  2002/12/23 07:10:59  joko
10    ##    + using MD5 for checksum generation again - the 32-bit integer hash from DBI seems to be too lazy
11    ##
12    ##    Revision 1.10  2002/12/19 01:07:16  joko
13    ##    + fixed output done via $logger
14    ##
15    ##    Revision 1.9  2002/12/16 07:02:34  jonen
16    ##    + added comment
17    ##
18    ##    Revision 1.8  2002/12/15 02:03:09  joko
19    ##    + fixed logging-messages
20    ##    + additional metadata-checks
21    ##
22    ##    Revision 1.7  2002/12/13 21:49:34  joko
23    ##    + sub configure
24    ##    + sub checkOptions
25    ##
26    ##    Revision 1.6  2002/12/06 04:49:10  jonen
27    ##    + disabled output-puffer here
28    ##
29  ##    Revision 1.5  2002/12/05 08:06:05  joko  ##    Revision 1.5  2002/12/05 08:06:05  joko
30  ##    + bugfix with determining empty fields (Null) with DBD::CSV  ##    + bugfix with determining empty fields (Null) with DBD::CSV
31  ##    + debugging  ##    + debugging
# Line 39  use strict; Line 59  use strict;
59  use warnings;  use warnings;
60    
61  use Data::Dumper;  use Data::Dumper;
62    #use Hash::Merge qw( merge );
63    
64  use misc::HashExt;  use misc::HashExt;
65  use libp qw( md5_base64 );  use libp qw( md5_base64 );
66  use libdb qw( quotesql hash2Sql );  use libdb qw( quotesql hash2Sql );
# Line 48  use Data::Compare::Struct qw( getDiffere Line 70  use Data::Compare::Struct qw( getDiffere
70  # get logger instance  # get logger instance
71  my $logger = Log::Dispatch::Config->instance;  my $logger = Log::Dispatch::Config->instance;
72    
73    $| = 1;
74    
75  sub new {  sub new {
76    my $invocant = shift;    my $invocant = shift;
77    my $class = ref($invocant) || $invocant;    my $class = ref($invocant) || $invocant;
78    my $self = { @_ };    my $self = {};
79    $logger->debug( __PACKAGE__ . "->new(@_)" );    $logger->debug( __PACKAGE__ . "->new(@_)" );
80    bless $self, $class;    bless $self, $class;
81    $self->_init();    $self->configure(@_);
82    return $self;    return $self;
83  }  }
84    
85    
86    sub configure {
87      my $self = shift;
88      my @args = @_;
89      if (!isEmpty(\@args)) {
90        my %properties = @_;
91        # merge args to properties
92        map { $self->{$_} = $properties{$_}; } keys %properties;
93        $self->_init();
94      } else {
95        #print "no args!", "\n";
96      }
97      #print Dumper($self);
98    }
99    
100  sub _init {  sub _init {
101    my $self = shift;    my $self = shift;
102    
103      $self->{configured} = 1;
104        
105    # build new container if necessary    # build new container if necessary
106    $self->{container} = Data::Storage::Container->new() if !$self->{container};    $self->{container} = Data::Storage::Container->new() if !$self->{container};
# Line 80  sub _init { Line 119  sub _init {
119  }  }
120    
121    
122    sub prepareOptions {
123    
124      my $self = shift;
125      my $opts = shift;
126    
127    #print Dumper($opts);
128    
129      $opts->{mode} ||= '';
130      $opts->{erase} ||= 0;
131      #$opts->{import} ||= 0;
132      
133      $logger->notice( __PACKAGE__ . "->prepareOptions( source_node $opts->{source_node} mode $opts->{mode} erase $opts->{erase} prepare $opts->{prepare} )");
134    
135      if (!$opts->{mapping} || !$opts->{mapping_module}) {
136        $logger->warning( __PACKAGE__ . "->prepareOptions: No mapping supplied - please check key 'mappings' in BizWorks/Config.pm");
137      }
138    
139      my $evstring = "use $opts->{mapping_module};";
140      eval($evstring);
141      if ($@) {
142        $logger->warning( __PACKAGE__ . "->prepareOptions: error while trying to access mapping - $@");
143        return;
144      }
145    
146      # resolve mapping metadata (returned from sub)
147      my $mapObject = $opts->{mapping_module}->new();
148      #print Dumper($map);
149      my $source_node_name = $opts->{source_node};
150      # check if mapping for certain node is contained in mapping object
151      if (!$mapObject->can($source_node_name)) {
152        $logger->warning( __PACKAGE__ . "->prepareOptions: Can't access mapping for node \"$source_node_name\" - please check $opts->{mapping_module}.");
153        return;
154      }
155      my $map = $mapObject->$source_node_name;
156    
157      # remove asymmetries from $map (patch keys)
158      $map->{source_node} = $map->{source}; delete $map->{source};
159      $map->{target_node} = $map->{target}; delete $map->{target};
160      $map->{mapping} = $map->{details}; delete $map->{details};
161      $map->{direction} = $map->{mode}; delete $map->{mode};
162    
163      # defaults (mostly for backward-compatibility)
164      $map->{source_node} ||= $source_node_name;
165      $map->{source_ident} ||= 'storage_method:id';
166      $map->{target_ident} ||= 'property:oid';
167      $map->{direction} ||= $opts->{mode};         # | PUSH | PULL | FULL
168      $map->{method} ||= 'checksum';                # | timestamp
169      $map->{source_exclude} ||= [qw( cs )];
170    
171      # merge map to opts
172      map { $opts->{$_} = $map->{$_}; } keys %$map;
173        
174    #print Dumper($opts);
175    
176      # TODO: move this to checkOptions...
177      
178      # check - do we have a target?
179      if (!$opts->{target_node}) {
180        $logger->warning( __PACKAGE__ . "->prepareOptions: No target given - please check metadata declaration.");
181        return;
182      }
183    
184    
185      #return $opts;
186      return 1;
187    
188    }
189    
190    
191    sub checkOptions {
192      my $self = shift;
193      my $opts = shift;
194      
195      my $result = 1;
196      
197      # check - do we have a target node?
198      if (!$opts->{target_node}) {
199        $logger->warning( __PACKAGE__ . "->checkOptions: Error while resolving resource metadata - no 'target node' could be determined.");
200        $result = 0;
201      }
202    
203      # check - do we have a mapping?
204      if (!$opts->{mapping} && !$opts->{mapping_module}) {
205        $logger->warning( __PACKAGE__ . "->checkOptions: Error while resolving resource metadata - no 'mapping' could be determined.");
206        $result = 0;
207      }
208      
209      return $result;
210      
211    }
212    
213    
214  # TODO: some feature to show off the progress of synchronization (cur/max * 100)  # TODO: some feature to show off the progress of synchronization (cur/max * 100)
215  sub syncNodes {  sub syncNodes {
216    
217    my $self = shift;    my $self = shift;
218    my $args = shift;    my $args = shift;
219    
220      if (!$self->{configured}) {
221        $logger->critical( __PACKAGE__ . "->syncNodes: Synchronization object is not configured/initialized correctly." );
222        return;
223      }
224    
225    # remember arguments through the whole processing    # remember arguments through the whole processing
226    $self->{args} = $args;    $self->{args} = $args;
227    
# Line 108  sub syncNodes { Line 244  sub syncNodes {
244    }    }
245    
246    # decompose identifiers for each partner    # decompose identifiers for each partner
247    # TODO: take this list from already established/given metadata    # TODO: refactor!!! take this list from already established/given metadata
248    foreach ('source', 'target') {    foreach ('source', 'target') {
249            
250      # get/set metadata for further processing      # get/set metadata for further processing
# Line 163  sub syncNodes { Line 299  sub syncNodes {
299      #print "iiiiisprov: ", Dumper($self->{meta}->{$_}->{storage}), "\n";      #print "iiiiisprov: ", Dumper($self->{meta}->{$_}->{storage}), "\n";
300    }    }
301    
302    #print Dumper($self->{meta});
303    
304    $logger->info( __PACKAGE__ . "->syncNodes: source=$self->{meta}->{source}->{dbkey}/$self->{meta}->{source}->{node} $direction_arrow target=$self->{meta}->{target}->{dbkey}/$self->{meta}->{target}->{node}" );    $logger->info( __PACKAGE__ . "->syncNodes: source=$self->{meta}->{source}->{dbkey}/$self->{meta}->{source}->{node} $direction_arrow target=$self->{meta}->{target}->{dbkey}/$self->{meta}->{target}->{node}" );
305    
306    # build mapping    # build mapping
# Line 190  sub syncNodes { Line 328  sub syncNodes {
328    
329    }    }
330    
331    #print Dumper($self->{meta});
332      
333    # check partners/nodes: does partner exist / is node available?    # check partners/nodes: does partner exist / is node available?
334    foreach my $partner (keys %{$self->{meta}}) {    foreach my $partner (keys %{$self->{meta}}) {
335      next if $self->{meta}->{$partner}->{storage}->{locator}->{type} eq 'DBI';    # for DBD::CSV - re-enable for others      
336        # 1. check partners & storages
337        if (!$self->{meta}->{$partner}) {
338          $logger->critical( __PACKAGE__ . "->syncNodes: Could not find partner '$partner' in configuration metadata." );
339          return;
340        }
341    
342        my $dbkey = $self->{meta}->{$partner}->{dbkey};
343    
344        if (!$self->{meta}->{$partner}->{storage}) {
345          $logger->critical( __PACKAGE__ . "->syncNodes: Could not access storage of partner '$partner' (named '$dbkey'), looks like a configuration-error." );
346          return;
347        }
348        
349        # TODO:
350        # 2. check if partners (and nodes?) are actually available....
351        # eventually pre-check mode of access-attempt (read/write) here to provide an "early-croak" if possible
352        
353        # 3. check nodes
354        next if $self->{meta}->{$partner}->{storage}->{locator}->{type} eq 'DBI';    # HACK for DBD::CSV - re-enable for others
355        # get node-name
356      my $node = $self->{meta}->{$partner}->{node};      my $node = $self->{meta}->{$partner}->{node};
357      if (!$self->{meta}->{$partner}->{storage}->existsChildNode($node)) {      if (!$self->{meta}->{$partner}->{storage}->existsChildNode($node)) {
358        $logger->critical( __PACKAGE__ . "->syncNodes: Could not reach \"$node\" at \"$partner\"." );        $logger->critical( __PACKAGE__ . "->syncNodes: Could not reach node \"$node\" at partner \"$partner\"." );
359        return;        return;
360      }      }
361        
362    }    }
363    
364    # TODO:    # TODO:
# Line 493  sub _syncNodes { Line 654  sub _syncNodes {
654  }  }
655    
656    
657    # refactor this as some core-function to do a generic dump resolving data-encapsulations of e.g. Set::Object
658  sub _dumpCompact {  sub _dumpCompact {
659    my $self = shift;    my $self = shift;
660    
# Line 556  sub _calcChecksum { Line 718  sub _calcChecksum {
718      #$logger->dump( __PACKAGE__ . ": " . $dump );      #$logger->dump( __PACKAGE__ . ": " . $dump );
719        
720    # calculate checksum from dump    # calculate checksum from dump
721        # note: the 32-bit integer hash from DBI seems
722        # to generate duplicates with small payloads already in ranges of hundreds of items/rows!!!
723        # try to avoid to use it or try to use it only for payloads greater than, hmmm, let's say 30 chars?
724        # (we had about 15 chars average per item (row))
725    
726      # md5-based fingerprint, base64 encoded (from Digest::MD5)      # md5-based fingerprint, base64 encoded (from Digest::MD5)
727        #my $checksum_cur = md5_base64($objdump) . '==';        $self->{node}->{$descent}->{checksum} = md5_base64($dump) . '==';
728      # 32-bit integer "hash" value (maybe faster?) (from DBI)      # 32-bit integer "hash" value (maybe faster?) (from DBI)
729        $self->{node}->{$descent}->{checksum} = DBI::hash($dump, 1);        #$self->{node}->{$descent}->{checksum} = DBI::hash($dump, 1);
730    
731    # signal good    # signal good
732    return 1;    return 1;
# Line 777  sub _modifyNode { Line 944  sub _modifyNode {
944      #print $action, "\n";      #print $action, "\n";
945  #$action = "anc";  #$action = "anc";
946  #print "yai", "\n";  #print "yai", "\n";
947    
948    #print Dumper($map);
949    #delete $map->{cs};
950    
951      if (lc($action) eq 'insert') {      if (lc($action) eq 'insert') {
952        $sql_main = hash2Sql($self->{meta}->{$descent}->{node}, $map, 'SQL_INSERT');        $sql_main = hash2Sql($self->{meta}->{$descent}->{node}, $map, 'SQL_INSERT');
953      } elsif (lc $action eq 'update') {      } elsif (lc $action eq 'update') {
# Line 784  sub _modifyNode { Line 955  sub _modifyNode {
955        $sql_main = hash2Sql($self->{meta}->{$descent}->{node}, $map, 'SQL_UPDATE', $crit);        $sql_main = hash2Sql($self->{meta}->{$descent}->{node}, $map, 'SQL_UPDATE', $crit);
956      }      }
957    
958      #print "sql: ", $sql_main, "\n";  #$sql_main = "UPDATE currencies_csv SET oid='abcdef' WHERE text='Australian Dollar' AND key='AUD';";
959      #exit;  #$sql_main = "UPDATE currencies_csv SET oid='huhu2' WHERE ekey='AUD'";
960    
961    #print "sql: ", $sql_main, "\n";
962    #exit;
963    
964      # transfer data      # transfer data
965      my $sqlHandle = $self->{meta}->{$descent}->{storage}->sendCommand($sql_main);      my $sqlHandle = $self->{meta}->{$descent}->{storage}->sendCommand($sql_main);
966    
967    #exit;
968    
969      # handle errors      # handle errors
970      if ($sqlHandle->err) {      if ($sqlHandle->err) {
971        #if ($self->{args}->{debug}) { print "sql-error with statement: $sql_main", "\n"; }        #if ($self->{args}->{debug}) { print "sql-error with statement: $sql_main", "\n"; }
# Line 978  sub _statloadNode { Line 1154  sub _statloadNode {
1154        return;        return;
1155      }      }
1156    
1157      my $result = $self->{meta}->{$descent}->{storage}->sendQuery({  #print "yai!", "\n";
1158    
1159        my $query = {
1160        node => $self->{meta}->{$descent}->{node},        node => $self->{meta}->{$descent}->{node},
1161        subnodes => [qw( cs )],        subnodes => [qw( cs )],
1162        criterias => [        criterias => [
# Line 986  sub _statloadNode { Line 1164  sub _statloadNode {
1164             op => 'eq',             op => 'eq',
1165             val => $ident },             val => $ident },
1166        ]        ]
1167      });      };
1168    
1169    #print Dumper($query);
1170    
1171        my $result = $self->{meta}->{$descent}->{storage}->sendQuery($query);
1172    
1173      my $entry = $result->getNextEntry();      my $entry = $result->getNextEntry();
1174    
1175    #print Dumper($entry);
1176    #print "pers: " . $self->{meta}->{$descent}->{storage}->is_persistent($entry), "\n";
1177    #my $state = $self->{meta}->{$descent}->{storage}->_fetch_object_state($entry, { name => 'TransactionHop' } );
1178    #print Dumper($state);
1179    
1180      my $status = $result->getStatus();      my $status = $result->getStatus();
1181    
1182  #print Dumper($status);  #print Dumper($status);
# Line 1003  sub _statloadNode { Line 1191  sub _statloadNode {
1191            
1192        # 1st level - hard error        # 1st level - hard error
1193        if ($status && $status->{err}) {        if ($status && $status->{err}) {
1194          $logger->debug( __PACKAGE__ . "->_statloadNode (ident=\"$ident\") failed - hard error (that's ok)" );          $logger->debug( __PACKAGE__ . "->_statloadNode (ident=\"$ident\") failed - hard error (that's ok): $status->{err}" );
1195          return;          return;
1196        }        }
1197        
Legend:
Removed from v.1.5  
changed lines
  Added in v.1.11
MailToCvsAdmin">MailToCvsAdmin
 ViewVC Help
Powered by ViewVC 1.1.26 RSS 2.0 feed