/[cvs]/nfo/perl/libs/Data/Transfer/Sync.pm
ViewVC logotype

Diff of /nfo/perl/libs/Data/Transfer/Sync.pm

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.5 by joko, Thu Dec 5 08:06:05 2002 UTC revision 1.10 by joko, Thu Dec 19 01:07:16 2002 UTC
# Line 6  Line 6 
6  ##  ##
7  ##    ----------------------------------------------------------------------------------------  ##    ----------------------------------------------------------------------------------------
8  ##    $Log$  ##    $Log$
9    ##    Revision 1.10  2002/12/19 01:07:16  joko
10    ##    + fixed output done via $logger
11    ##
12    ##    Revision 1.9  2002/12/16 07:02:34  jonen
13    ##    + added comment
14    ##
15    ##    Revision 1.8  2002/12/15 02:03:09  joko
16    ##    + fixed logging-messages
17    ##    + additional metadata-checks
18    ##
19    ##    Revision 1.7  2002/12/13 21:49:34  joko
20    ##    + sub configure
21    ##    + sub checkOptions
22    ##
23    ##    Revision 1.6  2002/12/06 04:49:10  jonen
24    ##    + disabled output-puffer here
25    ##
26  ##    Revision 1.5  2002/12/05 08:06:05  joko  ##    Revision 1.5  2002/12/05 08:06:05  joko
27  ##    + bugfix with determining empty fields (Null) with DBD::CSV  ##    + bugfix with determining empty fields (Null) with DBD::CSV
28  ##    + debugging  ##    + debugging
# Line 39  use strict; Line 56  use strict;
56  use warnings;  use warnings;
57    
58  use Data::Dumper;  use Data::Dumper;
59    #use Hash::Merge qw( merge );
60    
61  use misc::HashExt;  use misc::HashExt;
62  use libp qw( md5_base64 );  use libp qw( md5_base64 );
63  use libdb qw( quotesql hash2Sql );  use libdb qw( quotesql hash2Sql );
# Line 48  use Data::Compare::Struct qw( getDiffere Line 67  use Data::Compare::Struct qw( getDiffere
67  # get logger instance  # get logger instance
68  my $logger = Log::Dispatch::Config->instance;  my $logger = Log::Dispatch::Config->instance;
69    
70    $| = 1;
71    
72  sub new {  sub new {
73    my $invocant = shift;    my $invocant = shift;
74    my $class = ref($invocant) || $invocant;    my $class = ref($invocant) || $invocant;
75    my $self = { @_ };    my $self = {};
76    $logger->debug( __PACKAGE__ . "->new(@_)" );    $logger->debug( __PACKAGE__ . "->new(@_)" );
77    bless $self, $class;    bless $self, $class;
78    $self->_init();    $self->configure(@_);
79    return $self;    return $self;
80  }  }
81    
82    
83    sub configure {
84      my $self = shift;
85      my @args = @_;
86      if (!isEmpty(\@args)) {
87        my %properties = @_;
88        # merge args to properties
89        map { $self->{$_} = $properties{$_}; } keys %properties;
90        $self->_init();
91      } else {
92        #print "no args!", "\n";
93      }
94      #print Dumper($self);
95    }
96    
97  sub _init {  sub _init {
98    my $self = shift;    my $self = shift;
99    
100      $self->{configured} = 1;
101        
102    # build new container if necessary    # build new container if necessary
103    $self->{container} = Data::Storage::Container->new() if !$self->{container};    $self->{container} = Data::Storage::Container->new() if !$self->{container};
# Line 80  sub _init { Line 116  sub _init {
116  }  }
117    
118    
119    sub prepareOptions {
120    
121      my $self = shift;
122      my $opts = shift;
123    
124    #print Dumper($opts);
125    
126      $opts->{mode} ||= '';
127      $opts->{erase} ||= 0;
128      #$opts->{import} ||= 0;
129      
130      $logger->notice( __PACKAGE__ . "->prepareOptions( source_node $opts->{source_node} mode $opts->{mode} erase $opts->{erase} prepare $opts->{prepare} )");
131    
132      if (!$opts->{mapping} || !$opts->{mapping_module}) {
133        $logger->warning( __PACKAGE__ . "->prepareOptions: No mapping supplied - please check key 'mappings' in BizWorks/Config.pm");
134      }
135    
136      my $evstring = "use $opts->{mapping_module};";
137      eval($evstring);
138      if ($@) {
139        $logger->warning( __PACKAGE__ . "->prepareOptions: error while trying to access mapping - $@");
140        return;
141      }
142    
143      # resolve mapping metadata (returned from sub)
144      my $mapObject = $opts->{mapping_module}->new();
145      #print Dumper($map);
146      my $source_node_name = $opts->{source_node};
147      # check if mapping for certain node is contained in mapping object
148      if (!$mapObject->can($source_node_name)) {
149        $logger->warning( __PACKAGE__ . "->prepareOptions: Can't access mapping for node \"$source_node_name\" - please check $opts->{mapping_module}.");
150        return;
151      }
152      my $map = $mapObject->$source_node_name;
153    
154      # remove asymmetries from $map (patch keys)
155      $map->{source_node} = $map->{source}; delete $map->{source};
156      $map->{target_node} = $map->{target}; delete $map->{target};
157      $map->{mapping} = $map->{details}; delete $map->{details};
158      $map->{direction} = $map->{mode}; delete $map->{mode};
159    
160      # defaults (mostly for backward-compatibility)
161      $map->{source_node} ||= $source_node_name;
162      $map->{source_ident} ||= 'storage_method:id';
163      $map->{target_ident} ||= 'property:oid';
164      $map->{direction} ||= $opts->{mode};         # | PUSH | PULL | FULL
165      $map->{method} ||= 'checksum';                # | timestamp
166      $map->{source_exclude} ||= [qw( cs )];
167    
168      # merge map to opts
169      map { $opts->{$_} = $map->{$_}; } keys %$map;
170        
171    #print Dumper($opts);
172    
173      # TODO: move this to checkOptions...
174      
175      # check - do we have a target?
176      if (!$opts->{target_node}) {
177        $logger->warning( __PACKAGE__ . "->prepareOptions: No target given - please check metadata declaration.");
178        return;
179      }
180    
181    
182      #return $opts;
183      return 1;
184    
185    }
186    
187    
188    sub checkOptions {
189      my $self = shift;
190      my $opts = shift;
191      
192      my $result = 1;
193      
194      # check - do we have a target node?
195      if (!$opts->{target_node}) {
196        $logger->warning( __PACKAGE__ . "->checkOptions: Error while resolving resource metadata - no 'target node' could be determined.");
197        $result = 0;
198      }
199    
200      # check - do we have a mapping?
201      if (!$opts->{mapping} && !$opts->{mapping_module}) {
202        $logger->warning( __PACKAGE__ . "->checkOptions: Error while resolving resource metadata - no 'mapping' could be determined.");
203        $result = 0;
204      }
205      
206      return $result;
207      
208    }
209    
210    
211  # TODO: some feature to show off the progress of synchronization (cur/max * 100)  # TODO: some feature to show off the progress of synchronization (cur/max * 100)
212  sub syncNodes {  sub syncNodes {
213    
214    my $self = shift;    my $self = shift;
215    my $args = shift;    my $args = shift;
216    
217      if (!$self->{configured}) {
218        $logger->critical( __PACKAGE__ . "->syncNodes: Synchronization object is not configured/initialized correctly." );
219        return;
220      }
221    
222    # remember arguments through the whole processing    # remember arguments through the whole processing
223    $self->{args} = $args;    $self->{args} = $args;
224    
# Line 108  sub syncNodes { Line 241  sub syncNodes {
241    }    }
242    
243    # decompose identifiers for each partner    # decompose identifiers for each partner
244    # TODO: take this list from already established/given metadata    # TODO: refactor!!! take this list from already established/given metadata
245    foreach ('source', 'target') {    foreach ('source', 'target') {
246            
247      # get/set metadata for further processing      # get/set metadata for further processing
# Line 163  sub syncNodes { Line 296  sub syncNodes {
296      #print "iiiiisprov: ", Dumper($self->{meta}->{$_}->{storage}), "\n";      #print "iiiiisprov: ", Dumper($self->{meta}->{$_}->{storage}), "\n";
297    }    }
298    
299    #print Dumper($self->{meta});
300    
301    $logger->info( __PACKAGE__ . "->syncNodes: source=$self->{meta}->{source}->{dbkey}/$self->{meta}->{source}->{node} $direction_arrow target=$self->{meta}->{target}->{dbkey}/$self->{meta}->{target}->{node}" );    $logger->info( __PACKAGE__ . "->syncNodes: source=$self->{meta}->{source}->{dbkey}/$self->{meta}->{source}->{node} $direction_arrow target=$self->{meta}->{target}->{dbkey}/$self->{meta}->{target}->{node}" );
302    
303    # build mapping    # build mapping
# Line 190  sub syncNodes { Line 325  sub syncNodes {
325    
326    }    }
327    
328    #print Dumper($self->{meta});
329      
330    # check partners/nodes: does partner exist / is node available?    # check partners/nodes: does partner exist / is node available?
331    foreach my $partner (keys %{$self->{meta}}) {    foreach my $partner (keys %{$self->{meta}}) {
332      next if $self->{meta}->{$partner}->{storage}->{locator}->{type} eq 'DBI';    # for DBD::CSV - re-enable for others      
333        # 1. check partners & storages
334        if (!$self->{meta}->{$partner}) {
335          $logger->critical( __PACKAGE__ . "->syncNodes: Could not find partner '$partner' in configuration metadata." );
336          return;
337        }
338    
339        my $dbkey = $self->{meta}->{$partner}->{dbkey};
340    
341        if (!$self->{meta}->{$partner}->{storage}) {
342          $logger->critical( __PACKAGE__ . "->syncNodes: Could not access storage of partner '$partner' (named '$dbkey'), looks like a configuration-error." );
343          return;
344        }
345        
346        # TODO:
347        # 2. check if partners (and nodes?) are actually available....
348        # eventually pre-check mode of access-attempt (read/write) here to provide an "early-croak" if possible
349        
350        # 3. check nodes
351        next if $self->{meta}->{$partner}->{storage}->{locator}->{type} eq 'DBI';    # HACK for DBD::CSV - re-enable for others
352        # get node-name
353      my $node = $self->{meta}->{$partner}->{node};      my $node = $self->{meta}->{$partner}->{node};
354      if (!$self->{meta}->{$partner}->{storage}->existsChildNode($node)) {      if (!$self->{meta}->{$partner}->{storage}->existsChildNode($node)) {
355        $logger->critical( __PACKAGE__ . "->syncNodes: Could not reach \"$node\" at \"$partner\"." );        $logger->critical( __PACKAGE__ . "->syncNodes: Could not reach node \"$node\" at partner \"$partner\"." );
356        return;        return;
357      }      }
358        
359    }    }
360    
361    # TODO:    # TODO:
# Line 493  sub _syncNodes { Line 651  sub _syncNodes {
651  }  }
652    
653    
654    # refactor this as some core-function to do a generic dump resolving data-encapsulations of e.g. Set::Object
655  sub _dumpCompact {  sub _dumpCompact {
656    my $self = shift;    my $self = shift;
657    
# Line 777  sub _modifyNode { Line 936  sub _modifyNode {
936      #print $action, "\n";      #print $action, "\n";
937  #$action = "anc";  #$action = "anc";
938  #print "yai", "\n";  #print "yai", "\n";
939    
940    #print Dumper($map);
941    #delete $map->{cs};
942    
943      if (lc($action) eq 'insert') {      if (lc($action) eq 'insert') {
944        $sql_main = hash2Sql($self->{meta}->{$descent}->{node}, $map, 'SQL_INSERT');        $sql_main = hash2Sql($self->{meta}->{$descent}->{node}, $map, 'SQL_INSERT');
945      } elsif (lc $action eq 'update') {      } elsif (lc $action eq 'update') {
# Line 784  sub _modifyNode { Line 947  sub _modifyNode {
947        $sql_main = hash2Sql($self->{meta}->{$descent}->{node}, $map, 'SQL_UPDATE', $crit);        $sql_main = hash2Sql($self->{meta}->{$descent}->{node}, $map, 'SQL_UPDATE', $crit);
948      }      }
949    
950      #print "sql: ", $sql_main, "\n";  #$sql_main = "UPDATE currencies_csv SET oid='abcdef' WHERE text='Australian Dollar' AND key='AUD';";
951      #exit;  #$sql_main = "UPDATE currencies_csv SET oid='huhu2' WHERE ekey='AUD'";
952    
953    #print "sql: ", $sql_main, "\n";
954    #exit;
955    
956      # transfer data      # transfer data
957      my $sqlHandle = $self->{meta}->{$descent}->{storage}->sendCommand($sql_main);      my $sqlHandle = $self->{meta}->{$descent}->{storage}->sendCommand($sql_main);
958    
959    #exit;
960    
961      # handle errors      # handle errors
962      if ($sqlHandle->err) {      if ($sqlHandle->err) {
963        #if ($self->{args}->{debug}) { print "sql-error with statement: $sql_main", "\n"; }        #if ($self->{args}->{debug}) { print "sql-error with statement: $sql_main", "\n"; }
# Line 978  sub _statloadNode { Line 1146  sub _statloadNode {
1146        return;        return;
1147      }      }
1148    
1149      my $result = $self->{meta}->{$descent}->{storage}->sendQuery({  #print "yai!", "\n";
1150    
1151        my $query = {
1152        node => $self->{meta}->{$descent}->{node},        node => $self->{meta}->{$descent}->{node},
1153        subnodes => [qw( cs )],        subnodes => [qw( cs )],
1154        criterias => [        criterias => [
# Line 986  sub _statloadNode { Line 1156  sub _statloadNode {
1156             op => 'eq',             op => 'eq',
1157             val => $ident },             val => $ident },
1158        ]        ]
1159      });      };
1160    
1161    #print Dumper($query);
1162    
1163        my $result = $self->{meta}->{$descent}->{storage}->sendQuery($query);
1164    
1165      my $entry = $result->getNextEntry();      my $entry = $result->getNextEntry();
1166    
1167    #print Dumper($entry);
1168    #print "pers: " . $self->{meta}->{$descent}->{storage}->is_persistent($entry), "\n";
1169    #my $state = $self->{meta}->{$descent}->{storage}->_fetch_object_state($entry, { name => 'TransactionHop' } );
1170    #print Dumper($state);
1171    
1172      my $status = $result->getStatus();      my $status = $result->getStatus();
1173    
1174  #print Dumper($status);  #print Dumper($status);
# Line 1003  sub _statloadNode { Line 1183  sub _statloadNode {
1183            
1184        # 1st level - hard error        # 1st level - hard error
1185        if ($status && $status->{err}) {        if ($status && $status->{err}) {
1186          $logger->debug( __PACKAGE__ . "->_statloadNode (ident=\"$ident\") failed - hard error (that's ok)" );          $logger->debug( __PACKAGE__ . "->_statloadNode (ident=\"$ident\") failed - hard error (that's ok): $status->{err}" );
1187          return;          return;
1188        }        }
1189        
Legend:
Removed from v.1.5  
changed lines
  Added in v.1.10
MailToCvsAdmin">MailToCvsAdmin
 ViewVC Help
Powered by ViewVC 1.1.26 RSS 2.0 feed