--- nfo/perl/libs/Data/Transfer/Sync/API.pm 2003/01/20 16:59:48 1.2 +++ nfo/perl/libs/Data/Transfer/Sync/API.pm 2003/02/09 04:59:27 1.3 @@ -1,4 +1,4 @@ -## $Id: API.pm,v 1.2 2003/01/20 16:59:48 joko Exp $ +## $Id: API.pm,v 1.3 2003/02/09 04:59:27 joko Exp $ ## ## Copyright (c) 2002 Andreas Motl ## @@ -6,6 +6,11 @@ ## ## ---------------------------------------------------------------------------------------- ## $Log: API.pm,v $ +## Revision 1.3 2003/02/09 04:59:27 joko +## + api versioning mechanism +## + major structure changes +## - refactored code to sister modules +## ## Revision 1.2 2003/01/20 16:59:48 joko ## + cosmetics and debugging ## @@ -20,187 +25,239 @@ use strict; use warnings; +use base qw( DesignPattern::Bridge ); + use mixin::with qw( Data::Transfer::Sync ); # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - main use Data::Dumper; +use Hash::Merge qw( merge ); -use Data::Compare::Struct qw( getDifference isEmpty ); -use Data::Transform::Deep qw( merge ); +use Data::Compare::Struct qw( isEmpty ); # get logger instance my $logger = Log::Dispatch::Config->instance; -sub _init { + +sub api_constructor { my $self = shift; + $logger->debug( __PACKAGE__ . "->api_constructor: Loading API"); + $self->_loadVersionExtensions(); } -sub checkOptions { - my $self = shift; - my $opts = shift; - - my $result = 1; - - # check - do we have a target node? - if (!$opts->{target_node}) { - $logger->warning( __PACKAGE__ . "->checkOptions: Error while resolving resource metadata - no 'target node' could be determined."); - $result = 0; - } - # check - do we have a mapping? - if (!$opts->{mapping} && !$opts->{mapping_module}) { - $logger->warning( __PACKAGE__ . "->checkOptions: Error while resolving resource metadata - no 'mapping' could be determined."); - $result = 0; - } - - return $result; - +sub _loadVersionExtensions { + my $self = shift; + my $syncVersion = $self->{sync_version}; + $syncVersion ||= ''; + $logger->debug( __PACKAGE__ . "->loadVersionExtensions( version='$syncVersion' )"); + #print Dumper($self); + #exit; + my $module = "Version::$syncVersion"; + $self->load($module); } -sub checkOptionsV2 { + +sub configure { my $self = shift; -#print Dumper($self->{options}); +#print "YAI\n"; +#print Dumper(@_); +#exit; - my $result = 1; + $logger->debug( __PACKAGE__ . "->configure"); - # check - do we have a target node? - if (!$self->{options}->{target}->{nodeName}) { - $logger->warning( __PACKAGE__ . "->checkOptionsV2: No target given - please check metadata declaration."); - $result = 0; - } - - # check - do we have a mapping? - if (!$self->{options}->{fieldmap}) { - $logger->warning( __PACKAGE__ . "->checkOptionsV2: Error while resolving resource metadata - no 'fieldmap' could be determined."); - $result = 0; + my @args = @_; + +#print Dumper(@args); + + if (!isEmpty(\@args)) { + my %properties = @_; + # merge args to properties + #map { $self->{$_} = $properties{$_}; } keys %properties; +#print Dumper($self); +#print Dumper(\%properties); + if ($self->{options}) { + my $options_new = merge($self->{options}, \%properties); +#print Dumper($options_new); + $self->{options} = $options_new; +#print Dumper($self->{options}); + } else { + $self->{options} = \%properties; + } + $self->_init(); + #$self->_initV1(); + } else { + #print "no args!", "\n"; } - - # TODO: extend! - return $result; +#print Dumper($self); +#exit; + $self->{state}->{configured} = 1; + return 1; } -sub prepareOptions { +sub setArguments { + my $self = shift; + my $args_raw = shift; + $self->{args_raw} = $args_raw; +} +sub readArguments { my $self = shift; - my $opts = shift; -#print Dumper($opts); + my %syncConfig; + tie %syncConfig, 'Tie::IxHash'; + %syncConfig = ( + map => { + moduleName => $self->{args_raw}->{'mapping-module'}, + }, + source => { + dbKey => $self->{args_raw}->{source}, + nodeType => $self->{args_raw}->{'source-type'}, + nodeName => $self->{args_raw}->{'source-node'}, + }, + target => { + dbKey => $self->{args_raw}->{target}, + nodeName => $self->{args_raw}->{'target-node'}, + }, + process => { + mode => $self->{args_raw}->{mode}, + erase => $self->{args_raw}->{erase}, + import => $self->{args_raw}->{import}, + prepare => $self->{args_raw}->{prepare}, + }, +# metadata => { +# config => $self->{config_metadata}, +# } + ); + + $self->{args} = \%syncConfig; + +} + + +# TODO: some feature to show off the progress of synchronization (cur/max * 100) +sub syncNodes { + + my $self = shift; + my $args = shift; + + $logger->debug( __PACKAGE__ . "->syncNodes: starting" ); + +#print Dumper($self); #exit; - $opts->{mode} ||= ''; - $opts->{erase} ||= 0; - $opts->{prepare} ||= 0; - #$opts->{import} ||= 0; - - if (!$opts->{source_node}) { - $logger->error( __PACKAGE__ . "->prepareOptions failed: Please specify source-node!"); - return; - } - - $logger->notice( __PACKAGE__ . "->prepareOptions( source_node $opts->{source_node} mode $opts->{mode} erase $opts->{erase} prepare $opts->{prepare} )"); +#print Dumper($self->{options}); + $self->_prepareOptions(); - #if (!$opts->{mapping} || !$opts->{mapping_module}) { - if (!$opts->{mapping}) { - $logger->warning( __PACKAGE__ . "->prepareOptions: No mapping supplied - please check key 'mappings' in global configuration or specify additional argument '--mapping'."); - #return; - } +#print Dumper($self->{options}); - $opts->{mapping_module} ||= $opts->{mapping}; - my $evstring = "use $opts->{mapping_module};"; - eval($evstring); - if ($@) { - $logger->warning( __PACKAGE__ . "->prepareOptions: error while trying to access mapping - $@"); + if (!$self->checkOptions()) { + $logger->critical( __PACKAGE__ . "->syncNodes: 'Data::Transfer::Sync::checkOptions' failed."); return; } - # resolve mapping metadata (returned from sub) - my $mapObject = $opts->{mapping_module}->new(); - #print Dumper($map); - my $source_node_name = $opts->{source_node}; - # check if mapping for certain node is contained in mapping object - if (!$mapObject->can($source_node_name)) { - $logger->warning( __PACKAGE__ . "->prepareOptions: Can't access mapping for node \"$source_node_name\" - please check $opts->{mapping_module}."); + if (!$self->{state}->{configured}) { + $logger->critical( __PACKAGE__ . "->syncNodes: Synchronization object is not configured/initialized correctly." ); return; } - my $map = $mapObject->$source_node_name; -#print Dumper($map); + # remember arguments through the whole processing + $self->{args} = $args; - # check here if "target" is actually a CODEref - in this case: resolve it - deprecated!!! ??? - if (ref $map->{target} eq 'CODE') { - $map->{target} = $map->{target}->($source_node_name); - } + # hash to hold and/or fill in metadata required for the processing + $self->{meta} = {}; - # resolve expressions (on nodename-level) here - if ($map->{target} =~ m/^(code|expr):(.+?)$/) { - my $target_dynamic_type = $1; - my $target_dynamic_expression = $2; - if (lc $target_dynamic_type eq 'code') { -# $map->{target} = $mapObject->$target_dynamic_expression($map); - } - } + # hash to sum up results + # TODO: re-implement! (sync-statistics???) - # remove asymmetries from $map (patch keys) - $map->{source_node} = $map->{source}; delete $map->{source}; - $map->{target_node} = $map->{target}; delete $map->{target}; - $map->{mapping} = $map->{details}; delete $map->{details}; - $map->{direction} = $map->{mode}; delete $map->{mode}; - - # defaults (mostly for backward-compatibility) - $map->{source_node} ||= $source_node_name; - $map->{source_ident} ||= 'storage_method:id'; - $map->{target_ident} ||= 'property:oid'; - $map->{direction} ||= $opts->{mode}; # | PUSH | PULL | FULL - $map->{method} ||= 'checksum'; # | timestamp - $map->{source_exclude} ||= [qw( cs )]; + # detect synchronization method to determine which optical symbol (directed arrow) to use + my $mode = $self->{args}->{mode}; # V1 + $mode ||= $self->{options}->{process}->{mode}; # V2 + my $direction_arrow = $self->_getDirectedArrow($mode); - # merge map to opts - map { $opts->{$_} = $map->{$_}; } keys %$map; - -#print Dumper($opts); + # determine code versions + my $code_metadata_version; + # first, try to use version supplied by mapping-metadata + $code_metadata_version = $self->{options}->{metadata}->{version}; + # if not set, inherit from global 'sync_version' + $code_metadata_version ||= $self->{sync_version}; + + # load additional code from versioned namespace into current instance + my $dynModule = "Metadata::$code_metadata_version"; + $self->load($dynModule); + + # build metadata using versioned code and stuff + $self->options2metadata(); + $self->options2metadata_accessor(); + + # tracing + #print Dumper($self); + #exit; - # TODO: move this to checkOptions... - - # check - do we have a target? - if (!$opts->{target_node}) { - $logger->warning( __PACKAGE__ . "->prepareOptions: No target given - please check metadata declaration."); - return; - } + $logger->info( __PACKAGE__ . "->syncNodes: source=$self->{meta}->{source}->{dbKey}/$self->{meta}->{source}->{nodeName} [$self->{meta}->{source}->{nodeType}] $direction_arrow target=$self->{meta}->{target}->{dbKey}/$self->{meta}->{target}->{nodeName} [$self->{meta}->{target}->{nodeType}]" ); + return if !$self->buildFieldmapping(); + return if !$self->_touchNodeSet(); + return if !$self->_prepare_sync(); - #return $opts; - return 1; + # tracing + #print Dumper($self); + #print Dumper($self->{args}); + #print Dumper($self->{options}); + #print Dumper($self->{meta}); + #print Dumper($self->{metadata}); + #exit; + # finally, tell the core to start the synchronization process + $self->_run(); + } -sub prepareOptionsV2 { + +sub _prepareOptions { my $self = shift; - my $opts = shift; + + my $opts = $self->{args}; # patch options + $opts->{source}->{nodeName} ||= ''; + $opts->{target}->{nodeName} ||= ''; $opts->{process}->{mode} ||= ''; - $opts->{process}->{prepare} ||= 0; $opts->{process}->{erase} ||= 0; + $opts->{process}->{prepare} ||= 0; + + # defaults (mostly for backward-compatibility to V1 - + # but code mungled here out of prepareOptions_V1 from Version::V1) + $opts->{metadata}->{syncMethod} ||= 'checksum'; # | timestamp + $opts->{source}->{ident} ||= 'storage_method:id'; + $opts->{source}->{exclude} ||= [qw( cs )]; + $opts->{target}->{ident} ||= 'property:oid'; + #$map->{source_node} ||= $source_node_name; + #$map->{direction} ||= $opts->{mode}; # | PUSH | PULL | FULL # pre-check options if (!$self->_preCheckOptions($opts)) { - $logger->error( __PACKAGE__ . "->prepareOptionsV2: _preCheckOptions failed."); + $logger->error( __PACKAGE__ . "->_prepareOptions: _preCheckOptions failed."); return; } # inform user about option preparation - $logger->notice( __PACKAGE__ . "->prepareOptionsV2( source.node='$opts->{source}->{nodeName}', target.node='$opts->{target}->{nodeName}', mode='$opts->{process}->{mode}', e='$opts->{process}->{erase}', p='$opts->{process}->{prepare}' )"); + $logger->debug( __PACKAGE__ . "->_prepareOptions( source.node='$opts->{source}->{nodeName}', target.node='$opts->{target}->{nodeName}', mode='$opts->{process}->{mode}', e='$opts->{process}->{erase}', p='$opts->{process}->{prepare}' )"); # try to load mapping-metadata-container - #my $mapObject = getNewPerlObjectByPkgName($opts->{map}->{moduleName}); - my $mapObject = DesignPattern::Object->fromPackage($opts->{map}->{moduleName}); + # How? Create a new instance of the given + # perl module/package name in ->{...}->{moduleName}. + # This instance is used later in the innards of the sync, + # that's why the module (class) should have a certain layout + # enabling the core to use it for gathering metadata while processing. + my $mapObject = DesignPattern::Object->fromPackage($opts->{map}->{moduleName}); # try to resolve map from metadata-container @@ -208,36 +265,70 @@ my $source_nodeType = $opts->{source}->{nodeType}; # check if mapping for certain node is contained in mapping object - if (!$mapObject->can($source_nodeType)) { - $logger->warning( __PACKAGE__ . "->prepareOptionsV2: Can't access mapping for source-type=\"$source_nodeType\" - please check \"$opts->{map}->{moduleName}\"."); + if (!$mapObject || !$mapObject->can($source_nodeType)) { + $logger->warning( __PACKAGE__ . "->_prepareOptions: Can't access mapping for source-type=\"$source_nodeType\" - please check \"$opts->{map}->{moduleName}\"."); return; } - # get map my $map = $mapObject->$source_nodeType; #print Dumper($map); -=pod + my $map_version = $map->{metadata}->{version}; + # FIXME: backward compatibility + if (!$map_version) { + $self->options_to_V2($map); + } + + # trace + #print Dumper($map); + #exit; + #print "ref: ", ref $map->{target}, "\n"; + #print "target: ", $map->{target}, "\n"; + # check here if "target" is actually a CODEref - in this case: resolve it - deprecated!!! ??? if (ref $map->{target}->{address} eq 'CODE') { $map->{target}->{address} = $map->{target}->{address}->($source_nodeType); } # resolve expressions (on nodename-level) here - if ($map->{target}->{address} =~ m/^(code|expr):(.+?)$/) { + elsif ($map->{target}->{address} =~ m/^(code|expr):(.+?)$/) { my $target_dynamic_type = $1; my $target_dynamic_expression = $2; if (lc $target_dynamic_type eq 'code') { $map->{target} = $mapObject->$target_dynamic_expression($map); } } -=cut - #map { $opts->{$_} = $map->{$_}; } keys %$map; - my $opts_merged = merge( $opts, $map ); + # merging - V1 + #map { $opts->{$_} = $map->{$_}; } keys %$map; + # trace + #print Dumper($self->{options}); + + # merging - V2 + + # merge local map with local opts + # enable cloning + # FIXME: do we really need cloning here? trade safety/encapsulation for speed? + Hash::Merge::set_clone_behavior(1); + Hash::Merge::set_behavior( 'RIGHT_PRECEDENT' ); + my $locals_merged = merge( $opts, $map ); + + # trace + #print Dumper($opts); + #print Dumper($map); + #print Dumper($locals_merged); + #exit; + + # merge local-merged ones with instance-wide options + Hash::Merge::set_clone_behavior(0); + $self->{options} = merge( $self->{options}, $locals_merged ); + + # trace + #print Dumper($self->{options}); + #exit; + - $self->{options} = $opts_merged; $self->{state}->{options_ready} = 1; return 1; @@ -245,189 +336,49 @@ } -sub configure { - my $self = shift; - my @args = @_; - if (!isEmpty(\@args)) { - my %properties = @_; - # merge args to properties - map { $self->{$_} = $properties{$_}; } keys %properties; - $self->_init(); - $self->_initV1(); - } else { - #print "no args!", "\n"; - } - #print Dumper($self); - $self->{state}->{configured} = 1; - return 1; -} +sub _preCheckOptions { -sub configureV2 { my $self = shift; - my @args = @_; - if (!isEmpty(\@args)) { - my %properties = @_; - # merge args to properties - #map { $self->{$_} = $properties{$_}; } keys %properties; - $self->{options} = merge($self->{options}, \%properties); - $self->_init(); - #$self->_initV1(); - } else { - #print "no args!", "\n"; - } - -#print Dumper($self); - - $self->{state}->{configured} = 1; - return 1; -} + my $opts = shift; -sub _getDirectedArrow { - my $self = shift; - my $mode = shift; - $mode ||= ''; - - if (lc $mode eq 'push') { - return '->'; - } elsif (lc $mode eq 'pull') { - return '<-'; - } elsif (lc $mode eq 'full') { - return '<->'; - } else { - return ''; + # trace + #print Dumper($opts); + #exit; + + if (!$opts->{process}->{mode}) { + $logger->error( __PACKAGE__ . "->_preCheckOptions failed: Please specify \"--action=(load|save)\"."); + return; } -} - -# TODO: some feature to show off the progress of synchronization (cur/max * 100) -sub syncNodes { - - my $self = shift; - my $args = shift; - if (!$self->{state}->{configured}) { - $logger->critical( __PACKAGE__ . "->syncNodes: Synchronization object is not configured/initialized correctly." ); + # the type of the to-be-synced item + if (!$opts->{source}->{nodeType}) { + $logger->error( __PACKAGE__ . "->_preCheckOptions failed: Please specify \"source-type\"."); return; } - - # remember arguments through the whole processing - $self->{args} = $args; - - $logger->debug( __PACKAGE__ . "->syncNodes: starting" ); - - # hash to hold and/or fill in metadata required for the processing - $self->{meta} = {}; - - # hash to sum up results - # TODO: re-implement! (sync-statistics???) - - # detect synchronization method to determine which optical symbol (directed arrow) to use - my $mode = $self->{args}->{mode}; # V1 - $mode ||= $self->{options}->{process}->{mode}; # V2 - my $direction_arrow = $self->_getDirectedArrow($mode); - - if (!$self->{options}->{metadata}->{version} || $self->{options}->{metadata}->{version} < 0.2) { - $self->_buildMetadataV1(); - } else { - $self->_buildMetadataV2(); + # the name of the (container-) node the items are listed in + if (!$opts->{source}->{nodeName}) { + $logger->error( __PACKAGE__ . "->_preCheckOptions failed: Please specify \"source-node\"."); + return; } - $logger->info( __PACKAGE__ . "->syncNodes: source=$self->{meta}->{source}->{dbKey}/$self->{meta}->{source}->{nodeName} [$self->{meta}->{source}->{nodeType}] $direction_arrow target=$self->{meta}->{target}->{dbKey}/$self->{meta}->{target}->{nodeName} [$self->{meta}->{target}->{nodeType}]" ); - - return if !$self->_buildFieldmappingV1(); - return if !$self->_handleNodeContainers(); - return if !$self->_prepareSync(); + # a "map"-declaration which module to use for mapping- and/or lookup-purposes + if (!$opts->{map}) { + $logger->warning( __PACKAGE__ . "->_preCheckOptions: No mapping supplied - please check key 'map|mappings' in global configuration or specify additional argument '--mapping-module'."); + return; + } + if (!$opts->{map}->{moduleName}) { + $logger->warning( __PACKAGE__ . "->_preCheckOptions: Currently only perl-modules can provide mappings: Please specify one with '--mapping-module=My::Mapping::Module'."); + return; + } -#print Dumper($self); -#print Dumper($self->{args}); -#print Dumper($self->{options}); -#print Dumper($self->{meta}); -#print Dumper($self->{metadata}); + return 1; - $self->_syncNodes(); - } -sub _handleNodeContainers { - my $self = shift; - - # check partners/nodes: does partner exist / is node available? - foreach my $partner (keys %{$self->{meta}}) { - - # 1. check partners & storages - if (!$self->{meta}->{$partner}) { - $logger->critical( __PACKAGE__ . "->_handleNodeContainers: Could not find partner '$partner' in configuration metadata." ); - next; - } - - my $dbkey = $self->{meta}->{$partner}->{dbKey}; - -#print Dumper($self->{meta}); - - if (!$self->{meta}->{$partner}->{storage}) { - $logger->critical( __PACKAGE__ . "->_handleNodeContainers: Could not access storage ( partner='$partner', dbKey='$dbkey' ) - configuration-error?" ); - next; - } - - # TODO: - # 2. check if partners (and nodes?) are actually available.... - # eventually pre-check mode of access-attempt (read/write) here to provide an "early-croak" if possible - -# print Dumper($self->{meta}->{$partner}->{storage}->{locator}); - - my $dbType = $self->{meta}->{$partner}->{storage}->{locator}->{type}; -#print "dbType: $dbType", "\n"; - - # 3. check nodes - next if $dbType eq 'DBI'; # HACK for DBD::CSV - re-enable for others - # get node-name -# print Dumper($self); - #print Dumper($self->{meta}->{$partner}); - #print "öö", $self->{meta}->{$partner}->{node}, "\n"; - my $nodename = $self->{meta}->{$partner}->{node}; # V1 - $nodename ||= $self->{meta}->{$partner}->{nodeName}; # V2 - # check if nodename is actually a CODEref, execute it to get a mapped/filtered target-nodename - -#print "nodename: $nodename", "\n"; - - $logger->debug( __PACKAGE__ . "->_handleNodeContainers: Accessing dbType=\"$dbType\", nodename=\"$nodename\"." ); - -=pod - #print "----", ref $nodename, "\n"; - if ($nodename =~ m/CODE/) { - print Dumper($self); - #exit; - $nodename = $nodename->($nodename); - } -=cut -#print Dumper($self); - - #print "partner: $partner - nodename: $nodename", "\n"; - - if (!$self->{meta}->{$partner}->{storage}->existsChildNode($nodename)) { -#print "ex", "\n"; -#exit; +sub _prepare_sync { - if ($partner eq 'target' && $self->{options}->{target}->{autocreateFolders}) { - if (!$self->{meta}->{$partner}->{storage}->createChildNode($nodename)) { - $logger->critical( __PACKAGE__ . "->_handleNodeContainers: Could not create node '$self->{meta}->{$partner}->{nodeName}\@$self->{meta}->{$partner}->{dbKey}' [$self->{meta}->{$partner}->{nodeType}]." ); - next; - } - } else { - $logger->critical( __PACKAGE__ . "->_handleNodeContainers: Could not reach node \"$nodename\" at partner \"$partner\"." ); - next; - } - } - - } - - return 1; - -} - - -sub _prepareSync { my $self = shift; # TODO: @@ -435,16 +386,26 @@ # -+ if action == PULL: swap metadata and start processing # - if action == FULL: start processing, then swap metadata and (re-)start processing + #print "dir: ", $self->{args}->{direction}, "\n"; + # manipulate metainfo according to direction of synchronization - if (lc $self->{args}->{direction} eq 'push') { - # just do it ... - } elsif (lc $self->{args}->{direction} eq 'pull') { - #print "=======SWAP", "\n"; - # swap + if (lc $self->{options}->{process}->{mode} eq 'push') { + # just do it ... (don't modify any metadata) + + } elsif (lc $self->{options}->{process}->{mode} eq 'pull') { + # swap source and target metadata + # TODO: introduce different mechanism to make more then two partners (descents) possible ($self->{meta}->{source}, $self->{meta}->{target}) = ($self->{meta}->{target}, $self->{meta}->{source}); - } elsif (lc $self->{args}->{direction} eq 'full') { + + } elsif (lc $self->{options}->{process}->{mode} eq 'full') { + # TODO: + # do a pull first and a push afterwards + # this requires us to be called somehow recursively - just one recursion level ;-) + } else { + # TODO: are there any other synchronization modes besides PULL, PUSH, FULL? + } # import flag means: prepare the source node to be syncable @@ -468,4 +429,20 @@ } +sub _getDirectedArrow { + my $self = shift; + my $mode = shift; + $mode ||= ''; + + if (lc $mode eq 'push') { + return '->'; + } elsif (lc $mode eq 'pull') { + return '<-'; + } elsif (lc $mode eq 'full') { + return '<->'; + } else { + return ''; + } +} + 1;