libs/Data/Storage.pm

# $Id: Storage.pm,v 1.13 2002/12/17 21:54:12 joko Exp $
#
# Copyright (c) 2002  Andreas Motl <andreas.motl@ilo.de>
#
# See COPYRIGHT section in pod text below for usage and distribution rights.
#
############################################
#
#  $Log: Storage.pm,v $
#  Revision 1.13  2002/12/17 21:54:12  joko
#  + feature when using Tangram:
#    + what? each object created should delivered with a globally(!?) unique identifier (GUID) besides the native tangram object id (OID)
#        + patched Tangram::Storage (jonen)
#        + enhanced Data::Storage::Schema::Tangram (joko)
#        + enhanced Data::Storage::Handler::Tangram 'sub getObjectByGuid' (jonen)
#    + how?
#        + each concrete (non-abstract) class gets injected with an additional field/property called 'guid' - this is done (dynamically) on schema level
#        + this property ('guid') gets filled on object creation/insertion from 'sub Tangram::Storage::_insert' using Data::UUID from CPAN
#        + (as for now) this property can get accessed by calling 'getObjectByGuid' on the already known storage-handle used throughout the application
#
#  Revision 1.12  2002/12/12 02:50:15  joko
#  + this now (unfortunately) needs DBI for some helper functions
#  + TODO: these have to be refactored to another scope! (soon!)
#
#  Revision 1.11  2002/12/11 06:53:19  joko
#  + updated pod
#
#  Revision 1.10  2002/12/07 03:37:23  joko
#  + updated pod
#
#  Revision 1.9  2002/12/01 22:15:45  joko
#  - sub createDb: moved to handler
#
#  Revision 1.8  2002/11/29 04:48:23  joko
#  + updated pod
#
#  Revision 1.7  2002/11/17 06:07:18  joko
#  + creating the handler is easier than proposed first - for now :-)
#  + sub testAvailability
#
#  Revision 1.6  2002/11/09 01:04:58  joko
#  + updated pod
#
#  Revision 1.5  2002/10/29 19:24:18  joko
#  - reduced logging
#  + added some pod
#
#  Revision 1.4  2002/10/27 18:35:07  joko
#  + added pod
#
#  Revision 1.3  2002/10/25 11:40:37  joko
#  + enhanced robustness
#  + more logging for debug-levels
#  + sub dropDb
#
#  Revision 1.2  2002/10/17 00:04:29  joko
#  + sub createDb
#  + sub isConnected
#  + bugfixes regarding "deep recursion" stuff
#
#  Revision 1.1  2002/10/10 03:43:12  cvsjoko
#  + new
#
############################################


BEGIN {
  $Data::Storage::VERSION = 0.02;
}


=head1 NAME

  Data::Storage - Interface for accessing various Storage implementations for Perl in an independent way


=head1 AIMS

  - should encapsulate Tangram, DBI, DBD::CSV and LWP:: to access them in an unordinary (more convenient) way ;)
  - introduce a generic layered structure, refactor *SUBLAYER*-stuff, make (e.g.) this possible:
    Perl Data::Storage[DBD::CSV]  ->  Perl LWP::  ->  Internet HTTP/FTP/*  ->  Host Daemon  ->  csv-file
  - provide generic synchronization mechanisms across arbitrary/multiple storages based on ident/checksum
    maybe it's possible to have schema-, structural- and semantical modifications synchronized???


=head1 SYNOPSIS

=head2 BASIC ACCESS

=head2 ADVANCED ACCESS

  ... via inheritance:
  
    use Data::Storage;
    my $proxyObj = new HttpProxy;
    $proxyObj->{url} = $url;
    $proxyObj->{payload} = $content;
    $self->{storage}->insert($proxyObj);
    
    use Data::Storage;
    my $proxyObj = HttpProxy->new(
      url => $url,
      payload => $content,
    );
    $self->{storage}->insert($proxyObj);


=head2 SYNCHRONIZATION

  my $nodemapping = {
    'LangText' => 'langtexts.csv',
    'Currency' => 'currencies.csv',
    'Country'  => 'countries.csv',
  };

  my $propmapping = {
    'LangText' => [
      [ 'source:lcountrykey'  =>  'target:country' ],
      [ 'source:lkey'         =>  'target:key' ],
      [ 'source:lvalue'       =>  'target:text' ],
    ],
    'Currency' => [
      [ 'source:ckey'         =>  'target:key' ],
      [ 'source:cname'        =>  'target:text' ],
    ],
    'Country' => [
      [ 'source:ckey'         =>  'target:key' ],
      [ 'source:cname'        =>  'target:text' ],
    ],
  };

  sub syncResource {

    my $self = shift;
    my $node_source = shift;
    my $mode = shift;
    my $opts = shift;
    
    $mode ||= '';
    $opts->{erase} ||= 0;
    
    $logger->info( __PACKAGE__ . "->syncResource( node_source $node_source mode $mode erase $opts->{erase} )");
  
    # resolve metadata for syncing requested resource
    my $node_target = $nodemapping->{$node_source};
    my $mapping = $propmapping->{$node_source};
    
    if (!$node_target || !$mapping) {
      # loggger.... "no target, sorry!"
      print "error while resolving resource metadata", "\n";
      return;
    }
    
    if ($opts->{erase}) {
      $self->_erase_all($node_source);
    }
  
    # create new sync object
    my $sync = Data::Transfer::Sync->new( 
      storages => {
        L => $self->{bizWorks}->{backend},
        R => $self->{bizWorks}->{resources},
      },
      id_authorities        =>  [qw( L ) ],
      checksum_authorities  =>  [qw( L ) ],
      write_protected       =>  [qw( R ) ],
      verbose               =>  1,
    );
    
    # sync
    # todo: filter!?
    $sync->syncNodes( {
      direction       =>  $mode,                 # | +PUSH | +PULL | -FULL | +IMPORT | -EXPORT
      method          =>  'checksum',            # | -timestamp | -manual
      source          =>  "L:$node_source",
      source_ident    =>  'storage_method:id',
      source_exclude  =>  [qw( id cs )],
      target          =>  "R:$node_target",
      target_ident    =>  'property:oid',
      mapping         =>  $mapping,
    } );

  }


=head2 NOTE

  This module heavily relies on DBI and Tangram, but adds a lot of additional bugs and quirks. 
  Please look at their documentation and/or this code for additional information.


=head1 REQUIREMENTS

  For full functionality:
    DBI              from CPAN
    DBD::mysql       from CPAN
    Tangram 2.04     from CPAN         (hmmm, 2.04 won't do in some cases)
    Tangram 2.05     from http://...   (2.05 seems okay but there are also additional patches from our side)
    Class::Tangram   from CPAN
    DBD::CSV         from CPAN
    MySQL::Diff      from http://adamspiers.org/computing/mysqldiff/
    ... and all their dependencies

=cut

# The POD text continues at the end of the file.


package Data::Storage;

use strict;
use warnings;

use Data::Storage::Locator;
use Data::Dumper;

# TODO: wipe out!
use DBI;

# TODO: actually implement level (integrate with Log::Dispatch)
my $TRACELEVEL = 0;

# get logger instance
my $logger = Log::Dispatch::Config->instance;

sub new {
  my $invocant = shift;
  my $class = ref($invocant) || $invocant;
  #my @args = normalizeArgs(@_);

  my $arg_locator = shift;
  my $arg_options = shift;

  #my $self = { STORAGEHANDLE => undef, @_ };
  my $self = { STORAGEHANDLE => undef, locator => $arg_locator, options => $arg_options };
  #$logger->debug( __PACKAGE__ . "[$self->{locator}->{type}]" . "->new(@_)" );
  $logger->debug( __PACKAGE__ . "[$arg_locator->{type}]" . "->new(@_)" );
  return bless $self, $class;
}

sub AUTOLOAD {
  
  # since this is a core function acting as dispatcher to $self->{STORAGEHANDLE},
  # some sophisticated handling and filtering is needed to avoid things like
  #     - Deep recursion on subroutine "Data::Storage::AUTOLOAD"
  #     - Deep recursion on subroutine "Data::Storage::Handler::Abstract::AUTOLOAD"
  #     - Deep recursion on anonymous subroutine at [...]
  # we also might filter log messages caused by logging to itself in "advanced logging of AUTOLOAD calls"
  
  my $self = shift;
  our $AUTOLOAD;
  
  # ->DESTROY would - if not declared - trigger an AUTOLOAD also
  return if $AUTOLOAD =~ m/::DESTROY$/;

  my $method = $AUTOLOAD;
  $method =~ s/^.*:://;

  # advanced logging of AUTOLOAD calls ...
  # ... nice but do it only when TRACING (TODO) is enabled
    if ($TRACELEVEL) {
      my $logstring = "";
      $logstring .= __PACKAGE__ . "[$self->{locator}->{type}]" . "->" . $method;
      #print "count: ", $#_, "\n";
      #$logstring .= Dumper(@_) if ($#_ != -1);
      my $tabcount = int( (80 - length($logstring)) / 10 );
      $logstring .= "\t" x $tabcount . "(AUTOLOAD)";
      # TODO: only ok if logstring doesn't contain 
      #            e.g. "Data::Storage[Tangram]->insert(SystemEvent=HASH(0x5c0034c))          (AUTOLOAD)"
      # but that would be _way_ too specific as long as we don't have an abstract handler for this  ;)
      $logger->debug( $logstring );
      #print join('; ', @_);
    }
    
  # filtering AUTOLOAD calls and first-time-touch of the actual storage impl
  if ($self->_filter_AUTOLOAD($method)) {
    #print "_accessStorage\n";
    $self->_accessStorage();
    $self->{STORAGEHANDLE}->$method(@_);
  }
  
}

sub _filter_AUTOLOAD {
  my $self = shift;
  my $method = shift;
  if ($self->{options}->{protected}) {
    if ($method eq 'disconnect') {
      return;
    }
  }
  return 1;
}


sub normalizeArgs {
  my %args = @_;
  if (!$args{dsn} && $args{meta}{dsn}) {
    $args{dsn} = $args{meta}{dsn};
  }
  my @result = %args;
  return @result;
}

sub _accessStorage {
  my $self = shift;
  # TODO: to some tracelevel!
  if ($TRACELEVEL) {
    $logger->debug( __PACKAGE__ .  "[$self->{locator}->{type}]" . "->_accessStorage()" );
  }
  if (!$self->{STORAGEHANDLE}) {
    $self->_createStorageHandle();
  }
}

sub _createStorageHandle {
  my $self = shift;
  my $type = $self->{locator}->{type};
  $logger->debug( __PACKAGE__ .  "[$type]" . "->_createStorageHandle()" );

  my $pkg = "Data::Storage::Handler::" . $type . "";
  
  # try to load perl module at runtime
  my $evalstr = "use $pkg;";
  eval($evalstr);
  if ($@) {
    $logger->error( __PACKAGE__ .  "[$type]" . "->_createStorageHandle(): $@" );
    return;
  }
  
  # build up some additional arguments to pass on
  #my @args = %{$self->{locator}};
  my @args = ();

  # - create new storage handle object
  # - propagate arguments to handler
  # - pass locator by reference to be able to store status- or meta-information in it
  $self->{STORAGEHANDLE} = $pkg->new( locator => $self->{locator}, @args );

}

sub addLogDispatchHandler {

      my $self = shift;
      my $name = shift;
      my $package = shift;
      my $logger1 = shift;
      my $objectCreator = shift;
      
      #$logger->add( Log::Dispatch::Tangram->new( name => $name,
      $logger->add( $package->new( name => $name,
                                            #min_level => 'debug',
                                            min_level => 'info',
                                            storage => $self,
                                            objectCreator => $objectCreator,
                                            fields => {
                                              message => 'usermsg',
                                              timestamp => 'stamp',
                                              level => 'level',
                                              name => 'code',
                                            },
                                            filter_patterns => [ '->insert\(SystemEvent=' ],
                                            #filter_patterns => [ 'SystemEvent' ],
                                            
                                            #format  => '[%d] [%p] %m%n',
                                             ) );

}

sub removeLogDispatchHandler {
  my $self = shift;
  my $name = shift;
  #my $logger = shift;
  $logger->remove($name);
}

sub getDbName {
  my $self = shift;
  my $dsn = $self->{locator}->{dbi}->{dsn};
  $dsn =~ m/database=(.+?);/;
  my $database_name = $1;
  return $database_name;
}

sub testAvailability {
  my $self = shift;
  my $status = $self->testDsn();
  $self->{locator}->{status}->{available} = $status;
  return $status;
}

sub isConnected {
  my $self = shift;
  # TODO: REVIEW!
  return 1 if $self->{STORAGEHANDLE};
}

sub testDsn {
  my $self = shift;
  my $dsn = $self->{locator}->{dbi}->{dsn};
  my $result;
  if ( my $dbh = DBI->connect($dsn, '', '', {
                                                      PrintError => 0,
                                                      } ) ) {
    
    # TODO: REVIEW
    $dbh->disconnect();
    
    return 1;
  } else {
    $logger->warning( __PACKAGE__ .  "[$self->{locator}->{type}]" . "->testDsn(): " . "DBI-error: " . $DBI::errstr );
  }
}

1;
__END__


=head1 DESCRIPTION

=head2 Data::Storage

  Data::Storage is a module for accessing various "data structures / kinds of structured data" stored inside
  various "data containers".
  We tried to use the AdapterPattern (http://c2.com/cgi/wiki?AdapterPattern) to implement a wrapper-layer
  around core CPAN modules (Tangram, DBI).

=head2 Why?

  You will get a better code-structure (not bad for later maintenance) in growing Perl code projects,
  especially when using multiple database connections at the same time.
  You will be able to switch between different _kinds_ of implementations used for storing data.
  Your code will use the very same API to access these storage layers.
      ... implementation has to be changed for now
  Maybe you will be able to switch "on-the-fly" without changing any bits in code in the future.... 
      ... but that's not the focus

=head2 What else?

  Having this, we were able to do implement a generic data synchronization module more easy,
  please look at Data::Transfer.


=head1 AUTHORS / COPYRIGHT

  The Data::Storage module is Copyright (c) 2002 Andreas Motl.
  All rights reserved.
  You may distribute it under the terms of either the GNU General Public
  License or the Artistic License, as specified in the Perl README file.


=head1 ACKNOWLEDGEMENTS

  Larry Wall for Perl, Tim Bunce for DBI, Jean-Louis Leroy for Tangram and Set::Object, 
  Sam Vilain for Class::Tangram, Jochen Wiedmann and Jeff Zucker for DBD::CSV & Co.,
  Adam Spiers for MySQL::Diff and all contributors.


=head1 SUPPORT / WARRANTY

  Data::Storage is free software. IT COMES WITHOUT WARRANTY OF ANY KIND.


=head1 TODO


=head2 BUGS

"DBI-Error [Tangram]: DBD::mysql::st execute failed: Unknown column 't1.requestdump' in 'field list'"

  ... occours when operating on object-attributes not introduced yet:
  this should be detected and appended/replaced through:
  "Schema-Error detected, maybe (just) an inconsistency. 
  Please check if your declaration in schema-module "a" matches structure in database "b" or try to run"
  db_setup.pl --dbkey=import --action=deploy


Compare schema (structure diff) with database ... 

  ... when issuing "db_setup.pl --dbkey=import --action=deploy"
  on a database with an already deployed schema, use an additional "--update" then
  to lift the schema inside the database to the current declared schema.
  You will have to approve removals and changes on field-level while
  new objects and new fields are introduced silently without any interaction needed.
  In future versions there may be additional options to control silent processing of
  removals and changes.
  See this CRUD-table applying to the actions occouring on Classes and Class variables when deploying schemas,
  don't mix this up with CRUD-actions on Objects, these are already handled by (e.g.) Tangram itself.
  Classes:
    C create    ->  yes, handled automatically
    R retrieve  ->  no, not subject of this aspect since it is about deployment only
    U update    ->  yes, automatically for Class meta-attributes, yes/no for Class variables (look at the rules down here)
    D delete    ->  yes, just by user-interaction
  Class variables:
    C create    ->  yes, handled automatically
    R retrieve  ->  no, not subject of this aspect since it is about deployment only
    U update    ->  yes, just by user-interaction; maybe automatically if it can be determined that data wouldn't be lost
    D delete    ->  yes, just by user-interaction
  
  It's all about not to be able to loose data simply while this is in pre-alpha stage.
  And loosing data by being able to modify and redeploy schemas easily is definitely quite easy.
  
  As we can see, creations of Classes and new Class variables is handled 
  automatically and this is believed to be the most common case under normal circumstances.


=head2 FEATURES

  - Get this stuff together with UML (Unified Modeling Language) and/or standards from ODMG.
  - Make it possible to load/save schemas in XMI (XML Metadata Interchange), 
    which seems to be most commonly used today, perhaps handle objects with OIFML.
    Integrate/bundle this with a web-/html-based UML modeling tool or 
    some other interesting stuff like the "Co-operative UML Editor" from Uni Darmstadt. (web-/java-based)
  - Enable Round Trip Engineering. Keep code and diagrams in sync. Don't annoy/bother the programmers.
  - Add support for some more handlers/locators to be able to 
     access the following standards/protocols/interfaces/programs/apis transparently:
    +  DBD::CSV (via Data::Storage::Handler::DBI)
   (-) Text::CSV, XML::CSV, XML::Excel
    -  MAPI
    -  LDAP
    -  DAV (look at PerlDAV: http://www.webdav.org/perldav/)
    -  Mbox (use formail for seperating/splitting entries/nodes)
    -  Cyrus (cyrdeliver - what about cyrretrieve (export)???)
    -  use File::DiffTree, use File::Compare
    -  Hibernate
    -  "Win32::UserAccountDb"
    -  "*nix::UserAccountDb"
    -  .wab - files (Windows Address Book)
    -  .pst - files (Outlook Post Storage?)
    -  XML (e.g. via XML::Simple?)
  - Move to t3, look at InCASE
  - some kind of security layer for methods/objects
    - acls (stored via tangram/ldap?) for functions, methods and objects (entity- & data!?)
    - where are the hooks needed then?
      - is Data::Storage & Co. okay, or do we have to touch the innards of DBI and/or Tangram?
      - an attempt to start could be: 
         - 'sub getACLByObjectId($id, $context)'
         - 'sub getACLByMethodname($id, $context)'
         - 'sub getACLByName($id, $context)'
            ( would require a kinda registry to look up these very names pointing to arbitrary locations (code, data, ...) )
  - add more hooks and various levels
  - better integrate introduced 'getObjectByGuid'-mechanism from Data::Storage::Handler::Tangram


=head3 LINKS / REFERENCES

  Specs:
    UML 1.3 Spec: http://cgi.omg.org/cgi-bin/doc?ad/99-06-08.pdf
    XMI 1.1 Spec: http://cgi.omg.org/cgi-bin/doc?ad/99-10-02.pdf
    XMI 2.0 Spec: http://cgi.omg.org/docs/ad/01-06-12.pdf
    ODMG: http://odmg.org/
    OIFML: http://odmg.org/library/readingroom/oifml.pdf

  CASE Tools:
    Rational Rose (commercial): http://www.rational.com/products/rose/
    Together (commercial): http://www.oi.com/products/controlcenter/index.jsp
    InCASE - Tangram-based Universal Object Editor 
    Sybase PowerDesigner: http://www.sybase.com/powerdesigner
  
  UML Editors:
    Fujaba (free, university): http://www.fujaba.de/
    ArgoUML (free): http://argouml.tigris.org/
    Poseidon (commercial): http://www.gentleware.com/products/poseidonDE.php3
    Co-operative UML Editor (research): http://www.darmstadt.gmd.de/concert/activities/internal/umledit.html
    Metamill (commercial): http://www.metamill.com/
    Violet (university, research, education): http://www.horstmann.com/violet/
    PyUt (free): http://pyut.sourceforge.net/
    (Dia (free): http://www.lysator.liu.se/~alla/dia/)
    UMLet (free, university): http://www.swt.tuwien.ac.at/umlet/index.html
    Voodoo (free): http://voodoo.sourceforge.net/
    Umbrello UML Modeller: http://uml.sourceforge.net/

  UML Tools:
    http://www.objectsbydesign.com/tools/umltools_byPrice.html

  Further readings:
    http://www.google.com/search?q=web+based+uml+editor&hl=en&lr=&ie=UTF-8&oe=UTF-8&start=10&sa=N
    http://www.fernuni-hagen.de/DVT/Aktuelles/01FHHeidelberg.pdf
    http://www.enhyper.com/src/documentation/
    http://cis.cs.tu-berlin.de/Dokumente/Diplomarbeiten/2001/skinner.pdf
    http://citeseer.nj.nec.com/vilain00diagrammatic.html
    http://archive.devx.com/uml/articles/Smith01/Smith01-3.asp

1	joko	1.14	# $Id: Storage.pm,v 1.13 2002/12/17 21:54:12 joko Exp $
2	joko	1.4	#
3			# Copyright (c) 2002 Andreas Motl <andreas.motl@ilo.de>
4			#
5			# See COPYRIGHT section in pod text below for usage and distribution rights.
6			#
7	joko	1.8	############################################
8	cvsjoko	1.1	#
9	joko	1.4	# $Log: Storage.pm,v $
10	joko	1.14	# Revision 1.13 2002/12/17 21:54:12 joko
11			# + feature when using Tangram:
12			# + what? each object created should delivered with a globally(!?) unique identifier (GUID) besides the native tangram object id (OID)
13			# + patched Tangram::Storage (jonen)
14			# + enhanced Data::Storage::Schema::Tangram (joko)
15			# + enhanced Data::Storage::Handler::Tangram 'sub getObjectByGuid' (jonen)
16			# + how?
17			# + each concrete (non-abstract) class gets injected with an additional field/property called 'guid' - this is done (dynamically) on schema level
18			# + this property ('guid') gets filled on object creation/insertion from 'sub Tangram::Storage::_insert' using Data::UUID from CPAN
19			# + (as for now) this property can get accessed by calling 'getObjectByGuid' on the already known storage-handle used throughout the application
20			#
21	joko	1.13	# Revision 1.12 2002/12/12 02:50:15 joko
22			# + this now (unfortunately) needs DBI for some helper functions
23			# + TODO: these have to be refactored to another scope! (soon!)
24			#
25	joko	1.12	# Revision 1.11 2002/12/11 06:53:19 joko
26			# + updated pod
27			#
28	joko	1.11	# Revision 1.10 2002/12/07 03:37:23 joko
29			# + updated pod
30			#
31	joko	1.10	# Revision 1.9 2002/12/01 22:15:45 joko
32			# - sub createDb: moved to handler
33			#
34	joko	1.9	# Revision 1.8 2002/11/29 04:48:23 joko
35			# + updated pod
36			#
37	joko	1.8	# Revision 1.7 2002/11/17 06:07:18 joko
38			# + creating the handler is easier than proposed first - for now :-)
39			# + sub testAvailability
40			#
41	joko	1.7	# Revision 1.6 2002/11/09 01:04:58 joko
42			# + updated pod
43			#
44	joko	1.6	# Revision 1.5 2002/10/29 19:24:18 joko
45			# - reduced logging
46			# + added some pod
47			#
48	joko	1.5	# Revision 1.4 2002/10/27 18:35:07 joko
49			# + added pod
50			#
51	joko	1.4	# Revision 1.3 2002/10/25 11:40:37 joko
52			# + enhanced robustness
53			# + more logging for debug-levels
54			# + sub dropDb
55	joko	1.2	#
56	joko	1.3	# Revision 1.2 2002/10/17 00:04:29 joko
57			# + sub createDb
58			# + sub isConnected
59			# + bugfixes regarding "deep recursion" stuff
60			#
61	joko	1.2	# Revision 1.1 2002/10/10 03:43:12 cvsjoko
62			# + new
63	cvsjoko	1.1	#
64	joko	1.8	############################################
65	cvsjoko	1.1
66	joko	1.3
67	joko	1.4	BEGIN {
68	joko	1.8	$Data::Storage::VERSION = 0.02;
69	joko	1.4	}
70
71	joko	1.5
72	joko	1.4	=head1 NAME
73
74	joko	1.11	Data::Storage - Interface for accessing various Storage implementations for Perl in an independent way
75	joko	1.4
76	joko	1.8
77			=head1 AIMS
78
79			- should encapsulate Tangram, DBI, DBD::CSV and LWP:: to access them in an unordinary (more convenient) way ;)
80			- introduce a generic layered structure, refactor SUBLAYER-stuff, make (e.g.) this possible:
81			Perl Data::Storage[DBD::CSV] -> Perl LWP:: -> Internet HTTP/FTP/* -> Host Daemon -> csv-file
82			- provide generic synchronization mechanisms across arbitrary/multiple storages based on ident/checksum
83			maybe it's possible to have schema-, structural- and semantical modifications synchronized???
84
85
86	joko	1.4	=head1 SYNOPSIS
87
88	joko	1.8	=head2 BASIC ACCESS
89	joko	1.4
90	joko	1.8	=head2 ADVANCED ACCESS
91	joko	1.4
92			... via inheritance:
93
94			use Data::Storage;
95			my $proxyObj = new HttpProxy;
96			$proxyObj->{url} = $url;
97			$proxyObj->{payload} = $content;
98			$self->{storage}->insert($proxyObj);
99
100			use Data::Storage;
101			my $proxyObj = HttpProxy->new(
102			url => $url,
103			payload => $content,
104			);
105			$self->{storage}->insert($proxyObj);
106
107
108	joko	1.8	=head2 SYNCHRONIZATION
109
110			my $nodemapping = {
111			'LangText' => 'langtexts.csv',
112			'Currency' => 'currencies.csv',
113			'Country' => 'countries.csv',
114			};
115
116			my $propmapping = {
117			'LangText' => [
118			[ 'source:lcountrykey' => 'target:country' ],
119			[ 'source:lkey' => 'target:key' ],
120			[ 'source:lvalue' => 'target:text' ],
121			],
122			'Currency' => [
123			[ 'source:ckey' => 'target:key' ],
124			[ 'source:cname' => 'target:text' ],
125			],
126			'Country' => [
127			[ 'source:ckey' => 'target:key' ],
128			[ 'source:cname' => 'target:text' ],
129			],
130			};
131
132			sub syncResource {
133
134			my $self = shift;
135			my $node_source = shift;
136			my $mode = shift;
137			my $opts = shift;
138
139			$mode \|\|= '';
140			$opts->{erase} \|\|= 0;
141
142			$logger->info( __PACKAGE__ . "->syncResource( node_source $node_source mode $mode erase $opts->{erase} )");
143
144			# resolve metadata for syncing requested resource
145			my $node_target = $nodemapping->{$node_source};
146			my $mapping = $propmapping->{$node_source};
147
148			if (!$node_target \|\| !$mapping) {
149			# loggger.... "no target, sorry!"
150			print "error while resolving resource metadata", "\n";
151			return;
152			}
153
154			if ($opts->{erase}) {
155			$self->_erase_all($node_source);
156			}
157
158			# create new sync object
159			my $sync = Data::Transfer::Sync->new(
160			storages => {
161			L => $self->{bizWorks}->{backend},
162			R => $self->{bizWorks}->{resources},
163			},
164			id_authorities => [qw( L ) ],
165			checksum_authorities => [qw( L ) ],
166			write_protected => [qw( R ) ],
167			verbose => 1,
168			);
169
170			# sync
171			# todo: filter!?
172			$sync->syncNodes( {
173			direction => $mode, # \| +PUSH \| +PULL \| -FULL \| +IMPORT \| -EXPORT
174			method => 'checksum', # \| -timestamp \| -manual
175			source => "L:$node_source",
176			source_ident => 'storage_method:id',
177			source_exclude => [qw( id cs )],
178			target => "R:$node_target",
179			target_ident => 'property:oid',
180			mapping => $mapping,
181			} );
182
183			}
184
185
186	joko	1.4	=head2 NOTE
187
188	joko	1.11	This module heavily relies on DBI and Tangram, but adds a lot of additional bugs and quirks.
189			Please look at their documentation and/or this code for additional information.
190	joko	1.6
191
192			=head1 REQUIREMENTS
193	joko	1.4
194	joko	1.8	For full functionality:
195			DBI from CPAN
196			DBD::mysql from CPAN
197			Tangram 2.04 from CPAN (hmmm, 2.04 won't do in some cases)
198			Tangram 2.05 from http://... (2.05 seems okay but there are also additional patches from our side)
199			Class::Tangram from CPAN
200			DBD::CSV from CPAN
201			MySQL::Diff from http://adamspiers.org/computing/mysqldiff/
202			... and all their dependencies
203	joko	1.4
204			=cut
205
206			# The POD text continues at the end of the file.
207
208
209	cvsjoko	1.1	package Data::Storage;
210
211			use strict;
212			use warnings;
213
214			use Data::Storage::Locator;
215	joko	1.7	use Data::Dumper;
216	cvsjoko	1.1
217	joko	1.12	# TODO: wipe out!
218			use DBI;
219
220	joko	1.6	# TODO: actually implement level (integrate with Log::Dispatch)
221	joko	1.5	my $TRACELEVEL = 0;
222
223	cvsjoko	1.1	# get logger instance
224			my $logger = Log::Dispatch::Config->instance;
225
226			sub new {
227			my $invocant = shift;
228			my $class = ref($invocant) \|\| $invocant;
229			#my @args = normalizeArgs(@_);
230	joko	1.7
231	cvsjoko	1.1	my $arg_locator = shift;
232			my $arg_options = shift;
233	joko	1.7
234	cvsjoko	1.1	#my $self = { STORAGEHANDLE => undef, @_ };
235			my $self = { STORAGEHANDLE => undef, locator => $arg_locator, options => $arg_options };
236	joko	1.7	#$logger->debug( __PACKAGE__ . "[$self->{locator}->{type}]" . "->new(@_)" );
237			$logger->debug( __PACKAGE__ . "[$arg_locator->{type}]" . "->new(@_)" );
238	cvsjoko	1.1	return bless $self, $class;
239			}
240
241			sub AUTOLOAD {
242
243	joko	1.2	# since this is a core function acting as dispatcher to $self->{STORAGEHANDLE},
244			# some sophisticated handling and filtering is needed to avoid things like
245			# - Deep recursion on subroutine "Data::Storage::AUTOLOAD"
246			# - Deep recursion on subroutine "Data::Storage::Handler::Abstract::AUTOLOAD"
247			# - Deep recursion on anonymous subroutine at [...]
248	joko	1.8	# we also might filter log messages caused by logging to itself in "advanced logging of AUTOLOAD calls"
249	joko	1.2
250	cvsjoko	1.1	my $self = shift;
251			our $AUTOLOAD;
252
253			# ->DESTROY would - if not declared - trigger an AUTOLOAD also
254			return if $AUTOLOAD =~ m/::DESTROY$/;
255
256			my $method = $AUTOLOAD;
257			$method =~ s/^.*:://;
258
259	joko	1.5	# advanced logging of AUTOLOAD calls ...
260			# ... nice but do it only when TRACING (TODO) is enabled
261			if ($TRACELEVEL) {
262			my $logstring = "";
263			$logstring .= __PACKAGE__ . "[$self->{locator}->{type}]" . "->" . $method;
264			#print "count: ", $#_, "\n";
265			#$logstring .= Dumper(@_) if ($#_ != -1);
266			my $tabcount = int( (80 - length($logstring)) / 10 );
267			$logstring .= "\t" x $tabcount . "(AUTOLOAD)";
268			# TODO: only ok if logstring doesn't contain
269			# e.g. "Data::Storage[Tangram]->insert(SystemEvent=HASH(0x5c0034c)) (AUTOLOAD)"
270	joko	1.8	# but that would be _way_ too specific as long as we don't have an abstract handler for this ;)
271	joko	1.5	$logger->debug( $logstring );
272	joko	1.7	#print join('; ', @_);
273	joko	1.5	}
274
275	joko	1.8	# filtering AUTOLOAD calls and first-time-touch of the actual storage impl
276	cvsjoko	1.1	if ($self->_filter_AUTOLOAD($method)) {
277	joko	1.7	#print "_accessStorage\n";
278	cvsjoko	1.1	$self->_accessStorage();
279			$self->{STORAGEHANDLE}->$method(@_);
280			}
281
282			}
283
284			sub _filter_AUTOLOAD {
285			my $self = shift;
286			my $method = shift;
287			if ($self->{options}->{protected}) {
288			if ($method eq 'disconnect') {
289			return;
290			}
291			}
292			return 1;
293			}
294
295
296			sub normalizeArgs {
297			my %args = @_;
298			if (!$args{dsn} && $args{meta}{dsn}) {
299			$args{dsn} = $args{meta}{dsn};
300			}
301			my @result = %args;
302			return @result;
303			}
304
305			sub _accessStorage {
306			my $self = shift;
307			# TODO: to some tracelevel!
308	joko	1.5	if ($TRACELEVEL) {
309			$logger->debug( __PACKAGE__ . "[$self->{locator}->{type}]" . "->_accessStorage()" );
310			}
311	cvsjoko	1.1	if (!$self->{STORAGEHANDLE}) {
312			$self->_createStorageHandle();
313			}
314			}
315
316			sub _createStorageHandle {
317			my $self = shift;
318			my $type = $self->{locator}->{type};
319			$logger->debug( __PACKAGE__ . "[$type]" . "->_createStorageHandle()" );
320
321			my $pkg = "Data::Storage::Handler::" . $type . "";
322
323	joko	1.7	# try to load perl module at runtime
324			my $evalstr = "use $pkg;";
325			eval($evalstr);
326			if ($@) {
327			$logger->error( __PACKAGE__ . "[$type]" . "->_createStorageHandle(): $@" );
328			return;
329	cvsjoko	1.1	}
330	joko	1.7
331			# build up some additional arguments to pass on
332			#my @args = %{$self->{locator}};
333			my @args = ();
334
335	joko	1.8	# - create new storage handle object
336			# - propagate arguments to handler
337			# - pass locator by reference to be able to store status- or meta-information in it
338	joko	1.7	$self->{STORAGEHANDLE} = $pkg->new( locator => $self->{locator}, @args );
339	joko	1.3
340	cvsjoko	1.1	}
341
342			sub addLogDispatchHandler {
343
344			my $self = shift;
345			my $name = shift;
346			my $package = shift;
347	joko	1.3	my $logger1 = shift;
348	cvsjoko	1.1	my $objectCreator = shift;
349
350			#$logger->add( Log::Dispatch::Tangram->new( name => $name,
351			$logger->add( $package->new( name => $name,
352			#min_level => 'debug',
353			min_level => 'info',
354			storage => $self,
355			objectCreator => $objectCreator,
356			fields => {
357			message => 'usermsg',
358			timestamp => 'stamp',
359			level => 'level',
360			name => 'code',
361			},
362			filter_patterns => [ '->insert\(SystemEvent=' ],
363			#filter_patterns => [ 'SystemEvent' ],
364
365			#format => '[%d] [%p] %m%n',
366			) );
367
368			}
369
370			sub removeLogDispatchHandler {
371	joko	1.8	my $self = shift;
372			my $name = shift;
373			#my $logger = shift;
374			$logger->remove($name);
375	cvsjoko	1.1	}
376
377			sub getDbName {
378			my $self = shift;
379			my $dsn = $self->{locator}->{dbi}->{dsn};
380			$dsn =~ m/database=(.+?);/;
381			my $database_name = $1;
382			return $database_name;
383			}
384
385	joko	1.12	sub testAvailability {
386			my $self = shift;
387			my $status = $self->testDsn();
388			$self->{locator}->{status}->{available} = $status;
389			return $status;
390			}
391
392			sub isConnected {
393			my $self = shift;
394			# TODO: REVIEW!
395			return 1 if $self->{STORAGEHANDLE};
396			}
397
398	cvsjoko	1.1	sub testDsn {
399			my $self = shift;
400			my $dsn = $self->{locator}->{dbi}->{dsn};
401			my $result;
402			if ( my $dbh = DBI->connect($dsn, '', '', {
403			PrintError => 0,
404			} ) ) {
405	joko	1.9
406			# TODO: REVIEW
407	cvsjoko	1.1	$dbh->disconnect();
408	joko	1.9
409	cvsjoko	1.1	return 1;
410			} else {
411	joko	1.7	$logger->warning( __PACKAGE__ . "[$self->{locator}->{type}]" . "->testDsn(): " . "DBI-error: " . $DBI::errstr );
412	joko	1.2	}
413	cvsjoko	1.1	}
414
415	joko	1.4	1;
416			__END__
417
418
419			=head1 DESCRIPTION
420
421	joko	1.11	=head2 Data::Storage
422
423			Data::Storage is a module for accessing various "data structures / kinds of structured data" stored inside
424			various "data containers".
425			We tried to use the AdapterPattern (http://c2.com/cgi/wiki?AdapterPattern) to implement a wrapper-layer
426			around core CPAN modules (Tangram, DBI).
427
428			=head2 Why?
429
430			You will get a better code-structure (not bad for later maintenance) in growing Perl code projects,
431			especially when using multiple database connections at the same time.
432			You will be able to switch between different _kinds_ of implementations used for storing data.
433			Your code will use the very same API to access these storage layers.
434			... implementation has to be changed for now
435			Maybe you will be able to switch "on-the-fly" without changing any bits in code in the future....
436			... but that's not the focus
437
438			=head2 What else?
439
440			Having this, we were able to do implement a generic data synchronization module more easy,
441			please look at Data::Transfer.
442	joko	1.4
443
444			=head1 AUTHORS / COPYRIGHT
445
446	joko	1.11	The Data::Storage module is Copyright (c) 2002 Andreas Motl.
447			All rights reserved.
448			You may distribute it under the terms of either the GNU General Public
449			License or the Artistic License, as specified in the Perl README file.
450	joko	1.4
451
452			=head1 ACKNOWLEDGEMENTS
453
454	joko	1.11	Larry Wall for Perl, Tim Bunce for DBI, Jean-Louis Leroy for Tangram and Set::Object,
455			Sam Vilain for Class::Tangram, Jochen Wiedmann and Jeff Zucker for DBD::CSV & Co.,
456			Adam Spiers for MySQL::Diff and all contributors.
457	joko	1.4
458
459			=head1 SUPPORT / WARRANTY
460
461	joko	1.11	Data::Storage is free software. IT COMES WITHOUT WARRANTY OF ANY KIND.
462	joko	1.4
463
464			=head1 TODO
465
466
467	joko	1.8	=head2 BUGS
468
469			"DBI-Error [Tangram]: DBD::mysql::st execute failed: Unknown column 't1.requestdump' in 'field list'"
470	joko	1.4
471	joko	1.8	... occours when operating on object-attributes not introduced yet:
472			this should be detected and appended/replaced through:
473			"Schema-Error detected, maybe (just) an inconsistency.
474			Please check if your declaration in schema-module "a" matches structure in database "b" or try to run"
475			db_setup.pl --dbkey=import --action=deploy
476	joko	1.4
477
478	joko	1.8	Compare schema (structure diff) with database ...
479	joko	1.4
480			... when issuing "db_setup.pl --dbkey=import --action=deploy"
481			on a database with an already deployed schema, use an additional "--update" then
482			to lift the schema inside the database to the current declared schema.
483			You will have to approve removals and changes on field-level while
484			new objects and new fields are introduced silently without any interaction needed.
485			In future versions there may be additional options to control silent processing of
486			removals and changes.
487			See this CRUD-table applying to the actions occouring on Classes and Class variables when deploying schemas,
488			don't mix this up with CRUD-actions on Objects, these are already handled by (e.g.) Tangram itself.
489			Classes:
490			C create -> yes, handled automatically
491			R retrieve -> no, not subject of this aspect since it is about deployment only
492			U update -> yes, automatically for Class meta-attributes, yes/no for Class variables (look at the rules down here)
493			D delete -> yes, just by user-interaction
494			Class variables:
495			C create -> yes, handled automatically
496			R retrieve -> no, not subject of this aspect since it is about deployment only
497			U update -> yes, just by user-interaction; maybe automatically if it can be determined that data wouldn't be lost
498			D delete -> yes, just by user-interaction
499	joko	1.5
500			It's all about not to be able to loose data simply while this is in pre-alpha stage.
501			And loosing data by being able to modify and redeploy schemas easily is definitely quite easy.
502
503			As we can see, creations of Classes and new Class variables is handled
504			automatically and this is believed to be the most common case under normal circumstances.
505	joko	1.4
506
507	joko	1.8	=head2 FEATURES
508	joko	1.4
509	joko	1.5	- Get this stuff together with UML (Unified Modeling Language) and/or standards from ODMG.
510			- Make it possible to load/save schemas in XMI (XML Metadata Interchange),
511			which seems to be most commonly used today, perhaps handle objects with OIFML.
512			Integrate/bundle this with a web-/html-based UML modeling tool or
513			some other interesting stuff like the "Co-operative UML Editor" from Uni Darmstadt. (web-/java-based)
514			- Enable Round Trip Engineering. Keep code and diagrams in sync. Don't annoy/bother the programmers.
515	joko	1.8	- Add support for some more handlers/locators to be able to
516			access the following standards/protocols/interfaces/programs/apis transparently:
517			+ DBD::CSV (via Data::Storage::Handler::DBI)
518			(-) Text::CSV, XML::CSV, XML::Excel
519			- MAPI
520			- LDAP
521			- DAV (look at PerlDAV: http://www.webdav.org/perldav/)
522			- Mbox (use formail for seperating/splitting entries/nodes)
523			- Cyrus (cyrdeliver - what about cyrretrieve (export)???)
524			- use File::DiffTree, use File::Compare
525			- Hibernate
526			- "Win32::UserAccountDb"
527			- "*nix::UserAccountDb"
528			- .wab - files (Windows Address Book)
529			- .pst - files (Outlook Post Storage?)
530			- XML (e.g. via XML::Simple?)
531			- Move to t3, look at InCASE
532	joko	1.10	- some kind of security layer for methods/objects
533			- acls (stored via tangram/ldap?) for functions, methods and objects (entity- & data!?)
534			- where are the hooks needed then?
535			- is Data::Storage & Co. okay, or do we have to touch the innards of DBI and/or Tangram?
536			- an attempt to start could be:
537			- 'sub getACLByObjectId($id, $context)'
538			- 'sub getACLByMethodname($id, $context)'
539			- 'sub getACLByName($id, $context)'
540			( would require a kinda registry to look up these very names pointing to arbitrary locations (code, data, ...) )
541	joko	1.13	- add more hooks and various levels
542			- better integrate introduced 'getObjectByGuid'-mechanism from Data::Storage::Handler::Tangram
543	joko	1.4
544
545	joko	1.8	=head3 LINKS / REFERENCES
546	joko	1.4
547	joko	1.5	Specs:
548	joko	1.4	UML 1.3 Spec: http://cgi.omg.org/cgi-bin/doc?ad/99-06-08.pdf
549			XMI 1.1 Spec: http://cgi.omg.org/cgi-bin/doc?ad/99-10-02.pdf
550			XMI 2.0 Spec: http://cgi.omg.org/docs/ad/01-06-12.pdf
551			ODMG: http://odmg.org/
552			OIFML: http://odmg.org/library/readingroom/oifml.pdf
553
554	joko	1.5	CASE Tools:
555			Rational Rose (commercial): http://www.rational.com/products/rose/
556			Together (commercial): http://www.oi.com/products/controlcenter/index.jsp
557			InCASE - Tangram-based Universal Object Editor
558			Sybase PowerDesigner: http://www.sybase.com/powerdesigner
559
560			UML Editors:
561			Fujaba (free, university): http://www.fujaba.de/
562			ArgoUML (free): http://argouml.tigris.org/
563			Poseidon (commercial): http://www.gentleware.com/products/poseidonDE.php3
564			Co-operative UML Editor (research): http://www.darmstadt.gmd.de/concert/activities/internal/umledit.html
565			Metamill (commercial): http://www.metamill.com/
566			Violet (university, research, education): http://www.horstmann.com/violet/
567			PyUt (free): http://pyut.sourceforge.net/
568			(Dia (free): http://www.lysator.liu.se/~alla/dia/)
569			UMLet (free, university): http://www.swt.tuwien.ac.at/umlet/index.html
570			Voodoo (free): http://voodoo.sourceforge.net/
571	joko	1.6	Umbrello UML Modeller: http://uml.sourceforge.net/
572	joko	1.5
573			UML Tools:
574			http://www.objectsbydesign.com/tools/umltools_byPrice.html
575
576			Further readings:
577	joko	1.4	http://www.google.com/search?q=web+based+uml+editor&hl=en&lr=&ie=UTF-8&oe=UTF-8&start=10&sa=N
578			http://www.fernuni-hagen.de/DVT/Aktuelles/01FHHeidelberg.pdf
579			http://www.enhyper.com/src/documentation/
580			http://cis.cs.tu-berlin.de/Dokumente/Diplomarbeiten/2001/skinner.pdf
581			http://citeseer.nj.nec.com/vilain00diagrammatic.html
582			http://archive.devx.com/uml/articles/Smith01/Smith01-3.asp
583
MailToCvsAdmin">MailToCvsAdmin	ViewVC Help
Powered by ViewVC 1.1.26