/[cvs]/nfo/perl/libs/Data/Storage.pm
ViewVC logotype

Contents of /nfo/perl/libs/Data/Storage.pm

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.14 - (show annotations)
Thu Dec 19 16:27:59 2002 UTC (21 years, 6 months ago) by joko
Branch: MAIN
Changes since 1.13: +12 -28 lines
- moved 'sub dropDb' to Data::Storage::Handler::DBI

1 # $Id: Storage.pm,v 1.13 2002/12/17 21:54:12 joko Exp $
2 #
3 # Copyright (c) 2002 Andreas Motl <andreas.motl@ilo.de>
4 #
5 # See COPYRIGHT section in pod text below for usage and distribution rights.
6 #
7 ############################################
8 #
9 # $Log: Storage.pm,v $
10 # Revision 1.13 2002/12/17 21:54:12 joko
11 # + feature when using Tangram:
12 # + what? each object created should delivered with a globally(!?) unique identifier (GUID) besides the native tangram object id (OID)
13 # + patched Tangram::Storage (jonen)
14 # + enhanced Data::Storage::Schema::Tangram (joko)
15 # + enhanced Data::Storage::Handler::Tangram 'sub getObjectByGuid' (jonen)
16 # + how?
17 # + each concrete (non-abstract) class gets injected with an additional field/property called 'guid' - this is done (dynamically) on schema level
18 # + this property ('guid') gets filled on object creation/insertion from 'sub Tangram::Storage::_insert' using Data::UUID from CPAN
19 # + (as for now) this property can get accessed by calling 'getObjectByGuid' on the already known storage-handle used throughout the application
20 #
21 # Revision 1.12 2002/12/12 02:50:15 joko
22 # + this now (unfortunately) needs DBI for some helper functions
23 # + TODO: these have to be refactored to another scope! (soon!)
24 #
25 # Revision 1.11 2002/12/11 06:53:19 joko
26 # + updated pod
27 #
28 # Revision 1.10 2002/12/07 03:37:23 joko
29 # + updated pod
30 #
31 # Revision 1.9 2002/12/01 22:15:45 joko
32 # - sub createDb: moved to handler
33 #
34 # Revision 1.8 2002/11/29 04:48:23 joko
35 # + updated pod
36 #
37 # Revision 1.7 2002/11/17 06:07:18 joko
38 # + creating the handler is easier than proposed first - for now :-)
39 # + sub testAvailability
40 #
41 # Revision 1.6 2002/11/09 01:04:58 joko
42 # + updated pod
43 #
44 # Revision 1.5 2002/10/29 19:24:18 joko
45 # - reduced logging
46 # + added some pod
47 #
48 # Revision 1.4 2002/10/27 18:35:07 joko
49 # + added pod
50 #
51 # Revision 1.3 2002/10/25 11:40:37 joko
52 # + enhanced robustness
53 # + more logging for debug-levels
54 # + sub dropDb
55 #
56 # Revision 1.2 2002/10/17 00:04:29 joko
57 # + sub createDb
58 # + sub isConnected
59 # + bugfixes regarding "deep recursion" stuff
60 #
61 # Revision 1.1 2002/10/10 03:43:12 cvsjoko
62 # + new
63 #
64 ############################################
65
66
67 BEGIN {
68 $Data::Storage::VERSION = 0.02;
69 }
70
71
72 =head1 NAME
73
74 Data::Storage - Interface for accessing various Storage implementations for Perl in an independent way
75
76
77 =head1 AIMS
78
79 - should encapsulate Tangram, DBI, DBD::CSV and LWP:: to access them in an unordinary (more convenient) way ;)
80 - introduce a generic layered structure, refactor *SUBLAYER*-stuff, make (e.g.) this possible:
81 Perl Data::Storage[DBD::CSV] -> Perl LWP:: -> Internet HTTP/FTP/* -> Host Daemon -> csv-file
82 - provide generic synchronization mechanisms across arbitrary/multiple storages based on ident/checksum
83 maybe it's possible to have schema-, structural- and semantical modifications synchronized???
84
85
86 =head1 SYNOPSIS
87
88 =head2 BASIC ACCESS
89
90 =head2 ADVANCED ACCESS
91
92 ... via inheritance:
93
94 use Data::Storage;
95 my $proxyObj = new HttpProxy;
96 $proxyObj->{url} = $url;
97 $proxyObj->{payload} = $content;
98 $self->{storage}->insert($proxyObj);
99
100 use Data::Storage;
101 my $proxyObj = HttpProxy->new(
102 url => $url,
103 payload => $content,
104 );
105 $self->{storage}->insert($proxyObj);
106
107
108 =head2 SYNCHRONIZATION
109
110 my $nodemapping = {
111 'LangText' => 'langtexts.csv',
112 'Currency' => 'currencies.csv',
113 'Country' => 'countries.csv',
114 };
115
116 my $propmapping = {
117 'LangText' => [
118 [ 'source:lcountrykey' => 'target:country' ],
119 [ 'source:lkey' => 'target:key' ],
120 [ 'source:lvalue' => 'target:text' ],
121 ],
122 'Currency' => [
123 [ 'source:ckey' => 'target:key' ],
124 [ 'source:cname' => 'target:text' ],
125 ],
126 'Country' => [
127 [ 'source:ckey' => 'target:key' ],
128 [ 'source:cname' => 'target:text' ],
129 ],
130 };
131
132 sub syncResource {
133
134 my $self = shift;
135 my $node_source = shift;
136 my $mode = shift;
137 my $opts = shift;
138
139 $mode ||= '';
140 $opts->{erase} ||= 0;
141
142 $logger->info( __PACKAGE__ . "->syncResource( node_source $node_source mode $mode erase $opts->{erase} )");
143
144 # resolve metadata for syncing requested resource
145 my $node_target = $nodemapping->{$node_source};
146 my $mapping = $propmapping->{$node_source};
147
148 if (!$node_target || !$mapping) {
149 # loggger.... "no target, sorry!"
150 print "error while resolving resource metadata", "\n";
151 return;
152 }
153
154 if ($opts->{erase}) {
155 $self->_erase_all($node_source);
156 }
157
158 # create new sync object
159 my $sync = Data::Transfer::Sync->new(
160 storages => {
161 L => $self->{bizWorks}->{backend},
162 R => $self->{bizWorks}->{resources},
163 },
164 id_authorities => [qw( L ) ],
165 checksum_authorities => [qw( L ) ],
166 write_protected => [qw( R ) ],
167 verbose => 1,
168 );
169
170 # sync
171 # todo: filter!?
172 $sync->syncNodes( {
173 direction => $mode, # | +PUSH | +PULL | -FULL | +IMPORT | -EXPORT
174 method => 'checksum', # | -timestamp | -manual
175 source => "L:$node_source",
176 source_ident => 'storage_method:id',
177 source_exclude => [qw( id cs )],
178 target => "R:$node_target",
179 target_ident => 'property:oid',
180 mapping => $mapping,
181 } );
182
183 }
184
185
186 =head2 NOTE
187
188 This module heavily relies on DBI and Tangram, but adds a lot of additional bugs and quirks.
189 Please look at their documentation and/or this code for additional information.
190
191
192 =head1 REQUIREMENTS
193
194 For full functionality:
195 DBI from CPAN
196 DBD::mysql from CPAN
197 Tangram 2.04 from CPAN (hmmm, 2.04 won't do in some cases)
198 Tangram 2.05 from http://... (2.05 seems okay but there are also additional patches from our side)
199 Class::Tangram from CPAN
200 DBD::CSV from CPAN
201 MySQL::Diff from http://adamspiers.org/computing/mysqldiff/
202 ... and all their dependencies
203
204 =cut
205
206 # The POD text continues at the end of the file.
207
208
209 package Data::Storage;
210
211 use strict;
212 use warnings;
213
214 use Data::Storage::Locator;
215 use Data::Dumper;
216
217 # TODO: wipe out!
218 use DBI;
219
220 # TODO: actually implement level (integrate with Log::Dispatch)
221 my $TRACELEVEL = 0;
222
223 # get logger instance
224 my $logger = Log::Dispatch::Config->instance;
225
226 sub new {
227 my $invocant = shift;
228 my $class = ref($invocant) || $invocant;
229 #my @args = normalizeArgs(@_);
230
231 my $arg_locator = shift;
232 my $arg_options = shift;
233
234 #my $self = { STORAGEHANDLE => undef, @_ };
235 my $self = { STORAGEHANDLE => undef, locator => $arg_locator, options => $arg_options };
236 #$logger->debug( __PACKAGE__ . "[$self->{locator}->{type}]" . "->new(@_)" );
237 $logger->debug( __PACKAGE__ . "[$arg_locator->{type}]" . "->new(@_)" );
238 return bless $self, $class;
239 }
240
241 sub AUTOLOAD {
242
243 # since this is a core function acting as dispatcher to $self->{STORAGEHANDLE},
244 # some sophisticated handling and filtering is needed to avoid things like
245 # - Deep recursion on subroutine "Data::Storage::AUTOLOAD"
246 # - Deep recursion on subroutine "Data::Storage::Handler::Abstract::AUTOLOAD"
247 # - Deep recursion on anonymous subroutine at [...]
248 # we also might filter log messages caused by logging to itself in "advanced logging of AUTOLOAD calls"
249
250 my $self = shift;
251 our $AUTOLOAD;
252
253 # ->DESTROY would - if not declared - trigger an AUTOLOAD also
254 return if $AUTOLOAD =~ m/::DESTROY$/;
255
256 my $method = $AUTOLOAD;
257 $method =~ s/^.*:://;
258
259 # advanced logging of AUTOLOAD calls ...
260 # ... nice but do it only when TRACING (TODO) is enabled
261 if ($TRACELEVEL) {
262 my $logstring = "";
263 $logstring .= __PACKAGE__ . "[$self->{locator}->{type}]" . "->" . $method;
264 #print "count: ", $#_, "\n";
265 #$logstring .= Dumper(@_) if ($#_ != -1);
266 my $tabcount = int( (80 - length($logstring)) / 10 );
267 $logstring .= "\t" x $tabcount . "(AUTOLOAD)";
268 # TODO: only ok if logstring doesn't contain
269 # e.g. "Data::Storage[Tangram]->insert(SystemEvent=HASH(0x5c0034c)) (AUTOLOAD)"
270 # but that would be _way_ too specific as long as we don't have an abstract handler for this ;)
271 $logger->debug( $logstring );
272 #print join('; ', @_);
273 }
274
275 # filtering AUTOLOAD calls and first-time-touch of the actual storage impl
276 if ($self->_filter_AUTOLOAD($method)) {
277 #print "_accessStorage\n";
278 $self->_accessStorage();
279 $self->{STORAGEHANDLE}->$method(@_);
280 }
281
282 }
283
284 sub _filter_AUTOLOAD {
285 my $self = shift;
286 my $method = shift;
287 if ($self->{options}->{protected}) {
288 if ($method eq 'disconnect') {
289 return;
290 }
291 }
292 return 1;
293 }
294
295
296 sub normalizeArgs {
297 my %args = @_;
298 if (!$args{dsn} && $args{meta}{dsn}) {
299 $args{dsn} = $args{meta}{dsn};
300 }
301 my @result = %args;
302 return @result;
303 }
304
305 sub _accessStorage {
306 my $self = shift;
307 # TODO: to some tracelevel!
308 if ($TRACELEVEL) {
309 $logger->debug( __PACKAGE__ . "[$self->{locator}->{type}]" . "->_accessStorage()" );
310 }
311 if (!$self->{STORAGEHANDLE}) {
312 $self->_createStorageHandle();
313 }
314 }
315
316 sub _createStorageHandle {
317 my $self = shift;
318 my $type = $self->{locator}->{type};
319 $logger->debug( __PACKAGE__ . "[$type]" . "->_createStorageHandle()" );
320
321 my $pkg = "Data::Storage::Handler::" . $type . "";
322
323 # try to load perl module at runtime
324 my $evalstr = "use $pkg;";
325 eval($evalstr);
326 if ($@) {
327 $logger->error( __PACKAGE__ . "[$type]" . "->_createStorageHandle(): $@" );
328 return;
329 }
330
331 # build up some additional arguments to pass on
332 #my @args = %{$self->{locator}};
333 my @args = ();
334
335 # - create new storage handle object
336 # - propagate arguments to handler
337 # - pass locator by reference to be able to store status- or meta-information in it
338 $self->{STORAGEHANDLE} = $pkg->new( locator => $self->{locator}, @args );
339
340 }
341
342 sub addLogDispatchHandler {
343
344 my $self = shift;
345 my $name = shift;
346 my $package = shift;
347 my $logger1 = shift;
348 my $objectCreator = shift;
349
350 #$logger->add( Log::Dispatch::Tangram->new( name => $name,
351 $logger->add( $package->new( name => $name,
352 #min_level => 'debug',
353 min_level => 'info',
354 storage => $self,
355 objectCreator => $objectCreator,
356 fields => {
357 message => 'usermsg',
358 timestamp => 'stamp',
359 level => 'level',
360 name => 'code',
361 },
362 filter_patterns => [ '->insert\(SystemEvent=' ],
363 #filter_patterns => [ 'SystemEvent' ],
364
365 #format => '[%d] [%p] %m%n',
366 ) );
367
368 }
369
370 sub removeLogDispatchHandler {
371 my $self = shift;
372 my $name = shift;
373 #my $logger = shift;
374 $logger->remove($name);
375 }
376
377 sub getDbName {
378 my $self = shift;
379 my $dsn = $self->{locator}->{dbi}->{dsn};
380 $dsn =~ m/database=(.+?);/;
381 my $database_name = $1;
382 return $database_name;
383 }
384
385 sub testAvailability {
386 my $self = shift;
387 my $status = $self->testDsn();
388 $self->{locator}->{status}->{available} = $status;
389 return $status;
390 }
391
392 sub isConnected {
393 my $self = shift;
394 # TODO: REVIEW!
395 return 1 if $self->{STORAGEHANDLE};
396 }
397
398 sub testDsn {
399 my $self = shift;
400 my $dsn = $self->{locator}->{dbi}->{dsn};
401 my $result;
402 if ( my $dbh = DBI->connect($dsn, '', '', {
403 PrintError => 0,
404 } ) ) {
405
406 # TODO: REVIEW
407 $dbh->disconnect();
408
409 return 1;
410 } else {
411 $logger->warning( __PACKAGE__ . "[$self->{locator}->{type}]" . "->testDsn(): " . "DBI-error: " . $DBI::errstr );
412 }
413 }
414
415 1;
416 __END__
417
418
419 =head1 DESCRIPTION
420
421 =head2 Data::Storage
422
423 Data::Storage is a module for accessing various "data structures / kinds of structured data" stored inside
424 various "data containers".
425 We tried to use the AdapterPattern (http://c2.com/cgi/wiki?AdapterPattern) to implement a wrapper-layer
426 around core CPAN modules (Tangram, DBI).
427
428 =head2 Why?
429
430 You will get a better code-structure (not bad for later maintenance) in growing Perl code projects,
431 especially when using multiple database connections at the same time.
432 You will be able to switch between different _kinds_ of implementations used for storing data.
433 Your code will use the very same API to access these storage layers.
434 ... implementation has to be changed for now
435 Maybe you will be able to switch "on-the-fly" without changing any bits in code in the future....
436 ... but that's not the focus
437
438 =head2 What else?
439
440 Having this, we were able to do implement a generic data synchronization module more easy,
441 please look at Data::Transfer.
442
443
444 =head1 AUTHORS / COPYRIGHT
445
446 The Data::Storage module is Copyright (c) 2002 Andreas Motl.
447 All rights reserved.
448 You may distribute it under the terms of either the GNU General Public
449 License or the Artistic License, as specified in the Perl README file.
450
451
452 =head1 ACKNOWLEDGEMENTS
453
454 Larry Wall for Perl, Tim Bunce for DBI, Jean-Louis Leroy for Tangram and Set::Object,
455 Sam Vilain for Class::Tangram, Jochen Wiedmann and Jeff Zucker for DBD::CSV & Co.,
456 Adam Spiers for MySQL::Diff and all contributors.
457
458
459 =head1 SUPPORT / WARRANTY
460
461 Data::Storage is free software. IT COMES WITHOUT WARRANTY OF ANY KIND.
462
463
464 =head1 TODO
465
466
467 =head2 BUGS
468
469 "DBI-Error [Tangram]: DBD::mysql::st execute failed: Unknown column 't1.requestdump' in 'field list'"
470
471 ... occours when operating on object-attributes not introduced yet:
472 this should be detected and appended/replaced through:
473 "Schema-Error detected, maybe (just) an inconsistency.
474 Please check if your declaration in schema-module "a" matches structure in database "b" or try to run"
475 db_setup.pl --dbkey=import --action=deploy
476
477
478 Compare schema (structure diff) with database ...
479
480 ... when issuing "db_setup.pl --dbkey=import --action=deploy"
481 on a database with an already deployed schema, use an additional "--update" then
482 to lift the schema inside the database to the current declared schema.
483 You will have to approve removals and changes on field-level while
484 new objects and new fields are introduced silently without any interaction needed.
485 In future versions there may be additional options to control silent processing of
486 removals and changes.
487 See this CRUD-table applying to the actions occouring on Classes and Class variables when deploying schemas,
488 don't mix this up with CRUD-actions on Objects, these are already handled by (e.g.) Tangram itself.
489 Classes:
490 C create -> yes, handled automatically
491 R retrieve -> no, not subject of this aspect since it is about deployment only
492 U update -> yes, automatically for Class meta-attributes, yes/no for Class variables (look at the rules down here)
493 D delete -> yes, just by user-interaction
494 Class variables:
495 C create -> yes, handled automatically
496 R retrieve -> no, not subject of this aspect since it is about deployment only
497 U update -> yes, just by user-interaction; maybe automatically if it can be determined that data wouldn't be lost
498 D delete -> yes, just by user-interaction
499
500 It's all about not to be able to loose data simply while this is in pre-alpha stage.
501 And loosing data by being able to modify and redeploy schemas easily is definitely quite easy.
502
503 As we can see, creations of Classes and new Class variables is handled
504 automatically and this is believed to be the most common case under normal circumstances.
505
506
507 =head2 FEATURES
508
509 - Get this stuff together with UML (Unified Modeling Language) and/or standards from ODMG.
510 - Make it possible to load/save schemas in XMI (XML Metadata Interchange),
511 which seems to be most commonly used today, perhaps handle objects with OIFML.
512 Integrate/bundle this with a web-/html-based UML modeling tool or
513 some other interesting stuff like the "Co-operative UML Editor" from Uni Darmstadt. (web-/java-based)
514 - Enable Round Trip Engineering. Keep code and diagrams in sync. Don't annoy/bother the programmers.
515 - Add support for some more handlers/locators to be able to
516 access the following standards/protocols/interfaces/programs/apis transparently:
517 + DBD::CSV (via Data::Storage::Handler::DBI)
518 (-) Text::CSV, XML::CSV, XML::Excel
519 - MAPI
520 - LDAP
521 - DAV (look at PerlDAV: http://www.webdav.org/perldav/)
522 - Mbox (use formail for seperating/splitting entries/nodes)
523 - Cyrus (cyrdeliver - what about cyrretrieve (export)???)
524 - use File::DiffTree, use File::Compare
525 - Hibernate
526 - "Win32::UserAccountDb"
527 - "*nix::UserAccountDb"
528 - .wab - files (Windows Address Book)
529 - .pst - files (Outlook Post Storage?)
530 - XML (e.g. via XML::Simple?)
531 - Move to t3, look at InCASE
532 - some kind of security layer for methods/objects
533 - acls (stored via tangram/ldap?) for functions, methods and objects (entity- & data!?)
534 - where are the hooks needed then?
535 - is Data::Storage & Co. okay, or do we have to touch the innards of DBI and/or Tangram?
536 - an attempt to start could be:
537 - 'sub getACLByObjectId($id, $context)'
538 - 'sub getACLByMethodname($id, $context)'
539 - 'sub getACLByName($id, $context)'
540 ( would require a kinda registry to look up these very names pointing to arbitrary locations (code, data, ...) )
541 - add more hooks and various levels
542 - better integrate introduced 'getObjectByGuid'-mechanism from Data::Storage::Handler::Tangram
543
544
545 =head3 LINKS / REFERENCES
546
547 Specs:
548 UML 1.3 Spec: http://cgi.omg.org/cgi-bin/doc?ad/99-06-08.pdf
549 XMI 1.1 Spec: http://cgi.omg.org/cgi-bin/doc?ad/99-10-02.pdf
550 XMI 2.0 Spec: http://cgi.omg.org/docs/ad/01-06-12.pdf
551 ODMG: http://odmg.org/
552 OIFML: http://odmg.org/library/readingroom/oifml.pdf
553
554 CASE Tools:
555 Rational Rose (commercial): http://www.rational.com/products/rose/
556 Together (commercial): http://www.oi.com/products/controlcenter/index.jsp
557 InCASE - Tangram-based Universal Object Editor
558 Sybase PowerDesigner: http://www.sybase.com/powerdesigner
559
560 UML Editors:
561 Fujaba (free, university): http://www.fujaba.de/
562 ArgoUML (free): http://argouml.tigris.org/
563 Poseidon (commercial): http://www.gentleware.com/products/poseidonDE.php3
564 Co-operative UML Editor (research): http://www.darmstadt.gmd.de/concert/activities/internal/umledit.html
565 Metamill (commercial): http://www.metamill.com/
566 Violet (university, research, education): http://www.horstmann.com/violet/
567 PyUt (free): http://pyut.sourceforge.net/
568 (Dia (free): http://www.lysator.liu.se/~alla/dia/)
569 UMLet (free, university): http://www.swt.tuwien.ac.at/umlet/index.html
570 Voodoo (free): http://voodoo.sourceforge.net/
571 Umbrello UML Modeller: http://uml.sourceforge.net/
572
573 UML Tools:
574 http://www.objectsbydesign.com/tools/umltools_byPrice.html
575
576 Further readings:
577 http://www.google.com/search?q=web+based+uml+editor&hl=en&lr=&ie=UTF-8&oe=UTF-8&start=10&sa=N
578 http://www.fernuni-hagen.de/DVT/Aktuelles/01FHHeidelberg.pdf
579 http://www.enhyper.com/src/documentation/
580 http://cis.cs.tu-berlin.de/Dokumente/Diplomarbeiten/2001/skinner.pdf
581 http://citeseer.nj.nec.com/vilain00diagrammatic.html
582 http://archive.devx.com/uml/articles/Smith01/Smith01-3.asp
583

MailToCvsAdmin">MailToCvsAdmin
ViewVC Help
Powered by ViewVC 1.1.26 RSS 2.0 feed