/[cvs]/nfo/perl/libs/Data/Storage.pm
ViewVC logotype

Contents of /nfo/perl/libs/Data/Storage.pm

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.9 - (show annotations)
Sun Dec 1 22:15:45 2002 UTC (21 years, 7 months ago) by joko
Branch: MAIN
Changes since 1.8: +9 -26 lines
- sub createDb: moved to handler

1 # $Id: Storage.pm,v 1.8 2002/11/29 04:48:23 joko Exp $
2 #
3 # Copyright (c) 2002 Andreas Motl <andreas.motl@ilo.de>
4 #
5 # See COPYRIGHT section in pod text below for usage and distribution rights.
6 #
7 ############################################
8 #
9 # $Log: Storage.pm,v $
10 # Revision 1.8 2002/11/29 04:48:23 joko
11 # + updated pod
12 #
13 # Revision 1.7 2002/11/17 06:07:18 joko
14 # + creating the handler is easier than proposed first - for now :-)
15 # + sub testAvailability
16 #
17 # Revision 1.6 2002/11/09 01:04:58 joko
18 # + updated pod
19 #
20 # Revision 1.5 2002/10/29 19:24:18 joko
21 # - reduced logging
22 # + added some pod
23 #
24 # Revision 1.4 2002/10/27 18:35:07 joko
25 # + added pod
26 #
27 # Revision 1.3 2002/10/25 11:40:37 joko
28 # + enhanced robustness
29 # + more logging for debug-levels
30 # + sub dropDb
31 #
32 # Revision 1.2 2002/10/17 00:04:29 joko
33 # + sub createDb
34 # + sub isConnected
35 # + bugfixes regarding "deep recursion" stuff
36 #
37 # Revision 1.1 2002/10/10 03:43:12 cvsjoko
38 # + new
39 #
40 ############################################
41
42
43 BEGIN {
44 $Data::Storage::VERSION = 0.02;
45 }
46
47
48 =head1 NAME
49
50 Data::Storage - Interface for accessing various Storage implementations for Perl in an independent way
51
52
53 =head1 AIMS
54
55 - should encapsulate Tangram, DBI, DBD::CSV and LWP:: to access them in an unordinary (more convenient) way ;)
56 - introduce a generic layered structure, refactor *SUBLAYER*-stuff, make (e.g.) this possible:
57 Perl Data::Storage[DBD::CSV] -> Perl LWP:: -> Internet HTTP/FTP/* -> Host Daemon -> csv-file
58 - provide generic synchronization mechanisms across arbitrary/multiple storages based on ident/checksum
59 maybe it's possible to have schema-, structural- and semantical modifications synchronized???
60
61
62 =head1 SYNOPSIS
63
64 =head2 BASIC ACCESS
65
66 =head2 ADVANCED ACCESS
67
68 ... via inheritance:
69
70 use Data::Storage;
71 my $proxyObj = new HttpProxy;
72 $proxyObj->{url} = $url;
73 $proxyObj->{payload} = $content;
74 $self->{storage}->insert($proxyObj);
75
76 use Data::Storage;
77 my $proxyObj = HttpProxy->new(
78 url => $url,
79 payload => $content,
80 );
81 $self->{storage}->insert($proxyObj);
82
83
84 =head2 SYNCHRONIZATION
85
86 my $nodemapping = {
87 'LangText' => 'langtexts.csv',
88 'Currency' => 'currencies.csv',
89 'Country' => 'countries.csv',
90 };
91
92 my $propmapping = {
93 'LangText' => [
94 [ 'source:lcountrykey' => 'target:country' ],
95 [ 'source:lkey' => 'target:key' ],
96 [ 'source:lvalue' => 'target:text' ],
97 ],
98 'Currency' => [
99 [ 'source:ckey' => 'target:key' ],
100 [ 'source:cname' => 'target:text' ],
101 ],
102 'Country' => [
103 [ 'source:ckey' => 'target:key' ],
104 [ 'source:cname' => 'target:text' ],
105 ],
106 };
107
108 sub syncResource {
109
110 my $self = shift;
111 my $node_source = shift;
112 my $mode = shift;
113 my $opts = shift;
114
115 $mode ||= '';
116 $opts->{erase} ||= 0;
117
118 $logger->info( __PACKAGE__ . "->syncResource( node_source $node_source mode $mode erase $opts->{erase} )");
119
120 # resolve metadata for syncing requested resource
121 my $node_target = $nodemapping->{$node_source};
122 my $mapping = $propmapping->{$node_source};
123
124 if (!$node_target || !$mapping) {
125 # loggger.... "no target, sorry!"
126 print "error while resolving resource metadata", "\n";
127 return;
128 }
129
130 if ($opts->{erase}) {
131 $self->_erase_all($node_source);
132 }
133
134 # create new sync object
135 my $sync = Data::Transfer::Sync->new(
136 storages => {
137 L => $self->{bizWorks}->{backend},
138 R => $self->{bizWorks}->{resources},
139 },
140 id_authorities => [qw( L ) ],
141 checksum_authorities => [qw( L ) ],
142 write_protected => [qw( R ) ],
143 verbose => 1,
144 );
145
146 # sync
147 # todo: filter!?
148 $sync->syncNodes( {
149 direction => $mode, # | +PUSH | +PULL | -FULL | +IMPORT | -EXPORT
150 method => 'checksum', # | -timestamp | -manual
151 source => "L:$node_source",
152 source_ident => 'storage_method:id',
153 source_exclude => [qw( id cs )],
154 target => "R:$node_target",
155 target_ident => 'property:oid',
156 mapping => $mapping,
157 } );
158
159 }
160
161
162 =head2 NOTE
163
164 This module heavily relies on DBI and Tangram, but adds a lot of additional bugs and quirks.
165 Please look at their documentation and/or this code for additional information.
166
167
168 =head1 REQUIREMENTS
169
170 For full functionality:
171 DBI from CPAN
172 DBD::mysql from CPAN
173 Tangram 2.04 from CPAN (hmmm, 2.04 won't do in some cases)
174 Tangram 2.05 from http://... (2.05 seems okay but there are also additional patches from our side)
175 Class::Tangram from CPAN
176 DBD::CSV from CPAN
177 MySQL::Diff from http://adamspiers.org/computing/mysqldiff/
178 ... and all their dependencies
179
180 =cut
181
182 # The POD text continues at the end of the file.
183
184
185 package Data::Storage;
186
187 use strict;
188 use warnings;
189
190 use Data::Storage::Locator;
191 use Data::Dumper;
192
193 # TODO: actually implement level (integrate with Log::Dispatch)
194 my $TRACELEVEL = 0;
195
196 # get logger instance
197 my $logger = Log::Dispatch::Config->instance;
198
199 sub new {
200 my $invocant = shift;
201 my $class = ref($invocant) || $invocant;
202 #my @args = normalizeArgs(@_);
203
204 my $arg_locator = shift;
205 my $arg_options = shift;
206
207 #my $self = { STORAGEHANDLE => undef, @_ };
208 my $self = { STORAGEHANDLE => undef, locator => $arg_locator, options => $arg_options };
209 #$logger->debug( __PACKAGE__ . "[$self->{locator}->{type}]" . "->new(@_)" );
210 $logger->debug( __PACKAGE__ . "[$arg_locator->{type}]" . "->new(@_)" );
211 return bless $self, $class;
212 }
213
214 sub AUTOLOAD {
215
216 # since this is a core function acting as dispatcher to $self->{STORAGEHANDLE},
217 # some sophisticated handling and filtering is needed to avoid things like
218 # - Deep recursion on subroutine "Data::Storage::AUTOLOAD"
219 # - Deep recursion on subroutine "Data::Storage::Handler::Abstract::AUTOLOAD"
220 # - Deep recursion on anonymous subroutine at [...]
221 # we also might filter log messages caused by logging to itself in "advanced logging of AUTOLOAD calls"
222
223 my $self = shift;
224 our $AUTOLOAD;
225
226 # ->DESTROY would - if not declared - trigger an AUTOLOAD also
227 return if $AUTOLOAD =~ m/::DESTROY$/;
228
229 my $method = $AUTOLOAD;
230 $method =~ s/^.*:://;
231
232 # advanced logging of AUTOLOAD calls ...
233 # ... nice but do it only when TRACING (TODO) is enabled
234 if ($TRACELEVEL) {
235 my $logstring = "";
236 $logstring .= __PACKAGE__ . "[$self->{locator}->{type}]" . "->" . $method;
237 #print "count: ", $#_, "\n";
238 #$logstring .= Dumper(@_) if ($#_ != -1);
239 my $tabcount = int( (80 - length($logstring)) / 10 );
240 $logstring .= "\t" x $tabcount . "(AUTOLOAD)";
241 # TODO: only ok if logstring doesn't contain
242 # e.g. "Data::Storage[Tangram]->insert(SystemEvent=HASH(0x5c0034c)) (AUTOLOAD)"
243 # but that would be _way_ too specific as long as we don't have an abstract handler for this ;)
244 $logger->debug( $logstring );
245 #print join('; ', @_);
246 }
247
248 # filtering AUTOLOAD calls and first-time-touch of the actual storage impl
249 if ($self->_filter_AUTOLOAD($method)) {
250 #print "_accessStorage\n";
251 $self->_accessStorage();
252 $self->{STORAGEHANDLE}->$method(@_);
253 }
254
255 }
256
257 sub _filter_AUTOLOAD {
258 my $self = shift;
259 my $method = shift;
260 if ($self->{options}->{protected}) {
261 if ($method eq 'disconnect') {
262 return;
263 }
264 }
265 return 1;
266 }
267
268
269 sub normalizeArgs {
270 my %args = @_;
271 if (!$args{dsn} && $args{meta}{dsn}) {
272 $args{dsn} = $args{meta}{dsn};
273 }
274 my @result = %args;
275 return @result;
276 }
277
278 sub _accessStorage {
279 my $self = shift;
280 # TODO: to some tracelevel!
281 if ($TRACELEVEL) {
282 $logger->debug( __PACKAGE__ . "[$self->{locator}->{type}]" . "->_accessStorage()" );
283 }
284 if (!$self->{STORAGEHANDLE}) {
285 $self->_createStorageHandle();
286 }
287 }
288
289 sub _createStorageHandle {
290 my $self = shift;
291 my $type = $self->{locator}->{type};
292 $logger->debug( __PACKAGE__ . "[$type]" . "->_createStorageHandle()" );
293
294 my $pkg = "Data::Storage::Handler::" . $type . "";
295
296 # try to load perl module at runtime
297 my $evalstr = "use $pkg;";
298 eval($evalstr);
299 if ($@) {
300 $logger->error( __PACKAGE__ . "[$type]" . "->_createStorageHandle(): $@" );
301 return;
302 }
303
304 # build up some additional arguments to pass on
305 #my @args = %{$self->{locator}};
306 my @args = ();
307
308 # - create new storage handle object
309 # - propagate arguments to handler
310 # - pass locator by reference to be able to store status- or meta-information in it
311 $self->{STORAGEHANDLE} = $pkg->new( locator => $self->{locator}, @args );
312
313 }
314
315 sub addLogDispatchHandler {
316
317 my $self = shift;
318 my $name = shift;
319 my $package = shift;
320 my $logger1 = shift;
321 my $objectCreator = shift;
322
323 #$logger->add( Log::Dispatch::Tangram->new( name => $name,
324 $logger->add( $package->new( name => $name,
325 #min_level => 'debug',
326 min_level => 'info',
327 storage => $self,
328 objectCreator => $objectCreator,
329 fields => {
330 message => 'usermsg',
331 timestamp => 'stamp',
332 level => 'level',
333 name => 'code',
334 },
335 filter_patterns => [ '->insert\(SystemEvent=' ],
336 #filter_patterns => [ 'SystemEvent' ],
337
338 #format => '[%d] [%p] %m%n',
339 ) );
340
341 }
342
343 sub removeLogDispatchHandler {
344 my $self = shift;
345 my $name = shift;
346 #my $logger = shift;
347 $logger->remove($name);
348 }
349
350 sub getDbName {
351 my $self = shift;
352 my $dsn = $self->{locator}->{dbi}->{dsn};
353 $dsn =~ m/database=(.+?);/;
354 my $database_name = $1;
355 return $database_name;
356 }
357
358 sub testDsn {
359 my $self = shift;
360 my $dsn = $self->{locator}->{dbi}->{dsn};
361 my $result;
362 if ( my $dbh = DBI->connect($dsn, '', '', {
363 PrintError => 0,
364 } ) ) {
365
366 # TODO: REVIEW
367 $dbh->disconnect();
368
369 return 1;
370 } else {
371 $logger->warning( __PACKAGE__ . "[$self->{locator}->{type}]" . "->testDsn(): " . "DBI-error: " . $DBI::errstr );
372 }
373 }
374
375 sub testAvailability {
376 my $self = shift;
377 my $status = $self->testDsn();
378 $self->{locator}->{status}->{available} = $status;
379 return $status;
380 }
381
382
383 sub dropDb {
384 my $self = shift;
385 my $dsn = $self->{locator}->{dbi}->{dsn};
386
387 $logger->debug( __PACKAGE__ . "->dropDb( dsn $dsn )" );
388
389 $dsn =~ s/database=(.+?);//;
390 my $database_name = $1;
391
392 my $ok;
393
394 if ( my $dbh = DBI->connect($dsn, '', '', {
395 PrintError => 0,
396 } ) ) {
397 if ($database_name) {
398 if ($dbh->do("DROP DATABASE $database_name;")) {
399 $ok = 1;
400 }
401 }
402
403 $dbh->disconnect();
404
405 }
406
407 return $ok;
408 }
409
410 sub isConnected {
411 my $self = shift;
412 return 1 if $self->{STORAGEHANDLE};
413 }
414
415 1;
416 __END__
417
418
419 =head1 DESCRIPTION
420
421 Data::Storage is a module for accessing various "data structures" stored inside
422 various "data containers". It sits on top of DBI and/or Tangram.
423
424
425 =head1 AUTHORS / COPYRIGHT
426
427 The Data::Storage module is Copyright (c) 2002 Andreas Motl.
428 All rights reserved.
429
430 You may distribute it under the terms of either the GNU General Public
431 License or the Artistic License, as specified in the Perl README file.
432
433
434 =head1 ACKNOWLEDGEMENTS
435
436 Larry Wall for Perl, Tim Bunce for DBI, Jean-Louis Leroy for Tangram and Set::Object,
437 Sam Vilain for Class::Tangram, Jochen Wiedmann and Jeff Zucker for DBD::CSV and related,
438 Adam Spiers for MySQL::Diff and all contributors.
439
440
441 =head1 SUPPORT / WARRANTY
442
443 Data::Storage is free software. IT COMES WITHOUT WARRANTY OF ANY KIND.
444
445
446 =head1 TODO
447
448
449 =head2 BUGS
450
451 "DBI-Error [Tangram]: DBD::mysql::st execute failed: Unknown column 't1.requestdump' in 'field list'"
452
453 ... occours when operating on object-attributes not introduced yet:
454 this should be detected and appended/replaced through:
455 "Schema-Error detected, maybe (just) an inconsistency.
456 Please check if your declaration in schema-module "a" matches structure in database "b" or try to run"
457 db_setup.pl --dbkey=import --action=deploy
458
459
460 Compare schema (structure diff) with database ...
461
462 ... when issuing "db_setup.pl --dbkey=import --action=deploy"
463 on a database with an already deployed schema, use an additional "--update" then
464 to lift the schema inside the database to the current declared schema.
465 You will have to approve removals and changes on field-level while
466 new objects and new fields are introduced silently without any interaction needed.
467 In future versions there may be additional options to control silent processing of
468 removals and changes.
469 See this CRUD-table applying to the actions occouring on Classes and Class variables when deploying schemas,
470 don't mix this up with CRUD-actions on Objects, these are already handled by (e.g.) Tangram itself.
471 Classes:
472 C create -> yes, handled automatically
473 R retrieve -> no, not subject of this aspect since it is about deployment only
474 U update -> yes, automatically for Class meta-attributes, yes/no for Class variables (look at the rules down here)
475 D delete -> yes, just by user-interaction
476 Class variables:
477 C create -> yes, handled automatically
478 R retrieve -> no, not subject of this aspect since it is about deployment only
479 U update -> yes, just by user-interaction; maybe automatically if it can be determined that data wouldn't be lost
480 D delete -> yes, just by user-interaction
481
482 It's all about not to be able to loose data simply while this is in pre-alpha stage.
483 And loosing data by being able to modify and redeploy schemas easily is definitely quite easy.
484
485 As we can see, creations of Classes and new Class variables is handled
486 automatically and this is believed to be the most common case under normal circumstances.
487
488
489 =head2 FEATURES
490
491 - Get this stuff together with UML (Unified Modeling Language) and/or standards from ODMG.
492 - Make it possible to load/save schemas in XMI (XML Metadata Interchange),
493 which seems to be most commonly used today, perhaps handle objects with OIFML.
494 Integrate/bundle this with a web-/html-based UML modeling tool or
495 some other interesting stuff like the "Co-operative UML Editor" from Uni Darmstadt. (web-/java-based)
496 - Enable Round Trip Engineering. Keep code and diagrams in sync. Don't annoy/bother the programmers.
497 - Add support for some more handlers/locators to be able to
498 access the following standards/protocols/interfaces/programs/apis transparently:
499 + DBD::CSV (via Data::Storage::Handler::DBI)
500 (-) Text::CSV, XML::CSV, XML::Excel
501 - MAPI
502 - LDAP
503 - DAV (look at PerlDAV: http://www.webdav.org/perldav/)
504 - Mbox (use formail for seperating/splitting entries/nodes)
505 - Cyrus (cyrdeliver - what about cyrretrieve (export)???)
506 - use File::DiffTree, use File::Compare
507 - Hibernate
508 - "Win32::UserAccountDb"
509 - "*nix::UserAccountDb"
510 - .wab - files (Windows Address Book)
511 - .pst - files (Outlook Post Storage?)
512 - XML (e.g. via XML::Simple?)
513 - Move to t3, look at InCASE
514
515
516 =head3 LINKS / REFERENCES
517
518 Specs:
519 UML 1.3 Spec: http://cgi.omg.org/cgi-bin/doc?ad/99-06-08.pdf
520 XMI 1.1 Spec: http://cgi.omg.org/cgi-bin/doc?ad/99-10-02.pdf
521 XMI 2.0 Spec: http://cgi.omg.org/docs/ad/01-06-12.pdf
522 ODMG: http://odmg.org/
523 OIFML: http://odmg.org/library/readingroom/oifml.pdf
524
525 CASE Tools:
526 Rational Rose (commercial): http://www.rational.com/products/rose/
527 Together (commercial): http://www.oi.com/products/controlcenter/index.jsp
528 InCASE - Tangram-based Universal Object Editor
529 Sybase PowerDesigner: http://www.sybase.com/powerdesigner
530
531 UML Editors:
532 Fujaba (free, university): http://www.fujaba.de/
533 ArgoUML (free): http://argouml.tigris.org/
534 Poseidon (commercial): http://www.gentleware.com/products/poseidonDE.php3
535 Co-operative UML Editor (research): http://www.darmstadt.gmd.de/concert/activities/internal/umledit.html
536 Metamill (commercial): http://www.metamill.com/
537 Violet (university, research, education): http://www.horstmann.com/violet/
538 PyUt (free): http://pyut.sourceforge.net/
539 (Dia (free): http://www.lysator.liu.se/~alla/dia/)
540 UMLet (free, university): http://www.swt.tuwien.ac.at/umlet/index.html
541 Voodoo (free): http://voodoo.sourceforge.net/
542 Umbrello UML Modeller: http://uml.sourceforge.net/
543
544 UML Tools:
545 http://www.objectsbydesign.com/tools/umltools_byPrice.html
546
547 Further readings:
548 http://www.google.com/search?q=web+based+uml+editor&hl=en&lr=&ie=UTF-8&oe=UTF-8&start=10&sa=N
549 http://www.fernuni-hagen.de/DVT/Aktuelles/01FHHeidelberg.pdf
550 http://www.enhyper.com/src/documentation/
551 http://cis.cs.tu-berlin.de/Dokumente/Diplomarbeiten/2001/skinner.pdf
552 http://citeseer.nj.nec.com/vilain00diagrammatic.html
553 http://archive.devx.com/uml/articles/Smith01/Smith01-3.asp
554

MailToCvsAdmin">MailToCvsAdmin
ViewVC Help
Powered by ViewVC 1.1.26 RSS 2.0 feed