/[cvs]/nfo/perl/libs/Data/Storage.pod
ViewVC logotype

Annotation of /nfo/perl/libs/Data/Storage.pod

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.5 - (hide annotations)
Wed Nov 10 08:21:41 2004 UTC (19 years, 7 months ago) by joko
Branch: MAIN
CVS Tags: HEAD
Changes since 1.4: +7 -3 lines
U obfuscated email addresses

1 joko 1.1 =pod
2    
3    
4     =head1 NAME
5    
6     Data::Storage - Interface for accessing various Storage implementations for Perl in an independent way
7    
8    
9 joko 1.4 =head1 ABSTRACT ;-)
10    
11     Data Storage
12    
13     "Where is the wisdom? Lost in the knowledge.
14     Where is the knowledge? Lost in the information."
15     - T.S. Eliot
16    
17     "Where is the information? Lost in the data.
18     Where is the data? Lost in the #@$%?!& database."
19     - Joe Celko
20    
21    
22     from: MacPerl: Power and Ease - Chapter 15
23     url: http://www.macperl.com/ptf_book/r/MP/330.Data_Storage.html
24    
25    
26     =head1 INTENTIONS
27 joko 1.1
28     - should encapsulate Tangram, DBI, DBD::CSV and LWP:: to access them in an unordinary (more convenient) way ;)
29     - introduce a generic layered structure, refactor *SUBLAYER*-stuff, make (e.g.) this possible:
30     Perl Data::Storage[DBD::CSV] -> Perl LWP:: -> Internet HTTP/FTP/* -> Host Daemon -> csv-file
31     - provide generic synchronization mechanisms across arbitrary/multiple storages based on ident/checksum
32     maybe it's possible to have schema-, structural- and semantical modifications synchronized???
33 joko 1.4 - might be similar to http://sourceforge.net/projects/perl-repository
34    
35 joko 1.1
36     =head1 SYNOPSIS
37    
38 joko 1.2 =head2 ACCESS
39 joko 1.1
40 joko 1.2 # connect to LDAP
41     my $ldapLocator = Data::Storage::Locator->new(
42     ldap => {
43     type => "NetLDAP",
44     dsn => "ldap:host=192.168.10.150;binddn='cn=root, o=netfrag.org, c=de';pass=secret",
45     basedn => "o=netfrag.org, c=de",
46     want_transactions => 0,
47     syncable => 1,
48     },
49     );
50     my $ldapStorage = Data::Storage->new($ldapLocator);
51     $ldapStorage->connect();
52    
53     # connect to MAPI
54     my $mapiLocator = Data::Storage::Locator->new(
55     outlook => {
56     type => "MAPI",
57     showProfileChooser => $self->{config}->get("mapi_showProfileChooser"),
58     ProfileName => $self->{config}->get("mapi_ProfileName"),
59     ProfilePass => $self->{config}->get("mapi_ProfilePass"),
60     syncable => 1,
61     },
62     );
63     my $mapiStorage = Data::Storage->new($mapiLocator);
64     $mapiStorage->connect();
65 joko 1.1
66    
67 joko 1.2 =head2 SYNCHRONIZATION
68    
69     This functionality is (in the meanwhile) provided by the Data::Transfer::Sync module.
70 joko 1.1
71 joko 1.2 =head3 proposal V1
72 joko 1.1
73     my $nodemapping = {
74     'LangText' => 'langtexts.csv',
75     'Currency' => 'currencies.csv',
76     'Country' => 'countries.csv',
77     };
78    
79     my $propmapping = {
80     'LangText' => [
81     [ 'source:lcountrykey' => 'target:country' ],
82     [ 'source:lkey' => 'target:key' ],
83     [ 'source:lvalue' => 'target:text' ],
84     ],
85     'Currency' => [
86     [ 'source:ckey' => 'target:key' ],
87     [ 'source:cname' => 'target:text' ],
88     ],
89     'Country' => [
90     [ 'source:ckey' => 'target:key' ],
91     [ 'source:cname' => 'target:text' ],
92     ],
93     };
94    
95     s ub syncResource {
96    
97     my $self = shift;
98     my $node_source = shift;
99     my $mode = shift;
100     my $opts = shift;
101    
102     $mode ||= '';
103     $opts->{erase} ||= 0;
104    
105     $logger->info( __PACKAGE__ . "->syncResource( node_source $node_source mode $mode erase $opts->{erase} )");
106    
107     # resolve metadata for syncing requested resource
108     my $node_target = $nodemapping->{$node_source};
109     my $mapping = $propmapping->{$node_source};
110    
111     if (!$node_target || !$mapping) {
112     # loggger.... "no target, sorry!"
113     print "error while resolving resource metadata", "\n";
114     return;
115     }
116    
117     if ($opts->{erase}) {
118     $self->_erase_all($node_source);
119     }
120    
121     # create new sync object
122     my $sync = Data::Transfer::Sync->new(
123     storages => {
124 joko 1.2 L => $self->{storage}->{backend},
125     R => $self->{storage}->{resources},
126 joko 1.1 },
127     id_authorities => [qw( L ) ],
128     checksum_authorities => [qw( L ) ],
129     write_protected => [qw( R ) ],
130     verbose => 1,
131     );
132    
133     # sync
134     # todo: filter!?
135     $sync->syncNodes( {
136     direction => $mode, # | +PUSH | +PULL | -FULL | +IMPORT | -EXPORT
137     method => 'checksum', # | -timestamp | -manual
138     source => "L:$node_source",
139     source_ident => 'storage_method:id',
140     source_exclude => [qw( id cs )],
141     target => "R:$node_target",
142     target_ident => 'property:oid',
143     mapping => $mapping,
144     } );
145    
146     }
147 joko 1.2
148     =head3 proposal V2
149    
150     # create a new synchronization object
151     my $sync = Data::Transfer::Sync->new( 'sync_version' => $sync_version, __parent => $self );
152    
153     # configure the synchronization-object
154     $sync->configure(
155     source => {
156     storage => {
157     handle => $mapiStorage,
158     #isIdentAuthority => 1,
159     #isChecksumAuthority => 1,
160     #writeProtected => 1,
161     },
162     },
163     target => {
164     storage => {
165     handle => $ldapStorage,
166     #idAuthority => 1,
167     #isChecksumAuthority => 1,
168     #isWriteProtected => 0,
169     },
170     },
171     verbose => 1,
172     );
173 joko 1.1
174    
175     =head2 NOTE
176    
177     This module heavily relies on DBI and Tangram, but adds a lot of additional bugs and quirks.
178     Please look at their documentation and/or this code for additional information.
179    
180    
181     =head1 REQUIREMENTS
182    
183     For full functionality:
184     DBI from CPAN
185     DBD::mysql from CPAN
186     Tangram 2.04 from CPAN (hmmm, 2.04 won't do in some cases)
187     Tangram 2.05 from http://... (2.05 seems okay but there are also additional patches from our side)
188     Class::Tangram from CPAN
189     DBD::CSV from CPAN
190     MySQL::Diff from http://adamspiers.org/computing/mysqldiff/
191     ... and all their dependencies
192    
193    
194     =head1 DESCRIPTION
195    
196     =head2 Data::Storage
197    
198     Data::Storage is a module for accessing various "data structures / kinds of structured data" stored inside
199     various "data containers".
200     We tried to use the AdapterPattern to implement a wrapper-layer around known CPAN modules.
201     (e.g. DBI, Tangram, XML::Simple)
202     References:
203     - http://c2.com/cgi/wiki?AdapterPattern
204     - http://home.earthlink.net/~huston2/dp/adapter.html
205    
206     =head2 Why?
207    
208     You will get a better code-structure (not bad for later maintenance) in growing Perl code projects,
209     especially when using multiple database connections at the same time.
210     You will be able to switch between different _kinds_ of implementations used for storing data.
211     Your code will use the very same API to access these storage layers.
212     ... implementation has to be changed for now
213     Maybe you will be able to switch "on-the-fly" without changing any bits in code in the future....
214     ... but that's not the focus
215    
216     =head2 What else?
217    
218     Having this, we were able to do implement a generic data synchronization module more easy,
219     please look at Data::Transfer.
220    
221    
222     =head1 AUTHORS / COPYRIGHT
223    
224 joko 1.5 The Data::Storage module is Copyright (c) 2002-2004 Andreas Motl.
225 joko 1.1 All rights reserved.
226 joko 1.5 Mail to <andreas.motl (at) ilo.de> for improvements and suggestions.
227    
228 joko 1.1 You may distribute it under the terms of either the GNU General Public
229     License or the Artistic License, as specified in the Perl README file.
230    
231 joko 1.5 =head1 CONTRIBUTORS
232    
233     Sebastian Utz <su (at) rotamente.com>
234 joko 1.1
235     =head1 ACKNOWLEDGEMENTS
236    
237     Larry Wall for Perl, Tim Bunce for DBI, Jean-Louis Leroy for Tangram and Set::Object,
238     Sam Vilain for Class::Tangram, Jochen Wiedmann and Jeff Zucker for DBD::CSV & Co.,
239     Adam Spiers for MySQL::Diff and all contributors.
240 joko 1.5
241 joko 1.1 =head1 SUPPORT / WARRANTY
242    
243     Data::Storage is free software. IT COMES WITHOUT WARRANTY OF ANY KIND.
244    
245    
246     =head1 TODO
247    
248 joko 1.3 o interface with Jeff Zucker's AnyData:: modules, e.g. AnyData::Storage::RAM
249     o what about DBD::RAM? (DBD::RAM - a DBI driver for files and data structures)
250     o use DBD::Proxy!
251     o what about DBIx::AnyDBD?
252     o enhance schema information:
253     - DBIx::SystemCatalog
254     - DBIx::SystemCatalog::MSSQL?
255     - Data::Reporter
256    
257 joko 1.1
258     =head2 BUGS
259    
260     "DBI-Error [Tangram]: DBD::mysql::st execute failed: Unknown column 't1.requestdump' in 'field list'"
261    
262     ... occours when operating on object-attributes not introduced yet:
263     this should be detected and appended/replaced through:
264     "Schema-Error detected, maybe (just) an inconsistency.
265     Please check if your declaration in schema-module "a" matches structure in database "b" or try to run"
266     db_setup.pl --dbkey=import --action=deploy
267    
268    
269     Compare schema (structure diff) with database ...
270    
271     ... when issuing "db_setup.pl --dbkey=import --action=deploy"
272     on a database with an already deployed schema, use an additional "--update" then
273     to lift the schema inside the database to the current declared schema.
274     You will have to approve removals and changes on field-level while
275     new objects and new fields are introduced silently without any interaction needed.
276     In future versions there may be additional options to control silent processing of
277     removals and changes.
278     See this CRUD-table applying to the actions occouring on Classes and Class variables when deploying schemas,
279     don't mix this up with CRUD-actions on Objects, these are already handled by (e.g.) Tangram itself.
280     Classes:
281     C create -> yes, handled automatically
282     R retrieve -> no, not subject of this aspect since it is about deployment only
283     U update -> yes, automatically for Class meta-attributes, yes/no for Class variables (look at the rules down here)
284     D delete -> yes, just by user-interaction
285     Class variables:
286     C create -> yes, handled automatically
287     R retrieve -> no, not subject of this aspect since it is about deployment only
288     U update -> yes, just by user-interaction; maybe automatically if it can be determined that data wouldn't be lost
289     D delete -> yes, just by user-interaction
290    
291     It's all about not to be able to loose data simply while this is in pre-alpha stage.
292     And loosing data by being able to modify and redeploy schemas easily is definitely quite easy.
293    
294     As we can see, creations of Classes and new Class variables is handled
295     automatically and this is believed to be the most common case under normal circumstances.
296    
297    
298     =head2 FEATURES
299    
300     - Get this stuff together with UML (Unified Modeling Language) and/or standards from ODMG.
301     - Make it possible to load/save schemas in XMI (XML Metadata Interchange),
302     which seems to be most commonly used today, perhaps handle objects with OIFML.
303     Integrate/bundle this with a web-/html-based UML modeling tool or
304     some other interesting stuff like the "Co-operative UML Editor" from Uni Darmstadt. (web-/java-based)
305     - Enable Round Trip Engineering. Keep code and diagrams in sync. Don't annoy/bother the programmers.
306     - Add support for some more handlers/locators to be able to
307     access the following standards/protocols/interfaces/programs/apis transparently:
308     + DBD::CSV (via Data::Storage::Handler::DBI)
309     (-) Text::CSV, XML::CSV, XML::Excel
310     - MAPI
311     - LDAP
312     - DAV (look at PerlDAV: http://www.webdav.org/perldav/)
313     - Mbox (use formail for seperating/splitting entries/nodes)
314     - Cyrus (cyrdeliver - what about cyrretrieve (export)???)
315     - use File::DiffTree, use File::Compare
316     - Hibernate
317     - "Win32::UserAccountDb"
318     - "*nix::UserAccountDb"
319     - .wab - files (Windows Address Book)
320     - .pst - files (Outlook Post Storage?)
321     - XML (e.g. via XML::Simple?)
322     - Move to t3, look at InCASE
323     - some kind of security layer for methods/objects
324     - acls (stored via tangram/ldap?) for functions, methods and objects (entity- & data!?)
325     - where are the hooks needed then?
326     - is Data::Storage & Co. okay, or do we have to touch the innards of DBI and/or Tangram?
327     - an attempt to start could be:
328     - 'sub getACLByObjectId($id, $context)'
329     - 'sub getACLByMethodname($id, $context)'
330     - 'sub getACLByName($id, $context)'
331     ( would require a kinda registry to look up these very names pointing to arbitrary locations (code, data, ...) )
332     - add more hooks and various levels
333     - better integrate introduced 'getObjectByGuid'-mechanism from Data::Storage::Handler::Tangram
334    
335    
336     =head3 LINKS / REFERENCES
337    
338     Specs:
339     UML 1.3 Spec: http://cgi.omg.org/cgi-bin/doc?ad/99-06-08.pdf
340     XMI 1.1 Spec: http://cgi.omg.org/cgi-bin/doc?ad/99-10-02.pdf
341     XMI 2.0 Spec: http://cgi.omg.org/docs/ad/01-06-12.pdf
342     ODMG: http://odmg.org/
343     OIFML: http://odmg.org/library/readingroom/oifml.pdf
344    
345     CASE Tools:
346     Rational Rose (commercial): http://www.rational.com/products/rose/
347     Together (commercial): http://www.oi.com/products/controlcenter/index.jsp
348     InCASE - Tangram-based Universal Object Editor
349     Sybase PowerDesigner: http://www.sybase.com/powerdesigner
350    
351     UML Editors:
352     Fujaba (free, university): http://www.fujaba.de/
353     ArgoUML (free): http://argouml.tigris.org/
354     Poseidon (commercial): http://www.gentleware.com/products/poseidonDE.php3
355     Co-operative UML Editor (research): http://www.darmstadt.gmd.de/concert/activities/internal/umledit.html
356     Metamill (commercial): http://www.metamill.com/
357     Violet (university, research, education): http://www.horstmann.com/violet/
358     PyUt (free): http://pyut.sourceforge.net/
359     (Dia (free): http://www.lysator.liu.se/~alla/dia/)
360     UMLet (free, university): http://www.swt.tuwien.ac.at/umlet/index.html
361     Voodoo (free): http://voodoo.sourceforge.net/
362     Umbrello UML Modeller: http://uml.sourceforge.net/
363    
364     UML Tools:
365     http://www.objectsbydesign.com/tools/umltools_byPrice.html
366    
367     Further readings:
368     http://www.google.com/search?q=web+based+uml+editor&hl=en&lr=&ie=UTF-8&oe=UTF-8&start=10&sa=N
369     http://www.fernuni-hagen.de/DVT/Aktuelles/01FHHeidelberg.pdf
370     http://www.enhyper.com/src/documentation/
371     http://cis.cs.tu-berlin.de/Dokumente/Diplomarbeiten/2001/skinner.pdf
372     http://citeseer.nj.nec.com/vilain00diagrammatic.html
373     http://archive.devx.com/uml/articles/Smith01/Smith01-3.asp
374    
375    
376     =cut
377    

MailToCvsAdmin">MailToCvsAdmin
ViewVC Help
Powered by ViewVC 1.1.26 RSS 2.0 feed