/[cvs]/nfo/perl/libs/Data/Storage.pod
ViewVC logotype

Annotation of /nfo/perl/libs/Data/Storage.pod

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.4 - (hide annotations)
Tue May 13 07:27:56 2003 UTC (21 years, 6 months ago) by joko
Branch: MAIN
Changes since 1.3: +20 -2 lines
updated

1 joko 1.1 =pod
2    
3    
4     =head1 NAME
5    
6     Data::Storage - Interface for accessing various Storage implementations for Perl in an independent way
7    
8    
9 joko 1.4 =head1 ABSTRACT ;-)
10    
11     Data Storage
12    
13     "Where is the wisdom? Lost in the knowledge.
14     Where is the knowledge? Lost in the information."
15     - T.S. Eliot
16    
17     "Where is the information? Lost in the data.
18     Where is the data? Lost in the #@$%?!& database."
19     - Joe Celko
20    
21    
22     from: MacPerl: Power and Ease - Chapter 15
23     url: http://www.macperl.com/ptf_book/r/MP/330.Data_Storage.html
24    
25    
26     =head1 INTENTIONS
27 joko 1.1
28     - should encapsulate Tangram, DBI, DBD::CSV and LWP:: to access them in an unordinary (more convenient) way ;)
29     - introduce a generic layered structure, refactor *SUBLAYER*-stuff, make (e.g.) this possible:
30     Perl Data::Storage[DBD::CSV] -> Perl LWP:: -> Internet HTTP/FTP/* -> Host Daemon -> csv-file
31     - provide generic synchronization mechanisms across arbitrary/multiple storages based on ident/checksum
32     maybe it's possible to have schema-, structural- and semantical modifications synchronized???
33 joko 1.4 - might be similar to http://sourceforge.net/projects/perl-repository
34    
35 joko 1.1
36     =head1 SYNOPSIS
37    
38 joko 1.2 =head2 ACCESS
39 joko 1.1
40 joko 1.2 # connect to LDAP
41     my $ldapLocator = Data::Storage::Locator->new(
42     ldap => {
43     type => "NetLDAP",
44     dsn => "ldap:host=192.168.10.150;binddn='cn=root, o=netfrag.org, c=de';pass=secret",
45     basedn => "o=netfrag.org, c=de",
46     want_transactions => 0,
47     syncable => 1,
48     },
49     );
50     my $ldapStorage = Data::Storage->new($ldapLocator);
51     $ldapStorage->connect();
52    
53     # connect to MAPI
54     my $mapiLocator = Data::Storage::Locator->new(
55     outlook => {
56     type => "MAPI",
57     showProfileChooser => $self->{config}->get("mapi_showProfileChooser"),
58     ProfileName => $self->{config}->get("mapi_ProfileName"),
59     ProfilePass => $self->{config}->get("mapi_ProfilePass"),
60     syncable => 1,
61     },
62     );
63     my $mapiStorage = Data::Storage->new($mapiLocator);
64     $mapiStorage->connect();
65 joko 1.1
66    
67 joko 1.2 =head2 SYNCHRONIZATION
68    
69     This functionality is (in the meanwhile) provided by the Data::Transfer::Sync module.
70 joko 1.1
71 joko 1.2 =head3 proposal V1
72 joko 1.1
73     my $nodemapping = {
74     'LangText' => 'langtexts.csv',
75     'Currency' => 'currencies.csv',
76     'Country' => 'countries.csv',
77     };
78    
79     my $propmapping = {
80     'LangText' => [
81     [ 'source:lcountrykey' => 'target:country' ],
82     [ 'source:lkey' => 'target:key' ],
83     [ 'source:lvalue' => 'target:text' ],
84     ],
85     'Currency' => [
86     [ 'source:ckey' => 'target:key' ],
87     [ 'source:cname' => 'target:text' ],
88     ],
89     'Country' => [
90     [ 'source:ckey' => 'target:key' ],
91     [ 'source:cname' => 'target:text' ],
92     ],
93     };
94    
95     s ub syncResource {
96    
97     my $self = shift;
98     my $node_source = shift;
99     my $mode = shift;
100     my $opts = shift;
101    
102     $mode ||= '';
103     $opts->{erase} ||= 0;
104    
105     $logger->info( __PACKAGE__ . "->syncResource( node_source $node_source mode $mode erase $opts->{erase} )");
106    
107     # resolve metadata for syncing requested resource
108     my $node_target = $nodemapping->{$node_source};
109     my $mapping = $propmapping->{$node_source};
110    
111     if (!$node_target || !$mapping) {
112     # loggger.... "no target, sorry!"
113     print "error while resolving resource metadata", "\n";
114     return;
115     }
116    
117     if ($opts->{erase}) {
118     $self->_erase_all($node_source);
119     }
120    
121     # create new sync object
122     my $sync = Data::Transfer::Sync->new(
123     storages => {
124 joko 1.2 L => $self->{storage}->{backend},
125     R => $self->{storage}->{resources},
126 joko 1.1 },
127     id_authorities => [qw( L ) ],
128     checksum_authorities => [qw( L ) ],
129     write_protected => [qw( R ) ],
130     verbose => 1,
131     );
132    
133     # sync
134     # todo: filter!?
135     $sync->syncNodes( {
136     direction => $mode, # | +PUSH | +PULL | -FULL | +IMPORT | -EXPORT
137     method => 'checksum', # | -timestamp | -manual
138     source => "L:$node_source",
139     source_ident => 'storage_method:id',
140     source_exclude => [qw( id cs )],
141     target => "R:$node_target",
142     target_ident => 'property:oid',
143     mapping => $mapping,
144     } );
145    
146     }
147 joko 1.2
148     =head3 proposal V2
149    
150     # create a new synchronization object
151     my $sync = Data::Transfer::Sync->new( 'sync_version' => $sync_version, __parent => $self );
152    
153     # configure the synchronization-object
154     $sync->configure(
155     source => {
156     storage => {
157     handle => $mapiStorage,
158     #isIdentAuthority => 1,
159     #isChecksumAuthority => 1,
160     #writeProtected => 1,
161     },
162     },
163     target => {
164     storage => {
165     handle => $ldapStorage,
166     #idAuthority => 1,
167     #isChecksumAuthority => 1,
168     #isWriteProtected => 0,
169     },
170     },
171     verbose => 1,
172     );
173 joko 1.1
174    
175     =head2 NOTE
176    
177     This module heavily relies on DBI and Tangram, but adds a lot of additional bugs and quirks.
178     Please look at their documentation and/or this code for additional information.
179    
180    
181     =head1 REQUIREMENTS
182    
183     For full functionality:
184     DBI from CPAN
185     DBD::mysql from CPAN
186     Tangram 2.04 from CPAN (hmmm, 2.04 won't do in some cases)
187     Tangram 2.05 from http://... (2.05 seems okay but there are also additional patches from our side)
188     Class::Tangram from CPAN
189     DBD::CSV from CPAN
190     MySQL::Diff from http://adamspiers.org/computing/mysqldiff/
191     ... and all their dependencies
192    
193    
194     =head1 DESCRIPTION
195    
196     =head2 Data::Storage
197    
198     Data::Storage is a module for accessing various "data structures / kinds of structured data" stored inside
199     various "data containers".
200     We tried to use the AdapterPattern to implement a wrapper-layer around known CPAN modules.
201     (e.g. DBI, Tangram, XML::Simple)
202     References:
203     - http://c2.com/cgi/wiki?AdapterPattern
204     - http://home.earthlink.net/~huston2/dp/adapter.html
205    
206     =head2 Why?
207    
208     You will get a better code-structure (not bad for later maintenance) in growing Perl code projects,
209     especially when using multiple database connections at the same time.
210     You will be able to switch between different _kinds_ of implementations used for storing data.
211     Your code will use the very same API to access these storage layers.
212     ... implementation has to be changed for now
213     Maybe you will be able to switch "on-the-fly" without changing any bits in code in the future....
214     ... but that's not the focus
215    
216     =head2 What else?
217    
218     Having this, we were able to do implement a generic data synchronization module more easy,
219     please look at Data::Transfer.
220    
221    
222     =head1 AUTHORS / COPYRIGHT
223    
224     The Data::Storage module is Copyright (c) 2002 Andreas Motl.
225     All rights reserved.
226     You may distribute it under the terms of either the GNU General Public
227     License or the Artistic License, as specified in the Perl README file.
228    
229    
230     =head1 ACKNOWLEDGEMENTS
231    
232     Larry Wall for Perl, Tim Bunce for DBI, Jean-Louis Leroy for Tangram and Set::Object,
233     Sam Vilain for Class::Tangram, Jochen Wiedmann and Jeff Zucker for DBD::CSV & Co.,
234     Adam Spiers for MySQL::Diff and all contributors.
235    
236    
237     =head1 SUPPORT / WARRANTY
238    
239     Data::Storage is free software. IT COMES WITHOUT WARRANTY OF ANY KIND.
240    
241    
242     =head1 TODO
243    
244 joko 1.3 o interface with Jeff Zucker's AnyData:: modules, e.g. AnyData::Storage::RAM
245     o what about DBD::RAM? (DBD::RAM - a DBI driver for files and data structures)
246     o use DBD::Proxy!
247     o what about DBIx::AnyDBD?
248     o enhance schema information:
249     - DBIx::SystemCatalog
250     - DBIx::SystemCatalog::MSSQL?
251     - Data::Reporter
252    
253 joko 1.1
254     =head2 BUGS
255    
256     "DBI-Error [Tangram]: DBD::mysql::st execute failed: Unknown column 't1.requestdump' in 'field list'"
257    
258     ... occours when operating on object-attributes not introduced yet:
259     this should be detected and appended/replaced through:
260     "Schema-Error detected, maybe (just) an inconsistency.
261     Please check if your declaration in schema-module "a" matches structure in database "b" or try to run"
262     db_setup.pl --dbkey=import --action=deploy
263    
264    
265     Compare schema (structure diff) with database ...
266    
267     ... when issuing "db_setup.pl --dbkey=import --action=deploy"
268     on a database with an already deployed schema, use an additional "--update" then
269     to lift the schema inside the database to the current declared schema.
270     You will have to approve removals and changes on field-level while
271     new objects and new fields are introduced silently without any interaction needed.
272     In future versions there may be additional options to control silent processing of
273     removals and changes.
274     See this CRUD-table applying to the actions occouring on Classes and Class variables when deploying schemas,
275     don't mix this up with CRUD-actions on Objects, these are already handled by (e.g.) Tangram itself.
276     Classes:
277     C create -> yes, handled automatically
278     R retrieve -> no, not subject of this aspect since it is about deployment only
279     U update -> yes, automatically for Class meta-attributes, yes/no for Class variables (look at the rules down here)
280     D delete -> yes, just by user-interaction
281     Class variables:
282     C create -> yes, handled automatically
283     R retrieve -> no, not subject of this aspect since it is about deployment only
284     U update -> yes, just by user-interaction; maybe automatically if it can be determined that data wouldn't be lost
285     D delete -> yes, just by user-interaction
286    
287     It's all about not to be able to loose data simply while this is in pre-alpha stage.
288     And loosing data by being able to modify and redeploy schemas easily is definitely quite easy.
289    
290     As we can see, creations of Classes and new Class variables is handled
291     automatically and this is believed to be the most common case under normal circumstances.
292    
293    
294     =head2 FEATURES
295    
296     - Get this stuff together with UML (Unified Modeling Language) and/or standards from ODMG.
297     - Make it possible to load/save schemas in XMI (XML Metadata Interchange),
298     which seems to be most commonly used today, perhaps handle objects with OIFML.
299     Integrate/bundle this with a web-/html-based UML modeling tool or
300     some other interesting stuff like the "Co-operative UML Editor" from Uni Darmstadt. (web-/java-based)
301     - Enable Round Trip Engineering. Keep code and diagrams in sync. Don't annoy/bother the programmers.
302     - Add support for some more handlers/locators to be able to
303     access the following standards/protocols/interfaces/programs/apis transparently:
304     + DBD::CSV (via Data::Storage::Handler::DBI)
305     (-) Text::CSV, XML::CSV, XML::Excel
306     - MAPI
307     - LDAP
308     - DAV (look at PerlDAV: http://www.webdav.org/perldav/)
309     - Mbox (use formail for seperating/splitting entries/nodes)
310     - Cyrus (cyrdeliver - what about cyrretrieve (export)???)
311     - use File::DiffTree, use File::Compare
312     - Hibernate
313     - "Win32::UserAccountDb"
314     - "*nix::UserAccountDb"
315     - .wab - files (Windows Address Book)
316     - .pst - files (Outlook Post Storage?)
317     - XML (e.g. via XML::Simple?)
318     - Move to t3, look at InCASE
319     - some kind of security layer for methods/objects
320     - acls (stored via tangram/ldap?) for functions, methods and objects (entity- & data!?)
321     - where are the hooks needed then?
322     - is Data::Storage & Co. okay, or do we have to touch the innards of DBI and/or Tangram?
323     - an attempt to start could be:
324     - 'sub getACLByObjectId($id, $context)'
325     - 'sub getACLByMethodname($id, $context)'
326     - 'sub getACLByName($id, $context)'
327     ( would require a kinda registry to look up these very names pointing to arbitrary locations (code, data, ...) )
328     - add more hooks and various levels
329     - better integrate introduced 'getObjectByGuid'-mechanism from Data::Storage::Handler::Tangram
330    
331    
332     =head3 LINKS / REFERENCES
333    
334     Specs:
335     UML 1.3 Spec: http://cgi.omg.org/cgi-bin/doc?ad/99-06-08.pdf
336     XMI 1.1 Spec: http://cgi.omg.org/cgi-bin/doc?ad/99-10-02.pdf
337     XMI 2.0 Spec: http://cgi.omg.org/docs/ad/01-06-12.pdf
338     ODMG: http://odmg.org/
339     OIFML: http://odmg.org/library/readingroom/oifml.pdf
340    
341     CASE Tools:
342     Rational Rose (commercial): http://www.rational.com/products/rose/
343     Together (commercial): http://www.oi.com/products/controlcenter/index.jsp
344     InCASE - Tangram-based Universal Object Editor
345     Sybase PowerDesigner: http://www.sybase.com/powerdesigner
346    
347     UML Editors:
348     Fujaba (free, university): http://www.fujaba.de/
349     ArgoUML (free): http://argouml.tigris.org/
350     Poseidon (commercial): http://www.gentleware.com/products/poseidonDE.php3
351     Co-operative UML Editor (research): http://www.darmstadt.gmd.de/concert/activities/internal/umledit.html
352     Metamill (commercial): http://www.metamill.com/
353     Violet (university, research, education): http://www.horstmann.com/violet/
354     PyUt (free): http://pyut.sourceforge.net/
355     (Dia (free): http://www.lysator.liu.se/~alla/dia/)
356     UMLet (free, university): http://www.swt.tuwien.ac.at/umlet/index.html
357     Voodoo (free): http://voodoo.sourceforge.net/
358     Umbrello UML Modeller: http://uml.sourceforge.net/
359    
360     UML Tools:
361     http://www.objectsbydesign.com/tools/umltools_byPrice.html
362    
363     Further readings:
364     http://www.google.com/search?q=web+based+uml+editor&hl=en&lr=&ie=UTF-8&oe=UTF-8&start=10&sa=N
365     http://www.fernuni-hagen.de/DVT/Aktuelles/01FHHeidelberg.pdf
366     http://www.enhyper.com/src/documentation/
367     http://cis.cs.tu-berlin.de/Dokumente/Diplomarbeiten/2001/skinner.pdf
368     http://citeseer.nj.nec.com/vilain00diagrammatic.html
369     http://archive.devx.com/uml/articles/Smith01/Smith01-3.asp
370    
371    
372     =cut
373    

MailToCvsAdmin">MailToCvsAdmin
ViewVC Help
Powered by ViewVC 1.1.26 RSS 2.0 feed