/[cvs]/nfo/perl/libs/Data/Transfer/Sync/Compare/Checksum.pm

Diff of /nfo/perl/libs/Data/Transfer/Sync/Compare/Checksum.pm

Parent Directory | Revision Log | View Patch Patch

-revision 1.2 by joko,
Tue Feb 11 09:53:07 2003 UTC
+revision 1.6 by joko,
Sat Jun 19 01:48:03 2004 UTC
 Line 6
  ##
  ##    ----------------------------------------------------------------------------------------
  ##    $Log$
+ ##    Revision 1.6  2004/06/19 01:48:03  joko
+ ##    introduced infrastructure for "local checksum"-mechanism
+ ##    modified _calcChecksum
+ ##    added sub _dumpCompact
+ ##
+ ##    Revision 1.5  2004/05/11 19:43:02  joko
+ ##    don't use File::RsyncP::Digest on win32
+ ##
+ ##    Revision 1.4  2004/05/06 12:53:07  jonen
+ ##    + added use of File::RsycP::digest
+ ##
+ ##    Revision 1.3  2003/05/13 08:19:00  joko
+ ##    switched to crc32
+ ##
  ##    Revision 1.2  2003/02/11 09:53:07  joko
  ##    + metadata-structure-change, fixed some code here
  ##
-Line 15
+Line 29
  ##    ----------------------------------------------------------------------------------------
+ =pod
+ =head1 Todo
+   o Data::Transfer::Sync::Compare::Slot using Compare::Struct
+   o Load checksum algorithm on demand, do some negotiation
+ =cut
  package Data::Transfer::Sync::Compare::Checksum;
  use strict;
-Line 22 
 use warnings;
+Line 45 
 use warnings;
  use mixin::with qw( Data::Transfer::Sync );
+ use shortcuts qw( RUNNING_IN_HELL );
- # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -   main
  use Data::Dumper;
- use Digest::MD5 qw(md5 md5_hex md5_base64);
+ # TODO: Load these appropriatly at runtime.
+ use Digest::MD5 qw( md5 md5_hex md5_base64 );
+ use String::CRC32;
+ use Data::Mungle::Transform::Deep qw( deep_copy );
+ # don't use File::RsyncP::Digest on win32
+ # TODO: enhance here! (e.g. negotiate proper checksum-algorithm first, apply afterwards)
+ if (not RUNNING_IN_HELL()) {
+   eval "use File::RsyncP::Digest;";
+ }
  # get logger instance
  my $logger = Log::Dispatch::Config->instance;
+ # Maybe refactor to shortcuts::checksum?
+ # calculate checksum for current object
  sub _calcChecksum {
    my $self = shift;
    my $descent = shift;
-   my $specifier = shift;
+   my $options = shift;
-   # calculate checksum for current object
+   my $checksum;
      my $ident = $self->{node}->{$descent}->{ident};
    # build dump of this node
      my $payload = $self->{node}->{$descent}->{payload};
+     $payload = deep_copy($payload) if $options->{clone};
+     # new 2004-06-17: remove attributes if request before dumping/calculating
+     map { delete $payload->{$_} } @{$options->{exclude}} if $options->{exclude};
+     # various methods to dump this thing
      #my $dump = $ident . "\n" . $item->quickdump();
      #my $dump = $ident . "\n" . Dumper($item);
-     my $dump = $ident . "\n" . $self->_dumpCompact($payload);
+     my $dump = $self->_dumpCompact($payload);
+     $dump = $ident . $dump if $self->{options}->{ident};
    # TODO: $logger->dump( ... );
      #$logger->debug( __PACKAGE__ . ": " . $dump );
      #$logger->dump( __PACKAGE__ . ": " . $dump );
    # calculate checksum from dump
-     # note: the 32-bit integer hash from DBI seems
-     # to generate duplicates with small payloads already in ranges of hundreds of items/rows!!!
+     # 1. md5-based fingerprint, base64 encoded (from Digest::MD5)
-     # try to avoid to use it or try to use it only for payloads greater than, hmmm, let's say 30 chars?
+       #$checksum = md5_base64($dump) . '==';
-     # (we had about 15 chars average per item (row))
+     # 2. 32-bit integer "hash" value (maybe faster?) (from DBI)
-     # md5-based fingerprint, base64 encoded (from Digest::MD5)
+       # Note: The 32-bit integer hash from DBI seems to generate duplicates
-       $self->{node}->{$descent}->{checksum} = md5_base64($dump) . '==';
+       # with small payloads already in ranges of hundreds of items/rows!!!
-     # 32-bit integer "hash" value (maybe faster?) (from DBI)
+       # Try to avoid it or try to use it only for payloads greater than, hmmm, let's say 30 chars?
-       #$self->{node}->{$descent}->{checksum} = DBI::hash($dump, 1);
+       # (we had about 15 chars average per item (row))
+       # Possible (generic) solution: Just generate checksum, if length(checksum(payload)) < length(payload)
+       #$checksum = DBI::hash($dump, 1);
+     # 3. good old crc32???
+       #$checksum = crc32($dump);
+     # 4. File::RsyncP::Digest - Perl interface to rsync message digest algorithms
+     if (not RUNNING_IN_HELL()) {
+       my $rsDigest = new File::RsyncP::Digest;
+       $rsDigest->add($dump);
+       my $digest = $rsDigest->digest();
+       $checksum = unpack("H*", $digest);
+     } else {    # fallback to Digest::MD5 on win32
+       #print $dump, "  ==>  ";
+       $checksum = md5_base64($dump) . '==';
+       #print $checksum, "\n";
+     }
+     # 5. some more modern Digest::SHA1 or similar?
    # signal good
-     return 1;
+     return $checksum;
  }
- sub _readChecksum {
+ sub readChecksum {
    my $self = shift;
    my $descent = shift;
-Line 88 
 sub _readChecksum {
+Line 148 
 sub _readChecksum {
    # get checksum for current entry
      # TODO: don't have the checksum column/property hardcoded as "cs" here, make this configurable somehow
      if ($self->{meta}->{$descent}->{isChecksumAuthority}) {
-       $self->_calcChecksum($descent);
+       $self->{node}->{$descent}->{checksum} = $self->_calcChecksum($descent, { ident => 1 });
      } else {
        $self->{node}->{$descent}->{checksum} = $self->{node}->{$descent}->{payload}->{cs};
      }
-Line 98 
 sub _readChecksum {
+Line 158 
 sub _readChecksum {
  }
+ sub _readLocalChecksum {
+   my $self = shift;
+   my $descent = shift;
+   $self->{node}->{$descent}->{checksum_local_storage} = $self->{node}->{$descent}->{payload}->{cs_local};
+   $self->{node}->{$descent}->{checksum_local_storage} ||= "";
+ }
+ sub _calcLocalChecksum {
+   my $self = shift;
+   my $descent = shift;
+   # TODO: don't hardcode this (oid, cs, cs_local)
+   $self->{node}->{$descent}->{checksum_local_calculated} =
+     $self->_calcChecksum($descent, { clone => 1, exclude => [qw( oid cs cs_local )] });
+   $self->{node}->{$descent}->{checksum_local_calculated} ||= "";
+ }
+ sub handleLocalChecksum {
+   my $self = shift;
+   my $descent = shift;
+   $self->_readLocalChecksum($descent);
+   $self->_calcLocalChecksum($descent);
+ }
+ sub _dumpCompact {
+   my $self = shift;
+   my @data = ();
+   my $count = 0;
+   # walk through all arguments
+   foreach (@_) {
+     my $item = {};
+     # assume item is a hashref and walk through all entries
+     foreach my $key (keys %$_) {
+       my $val = $_->{$key};
+       # dive into tied/hidden types and resolve deep structure inside
+       if (ref $val eq 'Set::Object') {
+         $item->{$key} = $val->members();
+       } else {
+         $item->{$key} = $val;
+       }
+     }
+     push @data, $item;
+     $count++;
+   }
+   my $indent_save = $Data::Dumper::Indent;
+   $Data::Dumper::Indent = 0;
+   my $result = Dumper(@data);
+   $Data::Dumper::Indent = $indent_save;
+   return $result;
+ }
 ;
+ __END__

 Legend:



Removed from v.1.2
 


changed lines


 
Added in v.1.6
 Legend:



Removed from v.1.2
 


changed lines


 
Added in v.1.6
-Removed from v.1.2
+Added in v.1.6

MailToCvsAdmin">MailToCvsAdmin	ViewVC Help
Powered by ViewVC 1.1.26