| 1 |
joko |
1.1 |
## ------------------------------------------------------------------------ |
| 2 |
joko |
1.4 |
## $Id: WebCache.pm,v 1.3 2003/06/25 23:37:04 joko Exp $ |
| 3 |
joko |
1.1 |
## ------------------------------------------------------------------------ |
| 4 |
joko |
1.4 |
## $Log: WebCache.pm,v $ |
| 5 |
|
|
## Revision 1.3 2003/06/25 23:37:04 joko |
| 6 |
|
|
## + sub clearProxyCache |
| 7 |
|
|
## |
| 8 |
joko |
1.3 |
## Revision 1.2 2003/02/20 21:09:59 joko |
| 9 |
|
|
## modified runtime namespace hierarchy |
| 10 |
|
|
## |
| 11 |
joko |
1.2 |
## Revision 1.1 2003/02/11 09:46:09 joko |
| 12 |
|
|
## + initial commit |
| 13 |
|
|
## |
| 14 |
joko |
1.1 |
## ------------------------------------------------------------------------ |
| 15 |
|
|
|
| 16 |
|
|
|
| 17 |
joko |
1.4 |
package OEF::Component::WebCache; |
| 18 |
joko |
1.1 |
|
| 19 |
|
|
use strict; |
| 20 |
|
|
use warnings; |
| 21 |
|
|
|
| 22 |
|
|
use Data::Dumper; |
| 23 |
|
|
use LWP::UserAgent; |
| 24 |
|
|
use HTTP::Headers; |
| 25 |
|
|
|
| 26 |
|
|
use shortcuts qw( now ); |
| 27 |
|
|
|
| 28 |
|
|
# get logger instance |
| 29 |
|
|
my $logger = Log::Dispatch::Config->instance; |
| 30 |
|
|
|
| 31 |
|
|
|
| 32 |
|
|
sub makeInternetRequest { |
| 33 |
|
|
|
| 34 |
|
|
my $self = shift; |
| 35 |
|
|
my $url = shift; |
| 36 |
|
|
|
| 37 |
|
|
my $ua = LWP::UserAgent->new( |
| 38 |
|
|
agent => 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0)', |
| 39 |
|
|
env_proxy => 1, |
| 40 |
|
|
keep_alive => 1, |
| 41 |
|
|
timeout => 30, |
| 42 |
|
|
); |
| 43 |
|
|
|
| 44 |
|
|
my $taskArgs; |
| 45 |
|
|
|
| 46 |
|
|
$logger->debug( __PACKAGE__ . "->makeInternetRequest( url $url ) started" ); |
| 47 |
|
|
$logger->info( "Internet request sent" ); |
| 48 |
|
|
my $response = $ua->get($url); |
| 49 |
|
|
$logger->debug( __PACKAGE__ . "->makeInternetRequest( url $url ) ready" ); |
| 50 |
|
|
$logger->info( "Internet request ready" ); |
| 51 |
|
|
|
| 52 |
|
|
#return if ($response->code() != 200); |
| 53 |
|
|
return if (!$response->is_success); |
| 54 |
|
|
|
| 55 |
|
|
return $response; |
| 56 |
|
|
} |
| 57 |
|
|
|
| 58 |
|
|
|
| 59 |
|
|
sub fetchPageFromProxy { |
| 60 |
|
|
my $self = shift; |
| 61 |
|
|
my $url = shift; |
| 62 |
|
|
|
| 63 |
|
|
# trace |
| 64 |
|
|
#print Dumper($self); |
| 65 |
|
|
#exit; |
| 66 |
|
|
|
| 67 |
joko |
1.2 |
my $proxyObj = $self->{app}->{storage}->{import}->remote('HttpProxy'); |
| 68 |
|
|
my @results = $self->{app}->{storage}->{import}->select($proxyObj, $proxyObj->{url} eq $url); |
| 69 |
joko |
1.1 |
my $content = $results[0]->{content} if $results[0]->{content}; |
| 70 |
|
|
my $oktxt = "no"; |
| 71 |
|
|
if ($content) { |
| 72 |
|
|
$oktxt = "ok"; |
| 73 |
|
|
} |
| 74 |
|
|
$logger->debug( __PACKAGE__ . "->fetchPageFromProxy ...$oktxt" ); |
| 75 |
|
|
return $content; |
| 76 |
|
|
} |
| 77 |
|
|
|
| 78 |
|
|
sub cacheResponse { |
| 79 |
|
|
my $self = shift; |
| 80 |
|
|
my $url = shift; |
| 81 |
|
|
my $response = shift; |
| 82 |
|
|
return unless $response; |
| 83 |
|
|
my $proxyObj = HttpProxy->new( |
| 84 |
|
|
stamp => now(), |
| 85 |
|
|
url => $url, |
| 86 |
|
|
content => $response->content(), |
| 87 |
|
|
request => Dumper($response->request()), |
| 88 |
|
|
headers => Dumper($response->{_headers}), |
| 89 |
|
|
code => $response->code(), |
| 90 |
|
|
status => $response->status_line(), |
| 91 |
|
|
age => $response->current_age(), |
| 92 |
|
|
); |
| 93 |
|
|
my $oktxt = "no"; |
| 94 |
joko |
1.2 |
if ($self->{app}->{storage}->{import}->insert($proxyObj)) { |
| 95 |
joko |
1.1 |
$oktxt = "ok"; |
| 96 |
|
|
} |
| 97 |
|
|
$logger->debug( __PACKAGE__ . "->savePageToProxy ...$oktxt" ); |
| 98 |
|
|
} |
| 99 |
|
|
|
| 100 |
|
|
sub getUrl { |
| 101 |
|
|
my $self = shift; |
| 102 |
|
|
my $url = shift; |
| 103 |
|
|
my $force = shift; |
| 104 |
|
|
|
| 105 |
|
|
#print "force: ", $force, "\n"; |
| 106 |
|
|
#exit; |
| 107 |
|
|
|
| 108 |
|
|
if (!$url) { |
| 109 |
|
|
$logger->error( __PACKAGE__ . "->getUrl: no url given" ); |
| 110 |
|
|
return; |
| 111 |
|
|
} |
| 112 |
|
|
$logger->debug( __PACKAGE__ . "->getUrl( url $url )" ); |
| 113 |
|
|
my $content; |
| 114 |
|
|
if ( !$force && ($content = $self->fetchPageFromProxy($url)) ) { |
| 115 |
|
|
#$self->cachePage($url, $content); |
| 116 |
|
|
#$logger->info( __PACKAGE__ . "->getUrl: Proxy hit!" ); |
| 117 |
|
|
} else { |
| 118 |
|
|
if (my $response = $self->makeInternetRequest($url) ) { |
| 119 |
|
|
$self->cacheResponse($url, $response); |
| 120 |
|
|
$content = $response->as_string(); |
| 121 |
|
|
} else { |
| 122 |
|
|
$logger->error( __PACKAGE__ . "->getUrl( url $url ) failed" ); |
| 123 |
|
|
} |
| 124 |
|
|
} |
| 125 |
|
|
return $content; |
| 126 |
|
|
} |
| 127 |
|
|
|
| 128 |
joko |
1.3 |
sub clearProxyCache { |
| 129 |
|
|
my $self = shift; |
| 130 |
|
|
my $proxyObj = $self->{app}->{storage}->{import}->remote('HttpProxy'); |
| 131 |
|
|
#my @results = $self->{app}->{storage}->{import}->select($proxyObj, $proxyObj->{url} eq $url); |
| 132 |
|
|
my @results = $self->{app}->{storage}->{import}->select($proxyObj); |
| 133 |
|
|
$self->{app}->{storage}->{import}->erase( @results ); |
| 134 |
|
|
} |
| 135 |
|
|
|
| 136 |
joko |
1.1 |
1; |
| 137 |
joko |
1.3 |
__END__ |
Parent Directory
| 
Revision Log
