重寫代碼AnyEvent

Question

我寫了代碼：

---

use LWP::UserAgent; 
use HTTP::Cookies; 
use threads; 
use threads::shared; 
$| = 1; 

$threads = 50; 
my @urls : shared = loadf('url.txt'); 

my @thread_list =(); 
$thread_list[$_] = threads->create(\&thread) for 0 .. $threads - 1; 
$_->join for @thread_list; 
thread(); 

sub thread 
{ 
    my ($web, $ck) = browser(); 
    while(1) 
    { 
     my $url = shift @urls; 
     if(!$url) 
     { 
      last; 
     } 
     $code = $web->get($url)->code; 
     print "[+] $url - code: $code\n"; 
     if($code == 200) 
     { 
      open F, ">>200.txt"; 
      print F $url."\n"; 
      close F; 
     } 
     elsif($code == 301) 
     { 
      open F, ">>301.txt"; 
      print F $url."\n"; 
      close F; 
     } 
     else 
     { 
      open F, ">>else.txt"; 
      print F "$url code - $code\n"; 
      close F; 
     } 
    } 
} 


sub loadf { 
    open (F, "<".$_[0]) or erroropen($_[0]); 
    chomp(my @data = <F>); 
    close F; 
    return @data; 
} 

sub browser 
{ 
my $web = new LWP::UserAgent; 
my $ck = new HTTP::Cookies; 
    $web->cookie_jar($ck); 
    $web->agent('Opera/9.80 (Windows 7; U; en) Presto/2.9.168 Version/11.50'); 
    $web->timeout(5); 
    return $web, $ck; 
} 

---

其工作一段時間後的物理存儲已滿。 你能幫我用AnyEvent重寫它嗎？我試過了，但我的代碼沒有工作。我讀到它會幫助我安全地記憶。 非常感謝任何幫手。

Answer 1

線程是邪惡的;考慮使用異步AnyEvent::Net::Curl::Queued：

#!/usr/bin/env perl 
use common::sense; 

use File::Slurp; 
use YADA; 

my @url = read_file('url.txt'); 
YADA->new(50)->append(
    \@url => { 
     retry => 0, 
     timeout => 5, 
     opts => { 
      useragent => 'Opera/9.80 (Windows 7; U; en) Presto/2.9.168 Version/11.50', 
      #verbose  => 1, 
     }, 
    } => sub { 
     my $self = shift; 

     my $code = $self->getinfo('response_code'); 
     my $url = $self->final_url; 

     say "[+] $url - code: $code"; 
     given ($code) { 
      when (200) { 
       open my $fh, '>>', '200.txt'; 
       say {$fh} $url; 
       close $fh; 
      } when (301) { 
       open my $fh, '>>', '301.txt'; 
       say {$fh} $url; 
       close $fh; 
      } default { 
       open my $fh, '>>', 'else.txt'; 
       say {$fh} "$url code - $code"; 
       close $fh; 
      } 
     } 
    }, 
)->wait;