it* hacker news on gopher	Err	codevoid.de	70
i	Err	codevoid.de	70
hgit clone git://git.codevoid.de/hn-gopher	URL:git://git.codevoid.de/hn-gopher	codevoid.de	70
1Log	/git/hn-gopher/log.gph	codevoid.de	70
1Files	/git/hn-gopher/files.gph	codevoid.de	70
1Refs	/git/hn-gopher/refs.gph	codevoid.de	70
i---	Err	codevoid.de	70
1commit 74ea59bf91cf72e4d5e82ebd8bc852b7546c1a0a	/git/hn-gopher/commit/74ea59bf91cf72e4d5e82ebd8bc852b7546c1a0a.gph	codevoid.de	70
1parent 6533286bc1276e7584436c98fa4f88251da82bf9	/git/hn-gopher/commit/6533286bc1276e7584436c98fa4f88251da82bf9.gph	codevoid.de	70
hAuthor: Stefan Hagen <sh+git[at]codevoid[dot]de>	URL:mailto:sh+git[at]codevoid[dot]de	codevoid.de	70
iDate:   Mon, 30 Jul 2018 21:43:49 +0200	Err	codevoid.de	70
i	Err	codevoid.de	70
ibig update	Err	codevoid.de	70
i	Err	codevoid.de	70
i- pretty bars	Err	codevoid.de	70
i- pretty dates	Err	codevoid.de	70
i- parallel scraping	Err	codevoid.de	70
i- proper front page top stories	Err	codevoid.de	70
i- configurable front page story count	Err	codevoid.de	70
i- link parser update (still a bit wonky)	Err	codevoid.de	70
i- story file cache	Err	codevoid.de	70
i	Err	codevoid.de	70
iDiffstat:	Err	codevoid.de	70
i  M hn-scraper.pl                       |     252 ++++++++++++++++++++++++++-----	Err	codevoid.de	70
i	Err	codevoid.de	70
i1 file changed, 215 insertions(+), 37 deletions(-)	Err	codevoid.de	70
i---	Err	codevoid.de	70
1diff --git a/hn-scraper.pl b/hn-scraper.pl	/git/hn-gopher/file/hn-scraper.pl.gph	codevoid.de	70
it@@ -2,6 +2,10 @@	Err	codevoid.de	70
i 	Err	codevoid.de	70
i use strict;	Err	codevoid.de	70
i use warnings;	Err	codevoid.de	70
i+use Parallel::ForkManager;	Err	codevoid.de	70
i+use DateTime;	Err	codevoid.de	70
i+use DateTime::Duration;	Err	codevoid.de	70
i+use DateTime::Format::Duration;	Err	codevoid.de	70
i use LWP::UserAgent;	Err	codevoid.de	70
i use JSON;	Err	codevoid.de	70
i use HTML::LinkExtractor;	Err	codevoid.de	70
it@@ -10,21 +14,48 @@ use HTML::Entities;	Err	codevoid.de	70
i use Encode;	Err	codevoid.de	70
i use Text::Wrap;	Err	codevoid.de	70
i $Text::Wrap::columns=72;	Err	codevoid.de	70
i-use Data::Dumper;	Err	codevoid.de	70
i 	Err	codevoid.de	70
i ### CONFIGURATION	Err	codevoid.de	70
i-my $protocol   = "https";	Err	codevoid.de	70
i-my $server     = "hn.algolia.com";	Err	codevoid.de	70
i-my $api_uri    = "/api/v1";	Err	codevoid.de	70
i-my $go_root    = "/srv/codevoid-gopher";	Err	codevoid.de	70
i-my $go_path    = "/hn";	Err	codevoid.de	70
i-	Err	codevoid.de	70
i+my $protocol    = "https";	Err	codevoid.de	70
i+my $server      = "hn.algolia.com";	Err	codevoid.de	70
i+my $api_uri     = "/api/v1";	Err	codevoid.de	70
i+my $go_root     = "/srv/codevoid-gopher";	Err	codevoid.de	70
i+my $go_path     = "/hn";	Err	codevoid.de	70
i+my $index_count = 60;	Err	codevoid.de	70
i+	Err	codevoid.de	70
i+my $logo  =" _______               __                   _______\n";	Err	codevoid.de	70
i+   $logo .="|   |   |.---.-..----.|  |--..-----..----. |    |  |.-----..--.--.--..-----.\n";	Err	codevoid.de	70
i+   $logo .="|       ||  _  ||  __||    < |  -__||   _| |       ||  -__||  |  |  ||__ --|\n";	Err	codevoid.de	70
i+   $logo .="|___|___||___._||____||__|__||_____||__|   |__|____||_____||________||_____|\n";	Err	codevoid.de	70
i+   $logo .="                                                      on Gopher (inofficial)\n";	Err	codevoid.de	70
i+   $logo .= "[h|Visit Hacker News on the Internet|URL:https://news.ycombinator.com|server|port]\n\n";	Err	codevoid.de	70
i 	Err	codevoid.de	70
i ### FUNCTIONS	Err	codevoid.de	70
i+### SUB: $json = getTopStories();	Err	codevoid.de	70
i+sub getTopStories {	Err	codevoid.de	70
i+    # FIXME make this configurable, maybe.	Err	codevoid.de	70
i+    #print "Debug: getTopStories($protocol://hacker-news.firebaseio.com/v0/topstories.json)\n";	Err	codevoid.de	70
i+    my $REST= ({HOST => "hacker-news.firebaseio.com",	Err	codevoid.de	70
i+            URL  => "$protocol://hacker-news.firebaseio.com/v0/topstories.json" });	Err	codevoid.de	70
i+    $REST->{UA} = LWP::UserAgent->new(keep_alive => 0, timeout => 30);	Err	codevoid.de	70
i+    $REST->{UA}->agent("codevoid-hackernews-gopherproxy/0.1"); 	Err	codevoid.de	70
i+    $REST->{resource} = $REST->{URL};	Err	codevoid.de	70
i+    $REST->{request}  = HTTP::Request->new( GET => $REST->{resource} );	Err	codevoid.de	70
i+    $REST->{response} = $REST->{UA}->request( $REST->{request} );	Err	codevoid.de	70
i+    if(not $REST->{response}->is_success()) {	Err	codevoid.de	70
i+        my $delay = 0.5;	Err	codevoid.de	70
i+        #print "Debug: Got \"", $REST->{response}->status_line, "\" trying again in $delay seconds...\n";	Err	codevoid.de	70
i+        sleep $delay;	Err	codevoid.de	70
i+        return getTopStories();	Err	codevoid.de	70
i+    }    	Err	codevoid.de	70
i+    return decode_json($REST->{response}->content);	Err	codevoid.de	70
i+}	Err	codevoid.de	70
i+	Err	codevoid.de	70
i+	Err	codevoid.de	70
i ### SUB: $json = getApiData("/api/...");	Err	codevoid.de	70
i sub getApiData {	Err	codevoid.de	70
i     my ( $uri ) = @_;	Err	codevoid.de	70
i-    print "Debug: getApiData($protocol://$server$uri)\n";	Err	codevoid.de	70
i+    #print "Debug: getApiData($protocol://$server$uri)\n";	Err	codevoid.de	70
i     my $REST= ({HOST => "$server",	Err	codevoid.de	70
i                 URL  => "$protocol://$server$uri" });	Err	codevoid.de	70
i     $REST->{UA} = LWP::UserAgent->new(keep_alive => 0, timeout => 30);	Err	codevoid.de	70
it@@ -33,13 +64,14 @@ sub getApiData {	Err	codevoid.de	70
i     $REST->{request}  = HTTP::Request->new( GET => $REST->{resource} );	Err	codevoid.de	70
i     $REST->{response} = $REST->{UA}->request( $REST->{request} );	Err	codevoid.de	70
i     if(not $REST->{response}->is_success()) {	Err	codevoid.de	70
i-        print "Debug: Got \"", $REST->{response}->status_line, "\" trying again in 2 seconds...\n";	Err	codevoid.de	70
i+        #print "Debug: Got \"", $REST->{response}->status_line, "\" trying again in 2 seconds...\n";	Err	codevoid.de	70
i         sleep 2;	Err	codevoid.de	70
i         return getApiData ( $uri );	Err	codevoid.de	70
i     }    	Err	codevoid.de	70
i     return decode_json($REST->{response}->content);	Err	codevoid.de	70
i }	Err	codevoid.de	70
i 	Err	codevoid.de	70
i+	Err	codevoid.de	70
i ### SUB: $gph = scrapeSubComments($payload, $parentID, $lvl)	Err	codevoid.de	70
i sub scrapeSubComments {	Err	codevoid.de	70
i     my ( $payload, $parentID, $lvl ) = @_;	Err	codevoid.de	70
it@@ -50,7 +82,8 @@ sub scrapeSubComments {	Err	codevoid.de	70
i                 my $text     = encode("UTF-8", $comment->{'comment_text'});	Err	codevoid.de	70
i                 my $author   = encode("UTF-8", $comment->{'author'});	Err	codevoid.de	70
i                 my $objectID = $comment->{'objectID'};	Err	codevoid.de	70
i-                $output .= formatContent("$author:", $lvl);	Err	codevoid.de	70
i+                my $ago = parseDate($comment->{'created_at'});	Err	codevoid.de	70
i+                $output .= formatContent("$author wrote $ago:", $lvl);	Err	codevoid.de	70
i                 $output .= formatContent("$text", $lvl)."\n";	Err	codevoid.de	70
i                 $output .= scrapeSubComments( $payload, $objectID, ++$lvl );	Err	codevoid.de	70
i                 $lvl--;	Err	codevoid.de	70
it@@ -60,17 +93,96 @@ sub scrapeSubComments {	Err	codevoid.de	70
i     return $output;	Err	codevoid.de	70
i }	Err	codevoid.de	70
i 	Err	codevoid.de	70
i-### SUB: scrapeComments($objectID, $number)	Err	codevoid.de	70
i+### SUB: $datestr = parseDate($datestring)	Err	codevoid.de	70
i+sub parseDate {	Err	codevoid.de	70
i+    my ( $datestring ) = @_;	Err	codevoid.de	70
i+	Err	codevoid.de	70
i+    # set output (parse) pattern	Err	codevoid.de	70
i+    my $p = DateTime::Format::Duration->new(	Err	codevoid.de	70
i+        pattern => '%Y|%m|%e|%H|%M',	Err	codevoid.de	70
i+        normalize => 1	Err	codevoid.de	70
i+    );	Err	codevoid.de	70
i+	Err	codevoid.de	70
i+    # FIXME: DateTime::Duration can do the parsing	Err	codevoid.de	70
i+    # parse string and create datetime object	Err	codevoid.de	70
i+    $datestring =~ /(....)-(..)-(..)T(..):(..).*/;	Err	codevoid.de	70
i+    my $dt = DateTime->new(	Err	codevoid.de	70
i+        year       => $1,	Err	codevoid.de	70
i+        month      => $2,	Err	codevoid.de	70
i+        day        => $3,	Err	codevoid.de	70
i+        hour       => $4,	Err	codevoid.de	70
i+        minute     => $5,	Err	codevoid.de	70
i+        second     => 0,	Err	codevoid.de	70
i+        nanosecond => 0,	Err	codevoid.de	70
i+        time_zone  => 'UTC'	Err	codevoid.de	70
i+    );	Err	codevoid.de	70
i+	Err	codevoid.de	70
i+    # calculate difference	Err	codevoid.de	70
i+    my $dt_now = DateTime->now;	Err	codevoid.de	70
i+    my $dt_diff =  $dt_now - $dt;	Err	codevoid.de	70
i+	Err	codevoid.de	70
i+    # parse result	Err	codevoid.de	70
i+    my $o = $p->format_duration($dt_diff);	Err	codevoid.de	70
i+	Err	codevoid.de	70
i+    # parse output (FIXME: this is *so* ugly)	Err	codevoid.de	70
i+    my $dtstr = "";	Err	codevoid.de	70
i+    $o =~ /(\d+)\|(\d+)\|(\d+)\|(\d+)\|(\d+)/;	Err	codevoid.de	70
i+    my $Y = int($1);	Err	codevoid.de	70
i+    my $m = int($2);	Err	codevoid.de	70
i+    my $d = int($3);	Err	codevoid.de	70
i+    my $H = int($4);	Err	codevoid.de	70
i+    my $M = int($5);	Err	codevoid.de	70
i+    if($M) {	Err	codevoid.de	70
i+        $dtstr = "$M min ago";	Err	codevoid.de	70
i+    }	Err	codevoid.de	70
i+    if($H) {	Err	codevoid.de	70
i+        if($H == 1) {	Err	codevoid.de	70
i+            $dtstr = "$H hour $M min ago";	Err	codevoid.de	70
i+        } else {	Err	codevoid.de	70
i+            $dtstr = "$H hours $M min ago";	Err	codevoid.de	70
i+        }	Err	codevoid.de	70
i+    }	Err	codevoid.de	70
i+    if($d) {	Err	codevoid.de	70
i+        if($d == 1) {	Err	codevoid.de	70
i+            $dtstr = "$d day ago"; 	Err	codevoid.de	70
i+        } else {	Err	codevoid.de	70
i+            $dtstr = "$d days ago"; 	Err	codevoid.de	70
i+        }	Err	codevoid.de	70
i+    }	Err	codevoid.de	70
i+    if($m) {	Err	codevoid.de	70
i+        if($m == 1) {	Err	codevoid.de	70
i+            if($d == 1) {	Err	codevoid.de	70
i+                $dtstr = "$m month $d day ago"; 	Err	codevoid.de	70
i+            } else {	Err	codevoid.de	70
i+                $dtstr = "$m month $d days ago"; 	Err	codevoid.de	70
i+            }	Err	codevoid.de	70
i+        } else {	Err	codevoid.de	70
i+            if($d == 1) {	Err	codevoid.de	70
i+                $dtstr = "$m months $d day ago"; 	Err	codevoid.de	70
i+            } else {	Err	codevoid.de	70
i+                $dtstr = "$m months $d days ago"; 	Err	codevoid.de	70
i+            }	Err	codevoid.de	70
i+        }	Err	codevoid.de	70
i+    }	Err	codevoid.de	70
i+    if($Y) {	Err	codevoid.de	70
i+        $dtstr = "on $Y-$m-$d ($H:$M)"; 	Err	codevoid.de	70
i+    }	Err	codevoid.de	70
i+	Err	codevoid.de	70
i+    return $dtstr;	Err	codevoid.de	70
i+}	Err	codevoid.de	70
i+	Err	codevoid.de	70
i+### SUB: scrapeComments($objectID, $number, $title)	Err	codevoid.de	70
i sub scrapeComments {	Err	codevoid.de	70
i-    my ( $objectID, $number ) = @_;	Err	codevoid.de	70
i-    my $content = "";	Err	codevoid.de	70
i+    my ( $objectID, $number, $title ) = @_;	Err	codevoid.de	70
i+    my $content = "$logo\nCOMMENT PAGE FOR:\n  \"$title\"\n\n";	Err	codevoid.de	70
i     if($number) {	Err	codevoid.de	70
i         my $payload = getApiData("$api_uri/search?tags="."comment,story_$objectID&hitsPerPage=$number");	Err	codevoid.de	70
i-        $content = scrapeSubComments($payload, $objectID, 0);	Err	codevoid.de	70
i+        $content .= scrapeSubComments($payload, $objectID, 0);	Err	codevoid.de	70
i     } else {	Err	codevoid.de	70
i-        $content = "No comments available\n";	Err	codevoid.de	70
i+        $content .= "No comments available\n";	Err	codevoid.de	70
i     }	Err	codevoid.de	70
i-    saveFile($content, "story_$objectID.gph");	Err	codevoid.de	70
i+    $content .= "\n[1|<- back to front page|$go_path|server|port]";	Err	codevoid.de	70
i+    saveFile($content, "comments_$objectID.gph");	Err	codevoid.de	70
i }	Err	codevoid.de	70
i 	Err	codevoid.de	70
i ### SUB: formatContent($content, $lvl)	Err	codevoid.de	70
it@@ -90,8 +202,8 @@ sub formatContent {	Err	codevoid.de	70
i     # calculate padding	Err	codevoid.de	70
i     $Text::Wrap::columns=72-($lvl*2);	Err	codevoid.de	70
i     while($lvl > 0) {	Err	codevoid.de	70
i-      $pad="  ".$pad;	Err	codevoid.de	70
i-      $lvl--;	Err	codevoid.de	70
i+        $pad="  ".$pad;	Err	codevoid.de	70
i+        $lvl--;	Err	codevoid.de	70
i     }	Err	codevoid.de	70
i 	Err	codevoid.de	70
i     # Search for links	Err	codevoid.de	70
it@@ -125,13 +237,21 @@ sub formatContent {	Err	codevoid.de	70
i             # make sure there are no newlines/extra spaces around [0]	Err	codevoid.de	70
i             $content_clean =~ s/[\s\n]+\[$c\][\s\n]+/ \[$c\] /g;	Err	codevoid.de	70
i 	Err	codevoid.de	70
i+            # fix the [1] [1] situation (FIXME: how to do this properly?)	Err	codevoid.de	70
i+            $content_clean =~ s/\[1\][\.:\s\n]+\[1\]/\[1\]/g;	Err	codevoid.de	70
i+            $content_clean =~ s/\[2\][\.:\s\n]+\[2\]/\[2\]/g;	Err	codevoid.de	70
i+            $content_clean =~ s/\[3\][\.:\s\n]+\[3\]/\[3\]/g;	Err	codevoid.de	70
i+            $content_clean =~ s/\[4\][\.:\s\n]+\[3\]/\[4\]/g;	Err	codevoid.de	70
i+            $content_clean =~ s/\[5\][\.:\s\n]+\[3\]/\[5\]/g;	Err	codevoid.de	70
i+            $content_clean =~ s/ \[\d\] $//g;	Err	codevoid.de	70
i+	Err	codevoid.de	70
i             # shorten links	Err	codevoid.de	70
i             my $short = $linkitem->{href};	Err	codevoid.de	70
i             my $l = 62 - length($pad);	Err	codevoid.de	70
i             if(length($short) > $l) { $short = substr($short,0,$l)."..."; }	Err	codevoid.de	70
i 	Err	codevoid.de	70
i             # add link to output scalar	Err	codevoid.de	70
i-            $links .= sprintf("[h|${pad}\\|[%i]: %s|URL:%s|codevoid.de|70]\n", $c, $short, $linkitem->{href});	Err	codevoid.de	70
i+            $links .= sprintf("[h|${pad}║ [%i]: %s|URL:%s|codevoid.de|70]\n", $c, $short, $linkitem->{href});	Err	codevoid.de	70
i         }   	Err	codevoid.de	70
i     }	Err	codevoid.de	70
i 	Err	codevoid.de	70
it@@ -143,12 +263,12 @@ sub formatContent {	Err	codevoid.de	70
i     $content_clean =~ s/\n\n(\n)*/\n\n/g;	Err	codevoid.de	70
i 	Err	codevoid.de	70
i     # Add padding to the left	Err	codevoid.de	70
i-    $content_clean =~ s/^/$pad\|/g;	Err	codevoid.de	70
i-    $content_clean =~ s/\n/\n$pad\|/g;	Err	codevoid.de	70
i+    $content_clean =~ s/^/$pad║ /g;	Err	codevoid.de	70
i+    $content_clean =~ s/\n/\n$pad║ /g;	Err	codevoid.de	70
i 	Err	codevoid.de	70
i     # print links if there were any.	Err	codevoid.de	70
i     if($links) {	Err	codevoid.de	70
i-        $content_clean .= "\n$pad\|\n$links";	Err	codevoid.de	70
i+        $content_clean .= "\n$pad║ \n$links";	Err	codevoid.de	70
i     } else {	Err	codevoid.de	70
i         $content_clean .= "\n";    	Err	codevoid.de	70
i     }	Err	codevoid.de	70
it@@ -171,49 +291,107 @@ sub saveFile {	Err	codevoid.de	70
i 	Err	codevoid.de	70
i     # rename to temporary file to real file (atomic)	Err	codevoid.de	70
i     rename("$path/.$filename", "$path/$filename") || die "Cannot rename temporary file: $filename\n";	Err	codevoid.de	70
i-    print "Debug: saveFile(\$content, $filename);\n\n";	Err	codevoid.de	70
i+    #print "Debug: saveFile(\$content, $filename);\n\n";	Err	codevoid.de	70
i     return 0;	Err	codevoid.de	70
i }	Err	codevoid.de	70
i 	Err	codevoid.de	70
i 	Err	codevoid.de	70
i ### MAIN PROGRAM	Err	codevoid.de	70
i-	Err	codevoid.de	70
i my ($selected_story) = @ARGV;	Err	codevoid.de	70
i-my $json_fp = getApiData("$api_uri/search_by_date?tags=front_page&numericFilters=points>20,num_comments>5&hitsPerPage=100");	Err	codevoid.de	70
i-#my $json_fp = getApiData("$api_uri/search?tags=story");	Err	codevoid.de	70
i-	Err	codevoid.de	70
i my $content = "";	Err	codevoid.de	70
i+	Err	codevoid.de	70
i+# fetch top story IDs	Err	codevoid.de	70
i+my $json_top = getTopStories();	Err	codevoid.de	70
i+	Err	codevoid.de	70
i+# construct search query	Err	codevoid.de	70
i+my $query = "search?hitsPerPage=500&tags=story,(";	Err	codevoid.de	70
i+	Err	codevoid.de	70
i+# add stories to search query	Err	codevoid.de	70
i+my $count = 0;	Err	codevoid.de	70
i+for my $id (@$json_top) {	Err	codevoid.de	70
i+    $query .="story_$id,";	Err	codevoid.de	70
i+    $count++;	Err	codevoid.de	70
i+    if($count > $index_count) {	Err	codevoid.de	70
i+        last;	Err	codevoid.de	70
i+    }	Err	codevoid.de	70
i+}	Err	codevoid.de	70
i+	Err	codevoid.de	70
i+# remove trailing comma and close query	Err	codevoid.de	70
i+$query =~ s/,$/\)/g;	Err	codevoid.de	70
i+	Err	codevoid.de	70
i+# set up background tasks for parallel scraping	Err	codevoid.de	70
i+my $pm = new Parallel::ForkManager(50);	Err	codevoid.de	70
i+	Err	codevoid.de	70
i+my $json_fp = getApiData("$api_uri/$query");	Err	codevoid.de	70
i for my $hit ($json_fp->{"hits"}) {	Err	codevoid.de	70
i     foreach my $story (@$hit) {	Err	codevoid.de	70
i+	Err	codevoid.de	70
i+        # do everything from here in background	Err	codevoid.de	70
i+        $pm->start and next;	Err	codevoid.de	70
i+	Err	codevoid.de	70
i+        # title is a link, escape "|"	Err	codevoid.de	70
i         my $title  = encode("UTF-8", $story->{'title'});	Err	codevoid.de	70
i         $title =~ s/\|/\\|/g;	Err	codevoid.de	70
i+	Err	codevoid.de	70
i+        # URL is either a HTML link line or a gopher dir	Err	codevoid.de	70
i         my $url = "";	Err	codevoid.de	70
i         if($story->{'url'}) {	Err	codevoid.de	70
i             $url    = encode("UTF-8", $story->{'url'});	Err	codevoid.de	70
i             $content .= "[h|  $title|URL:$url|server|port]\n";	Err	codevoid.de	70
i         } else {	Err	codevoid.de	70
i-            $url    = "$go_path/story_$story->{'objectID'}.gph";	Err	codevoid.de	70
i+            $url    = "$go_path/comments_$story->{'objectID'}.gph";	Err	codevoid.de	70
i             $content .= "[1|  $title|$url|server|port]\n";	Err	codevoid.de	70
i         }	Err	codevoid.de	70
i+	Err	codevoid.de	70
i+        # 	Err	codevoid.de	70
i         my $author = encode("UTF-8", $story->{'author'});	Err	codevoid.de	70
i+        my $objectID = $story->{'objectID'};	Err	codevoid.de	70
i+	Err	codevoid.de	70
i+        # parse date	Err	codevoid.de	70
i+        my $ago = parseDate($story->{'created_at'});	Err	codevoid.de	70
i 	Err	codevoid.de	70
i-        $story->{'created_at'} =~ /(....-..-..)T(..:..).*/;	Err	codevoid.de	70
i-        my $date = $1;	Err	codevoid.de	70
i-        my $time = $2;	Err	codevoid.de	70
i         my $number = 0;	Err	codevoid.de	70
i         if($story->{'num_comments'}) {	Err	codevoid.de	70
i             $number = $story->{'num_comments'};	Err	codevoid.de	70
i         }	Err	codevoid.de	70
i 	Err	codevoid.de	70
i-        $content .= "  by $author ($story->{'points'} points) at $time ($date)\n";	Err	codevoid.de	70
i-        $content .= "[1|  read $number comments|$go_path/story_$story->{'objectID'}.gph|server|port]\n";	Err	codevoid.de	70
i+        # build content	Err	codevoid.de	70
i+        $content .= "  by $author ($story->{'points'} points) $ago\n";	Err	codevoid.de	70
i+        $content .= "[1|  read $number comments|$go_path/comments_$objectID.gph|server|port]\n";	Err	codevoid.de	70
i         $content .= "\n";	Err	codevoid.de	70
i-        print "Debug: scrapeComments($story->{'objectID'}, $number);\n";	Err	codevoid.de	70
i 	Err	codevoid.de	70
i-        scrapeComments($story->{'objectID'}, $number);	Err	codevoid.de	70
i+        # Save (if not already done - assuming the story doesn't change)	Err	codevoid.de	70
i+        # FIXME: the title could be changed by the staff	Err	codevoid.de	70
i+        if (not -e "$go_root$go_path/story_$objectID.gph") {	Err	codevoid.de	70
i+            saveFile($content, "story_$objectID.gph");	Err	codevoid.de	70
i+        }	Err	codevoid.de	70
i+	Err	codevoid.de	70
i+        # Fire up the comment scraper	Err	codevoid.de	70
i+        #print "Debug: scrapeComments($objectID, $number, $title);\n";	Err	codevoid.de	70
i+        scrapeComments($story->{'objectID'}, $number, $title);	Err	codevoid.de	70
i+	Err	codevoid.de	70
i+        # background task stopps here	Err	codevoid.de	70
i+        $pm->finish	Err	codevoid.de	70
i+    }	Err	codevoid.de	70
i+}	Err	codevoid.de	70
i+	Err	codevoid.de	70
i+# wait for all scraping be done and all cache files be present	Err	codevoid.de	70
i+$pm->wait_all_children;	Err	codevoid.de	70
i+	Err	codevoid.de	70
i+# construct index from cached files	Err	codevoid.de	70
i+$count = 0;	Err	codevoid.de	70
i+my $index_out = "$logo";	Err	codevoid.de	70
i+for my $id (@$json_top) {	Err	codevoid.de	70
i+    if (-e "$go_root$go_path/story_$id.gph") {	Err	codevoid.de	70
i+        open(my $fh, '<', "$go_root$go_path/story_$id.gph");	Err	codevoid.de	70
i+        while (my $row = <$fh>) {	Err	codevoid.de	70
i+            $index_out .= $row;	Err	codevoid.de	70
i+        }	Err	codevoid.de	70
i+        close($fh);	Err	codevoid.de	70
i     }	Err	codevoid.de	70
i+    $count++;	Err	codevoid.de	70
i+    if($count > $index_count) { last; }	Err	codevoid.de	70
i }	Err	codevoid.de	70
i-# saving index last to avoid broken links while scraper is running.	Err	codevoid.de	70
i-saveFile($content, "index.gph");	Err	codevoid.de	70
i+saveFile($index_out, "index.gph");	Err	codevoid.de	70
i 	Err	codevoid.de	70
i exit 0;	Err	codevoid.de	70
.