* hacker news on gopher
       
   URI git clone git://git.codevoid.de/hn-gopher
   DIR Log
   DIR Files
   DIR Refs
       ---
   DIR commit 23a060ffd2c3ed13af0c3c49a8938c4802e3e5da
   URI Author: Stefan Hagen <sh+git[at]codevoid[dot]de>
       Date:   Sun, 29 Jul 2018 22:50:47 +0200
       
       Initial Commit
       
       Diffstat:
         A hn-scraper.pl                       |     214 +++++++++++++++++++++++++++++++
       
       1 file changed, 214 insertions(+), 0 deletions(-)
       ---
   DIR diff --git a/hn-scraper.pl b/hn-scraper.pl
       @@ -0,0 +1,214 @@
       +#!/usr/bin/env perl
       +
       +use strict;
       +use warnings;
       +use LWP::UserAgent;
       +use JSON;
       +use HTML::LinkExtractor;
       +use HTML::Restrict;
       +use HTML::Entities;
       +use Encode;
       +use Text::Wrap;
       +$Text::Wrap::columns=72;
       +use Data::Dumper;
       +
       +### CONFIGURATION
       +my $protocol   = "https";
       +my $server     = "hn.algolia.com";
       +my $api_taguri = "/api/v1/search?tags=";
       +my $go_root    = "/srv/codevoid-gopher";
       +my $go_path    = "/hn";
       +
       +
       +### FUNCTIONS
       +### SUB: $json = getApiData("/api/...");
       +sub getApiData {
       +    my ( $uri ) = @_;
       +    print "Debug: getApiData($protocol://$server$uri)\n";
       +    my $REST= ({HOST => "$server",
       +                URL  => "$protocol://$server$uri" });
       +    $REST->{UA} = LWP::UserAgent->new(keep_alive => 0, timeout => 30);
       +    $REST->{UA}->agent("codevoid-hackernews-gopherproxy/0.1"); 
       +    $REST->{resource} = $REST->{URL};
       +    $REST->{request}  = HTTP::Request->new( GET => $REST->{resource} );
       +    $REST->{response} = $REST->{UA}->request( $REST->{request} );
       +    if(not $REST->{response}->is_success()) {
       +        print "Debug: Got \"", $REST->{response}->status_line, "\" trying again in 2 seconds...\n";
       +        sleep 2;
       +        return getApiData ( $uri );
       +    }    
       +    return decode_json($REST->{response}->content);
       +}
       +
       +### SUB: $gph = scrapeSubComments($payload, $parentID, $lvl)
       +sub scrapeSubComments {
       +    my ( $payload, $parentID, $lvl ) = @_;
       +    my $output = "";
       +    for my $hit ($payload->{"hits"}) {
       +        foreach my $comment (@$hit) {
       +            if ($comment->{'parent_id'} == $parentID) {
       +                my $text     = encode("UTF-8", $comment->{'comment_text'});
       +                my $author   = encode("UTF-8", $comment->{'author'});
       +                my $objectID = $comment->{'objectID'};
       +                $output .= formatContent("$author:", $lvl);
       +                $output .= formatContent("$text", $lvl)."\n\n";
       +                $output .= scrapeSubComments( $payload, $objectID, ++$lvl );
       +            }
       +        }
       +    }
       +    return $output;
       +}
       +
       +### SUB: scrapeComments($objectID, $number)
       +sub scrapeComments {
       +    my ( $objectID, $number ) = @_;
       +    my $payload = getApiData("$api_taguri"."comment,story_$objectID&hitsPerPage=$number");
       +    my $content = scrapeSubComments($payload, $objectID, 0);
       +    saveFile($content, "story_$objectID.gph");
       +}
       +
       +### SUB: formatContent($content, $lvl)
       +sub formatContent {
       +    my ( $content, $lvl ) = @_;
       +    $content = decode_entities($content);
       +
       +    # remove trailing space before wrapping
       +    $content =~ s/ $/\n/g;
       +
       +    # calculate padding
       +    $Text::Wrap::columns=72-($lvl*2);
       +    my $pad="";
       +    while($lvl > 0) {
       +      $pad="  ".$pad;
       +      $lvl--;
       +    }
       +
       +    # Search for links
       +    my $LX = new HTML::LinkExtractor();
       +    $LX->strip(1);
       +    $LX->parse(\$content);
       +
       +    # Replace some HTML elements
       +    my $HR = HTML::Restrict->new();
       +    $content =~ s/<p>/\n\n/g;
       +    $content =~ s/<li>/\n\n\* /g;
       +    $content =~ s/<blockquote>/\n\n--- QUOTE ---\n/g;
       +    $content =~ s/<\/blockquote>/\n---- END ----\n\n/g;
       +    my $content_clean = $HR->process($content);
       +
       +    # nobody needs more that one newline.
       +    $content_clean =~ s/\n\n(\n)*/\n\n/g;
       +
       +    # Loop at links, match text, add [counter] and generate output.
       +    my $c = 0;
       +    my $links = "";
       +    foreach my $link ($LX->links) {
       +        foreach my $linkitem (@$link) {
       +
       +            # skip empty links (image links for example)
       +            if(!$linkitem->{_TEXT}) { next; }
       +
       +            $c++;
       +            $content_clean =~ s/(\Q$linkitem->{_TEXT}\E)/ \[$c\] /g;
       +
       +            # FIXME FIXME FIXME
       +            # It's late and the below works.
       +            $content_clean =~ s/\n \[$c\] / \[$c\] /g;
       +            $content_clean =~ s/\n \[$c\] / \[$c\] /g;
       +
       +            $content_clean =~ s/\[$c\] \n/ \[$c\] /g;
       +            $content_clean =~ s/\[$c\] \n/ \[$c\] /g;
       +
       +            $content_clean =~ s/\n \[$c\] \n/ \[$c\] /g;
       +            $content_clean =~ s/\n \[$c\] \n/ \[$c\] /g;
       +
       +            $content_clean =~ s/  / /g;
       +            $content_clean =~ s/  / /g;
       +            $content_clean =~ s/  / /g;
       +            # FIXME FIXME FIXME
       +
       +            # shorten links
       +            my $short = $linkitem->{href};
       +            my $l = 63 - length($pad);
       +            if(length($short) > $l) { $short = substr($short,0,$l)."..."; }
       +
       +            # add link to output scalar
       +            $links .= sprintf("[h|${pad}[%i]: %s|URL:%s|codevoid.de|70]\n", $c, $short, $linkitem->{href});
       +        }   
       +    }
       +
       +
       +    # Wrap content 72 - padding
       +    $content_clean = wrap("","",$content_clean);
       +
       +    # shrink multiple newlines
       +    $content_clean =~ s/\n\n(\n)*/\n\n/g;
       +
       +    # Add padding to the left
       +    $content_clean =~ s/^/$pad/g;
       +    $content_clean =~ s/\n/\n$pad/g;
       +
       +    # print links if there were any.
       +    if($links) {
       +        $content_clean .= "\n\n$links";
       +    } else {
       +        $content_clean .= "\n";    
       +    }
       +
       +    $content_clean =~ s/\t/    /g;
       +    $content_clean =~ s/\nt/\ntt/g;
       +
       +    return $content_clean;
       +}
       +
       +### SUB: saveFile($content, $filename)
       +sub saveFile {
       +    my ( $content, $filename ) = @_;
       +    my $path = "$go_root$go_path";
       +
       +    # save temporary file
       +    open (FH, ">> $path/.$filename") || die "Cannot open file temporary file: $filename\n";
       +      print FH $content;
       +    close(FH);
       +
       +    # rename to temporary file to real file (atomic)
       +    rename("$path/.$filename", "$path/$filename") || die "Cannot rename temporary file: $filename\n";
       +    print "Debug: saveFile(\$content, $filename);\n\n";
       +    return 0;
       +}
       +
       +
       +### MAIN PROGRAM
       +
       +my ($selected_story) = @ARGV;
       +my $json_fp = getApiData("$api_taguri"."front_page");
       +
       +my $content = "";
       +for my $hit ($json_fp->{"hits"}) {
       +    foreach my $story (@$hit) {
       +        my $title  = encode("UTF-8", $story->{'title'});
       +        my $url = "";
       +        if($story->{'url'}) {
       +            $url    = encode("UTF-8", $story->{'url'});
       +        } else {
       +            $url    = "/hn/story_$story->{'objectID'}.gph";
       +        }
       +        my $author = encode("UTF-8", $story->{'author'});
       +
       +        $title =~ s/\|/\\|/g;
       +        $story->{'created_at'} =~ /(....-..-..)T(..:..).*/;
       +        my $date = $1;
       +        my $time = $2;
       +
       +        $content .= "[h|  $title|URL:$url|server|port]\n";
       +        $content .= "  by $author ($story->{'points'} points) at $time ($date)\n";
       +        $content .= "[1|  read $story->{'num_comments'} comments|$go_path/story_$story->{'objectID'}.gph|server|port]\n";
       +        $content .= "\n";
       +        print "Debug: scrapeComments($story->{'objectID'}, $story->{'num_comments'});\n";
       +        scrapeComments($story->{'objectID'}, $story->{'num_comments'});
       +    }
       +}
       +# saving index last to avoid broken links while scraper is running.
       +saveFile($content, "index.gph");
       +
       +exit 0;