t* hacker news on gopher URI git clone git://git.codevoid.de/hn-gopher DIR Log DIR Files DIR Refs --- DIR commit 2b7e9e0f8bb755628f7f9e77174c93b95e299da6 DIR parent 23a060ffd2c3ed13af0c3c49a8938c4802e3e5da URI Author: Stefan Hagen <sh+git[at]codevoid[dot]de> Date: Sun, 29 Jul 2018 23:44:21 +0200 Handle 0 comments and filter frontpage a bit Diffstat: M hn-scraper.pl | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) --- DIR diff --git a/hn-scraper.pl b/hn-scraper.pl t@@ -15,7 +15,7 @@ use Data::Dumper; ### CONFIGURATION my $protocol = "https"; my $server = "hn.algolia.com"; -my $api_taguri = "/api/v1/search?tags="; +my $api_uri = "/api/v1"; my $go_root = "/srv/codevoid-gopher"; my $go_path = "/hn"; t@@ -62,8 +62,13 @@ sub scrapeSubComments { ### SUB: scrapeComments($objectID, $number) sub scrapeComments { my ( $objectID, $number ) = @_; - my $payload = getApiData("$api_taguri"."comment,story_$objectID&hitsPerPage=$number"); - my $content = scrapeSubComments($payload, $objectID, 0); + my $content = ""; + if($number) { + my $payload = getApiData("$api_uri/search?tags="."comment,story_$objectID&hitsPerPage=$number"); + $content = scrapeSubComments($payload, $objectID, 0); + } else { + $content = "No comments available\n"; + } saveFile($content, "story_$objectID.gph"); } t@@ -181,7 +186,8 @@ sub saveFile { ### MAIN PROGRAM my ($selected_story) = @ARGV; -my $json_fp = getApiData("$api_taguri"."front_page"); +my $json_fp = getApiData("$api_uri/search_by_date?tags=front_page&numericFilters=points>60"); +#my $json_fp = getApiData("$api_uri/search?tags=story"); my $content = ""; for my $hit ($json_fp->{"hits"}) { t@@ -199,13 +205,18 @@ for my $hit ($json_fp->{"hits"}) { $story->{'created_at'} =~ /(....-..-..)T(..:..).*/; my $date = $1; my $time = $2; + my $number = 0; + if($story->{'num_comments'}) { + $number = $story->{'num_comments'}; + } $content .= "[h| $title|URL:$url|server|port]\n"; $content .= " by $author ($story->{'points'} points) at $time ($date)\n"; - $content .= "[1| read $story->{'num_comments'} comments|$go_path/story_$story->{'objectID'}.gph|server|port]\n"; + $content .= "[1| read $number comments|$go_path/story_$story->{'objectID'}.gph|server|port]\n"; $content .= "\n"; - print "Debug: scrapeComments($story->{'objectID'}, $story->{'num_comments'});\n"; - scrapeComments($story->{'objectID'}, $story->{'num_comments'}); + print "Debug: scrapeComments($story->{'objectID'}, $number);\n"; + + scrapeComments($story->{'objectID'}, $number); } } # saving index last to avoid broken links while scraper is running.