Index: head/ports-mgmt/portscout/Makefile =================================================================== --- head/ports-mgmt/portscout/Makefile (revision 405249) +++ head/ports-mgmt/portscout/Makefile (revision 405250) @@ -1,117 +1,117 @@ # Created by: Shaun Amott # $FreeBSD$ PORTNAME= portscout PORTVERSION= 0.8.1 -PORTREVISION= 4 +PORTREVISION= 5 CATEGORIES= ports-mgmt MASTER_SITES= http://mirror.inerd.com/FreeBSD/distfiles/${PORTNAME}/ \ http://www.atarininja.org/~wxs/distfiles/ \ http://www.inerd.com/software/${PORTNAME}/ MAINTAINER= koobs@FreeBSD.org COMMENT= Tool to scan for new versions of FreeBSD ports LICENSE= BSD2CLAUSE RUN_DEPENDS= p5-DBI>=0:${PORTSDIR}/databases/p5-DBI \ p5-Proc-Queue>=0:${PORTSDIR}/devel/p5-Proc-Queue \ p5-Net>=0:${PORTSDIR}/net/p5-Net \ p5-URI>=0:${PORTSDIR}/net/p5-URI \ p5-XML-XPath>=0:${PORTSDIR}/textproc/p5-XML-XPath \ p5-MIME-Lite>=0:${PORTSDIR}/mail/p5-MIME-Lite \ p5-libwww>=0:${PORTSDIR}/www/p5-libwww \ p5-JSON>0:${PORTSDIR}/converters/p5-JSON OPTIONS_DEFINE= DOCS HTTPS OPTIONS_DEFAULT= HTTPS PGSQL OPTIONS_MULTI= DATABASE OPTIONS_MULTI_DATABASE= SQLITE PGSQL DATABASE_DESC= Database Backends USES= perl5 shebangfix NO_BUILD= yes SHEBANG_FILES= ${WRKSRC}/portscout.pl PORTDOCS= UPDATING portscout-portconfig.txt xml-datasrc-example.xml HTTPS_RUN_DEPENDS= p5-LWP-Protocol-https>=0:${PORTSDIR}/www/p5-LWP-Protocol-https SQLITE_USE= SQLITE=3 SQLITE_RUN_DEPENDS= p5-DBD-SQLite>=0:${PORTSDIR}/databases/p5-DBD-SQLite PGSQL_USES= pgsql PGSQL_RUN_DEPENDS= p5-DBD-Pg>=0:${PORTSDIR}/databases/p5-DBD-Pg .include pre-everything:: .if ${PORT_OPTIONS:MSQLITE} @${ECHO_MSG} "+-------------------------------------------------------------+" @${ECHO_MSG} "| Warning! although SQLite is supported, portscout will only |" @${ECHO_MSG} "| operate in non-forking mode with this database backend. It |" @${ECHO_MSG} "| is therefore recommended only for use with light workloads. |" @${ECHO_MSG} "+-------------------------------------------------------------+" .endif post-extract: @${CP} ${FILESDIR}/files-Portscout-SiteHandler-GitHub.pm ${WRKSRC}/Portscout/SiteHandler/GitHub.pm @${CP} ${FILESDIR}/files-Portscout-SiteHandler-PyPI.pm ${WRKSRC}/Portscout/SiteHandler/PyPI.pm post-patch: .if ${PORT_OPTIONS:MSQLITE} @${REINPLACE_CMD} 's/^\([^#]*DBI:Pg.*\)$$/#\1/g' ${WRKSRC}/portscout.conf @${REINPLACE_CMD} 's/^#\(.*DBI:SQLite.*\)$$/\1/g' ${WRKSRC}/portscout.conf .endif @${REINPLACE_CMD} -e "s#^\(templates .*\)/etc#\1/share#" \ -e "s#^prefix\( *= *\).*#prefix\1${PREFIX}#" \ ${WRKSRC}/portscout.conf @${REINPLACE_CMD} -e "s#^\(.*PREFIX.*=> *\)'.*'#\1'${PREFIX}'#" \ ${WRKSRC}/Portscout/Const.pm @${MV} ${WRKSRC}/docs/*.* ${WRKSRC} do-install: @${MKDIR} ${STAGEDIR}${PREFIX}/${SITE_PERL_REL}/Portscout @${MKDIR} ${STAGEDIR}${PREFIX}/${SITE_PERL_REL}/Portscout/DataSrc @${MKDIR} ${STAGEDIR}${PREFIX}/${SITE_PERL_REL}/Portscout/SiteHandler @${MKDIR} ${STAGEDIR}${PREFIX}/${SITE_PERL_REL}/Portscout/SQL ${INSTALL_SCRIPT} ${WRKSRC}/Portscout/*.pm ${STAGEDIR}${PREFIX}/${SITE_PERL_REL}/Portscout/ ${INSTALL_SCRIPT} ${WRKSRC}/Portscout/DataSrc/*.pm ${STAGEDIR}${PREFIX}/${SITE_PERL_REL}/Portscout/DataSrc/ ${INSTALL_SCRIPT} ${WRKSRC}/Portscout/SiteHandler/*.pm ${STAGEDIR}${PREFIX}/${SITE_PERL_REL}/Portscout/SiteHandler/ ${INSTALL_SCRIPT} ${WRKSRC}/Portscout/SQL/*.pm ${STAGEDIR}${PREFIX}/${SITE_PERL_REL}/Portscout/SQL/ ${INSTALL_SCRIPT} ${WRKSRC}/Portscout.pm ${STAGEDIR}${PREFIX}/${SITE_PERL_REL}/Portscout.pm ${INSTALL_SCRIPT} ${WRKSRC}/portscout.pl ${STAGEDIR}${PREFIX}/bin/portscout ${INSTALL_DATA} ${WRKSRC}/portscout.pod ${STAGEDIR}${PREFIX}/${SITE_PERL_REL} pod2man ${WRKSRC}/portscout.pod ${WRKSRC}/portscout.1 ${INSTALL_MAN} ${WRKSRC}/portscout.1 ${STAGEDIR}${MAN1PREFIX}/man/man1/ @${MKDIR} ${STAGEDIR}${DATADIR}/templates cd ${WRKSRC}/templates \ && ${INSTALL_DATA} * ${STAGEDIR}${DATADIR}/templates @${MKDIR} ${STAGEDIR}${DATADIR}/sql cd ${WRKSRC}/sql && ${INSTALL_DATA} *.sql ${STAGEDIR}${DATADIR}/sql ${INSTALL_DATA} ${WRKSRC}/portscout.conf ${STAGEDIR}${PREFIX}/etc/portscout.conf.sample .if ${PORT_OPTIONS:MDOCS} @${MKDIR} ${STAGEDIR}${DOCSDIR} . for f in ${PORTDOCS} ${INSTALL_DATA} ${WRKSRC}/${f} ${STAGEDIR}${DOCSDIR} . endfor .endif test: ${PERL} ${WRKSRC}/t/00-use.t ${PERL} ${WRKSRC}/t/01-vercompare.t .if ${PORT_OPTIONS:MSQLITE} ${PERL} ${WRKSRC}/t/10-sqlite.t #.else # ${PERL} ${WRKSRC}/10-postgresql.t .endif .include Index: head/ports-mgmt/portscout/files/patch-Portscout_SiteHandler_GitHub.pm =================================================================== --- head/ports-mgmt/portscout/files/patch-Portscout_SiteHandler_GitHub.pm (revision 405249) +++ head/ports-mgmt/portscout/files/patch-Portscout_SiteHandler_GitHub.pm (revision 405250) @@ -1,26 +1,38 @@ ---- Portscout/SiteHandler/GitHub.pm.orig 2015-10-25 05:00:48 UTC +--- Portscout/SiteHandler/GitHub.pm.orig 2016-01-04 10:46:49 UTC +++ Portscout/SiteHandler/GitHub.pm @@ -97,7 +97,9 @@ sub GetFiles my ($url, $port, $files) = @_; my $projname; - if ($url =~ /https:\/\/github\.com\/(.*?)\/archive\//) { + if ($url =~ /https?:\/\/codeload\.github\.com\/(.*?)\/tar.gz\//) { + $projname = $1; + } elsif ($url =~ /https:\/\/github\.com\/(.*?)\/archive\//) { $projname = $1; } elsif ($url =~ /https:\/\/github.com\/downloads\/(.*)\//) { $projname = $1; @@ -108,7 +110,11 @@ sub GetFiles # First check if there's a latest releases endpoint $query = 'https://api.github.com/repos/' . $projname . '/releases/latest'; - + # Add GitHub Client ID & Secret if they are set in settings + # https://developer.github.com/v3/#authentication -+ if ($settings{github_client_id} && $settings{github_client_id}) { ++ if ($settings{github_client_id} && $settings{github_client_secret}) { + $query = $query . "?client_id=$settings{github_client_id}&client_secret=$settings{github_client_secret}"; + } _debug("GET $query"); $ua = LWP::UserAgent->new; $ua->agent(USER_AGENT); +@@ -120,6 +126,11 @@ sub GetFiles + _debug('GET failed: ' . $response->status_line); + # Project didn't do any releases, so let's try tags instead. + $query = 'https://api.github.com/repos/' . $projname . '/tags'; ++ # Add GitHub Client ID & Secret if they are set in settings ++ # https://developer.github.com/v3/#authentication ++ if ($settings{github_client_id} && $settings{github_client_secret}) { ++ $query = $query . "?client_id=$settings{github_client_id}&client_secret=$settings{github_client_secret}"; ++ } + _debug("GET $query"); + $ua = LWP::UserAgent->new; + $ua->agent(USER_AGENT); Index: head/ports-mgmt/portscout/files/patch-portscout.pl =================================================================== --- head/ports-mgmt/portscout/files/patch-portscout.pl (revision 405249) +++ head/ports-mgmt/portscout/files/patch-portscout.pl (revision 405250) @@ -1,55 +1,217 @@ ---- portscout.pl.orig 2015-10-25 05:00:48 UTC +--- portscout.pl.orig 2016-01-04 10:46:49 UTC +++ portscout.pl @@ -463,7 +463,7 @@ sub VersionCheck $i++; - info($k, 'Checking site: ' . strchop($site, 60)); + info($k, 'Checking site: ' . strchop($site, 200)); # Look to see if the URL contains the distfile version. # This will affect our checks and guesses later on. @@ -493,15 +493,23 @@ sub VersionCheck } # Check for special handler for this site first + print "Does site handler exist ... " + unless ($settings{quiet}); if (my $sh = Portscout::SiteHandler->FindHandler($site)) { - info($k, $site, 'Using dedicated site handler for site.'); + print "Yes \n" + unless ($settings{quiet}); if (!$sh->GetFiles($site, $port, \@files)) { info($k, $site, 'SiteHandler::GetFiles() failed for ' . $site); next; } } + elsif (!$sh) + { + print "No \n" + unless ($settings{quiet}); + } elsif ($site->scheme eq 'ftp') { my $ftp; @@ -713,7 +721,8 @@ sub VersionCheck # Got a response which wasn't HTTP 4xx -> bail out if ($response->is_success && $response->status_line !~ /^4/) { - info($k, $site, 'Not doing any guessing; site is lieing to us.'); + print "URL: $url\n"; + info($k, $site, "Skip guessing: Response not 4xx to a file that shouldnt exist (". $response->status_line .")"); $sths->{sitedata_initliecount}->execute($sitedata->{host}) unless($settings{precious_data}); next; @@ -797,7 +806,7 @@ sub VersionCheck $new_found = 1; last; } else { - info($k, $site, "Guess failed $port->{ver} -> $guess_v"); + info($k, $site, "Guess failed $port->{ver} -> $guess_v (". $response->status_line .")"); } last if ($new_found); +@@ -867,7 +876,10 @@ sub FindNewestFile + + foreach my $file (@$files) + { +- my $poss_path; ++ my ($poss_path, $github); ++ ++ print "FindNewest: Checking $file ... against port DISTFILES. \n" ++ if ($settings{debug}); + + if ($file =~ /^(.*)\/(.*?)$/) { + # Files from SiteHandlers can come with paths +@@ -888,6 +900,9 @@ sub FindNewestFile + + my $skip = 0; + ++ print "FindNewest: Checking DISTFILE ... $distfile (ver: $v, sufx: $s)\n" ++ if ($settings{debug}); ++ + if ($poss_path) { + # Do a full-URL comparison for $old_found + # if we're dealing with paths too. +@@ -908,14 +923,20 @@ sub FindNewestFile + } else { + if ($file eq $distfile) { + $old_found = 1; ++ print "FindNewest: File matches existing port DISTFILE. Old found. \n" ++ if ($settings{debug}); + next; + } + } + ++ $github = 1 if ($site->clone =~ /^https?:\/\/([^\/.]+\.)?github\.com\/(.*?)\/tar.gz/); ++ + # Skip beta versions if requested + + if ($port->{skipbeta}) { + if (isbeta($file) && !isbeta($distfile)) { ++ print "FindNewest: File is beta, skipbeta is defined. Skipping ...\n" ++ if ($settings{debug}); + next; + } + } +@@ -940,31 +961,64 @@ sub FindNewestFile + + # Possible candidate - extract version + +- if ($file =~ /^($distfile)$/ && $2) ++ #warn "distfile = $distfile 2 = $2"; ++ if (($file =~ /^($distfile)$/ && $2) or $github) + { +- my $version = $2; +- my $new_v = lc $version; ++ my ($version, $new_v, $matchver); + +- # Catch a few missed cases ++ unless ($github) { ++ $version = $2; ++ $new_v = lc $version; + +- $new_v =~ s/(?:$ext_regex)$//; ++ # Catch a few missed cases ++ $new_v =~ s/(?:$ext_regex)$//; + +- # Version is much longer than original - skip it ++ # Version is much longer than original - skip it ++ next if (length $new_v > (12 + length $old_v)); + +- next if (length $new_v > (12 + length $old_v)); ++ # New version is in date format (or contains a date-like ++ # string) - old one is not. Probably best to ignore. + +- # New version is in date format (or contains a date-like +- # string) - old one is not. Probably best to ignore. ++ next if ( ++ $new_v =~ /$date_regex/i && ++ $old_v !~ /$date_regex/i ++ ); + +- next if ( +- $new_v =~ /$date_regex/i && +- $old_v !~ /$date_regex/i +- ); ++ # Skip a few strange version format change cases ++ # (formatted -> "just a number") ++ next if ($new_v !~ /\./ && $old_v =~ /\./); ++ } else { ++ # Github is "special" since the actual URI we get back from the ++ # handler isn't the same as what is actually being retrieved. ++ # So fall back on comparing tags instead. + +- # Skip a few strange version format change cases +- # (formatted -> "just a number") ++ $new_v = $file; ++ $version = lc $new_v; + +- next if ($new_v !~ /\./ && $old_v =~ /\./); ++ # Only match version-esque looking strings from the tag ++ # because portscout currently only uses/stores a normalized ++ # 'ver', not PORTVERSION or DISTVERSION{FULL} so we have nothing ++ # canonical to compare agains. Currently the match is for digits ++ # with any non-digit separators, without non-digit prefixes and ++ # suffixes. This is likely the best we can get for now, without ++ # introducing false positives. ++ ++ ($matchver) = ($version =~ m/((?:\d+.)*\d+)/); ++ ++ print "FindNewestFile::Github: Input: $file Matchver: $matchver \n" ++ if ($settings{debug}); ++ ++ # Replace non-digit separators of digits with dots. ++ ++ $matchver =~ s/[^\d+]/\./g; ++ ++ if (defined($matchver)) { ++ $new_v = $matchver; ++ } ++ ++ print "FindNewestFile::Github: Matchver: $matchver Normalized: $new_v \n" ++ if ($settings{debug}); ++ } + + # Skip any specific versions if requested + +@@ -973,6 +1027,8 @@ sub FindNewestFile + + foreach (split (/\s+/, $port->{skipversions})) { + if ($new_v eq $_) { ++ print "FindNewest: skipversions is defined. Matched $_. Skipping...\n" ++ if ($settings{debug}); + $skip = 1; + last; + } +@@ -981,7 +1037,7 @@ sub FindNewestFile + next if ($skip); + } + +- unless ($settings{sillystrings_enable}) { ++ unless ($settings{sillystrings_enable} or $github) { + if ($new_v =~ /[-_.]([A-Za-z]+[A-Za-z_-]{2,})$/) { + my $str = $1; + next if ( +@@ -1026,12 +1082,19 @@ sub FindNewestFile + + # Test our new version string + ++ print "FindNewest: Comparing Old ($old_v) vs New ($new_v) version strings... \n" ++ if ($settings{debug}); ++ + if ($new_v eq $old_v) + { ++ print "FindNewest: Old ($old_v) == New ($new_v). Old found.\n" ++ if ($settings{debug}); + $old_found = 1; + } + elsif (vercompare($new_v, $old_v)) + { ++ print "FindNewest: Old ($old_v) < New ($new_v). New found.\n" ++ if ($settings{debug}); + $new_found = 1; + + # Keep going until we find the newest version