Index: head/ports-mgmt/portscout/Makefile =================================================================== --- head/ports-mgmt/portscout/Makefile (revision 405731) +++ head/ports-mgmt/portscout/Makefile (revision 405732) @@ -1,117 +1,117 @@ # Created by: Shaun Amott # $FreeBSD$ PORTNAME= portscout PORTVERSION= 0.8.1 -PORTREVISION= 5 +PORTREVISION= 6 CATEGORIES= ports-mgmt MASTER_SITES= http://mirror.inerd.com/FreeBSD/distfiles/${PORTNAME}/ \ http://www.atarininja.org/~wxs/distfiles/ \ http://www.inerd.com/software/${PORTNAME}/ MAINTAINER= koobs@FreeBSD.org COMMENT= Tool to scan for new versions of FreeBSD ports LICENSE= BSD2CLAUSE RUN_DEPENDS= p5-DBI>=0:${PORTSDIR}/databases/p5-DBI \ p5-Proc-Queue>=0:${PORTSDIR}/devel/p5-Proc-Queue \ p5-Net>=0:${PORTSDIR}/net/p5-Net \ p5-URI>=0:${PORTSDIR}/net/p5-URI \ p5-XML-XPath>=0:${PORTSDIR}/textproc/p5-XML-XPath \ p5-MIME-Lite>=0:${PORTSDIR}/mail/p5-MIME-Lite \ p5-libwww>=0:${PORTSDIR}/www/p5-libwww \ p5-JSON>0:${PORTSDIR}/converters/p5-JSON OPTIONS_DEFINE= DOCS HTTPS OPTIONS_DEFAULT= HTTPS PGSQL OPTIONS_MULTI= DATABASE OPTIONS_MULTI_DATABASE= SQLITE PGSQL DATABASE_DESC= Database Backends USES= perl5 shebangfix NO_BUILD= yes SHEBANG_FILES= ${WRKSRC}/portscout.pl PORTDOCS= UPDATING portscout-portconfig.txt xml-datasrc-example.xml HTTPS_RUN_DEPENDS= p5-LWP-Protocol-https>=0:${PORTSDIR}/www/p5-LWP-Protocol-https SQLITE_USE= SQLITE=3 SQLITE_RUN_DEPENDS= p5-DBD-SQLite>=0:${PORTSDIR}/databases/p5-DBD-SQLite PGSQL_USES= pgsql PGSQL_RUN_DEPENDS= p5-DBD-Pg>=0:${PORTSDIR}/databases/p5-DBD-Pg .include pre-everything:: .if ${PORT_OPTIONS:MSQLITE} @${ECHO_MSG} "+-------------------------------------------------------------+" @${ECHO_MSG} "| Warning! although SQLite is supported, portscout will only |" @${ECHO_MSG} "| operate in non-forking mode with this database backend. It |" @${ECHO_MSG} "| is therefore recommended only for use with light workloads. |" @${ECHO_MSG} "+-------------------------------------------------------------+" .endif post-extract: @${CP} ${FILESDIR}/files-Portscout-SiteHandler-GitHub.pm ${WRKSRC}/Portscout/SiteHandler/GitHub.pm @${CP} ${FILESDIR}/files-Portscout-SiteHandler-PyPI.pm ${WRKSRC}/Portscout/SiteHandler/PyPI.pm post-patch: .if ${PORT_OPTIONS:MSQLITE} @${REINPLACE_CMD} 's/^\([^#]*DBI:Pg.*\)$$/#\1/g' ${WRKSRC}/portscout.conf @${REINPLACE_CMD} 's/^#\(.*DBI:SQLite.*\)$$/\1/g' ${WRKSRC}/portscout.conf .endif @${REINPLACE_CMD} -e "s#^\(templates .*\)/etc#\1/share#" \ -e "s#^prefix\( *= *\).*#prefix\1${PREFIX}#" \ ${WRKSRC}/portscout.conf @${REINPLACE_CMD} -e "s#^\(.*PREFIX.*=> *\)'.*'#\1'${PREFIX}'#" \ ${WRKSRC}/Portscout/Const.pm @${MV} ${WRKSRC}/docs/*.* ${WRKSRC} do-install: @${MKDIR} ${STAGEDIR}${PREFIX}/${SITE_PERL_REL}/Portscout @${MKDIR} ${STAGEDIR}${PREFIX}/${SITE_PERL_REL}/Portscout/DataSrc @${MKDIR} ${STAGEDIR}${PREFIX}/${SITE_PERL_REL}/Portscout/SiteHandler @${MKDIR} ${STAGEDIR}${PREFIX}/${SITE_PERL_REL}/Portscout/SQL ${INSTALL_SCRIPT} ${WRKSRC}/Portscout/*.pm ${STAGEDIR}${PREFIX}/${SITE_PERL_REL}/Portscout/ ${INSTALL_SCRIPT} ${WRKSRC}/Portscout/DataSrc/*.pm ${STAGEDIR}${PREFIX}/${SITE_PERL_REL}/Portscout/DataSrc/ ${INSTALL_SCRIPT} ${WRKSRC}/Portscout/SiteHandler/*.pm ${STAGEDIR}${PREFIX}/${SITE_PERL_REL}/Portscout/SiteHandler/ ${INSTALL_SCRIPT} ${WRKSRC}/Portscout/SQL/*.pm ${STAGEDIR}${PREFIX}/${SITE_PERL_REL}/Portscout/SQL/ ${INSTALL_SCRIPT} ${WRKSRC}/Portscout.pm ${STAGEDIR}${PREFIX}/${SITE_PERL_REL}/Portscout.pm ${INSTALL_SCRIPT} ${WRKSRC}/portscout.pl ${STAGEDIR}${PREFIX}/bin/portscout ${INSTALL_DATA} ${WRKSRC}/portscout.pod ${STAGEDIR}${PREFIX}/${SITE_PERL_REL} pod2man ${WRKSRC}/portscout.pod ${WRKSRC}/portscout.1 ${INSTALL_MAN} ${WRKSRC}/portscout.1 ${STAGEDIR}${MAN1PREFIX}/man/man1/ @${MKDIR} ${STAGEDIR}${DATADIR}/templates cd ${WRKSRC}/templates \ && ${INSTALL_DATA} * ${STAGEDIR}${DATADIR}/templates @${MKDIR} ${STAGEDIR}${DATADIR}/sql cd ${WRKSRC}/sql && ${INSTALL_DATA} *.sql ${STAGEDIR}${DATADIR}/sql ${INSTALL_DATA} ${WRKSRC}/portscout.conf ${STAGEDIR}${PREFIX}/etc/portscout.conf.sample .if ${PORT_OPTIONS:MDOCS} @${MKDIR} ${STAGEDIR}${DOCSDIR} . for f in ${PORTDOCS} ${INSTALL_DATA} ${WRKSRC}/${f} ${STAGEDIR}${DOCSDIR} . endfor .endif test: ${PERL} ${WRKSRC}/t/00-use.t ${PERL} ${WRKSRC}/t/01-vercompare.t .if ${PORT_OPTIONS:MSQLITE} ${PERL} ${WRKSRC}/t/10-sqlite.t #.else # ${PERL} ${WRKSRC}/10-postgresql.t .endif .include Index: head/ports-mgmt/portscout/files/patch-portscout.pl =================================================================== --- head/ports-mgmt/portscout/files/patch-portscout.pl (revision 405731) +++ head/ports-mgmt/portscout/files/patch-portscout.pl (revision 405732) @@ -1,217 +1,226 @@ ---- portscout.pl.orig 2016-01-04 10:46:49 UTC +--- portscout.pl.orig 2016-01-10 17:42:32 UTC +++ portscout.pl @@ -463,7 +463,7 @@ sub VersionCheck $i++; - info($k, 'Checking site: ' . strchop($site, 60)); + info($k, 'Checking site: ' . strchop($site, 200)); # Look to see if the URL contains the distfile version. # This will affect our checks and guesses later on. -@@ -493,15 +493,23 @@ sub VersionCheck +@@ -493,9 +493,12 @@ sub VersionCheck } # Check for special handler for this site first + print "Does site handler exist ... " + unless ($settings{quiet}); if (my $sh = Portscout::SiteHandler->FindHandler($site)) { - info($k, $site, 'Using dedicated site handler for site.'); + print "Yes \n" + unless ($settings{quiet}); if (!$sh->GetFiles($site, $port, \@files)) { info($k, $site, 'SiteHandler::GetFiles() failed for ' . $site); - next; - } - } -+ elsif (!$sh) -+ { -+ print "No \n" -+ unless ($settings{quiet}); -+ } - elsif ($site->scheme eq 'ftp') +@@ -506,6 +509,9 @@ sub VersionCheck { my $ftp; -@@ -713,7 +721,8 @@ sub VersionCheck ++ print "No. Trying FTP...\n" ++ unless ($settings{quiet}); ++ + $ftp = Net::FTP->new( + $site->host, + Port => $site->port, +@@ -609,6 +615,9 @@ sub VersionCheck + { + my ($ua, $response); + ++ print "No. Trying HTTP...\n" ++ unless ($settings{quiet}); ++ + unless (robotsallowed($dbh, $site, $sitedata)) { + info($k, $site, 'Ignoring site as per rules in robots.txt.'); + +@@ -713,7 +722,8 @@ sub VersionCheck + # Got a response which wasn't HTTP 4xx -> bail out if ($response->is_success && $response->status_line !~ /^4/) { - info($k, $site, 'Not doing any guessing; site is lieing to us.'); + print "URL: $url\n"; + info($k, $site, "Skip guessing: Response not 4xx to a file that shouldnt exist (". $response->status_line .")"); $sths->{sitedata_initliecount}->execute($sitedata->{host}) unless($settings{precious_data}); next; -@@ -797,7 +806,7 @@ sub VersionCheck +@@ -797,7 +807,7 @@ sub VersionCheck $new_found = 1; last; } else { - info($k, $site, "Guess failed $port->{ver} -> $guess_v"); + info($k, $site, "Guess failed $port->{ver} -> $guess_v (". $response->status_line .")"); } last if ($new_found); -@@ -867,7 +876,10 @@ sub FindNewestFile +@@ -867,7 +877,10 @@ sub FindNewestFile foreach my $file (@$files) { - my $poss_path; + my ($poss_path, $github); + + print "FindNewest: Checking $file ... against port DISTFILES. \n" + if ($settings{debug}); if ($file =~ /^(.*)\/(.*?)$/) { # Files from SiteHandlers can come with paths -@@ -888,6 +900,9 @@ sub FindNewestFile +@@ -888,6 +901,9 @@ sub FindNewestFile my $skip = 0; + print "FindNewest: Checking DISTFILE ... $distfile (ver: $v, sufx: $s)\n" + if ($settings{debug}); + if ($poss_path) { # Do a full-URL comparison for $old_found # if we're dealing with paths too. -@@ -908,14 +923,20 @@ sub FindNewestFile +@@ -908,14 +924,20 @@ sub FindNewestFile } else { if ($file eq $distfile) { $old_found = 1; + print "FindNewest: File matches existing port DISTFILE. Old found. \n" + if ($settings{debug}); next; } } + $github = 1 if ($site->clone =~ /^https?:\/\/([^\/.]+\.)?github\.com\/(.*?)\/tar.gz/); + # Skip beta versions if requested if ($port->{skipbeta}) { if (isbeta($file) && !isbeta($distfile)) { + print "FindNewest: File is beta, skipbeta is defined. Skipping ...\n" + if ($settings{debug}); next; } } -@@ -940,31 +961,64 @@ sub FindNewestFile +@@ -940,31 +962,64 @@ sub FindNewestFile # Possible candidate - extract version - if ($file =~ /^($distfile)$/ && $2) + #warn "distfile = $distfile 2 = $2"; + if (($file =~ /^($distfile)$/ && $2) or $github) { - my $version = $2; - my $new_v = lc $version; + my ($version, $new_v, $matchver); - # Catch a few missed cases + unless ($github) { + $version = $2; + $new_v = lc $version; - $new_v =~ s/(?:$ext_regex)$//; + # Catch a few missed cases + $new_v =~ s/(?:$ext_regex)$//; - # Version is much longer than original - skip it + # Version is much longer than original - skip it + next if (length $new_v > (12 + length $old_v)); - next if (length $new_v > (12 + length $old_v)); + # New version is in date format (or contains a date-like + # string) - old one is not. Probably best to ignore. - # New version is in date format (or contains a date-like - # string) - old one is not. Probably best to ignore. + next if ( + $new_v =~ /$date_regex/i && + $old_v !~ /$date_regex/i + ); - next if ( - $new_v =~ /$date_regex/i && - $old_v !~ /$date_regex/i - ); + # Skip a few strange version format change cases + # (formatted -> "just a number") + next if ($new_v !~ /\./ && $old_v =~ /\./); + } else { + # Github is "special" since the actual URI we get back from the + # handler isn't the same as what is actually being retrieved. + # So fall back on comparing tags instead. - # Skip a few strange version format change cases - # (formatted -> "just a number") + $new_v = $file; + $version = lc $new_v; - next if ($new_v !~ /\./ && $old_v =~ /\./); + # Only match version-esque looking strings from the tag + # because portscout currently only uses/stores a normalized + # 'ver', not PORTVERSION or DISTVERSION{FULL} so we have nothing + # canonical to compare agains. Currently the match is for digits + # with any non-digit separators, without non-digit prefixes and + # suffixes. This is likely the best we can get for now, without + # introducing false positives. + + ($matchver) = ($version =~ m/((?:\d+.)*\d+)/); + + print "FindNewestFile::Github: Input: $file Matchver: $matchver \n" + if ($settings{debug}); + + # Replace non-digit separators of digits with dots. + + $matchver =~ s/[^\d+]/\./g; + + if (defined($matchver)) { + $new_v = $matchver; + } + + print "FindNewestFile::Github: Matchver: $matchver Normalized: $new_v \n" + if ($settings{debug}); + } # Skip any specific versions if requested -@@ -973,6 +1027,8 @@ sub FindNewestFile +@@ -973,6 +1028,8 @@ sub FindNewestFile foreach (split (/\s+/, $port->{skipversions})) { if ($new_v eq $_) { + print "FindNewest: skipversions is defined. Matched $_. Skipping...\n" + if ($settings{debug}); $skip = 1; last; } -@@ -981,7 +1037,7 @@ sub FindNewestFile +@@ -981,7 +1038,7 @@ sub FindNewestFile next if ($skip); } - unless ($settings{sillystrings_enable}) { + unless ($settings{sillystrings_enable} or $github) { if ($new_v =~ /[-_.]([A-Za-z]+[A-Za-z_-]{2,})$/) { my $str = $1; next if ( -@@ -1026,12 +1082,19 @@ sub FindNewestFile +@@ -1026,12 +1083,19 @@ sub FindNewestFile # Test our new version string + print "FindNewest: Comparing Old ($old_v) vs New ($new_v) version strings... \n" + if ($settings{debug}); + if ($new_v eq $old_v) { + print "FindNewest: Old ($old_v) == New ($new_v). Old found.\n" + if ($settings{debug}); $old_found = 1; } elsif (vercompare($new_v, $old_v)) { + print "FindNewest: Old ($old_v) < New ($new_v). New found.\n" + if ($settings{debug}); $new_found = 1; # Keep going until we find the newest version