Index: head/irc/py-limnoria/Makefile =================================================================== --- head/irc/py-limnoria/Makefile (revision 513452) +++ head/irc/py-limnoria/Makefile (revision 513453) @@ -1,74 +1,73 @@ # Created by: Kubilay Kocak # $FreeBSD$ PORTNAME= limnoria -PORTVERSION= 2018.09.09 -PORTREVISION= 2 +PORTVERSION= 2019.09.08 CATEGORIES= irc python PKGNAMEPREFIX= ${PYTHON_PKGNAMEPREFIX} MAINTAINER= koobs@FreeBSD.org COMMENT= Modified version of Supybot (IRC bot and framework) LICENSE= BSD3CLAUSE LICENSE_FILE= ${WRKSRC}/LICENSE.md TEST_DEPENDS= ${PYTHON_PKGNAMEPREFIX}feedparser>=0:textproc/py-feedparser@${PY_FLAVOR} \ ${PYTHON_PKGNAMEPREFIX}sqlite3>0:databases/py-sqlite3@${PY_FLAVOR} \ ${PYTHON_PKGNAMEPREFIX}dateutil>=0:devel/py-dateutil@${PY_FLAVOR} \ ${PYTHON_PKGNAMEPREFIX}pytz>=0:devel/py-pytz@${PY_FLAVOR} \ ${PYTHON_PKGNAMEPREFIX}mock>=0:devel/py-mock@${PY_FLAVOR} USES= python # 2.6+ USE_GITHUB= yes USE_PYTHON= autoplist concurrent distutils GH_ACCOUNT= ProgVal GH_PROJECT= Limnoria -GH_TAGNAME= master-2018-09-10 +GH_TAGNAME= master-2019-09-08 OPTIONS_DEFINE= ECDSA PROXY OPTIONS_DEFAULT= ECDSA ENCODING RSS TIME OPTIONS_GROUP= EXTRAS OPTIONS_GROUP_EXTRAS= ENCODING GPG RSS TIME OPTIONS_RADIO= AKA OPTIONS_RADIO_AKA= SQLITE SQLALCHEMY EXTRAS_DESC= Plugin Extras AKA_DESC= AKA Plugin: SQL Backend (none = plugin disabled) ECDSA_DESC= ECDSA SASL authentication PROXY_DESC= SOCKS Proxy connections ENCODING_DESC= Web Plugin: Better encoding detection (via chardet) GPG_DESC= GPG Plugin: GnuPG Authentication RSS_DESC= RSS Plugin: Feed parsing (via feedparser) TIME_DESC= Time Plugin: Extra time features (via dateutil/pytz) SQLALCHEMY_DESC= SQLAlchemy database support ECDSA_RUN_DEPENDS= ${PYTHON_PKGNAMEPREFIX}ecdsa>0:security/py-ecdsa@${PY_FLAVOR} PROXY_RUN_DEPENDS= ${PYTHON_PKGNAMEPREFIX}pysocks>=0:net/py-pysocks@${PY_FLAVOR} ENCODING_RUN_DEPENDS= ${PYTHON_PKGNAMEPREFIX}chardet>=0:textproc/py-chardet@${PY_FLAVOR} GPG_RUN_DEPENDS= ${PYTHON_PKGNAMEPREFIX}python-gnupg>=0:security/py-python-gnupg@${PY_FLAVOR} RSS_RUN_DEPENDS= ${PYTHON_PKGNAMEPREFIX}feedparser>=0:textproc/py-feedparser@${PY_FLAVOR} TIME_RUN_DEPENDS= ${PYTHON_PKGNAMEPREFIX}dateutil>=0:devel/py-dateutil@${PY_FLAVOR} \ ${PYTHON_PKGNAMEPREFIX}pytz>=0:devel/py-pytz@${PY_FLAVOR} SQLITE_RUN_DEPENDS= ${PYTHON_PKGNAMEPREFIX}sqlite3>0:databases/py-sqlite3@${PY_FLAVOR} SQLALCHEMY_RUN_DEPENDS= ${PYTHON_PKGNAMEPREFIX}sqlalchemy10>0:databases/py-sqlalchemy10@${PY_FLAVOR} NO_ARCH= yes # Needs to be installed first to work. do-test: @cd ${WRKSRC} && ${PYTHON_CMD} ${LOCALBASE}/bin/supybot-test \ --no-network \ --plugins-dir="./build/lib*/supybot/plugins/" \ --disable-multiprocessing .include Index: head/irc/py-limnoria/distinfo =================================================================== --- head/irc/py-limnoria/distinfo (revision 513452) +++ head/irc/py-limnoria/distinfo (revision 513453) @@ -1,3 +1,3 @@ -TIMESTAMP = 1538799817 -SHA256 (ProgVal-Limnoria-2018.09.09-master-2018-09-10_GH0.tar.gz) = 66af1f9bf02ae8a614451445efdcd5c782cc9881a3775de1f74d748c65626a6d -SIZE (ProgVal-Limnoria-2018.09.09-master-2018-09-10_GH0.tar.gz) = 963326 +TIMESTAMP = 1569902980 +SHA256 (ProgVal-Limnoria-2019.09.08-master-2019-09-08_GH0.tar.gz) = e7b8dec6befa6381d6f1ce75c240e132a0c020bbef15bb0e7b943a01d6790875 +SIZE (ProgVal-Limnoria-2019.09.08-master-2019-09-08_GH0.tar.gz) = 966922 Index: head/irc/py-limnoria/files/patch-plugins_Web_plugin.py =================================================================== --- head/irc/py-limnoria/files/patch-plugins_Web_plugin.py (revision 513452) +++ head/irc/py-limnoria/files/patch-plugins_Web_plugin.py (revision 513453) @@ -1,62 +1,179 @@ # https://github.com/ProgVal/Limnoria/pull/1371 # https://github.com/ProgVal/Limnoria/issues/1362 # https://github.com/ProgVal/Limnoria/issues/1359 ---- plugins/Web/plugin.py.orig 2019-09-29 03:00:58 UTC +From 7cf91ad703ab324e38c37fa2976626505f1d569a Mon Sep 17 00:00:00 2001 +From: Rodrigo +Date: Fri, 27 Sep 2019 01:48:36 -0300 +Subject: [PATCH 1/3] make clear in logs about charade dep|make behave the same + way in py3 or py2 without errors in any case (e.g. without charade installed) + +--- + plugins/Web/plugin.py | 44 +++++++++++++++++-------------------------- + 1 file changed, 17 insertions(+), 27 deletions(-) + +diff --git plugins/Web/plugin.py plugins/Web/plugin.py +index 1ac362714..0bdb68a4a 100644 +--- plugins/Web/plugin.py +++ plugins/Web/plugin.py -@@ -149,32 +149,31 @@ class Web(callbacks.PluginRegexp): - def getTitle(self, irc, url, raiseErrors): +@@ -150,37 +150,27 @@ def getTitle(self, irc, url, raiseErrors): size = conf.supybot.protocols.http.peekSize() timeout = self.registryValue('timeout') -- (target, text) = utils.web.getUrlTargetAndContent(url, size=size, + (target, text) = utils.web.getUrlTargetAndContent(url, size=size, - timeout=timeout) - try: +- try: - text = text.decode(utils.web.getEncoding(text) or 'utf8', - 'replace') - except UnicodeDecodeError: - pass -- parser = Title() - if minisix.PY3 and isinstance(text, bytes): - if raiseErrors: - irc.error(_('Could not guess the page\'s encoding. (Try ' - 'installing python-charade.)'), Raise=True) -+ (target, text) = utils.web.getUrlTargetAndContent(url, size=size,timeout=timeout) -+ encoding = utils.web.getEncoding(text) -+ if encoding is None: # Condition if charade not installed -+ self.log.info('Web plugin TitleSnarfer: Could not guess the page\'s' -+ ' encoding. (Try installing python-charade.)') -+ encoding = 'utf-8' # Assume UTF-8 and replace unknown chars to the UTF-8 codec for U+FFFD in the next hop -+ text = text.decode(utils.web.getEncoding(text) or 'utf-8','replace') -+ parser = Title() -+ try: -+ parser.feed(text) -+ except: -+ parser = Title() -+ parser.feed(bytes(text)) # Explicitly pack to bytes in encoding errors for (more) python2 compatibility -+ parser.close() -+ title = utils.str.normalizeWhitespace(''.join(parser.data).strip()) -+ if title: -+ return (target, title) - else: +- else: - return None -- parser.feed(text) ++ timeout=timeout) ++ encoding = utils.web.getEncoding(text) ++ if encoding is None: # Condition if charade not installed ++ self.log.info('Web plugin TitleSnarfer: Could not guess the page\'s' ++ ' encoding. (Try installing python-charade.)') ++ encoding = 'utf-8' # Assume UTF-8 and replace unknown chars to the UTF-8 codec for U+FFFD in the next hop + try: ++ text = text.decode(utils.web.getEncoding(text) or 'utf-8', 'replace') + parser = Title() + parser.feed(text) +- except UnicodeDecodeError: +- # Workaround for Python 2 +- # https://github.com/ProgVal/Limnoria/issues/1359 +- parser = Title() +- parser.feed(text.encode('utf8')) - parser.close() - title = utils.str.normalizeWhitespace(''.join(parser.data).strip()) - if title: - return (target, title) - elif raiseErrors: - if len(text) < size: - irc.error(_('That URL appears to have no HTML title.'), - Raise=True) -- else: ++ parser.close() ++ title = utils.str.normalizeWhitespace(''.join(parser.data).strip()) ++ if title: ++ return (target, title) + else: - irc.error(format(_('That URL appears to have no HTML title ' - 'within the first %S.'), size), Raise=True) + if len(text) < size: ++ self.log.info('Web plugin TitleSnarfer: ' ++ 'That URL appears to have no HTML title.') ++ except raiseErrors: # Can use raiseErrors here ? ++ irc.error(_('Web plugin TitleSnarfer encoding errors'), ++ Raise=True) + + @fetch_sandbox + def titleSnarfer(self, irc, msg, match): + +From 16247c5caf59f307b3d6910e31832fe30d0d583a Mon Sep 17 00:00:00 2001 +From: Rodrigo +Date: Fri, 27 Sep 2019 20:12:00 -0300 +Subject: [PATCH 2/3] more python2 compatibility + +--- + plugins/Web/plugin.py | 18 +++++++++++------- + 1 file changed, 11 insertions(+), 7 deletions(-) + +diff --git plugins/Web/plugin.py plugins/Web/plugin.py +index 0bdb68a4a..85c129de3 100644 +--- plugins/Web/plugin.py ++++ plugins/Web/plugin.py +@@ -150,16 +150,20 @@ def getTitle(self, irc, url, raiseErrors): + size = conf.supybot.protocols.http.peekSize() + timeout = self.registryValue('timeout') + (target, text) = utils.web.getUrlTargetAndContent(url, size=size, +- timeout=timeout) ++ timeout=timeout) + encoding = utils.web.getEncoding(text) +- if encoding is None: # Condition if charade not installed ++ if encoding is None: # Condition if charade not installed + self.log.info('Web plugin TitleSnarfer: Could not guess the page\'s' + ' encoding. (Try installing python-charade.)') +- encoding = 'utf-8' # Assume UTF-8 and replace unknown chars to the UTF-8 codec for U+FFFD in the next hop ++ encoding = 'utf-8' # Assume UTF-8 and replace unknown chars to the UTF-8 codec for U+FFFD in the next hop + try: +- text = text.decode(utils.web.getEncoding(text) or 'utf-8', 'replace') ++ text = text.decode(utils.web.getEncoding(text) or 'utf-8','replace') + parser = Title() +- parser.feed(text) ++ try: ++ parser.feed(text) ++ except: ++ parser = Title() ++ parser.feed(bytes(text)) # Explicitly pack to bytes in encoding errors for (more) python2 compatibility + parser.close() + title = utils.str.normalizeWhitespace(''.join(parser.data).strip()) + if title: +@@ -168,9 +172,9 @@ def getTitle(self, irc, url, raiseErrors): + if len(text) < size: + self.log.info('Web plugin TitleSnarfer: ' + 'That URL appears to have no HTML title.') +- except raiseErrors: # Can use raiseErrors here ? ++ except: + irc.error(_('Web plugin TitleSnarfer encoding errors'), +- Raise=True) ++ Raise=True) + + @fetch_sandbox + def titleSnarfer(self, irc, msg, match): + +From 5cac4a3cbda26186a330709901da2633914de415 Mon Sep 17 00:00:00 2001 +From: Rodrigo +Date: Sat, 28 Sep 2019 23:40:44 -0300 +Subject: [PATCH 3/3] Include '(target, text) = + utils.web.getUrlTargetAndContent(url, size=size,timeout=timeout)' in + try/catch block to avoid thread blocking | send http errors to log + +--- + plugins/Web/plugin.py | 23 +++++++++++------------ + 1 file changed, 11 insertions(+), 12 deletions(-) + +diff --git plugins/Web/plugin.py plugins/Web/plugin.py +index 85c129de3..e67944f29 100644 +--- plugins/Web/plugin.py ++++ plugins/Web/plugin.py +@@ -149,14 +149,13 @@ def noIgnore(self, irc, msg): + def getTitle(self, irc, url, raiseErrors): + size = conf.supybot.protocols.http.peekSize() + timeout = self.registryValue('timeout') +- (target, text) = utils.web.getUrlTargetAndContent(url, size=size, +- timeout=timeout) +- encoding = utils.web.getEncoding(text) +- if encoding is None: # Condition if charade not installed +- self.log.info('Web plugin TitleSnarfer: Could not guess the page\'s' +- ' encoding. (Try installing python-charade.)') +- encoding = 'utf-8' # Assume UTF-8 and replace unknown chars to the UTF-8 codec for U+FFFD in the next hop + try: ++ (target, text) = utils.web.getUrlTargetAndContent(url, size=size,timeout=timeout) ++ encoding = utils.web.getEncoding(text) ++ if encoding is None: # Condition if charade not installed ++ self.log.info('Web plugin TitleSnarfer: Could not guess the page\'s' ++ ' encoding. (Try installing python-charade.)') ++ encoding = 'utf-8' # Assume UTF-8 and replace unknown chars to the UTF-8 codec for U+FFFD in the next hop + text = text.decode(utils.web.getEncoding(text) or 'utf-8','replace') + parser = Title() + try: +@@ -170,11 +169,11 @@ def getTitle(self, irc, url, raiseErrors): + return (target, title) + else: + if len(text) < size: +- self.log.info('Web plugin TitleSnarfer: ' +- 'That URL appears to have no HTML title.') +- except: +- irc.error(_('Web plugin TitleSnarfer encoding errors'), +- Raise=True) + self.log.info('Web plugin TitleSnarfer: <' + url + '> appears to have no HTML title.') + else: + self.log.info('Web plugin TitleSnarfer: Could not retrieve title of <' + url + '>') + except Exception as e: + self.log.info('Web plugin TitleSnarfer: <' + str(e) + '> while trying to process <' + url +'>') @fetch_sandbox def titleSnarfer(self, irc, msg, match):