From bbb60ed077833599607af2bae6cd7a3372ecee3f Mon Sep 17 00:00:00 2001 From: Richard Harding Date: Tue, 17 Apr 2012 13:59:02 -0400 Subject: [PATCH 01/31] Update the name of the summary option to --- readability/readability.py | 12 ++++++------ tests/test_article_only.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/readability/readability.py b/readability/readability.py index ef8480df..168bc950 100755 --- a/readability/readability.py +++ b/readability/readability.py @@ -123,10 +123,10 @@ def title(self): def short_title(self): return shorten_title(self._html(True)) - def summary(self, html_partial=False): + def summary(self, enclose_with_html_tag=False): """Generate the summary of the html docuemnt - :param html_partial: return only the div of the document, don't wrap + :param enclose_with_html_tag: return only the div of the document, don't wrap in html and body tags. """ @@ -147,7 +147,7 @@ def summary(self, html_partial=False): if best_candidate: article = self.get_article(candidates, best_candidate, - html_partial=html_partial) + enclose_with_html_tag=enclose_with_html_tag) else: if ruthless: log.debug("ruthless removal did not work. ") @@ -180,7 +180,7 @@ def summary(self, html_partial=False): log.exception('error getting summary: ') raise Unparseable(str(e)), None, sys.exc_info()[2] - def get_article(self, candidates, best_candidate, html_partial=False): + def get_article(self, candidates, best_candidate, enclose_with_html_tag=False): # Now that we have the top candidate, look through its siblings for # content that might also be related. # Things like preambles, content split by ads that we removed, etc. @@ -188,7 +188,7 @@ def get_article(self, candidates, best_candidate, html_partial=False): 10, best_candidate['content_score'] * 0.2]) # create a new html document with a html->body->div - if html_partial: + if enclose_with_html_tag: output = fragment_fromstring('
') else: output = document_fromstring('
') @@ -219,7 +219,7 @@ def get_article(self, candidates, best_candidate, html_partial=False): if append: # We don't want to append directly to output, but the div # in html->body->div - if html_partial: + if enclose_with_html_tag: output.append(sibling) else: output.getchildren()[0].getchildren()[0].append(sibling) diff --git a/tests/test_article_only.py b/tests/test_article_only.py index 3a8f1c6e..a45cba5f 100644 --- a/tests/test_article_only.py +++ b/tests/test_article_only.py @@ -34,6 +34,6 @@ def test_si_sample_html_partial(self): """Using the si sample, make sure we can get the article alone.""" sample = load_sample('si-game.sample.html') doc = Document(sample, url='http://sportsillustrated.cnn.com/baseball/mlb/gameflash/2012/04/16/40630_preview.html') - res = doc.summary(html_partial=True) + res = doc.summary(enclose_with_html_tag=True) self.assertEqual('
]+' -htmlstrip = re.compile("<" # open - "([^>]+) " # prefix - "(?:%s) *" % ('|'.join(bad_attrs),) + # undesirable attributes - '= *(?:%s|%s|%s)' % (non_space, single_quoted, double_quoted) + # value +htmlstrip = re.compile("<" # open + "([^>]+) " # prefix + "(?:%s) *" % ('|'.join(bad_attrs),) + # undesirable attributes + '= *(?:%s|%s|%s)' % (non_space, single_quoted, double_quoted) + # value "([^>]*)" # postfix - ">" # end -, re.I) + ">", # end + re.I) + def clean_attributes(html): while htmlstrip.search(html): html = htmlstrip.sub('<\\1\\2>', html) return html + def normalize_spaces(s): - if not s: return '' - """replace any sequence of whitespace - characters with a single space""" + """replace any sequence of whitespace characters with a single space""" + if not s: + return '' return ' '.join(s.split()) + html_cleaner = Cleaner(scripts=True, javascript=True, comments=True, style=True, links=True, meta=False, add_nofollow=False, - page_structure=False, processing_instructions=True, embedded=False, - frames=False, forms=False, annoying_tags=False, remove_tags=None, + page_structure=False, processing_instructions=True, + embedded=False, frames=False, forms=False, + annoying_tags=False, remove_tags=None, remove_unknown_tags=False, safe_attrs_only=False) diff --git a/readability/debug.py b/readability/debug.py index a5e644d8..fbf13c06 100644 --- a/readability/debug.py +++ b/readability/debug.py @@ -1,25 +1,32 @@ +uids = {} + + def save_to_file(text, filename): f = open(filename, 'wt') - f.write('') + f.write(""" + """) f.write(text.encode('utf-8')) f.close() -uids = {} + def describe(node, depth=2): if not hasattr(node, 'tag'): return "[%s]" % type(node) name = node.tag - if node.get('id', ''): name += '#'+node.get('id') - if node.get('class', ''): - name += '.' + node.get('class').replace(' ','.') + if node.get('id', ''): + name += '#' + node.get('id') + if node.get('class', ''): + name += '.' + node.get('class').replace(' ', '.') if name[:4] in ['div#', 'div.']: name = name[3:] if name in ['tr', 'td', 'div', 'p']: if not node in uids: - uid = uids[node] = len(uids)+1 + uid = uids[node] = len(uids) + 1 else: uid = uids.get(node) name += "%02d" % (uid) if depth and node.getparent() is not None: - return name+' - '+describe(node.getparent(), depth-1) + return name + ' - ' + describe(node.getparent(), depth - 1) return name diff --git a/readability/encoding.py b/readability/encoding.py index d05b7f44..2207495a 100644 --- a/readability/encoding.py +++ b/readability/encoding.py @@ -1,21 +1,23 @@ import re import chardet + def get_encoding(page): text = re.sub(']*>\s*', ' ', page) enc = 'utf-8' if not text.strip() or len(text) < 10: - return enc # can't guess + return enc # can't guess try: diff = text.decode(enc, 'ignore').encode(enc) sizes = len(diff), len(text) - if abs(len(text) - len(diff)) < max(sizes) * 0.01: # 99% of utf-8 + # 99% of utf-8 + if abs(len(text) - len(diff)) < max(sizes) * 0.01: return enc except UnicodeDecodeError: pass res = chardet.detect(text) enc = res['encoding'] - #print '->', enc, "%.2f" % res['confidence'] + # print '->', enc, "%.2f" % res['confidence'] if enc == 'MacCyrillic': enc = 'cp1251' return enc diff --git a/readability/htmls.py b/readability/htmls.py index 97aa55b7..1f8b5e36 100644 --- a/readability/htmls.py +++ b/readability/htmls.py @@ -1,13 +1,17 @@ -from cleaners import normalize_spaces, clean_attributes -from encoding import get_encoding -from lxml.html import tostring import logging -import lxml.html import re +from lxml.html import document_fromstring +from lxml.html import HTMLParser +from lxml.html import tostring + +from cleaners import clean_attributes +from cleaners import normalize_spaces +from encoding import get_encoding + logging.getLogger().setLevel(logging.DEBUG) +utf8_parser = HTMLParser(encoding='utf-8') -utf8_parser = lxml.html.HTMLParser(encoding='utf-8') def build_doc(page): if isinstance(page, unicode): @@ -15,17 +19,20 @@ def build_doc(page): else: enc = get_encoding(page) page_unicode = page.decode(enc, 'replace') - doc = lxml.html.document_fromstring(page_unicode.encode('utf-8', 'replace'), parser=utf8_parser) + doc = document_fromstring( + page_unicode.encode('utf-8', 'replace'), + parser=utf8_parser) return doc + def js_re(src, pattern, flags, repl): return re.compile(pattern, flags).sub(src, repl.replace('$', '\\')) def normalize_entities(cur_title): entities = { - u'\u2014':'-', - u'\u2013':'-', + u'\u2014': '-', + u'\u2013': '-', u'—': '-', u'–': '-', u'\u00A0': ' ', @@ -39,27 +46,31 @@ def normalize_entities(cur_title): return cur_title + def norm_title(title): return normalize_entities(normalize_spaces(title)) + def get_title(doc): title = doc.find('.//title').text if not title: return '[no-title]' - + return norm_title(title) + def add_match(collection, text, orig): text = norm_title(text) if len(text.split()) >= 2 and len(text) >= 15: if text.replace('"', '') in orig.replace('"', ''): collection.add(text) + def shorten_title(doc): title = doc.find('.//title').text if not title: return '' - + title = orig = norm_title(title) candidates = set() @@ -71,13 +82,14 @@ def shorten_title(doc): if e.text_content(): add_match(candidates, e.text_content(), orig) - for item in ['#title', '#head', '#heading', '.pageTitle', '.news_title', '.title', '.head', '.heading', '.contentheading', '.small_header_red']: + for item in ['#title', '#head', '#heading', '.pageTitle', '.news_title', + '.title', '.head', '.heading', '.contentheading', '.small_header_red']: for e in doc.cssselect(item): if e.text: add_match(candidates, e.text, orig) if e.text_content(): add_match(candidates, e.text_content(), orig) - + if candidates: title = sorted(candidates, key=len)[-1] else: @@ -103,13 +115,16 @@ def shorten_title(doc): return title + def get_body(doc): - [ elem.drop_tree() for elem in doc.xpath('.//script | .//link | .//style') ] + [elem.drop_tree() for elem in doc.xpath('.//script | .//link | .//style')] raw_html = unicode(tostring(doc.body or doc)) cleaned = clean_attributes(raw_html) try: #BeautifulSoup(cleaned) #FIXME do we really need to try loading it? return cleaned - except Exception: #FIXME find the equivalent lxml error - logging.error("cleansing broke html content: %s\n---------\n%s" % (raw_html, cleaned)) + except Exception: # FIXME find the equivalent lxml error + logging.error("cleansing broke html content: %s\n---------\n%s" % ( + raw_html, + cleaned)) return raw_html From a6361854a963414518714767b349634a36e88f08 Mon Sep 17 00:00:00 2001 From: Richard Harding Date: Tue, 17 Apr 2012 14:35:50 -0400 Subject: [PATCH 03/31] Update the setup.py, tweak makefile for building/cleaning/tests --- Makefile | 12 ++++++++---- setup.py | 23 +++++++++++++++++------ 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 222f8507..e9194438 100644 --- a/Makefile +++ b/Makefile @@ -10,11 +10,11 @@ NOSE := bin/nosetests # Tests rule! # ########### .PHONY: test -test: venv develop $(NOSE) +test: venv $(NOSE) $(NOSE) --with-id -s tests $(NOSE): - $(PIP) install nose pep8 coverage + $(PY) setup.py test # ####### # INSTALL @@ -30,8 +30,9 @@ bin/python: clean_venv: rm -rf bin include lib local man -develop: lib/python*/site-packages/bookie-api.egg-link -lib/python*/site-packages/bookie-api.egg-link: +.PHONY: develop +develop: lib/python*/site-packages/readability_lxml.egg-link +lib/python*/site-packages/readability_lxml.egg-link: $(PY) setup.py develop @@ -40,6 +41,9 @@ lib/python*/site-packages/bookie-api.egg-link: # ########### .PHONY: clean_all clean_all: clean_venv + if [ -d dist ]; then \ + rm -r dist; \ + fi # ########### diff --git a/setup.py b/setup.py index 02251fcb..fd68a814 100755 --- a/setup.py +++ b/setup.py @@ -1,22 +1,33 @@ #!/usr/bin/env python from setuptools import setup, find_packages +version = "0.2.5" +install_requires = [ + "chardet", + "lxml", +] +tests_require = [ + 'coverage', + 'nose', + 'pep8', +] + setup( name="readability-lxml", - version="0.2.5", + version=version, author="Yuri Baburov", author_email="burchik@gmail.com", description="fast python port of arc90's readability tool", - test_suite = "tests.test_article_only", + keywords='readable read parse html document readability', long_description=open("README").read(), license="Apache License 2.0", url="http://github.com/buriy/python-readability", package_dir={'': 'readability'}, packages=find_packages('readability', exclude=["*.tests", "*.tests.*"]), - install_requires=[ - "chardet", - "lxml" - ], + install_requires=install_requires, + extras_require={'test': tests_require}, + test_suite = "nose.collector", + # test_suite="tests.test_article_only", classifiers=[ "Environment :: Web Environment", "Intended Audience :: Developers", From d11b928504aea7787eb7039edc8e63edf224b1ba Mon Sep 17 00:00:00 2001 From: Richard Harding Date: Tue, 17 Apr 2012 14:37:34 -0400 Subject: [PATCH 04/31] Add credits file --- CREDITS | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 CREDITS diff --git a/CREDITS b/CREDITS new file mode 100644 index 00000000..1a886657 --- /dev/null +++ b/CREDITS @@ -0,0 +1,10 @@ +Yuri Baburov +facundo +gfxmonk +Jan Weiß +Jerry Charumilind +Laurent Peuch +Lee Semel +Rick Harding +Sean Brant +Tim Cuthbertson From 62e153eaf8acdf1544eb0534ca648427458dd244 Mon Sep 17 00:00:00 2001 From: Richard Harding Date: Tue, 17 Apr 2012 21:28:25 -0400 Subject: [PATCH 05/31] Start to update the module for a better directory layout --- Makefile | 4 +- setup.py | 22 +- tests/__init__.py | 0 tests/samples/si-game.sample.html | 762 ------------------------------ tests/test_article_only.py | 39 -- 5 files changed, 16 insertions(+), 811 deletions(-) delete mode 100644 tests/__init__.py delete mode 100644 tests/samples/si-game.sample.html delete mode 100644 tests/test_article_only.py diff --git a/Makefile b/Makefile index e9194438..2472d1bc 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ NOSE := bin/nosetests # ########### .PHONY: test test: venv $(NOSE) - $(NOSE) --with-id -s tests + $(NOSE) --with-id -s src/tests $(NOSE): $(PY) setup.py test @@ -28,7 +28,7 @@ bin/python: .PHONY: clean_venv clean_venv: - rm -rf bin include lib local man + rm -rf bin include lib local man share .PHONY: develop develop: lib/python*/site-packages/readability_lxml.egg-link diff --git a/setup.py b/setup.py index fd68a814..178f92a8 100755 --- a/setup.py +++ b/setup.py @@ -21,17 +21,23 @@ keywords='readable read parse html document readability', long_description=open("README").read(), license="Apache License 2.0", - url="http://github.com/buriy/python-readability", - package_dir={'': 'readability'}, - packages=find_packages('readability', exclude=["*.tests", "*.tests.*"]), - install_requires=install_requires, - extras_require={'test': tests_require}, - test_suite = "nose.collector", - # test_suite="tests.test_article_only", classifiers=[ "Environment :: Web Environment", "Intended Audience :: Developers", "Operating System :: OS Independent", "Programming Language :: Python", - ], + ], + url="http://github.com/buriy/python-readability", + packages=find_packages('src'), + package_dir = {'': 'src'}, + include_package_data=True, + zip_safe=False, + install_requires=install_requires, + extras_require={'test': tests_require}, + test_suite = "nose.collector", + entry_points={ + 'console_scripts': + ['readability=readability_lxml:client.main'] + }, + ) diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/samples/si-game.sample.html b/tests/samples/si-game.sample.html deleted file mode 100644 index fab4f4fe..00000000 --- a/tests/samples/si-game.sample.html +++ /dev/null @@ -1,762 +0,0 @@ - - - - - - Detroit Tigers vs. Kansas City Royals - Preview - April 16, 2012 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - - - - -
- - - -
-
    -
  • - - - -
    Get the Wildcats Championship Package
    -
    Get the Wildcats Championship Package
    - - - - - -
    • -
  • -
    • -
  • - - - - -
    Get MLB 2K 12 FREE
    -
    Get MLB 2K 12 FREE
    - - - -
    • -
-
-
-
- - - -
-
SI.com Home -
-
-
-
MLB GAMEFLASH
-
Scores | Teams | Players | Player News | Standings | Probables | Schedules | Stats | Transactions | Injuries | Tickets | MLB.TV -
-
-
-
- - - -
- -
- - - - -
-
- -
-
- - - -
- - -
- -
- - -
-
- -
- - - - - -
-
-
-
  - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 123456789RHE
TIGERS            
ROYALS            
-
-
-
    -
  1. 8:10 PM ET - -
    2. -
- -
-
-
- - - -
- - - - - - -
PREVIEWMATCHUPFAN COMMENTS
-
- - - -
-
-
- - -

Tigers-Royals Preview

-

- - Justin Verlander - has pitched well in each of his first two starts, though he doesn't have a win to show for those efforts. - -

-

- He hasn't had much trouble earning victories against the - Kansas City Royals - . - -

-

- Verlander looks to continue his mastery of the Royals when the - Detroit Tigers - visit Kauffman Stadium in the opener of a three-game series Monday night. - -

-

- The reigning AL - Cy Young - winner and MVP had a 2-0 lead through eight innings in both of his outings, but the Tigers weren't able to hold the lead. - -

-

Verlander (0-1, 2.20 ERA) allowed two hits before running into trouble in the ninth against Tampa Bay on Wednesday, getting - charged with four runs in 8 1-3 innings of a 4-2 defeat. -

"Once a couple guys got on, really the first time I've cranked it up like that - and lost a little bit of my consistency that - I'd had all day," Verlander said. "It's inexcusable. This loss rests solely on my shoulders." -

The right-hander did his part in his opening-day start against Boston on April 5, allowing two hits before the bullpen faltered. - Detroit ended up winning 3-2 with a run in the bottom of the ninth, though Verlander didn't earn a decision. -

-

That hasn't been the case in his last four starts against the Royals, winning each with a 1.82 ERA. Verlander is 13-2 with - a 2.40 ERA in 19 career starts versus Kansas City, and another win will give him more victories than he has against any other - team. He's also beaten Cleveland 13 times. -

-

Verlander is 8-2 with a 1.82 ERA lifetime at Kauffman Stadium, where the Royals (3-6) were swept in a three-game series against - the Indians with Sunday's 13-7 loss. -

-

- - Billy Butler - , who is 14 for 39 (.359) with two homers off Verlander, had an RBI single and is hitting .364 with four doubles and a homer - during a five-game hitting streak. - -

-

- Royals pitchers allowed seven home runs, 17 extra-base hits and 32 runs in the series, and manager - Ned Yost - turned to outfielder - Mitch Maier - in the ninth to pitched a scoreless inning Sunday. - -

"Let's hope it doesn't happen again," Maier said. "I don't like to be put in that situation, but we needed an inning." -

- Kansas City will look to bounce back with the help of another solid outing from - Danny Duffy - (1-0, 0.00), who allowed one hit and struck out eight in six innings of a 3-0 win over Oakland on Tuesday. - -

-

The left-hander will be seeking his first win against Detroit after going 0-2 with a 5.63 ERA in three starts versus the Tigers - as a rookie. -

-

- - Gerald Laird - was a triple short of the cycle and helped the Tigers (6-3) salvage the finale of a three-game series with a 5-2 victory over - Chicago on Sunday. - -

-

- - Rick Porcello - allowed one run in 7 2-3 innings to give Detroit's starting rotation its first victory. - -

"All the other starters have pitched well," Porcello said. "It's just the way it's happened so far." -

Verlander allowed three runs in seven innings of a 4-3 win over the Royals on Aug. 6, beating Duffy, who gave up three runs - over five. -

- -

- © 2011 STATS LLC STATS, Inc - -

-
-
- -
-
-
-
- -
-
-
-
-
TURN COMMENTS OFF
-
TURN COMMENTS ON
-
-
-
-
-
- - - -
- -
-
-
-
-
SI.com
-
Hot Topics: Peter King: MMQB NHL Playoffs Bobby Petrino Bobby Valentine Roger Clemens MLB Power Rankings Jackie Robinson
-
-
- -
-
- - Turner - SI Digital - -
Terms under which this service is provided to you. Read our privacy guidelines, your California privacy rights, and ad choices. -
-
-
SI CoverRead All ArticlesBuy Cover Reprint -
-
-
- - - - -
-
-
- - - - - - - -
-
- - - - - - - - - - -
- -
-
-
- - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/tests/test_article_only.py b/tests/test_article_only.py deleted file mode 100644 index a45cba5f..00000000 --- a/tests/test_article_only.py +++ /dev/null @@ -1,39 +0,0 @@ -import os -import unittest - -from readability import Document - - -SAMPLES = os.path.join(os.path.dirname(__file__), 'samples') - - -def load_sample(filename): - """Helper to get the content out of the sample files""" - return open(os.path.join(SAMPLES, filename)).read() - - -class TestArticleOnly(unittest.TestCase): - """The option to not get back a full html doc should work - - Given a full html document, the call can request just divs of processed - content. In this way the developer can then wrap the article however they - want in their own view or application. - - """ - - def test_si_sample(self): - """Using the si sample, load article with only opening body element""" - sample = load_sample('si-game.sample.html') - doc = Document( - sample, - url='http://sportsillustrated.cnn.com/baseball/mlb/gameflash/2012/04/16/40630_preview.html') - res = doc.summary() - self.assertEqual('
+ + + + + Detroit Tigers vs. Kansas City Royals - Preview - April 16, 2012 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + +
+ + + +
+
    +
  • + + + +
    Get the Wildcats Championship Package
    +
    Get the Wildcats Championship Package
    + + + + + +
    • +
  • +
    • +
  • + + + + +
    Get MLB 2K 12 FREE
    +
    Get MLB 2K 12 FREE
    + + + +
    • +
+
+
+
+ + + +
+
SI.com Home +
+
+
+
MLB GAMEFLASH
+
Scores | Teams | Players | Player News | Standings | Probables | Schedules | Stats | Transactions | Injuries | Tickets | MLB.TV +
+
+
+
+ + + +
+ +
+ + + + +
+
+ +
+
+ + + +
+ + +
+ +
+ + +
+
+ +
+ + + + + +
+
+
+
  + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
 123456789RHE
TIGERS            
ROYALS            
+
+
+
    +
  1. 8:10 PM ET + +
    2. +
+ +
+
+
+ + + +
+ + + + + + +
PREVIEWMATCHUPFAN COMMENTS
+
+ + + +
+
+
+ + +

Tigers-Royals Preview

+

+ + Justin Verlander + has pitched well in each of his first two starts, though he doesn't have a win to show for those efforts. + +

+

+ He hasn't had much trouble earning victories against the + Kansas City Royals + . + +

+

+ Verlander looks to continue his mastery of the Royals when the + Detroit Tigers + visit Kauffman Stadium in the opener of a three-game series Monday night. + +

+

+ The reigning AL + Cy Young + winner and MVP had a 2-0 lead through eight innings in both of his outings, but the Tigers weren't able to hold the lead. + +

+

Verlander (0-1, 2.20 ERA) allowed two hits before running into trouble in the ninth against Tampa Bay on Wednesday, getting + charged with four runs in 8 1-3 innings of a 4-2 defeat. +

"Once a couple guys got on, really the first time I've cranked it up like that - and lost a little bit of my consistency that + I'd had all day," Verlander said. "It's inexcusable. This loss rests solely on my shoulders." +

The right-hander did his part in his opening-day start against Boston on April 5, allowing two hits before the bullpen faltered. + Detroit ended up winning 3-2 with a run in the bottom of the ninth, though Verlander didn't earn a decision. +

+

That hasn't been the case in his last four starts against the Royals, winning each with a 1.82 ERA. Verlander is 13-2 with + a 2.40 ERA in 19 career starts versus Kansas City, and another win will give him more victories than he has against any other + team. He's also beaten Cleveland 13 times. +

+

Verlander is 8-2 with a 1.82 ERA lifetime at Kauffman Stadium, where the Royals (3-6) were swept in a three-game series against + the Indians with Sunday's 13-7 loss. +

+

+ + Billy Butler + , who is 14 for 39 (.359) with two homers off Verlander, had an RBI single and is hitting .364 with four doubles and a homer + during a five-game hitting streak. + +

+

+ Royals pitchers allowed seven home runs, 17 extra-base hits and 32 runs in the series, and manager + Ned Yost + turned to outfielder + Mitch Maier + in the ninth to pitched a scoreless inning Sunday. + +

"Let's hope it doesn't happen again," Maier said. "I don't like to be put in that situation, but we needed an inning." +

+ Kansas City will look to bounce back with the help of another solid outing from + Danny Duffy + (1-0, 0.00), who allowed one hit and struck out eight in six innings of a 3-0 win over Oakland on Tuesday. + +

+

The left-hander will be seeking his first win against Detroit after going 0-2 with a 5.63 ERA in three starts versus the Tigers + as a rookie. +

+

+ + Gerald Laird + was a triple short of the cycle and helped the Tigers (6-3) salvage the finale of a three-game series with a 5-2 victory over + Chicago on Sunday. + +

+

+ + Rick Porcello + allowed one run in 7 2-3 innings to give Detroit's starting rotation its first victory. + +

"All the other starters have pitched well," Porcello said. "It's just the way it's happened so far." +

Verlander allowed three runs in seven innings of a 4-3 win over the Royals on Aug. 6, beating Duffy, who gave up three runs + over five. +

+ +

+ © 2011 STATS LLC STATS, Inc + +

+
+
+ +
+
+
+
+ +
+
+
+
+
TURN COMMENTS OFF
+
TURN COMMENTS ON
+
+
+
+
+
+ + + +
+ +
+
+
+
+
SI.com
+
Hot Topics: Peter King: MMQB NHL Playoffs Bobby Petrino Bobby Valentine Roger Clemens MLB Power Rankings Jackie Robinson
+
+
+ +
+
+ + Turner - SI Digital + +
Terms under which this service is provided to you. Read our privacy guidelines, your California privacy rights, and ad choices. +
+
+
SI CoverRead All ArticlesBuy Cover Reprint +
+
+
+ + + + +
+
+
+ + + + + + + +
+
+ + + + + + + + + + +
+ +
+
+
+ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/tests/test_article_only.py b/src/tests/test_article_only.py new file mode 100644 index 00000000..a45cba5f --- /dev/null +++ b/src/tests/test_article_only.py @@ -0,0 +1,39 @@ +import os +import unittest + +from readability import Document + + +SAMPLES = os.path.join(os.path.dirname(__file__), 'samples') + + +def load_sample(filename): + """Helper to get the content out of the sample files""" + return open(os.path.join(SAMPLES, filename)).read() + + +class TestArticleOnly(unittest.TestCase): + """The option to not get back a full html doc should work + + Given a full html document, the call can request just divs of processed + content. In this way the developer can then wrap the article however they + want in their own view or application. + + """ + + def test_si_sample(self): + """Using the si sample, load article with only opening body element""" + sample = load_sample('si-game.sample.html') + doc = Document( + sample, + url='http://sportsillustrated.cnn.com/baseball/mlb/gameflash/2012/04/16/40630_preview.html') + res = doc.summary() + self.assertEqual('
Date: Tue, 17 Apr 2012 21:40:32 -0400 Subject: [PATCH 09/31] Move version to package --- src/readability_lxml/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/readability_lxml/__init__.py b/src/readability_lxml/__init__.py index 8822a512..7e737968 100644 --- a/src/readability_lxml/__init__.py +++ b/src/readability_lxml/__init__.py @@ -1 +1,3 @@ -from .readability import Document +VERSION = '0.2.5' + +import client From 509aed0d9f201065a2a5b8dc1174bbc6590863a0 Mon Sep 17 00:00:00 2001 From: Richard Harding Date: Tue, 17 Apr 2012 21:49:59 -0400 Subject: [PATCH 10/31] Move the module into the readable_lxml space so that we can actually import it nicely. --- .gitignore | 1 + setup.py | 1 + src/readability_lxml/client.py | 2 +- src/tests/test_article_only.py | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 84fca1f2..fb5febac 100644 --- a/.gitignore +++ b/.gitignore @@ -7,5 +7,6 @@ dist /lib /local /man +/share nosetests.xml .coverage diff --git a/setup.py b/setup.py index 178f92a8..6060bf32 100755 --- a/setup.py +++ b/setup.py @@ -33,6 +33,7 @@ include_package_data=True, zip_safe=False, install_requires=install_requires, + tests_require=tests_require, extras_require={'test': tests_require}, test_suite = "nose.collector", entry_points={ diff --git a/src/readability_lxml/client.py b/src/readability_lxml/client.py index 5a1e3716..e0466ad8 100644 --- a/src/readability_lxml/client.py +++ b/src/readability_lxml/client.py @@ -1,7 +1,7 @@ import argparse import sys -from readability_lxmly import VERSION +from readability_lxml import VERSION from readability_lxml.readability import Document diff --git a/src/tests/test_article_only.py b/src/tests/test_article_only.py index a45cba5f..0d7ddc37 100644 --- a/src/tests/test_article_only.py +++ b/src/tests/test_article_only.py @@ -1,7 +1,7 @@ import os import unittest -from readability import Document +from readability_lxml.readability import Document SAMPLES = os.path.join(os.path.dirname(__file__), 'samples') From f5451356ee72b88232be6748ecc4c7f234aa6fc7 Mon Sep 17 00:00:00 2001 From: Richard Harding Date: Tue, 17 Apr 2012 22:05:27 -0400 Subject: [PATCH 11/31] Make sure we update both version strings until we can figure out how to pull it into the setup.py by magic --- Makefile | 2 +- setup.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2472d1bc..e2e69416 100644 --- a/Makefile +++ b/Makefile @@ -59,4 +59,4 @@ upload: .PHONY: version_update version_update: - $(EDITOR) setup.py + $(EDITOR) setup.py src/readability_lxml/__init__.py diff --git a/setup.py b/setup.py index 6060bf32..637813e5 100755 --- a/setup.py +++ b/setup.py @@ -12,6 +12,7 @@ 'pep8', ] + setup( name="readability-lxml", version=version, From ac5ef73e710acc61b54435d53a1060c5ce30bde7 Mon Sep 17 00:00:00 2001 From: Richard Harding Date: Wed, 18 Apr 2012 14:45:46 -0400 Subject: [PATCH 12/31] Update cli client commands, add debugging to test server --- src/readability_lxml/client.py | 23 +++++++++++++---------- src/readability_lxml/encoding.py | 6 ++++++ src/readability_lxml/htmls.py | 5 +++++ 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/readability_lxml/client.py b/src/readability_lxml/client.py index e0466ad8..22ba55e5 100644 --- a/src/readability_lxml/client.py +++ b/src/readability_lxml/client.py @@ -16,11 +16,6 @@ def parse_args(): default=False, help="Increase logging verbosity to DEBUG.") - parser.add_argument('-u', '--url', - action='store', - default=None, - help="Indicate that this is a url path.") - parser.add_argument('path', metavar='P', type=str, nargs=1, help="The url or file path to process in readable form.") @@ -31,19 +26,27 @@ def parse_args(): def main(): args = parse_args() - target = None - if args.url: + target = args.path[0] + + if target.startswith('http') or target.startswith('www'): + is_url = True + url = target + else: + is_url = False + url = None + + if is_url: import urllib - target = urllib.urlopen(args.path[0]) + target = urllib.urlopen(target) else: - target = open(args.path[0], 'rt') + target = open(target, 'rt') enc = sys.__stdout__.encoding or 'utf-8' try: doc = Document(target.read(), debug=args.verbose, - url=args.url) + url=url) print doc.summary().encode(enc, 'replace') finally: diff --git a/src/readability_lxml/encoding.py b/src/readability_lxml/encoding.py index 2207495a..b80a354c 100644 --- a/src/readability_lxml/encoding.py +++ b/src/readability_lxml/encoding.py @@ -1,8 +1,14 @@ +import logging import re import chardet +LOG = logging.getLogger() + + def get_encoding(page): + LOG.info('GET ENCODING') + LOG.info(type(page)) text = re.sub(']*>\s*', ' ', page) enc = 'utf-8' if not text.strip() or len(text) < 10: diff --git a/src/readability_lxml/htmls.py b/src/readability_lxml/htmls.py index 1f8b5e36..97414277 100644 --- a/src/readability_lxml/htmls.py +++ b/src/readability_lxml/htmls.py @@ -13,7 +13,12 @@ utf8_parser = HTMLParser(encoding='utf-8') +LOG = logging.getLogger() + + def build_doc(page): + LOG.info('BUILD DOC') + LOG.info(type(page)) if isinstance(page, unicode): page_unicode = page else: From 2ee2fe953675c45bff3a9dbea6b028cf1bbd8115 Mon Sep 17 00:00:00 2001 From: Richard Harding Date: Wed, 18 Apr 2012 15:02:01 -0400 Subject: [PATCH 13/31] Throw some checking aroud the build_doc --- src/readability_lxml/encoding.py | 2 -- src/readability_lxml/htmls.py | 6 ++++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/readability_lxml/encoding.py b/src/readability_lxml/encoding.py index b80a354c..b8579558 100644 --- a/src/readability_lxml/encoding.py +++ b/src/readability_lxml/encoding.py @@ -7,8 +7,6 @@ def get_encoding(page): - LOG.info('GET ENCODING') - LOG.info(type(page)) text = re.sub(']*>\s*', ' ', page) enc = 'utf-8' if not text.strip() or len(text) < 10: diff --git a/src/readability_lxml/htmls.py b/src/readability_lxml/htmls.py index 97414277..a4016256 100644 --- a/src/readability_lxml/htmls.py +++ b/src/readability_lxml/htmls.py @@ -17,8 +17,10 @@ def build_doc(page): - LOG.info('BUILD DOC') - LOG.info(type(page)) + """Requires that the `page` not be None""" + if page is None: + LOG.error("Page content is None, can't build_doc") + return '' if isinstance(page, unicode): page_unicode = page else: From dc86283d83ecdfab359c786546745924b5cdf14c Mon Sep 17 00:00:00 2001 From: Richard Harding Date: Wed, 18 Apr 2012 20:12:54 -0400 Subject: [PATCH 14/31] Add a sample articler tester and a nyt sample to process --- src/tests/samples/nyt.sample.html | 944 ++++++++++++++++++++++++++++++ src/tests/test_sample_articles.py | 31 + 2 files changed, 975 insertions(+) create mode 100644 src/tests/samples/nyt.sample.html create mode 100644 src/tests/test_sample_articles.py diff --git a/src/tests/samples/nyt.sample.html b/src/tests/samples/nyt.sample.html new file mode 100644 index 00000000..19caa35f --- /dev/null +++ b/src/tests/samples/nyt.sample.html @@ -0,0 +1,944 @@ + + + + + + + + + + +Watches Are Rediscovered by the Cellphone Generation - NYTimes.com + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+
+
+ +
+ +Click Here +
+ +
+
+
+ + + + + +
+
+
+
+ + +
+ +

+ +Fashion & Style +

+ +
+
+ +
+ + + +
+ + +
+ + +
+
+
+
+ +
+

Watches Are Rediscovered by the Cellphone Generation

+
+
Elizabeth Lippman for The New York Times
+
+
By ALEX WILLIAMS
+ +
Published: July 6, 2011
+ +
+
+
+
    + +
  • +Print +
    • +
  • + Single Page +
    • + + + +
  • + + + + + + + + +
    +Reprints +
    • +     +
+
+ +
+
+
+
+ + + + + + + + + + + + + +

+MICHAEL WILLIAMS, who runs A Continuous Lean, a men’s style blog, ditched his Timex when he got his first cellphone in 2001.

+
+
+ + +
+

The Collection: A New Fashion App for the iPad

+
The Collection
+

A one-stop destination for Times fashion coverage and the latest from the runways.

+ +
+ +
+
+ +

Follow Us on Twitter

+
NYTimesFashion on Twitter
+

Follow @NYTimesfashion for fashion, beauty and lifestyle news and headlines.

+ +
+ + +
+
+
+
Enlarge This Image
+ + + +
+
Peter DaSilva for The New York Times
+

ANALOG LOVE Andy Greenblatt of Watchismo, an online retailer that has seen more interest from a generation of men who rely on their  cellphones for the time.

+
+ +
+
+

+Tyler Thoreson, the head of men’s editorial for Gilt Man, the flash sale Web site, often kept his forgettable watches stashed in a drawer.

+And Eddy Chai, an owner of Odin New York, a downtown men’s boutique, gave up wearing watches regularly in his mid-20s, when he outgrew his Casio.

+But after going watch-free for much of the last decade, the three men — all in their 30s and considered style influencers — are turning back time. Mr. Thoreson, 38, is shopping for a vintage gold IWC with a white dial or a Rolex GMT-Master. Mr. Chai, 38, has been wearing a vintage Rolex, loosely dangling around his wrist, “not as a timepiece, but as a piece of jewelry,” he said.

+And Mr. Williams, 32, splurged on three watches: an IWC Portuguese, a Rolex GMT-Master II and an Omega Speedmaster, also known as the “moon watch,” since that is what Apollo astronauts wore.

+“The men’s-wear set has recently rediscovered the joy of proper mechanical timepieces,” Mr. Williams said. “Right now there is no clearer indication of cool than wearing a watch. If it was your grandfather’s bubbleback Rolex, even better.”

+As recently as a half-decade ago, time seemed to be running out for the wristwatch. With cellphones, iPods and other clock-equipped devices becoming ubiquitous, armchair sociologists were writing off the wristwatch as an antique, joining VHS tapes, Walkman players and pocket calculators on the slag heap of outmoded gadgets.

+The wristwatch “may be going the way of the abacus,” declared a news article in The Sacramento Bee in 2006. The Times of London had it “going the same way as the sundial.” The Boston Globe, in a 2005 lifestyle feature, was more definitive: “Anyone who needs to know the time these days would be wise to ask someone over the age of 30. To most young people, the wristwatch is an obsolete artifact.”

+Or, not.

+The “sundial” of the wrist is experiencing an uptick among members of the supposed lost generation, particularly by heritage-macho types in their 20s and 30s who are drawn to the wristwatch’s retro appeal, just as they have seized on straight razors, selvedge denim and vintage vinyl.

+"It’s an understated statement about your station in life, your taste level,” Mr. Thoreson said.

+He got a taste of the pent-up demand last fall, when Gilt organized a high-end vintage watch sale with Benjamin Clymer, 28, who runs an online magazine for watch enthusiasts called Hodinkee.com. (Mr. Clymer, a former UBS manager, said his site attracts 250,000 unique visitors a month, more than half of them under 40.)

+Fourteen of the 17 watches, with an average price of $4,800, sold in the first six hours. Gilt now holds a watch sale every month. “In certain circles,” Mr. Thoreson said, “if you don’t have a substantial timepiece with some pedigree, you feel like you’re missing out on something.”

+To be fair, the doomsayers were not entirely wrong. Few people actually need a watch to tell time anymore. Melanie Shreffler, editor in chief of Ypulse, a Web site and market research company that tracks youth trends, observed, “even the high school and college students who wear watches usually pull out their cellphones to check the time.”

+But that’s the point. A watch these days may strike some people as an impractical, frivolous and often costly way to express individual style. But that is just another way of saying that it’s fashion.

+“Considering how casual most people dress on a day-to-day basis, a glamorous watch is one of the few accessories that can be at once sporty, luxurious and utilitarian,” the designer Michael Kors wrote in an e-mail. Mr. Kors has a line of oversize chronographs, manufactured by Fossil, that is popular among women (they are a current must-have accessory among under-30 fashion assistant types in Manhattan).

+For a generation raised on Game Boys, however, the appeal seems to go a little deeper than just a desire for another fashion accessory. In a world surrounded by ever-glowing LCD screens, there’s an analog chic to wearing a mechanical instrument.

+“A cool machine that is all moving parts has got to be intrinsically interesting to someone born into this generation, because there’s just nothing like that in their life,” said Mitch Greenblatt, a founder, with his brother, Andy, of Watchismo, a California online retailer of design-forward watches.

+
+ + + +
+ +
+
+
+
+
+
A version of this article appeared in print on July 7, 2011, on page E1 of the New York edition with the headline: Tick, Tick, Tick, Chic.
+
+
+
+
+ +
+
+
+
+
+
+
    + +
  • +Print +
    • +
  • + Single Page +
    • + + + +
  • + + + + + + + + +
    +Reprints +
    • +     +
+
+
+
+ +
+
+ + + +
+ +
+
+
+
+
+
+
+
Get Free E-mail Alerts on These Topics
+
+ + + + + +
+
+
+
+ +
+ + + + + +
+
+
+
+ +
+ +
+
+ +
+
+ +
+ + + +
+ +
+
+ +
+ + +
+ +
+
+ +
+
+ +
+
+
+ +
+
+ +
+
+ + + +
+
+ +
+

+Advertisements

+
+
+
+
+ +
+
+
+
+
+
+
+
+
+ +
+
+ +
+ + + +
+ +
+
+ +
+
+ + + +
+ +
+ +
+
+ +
+ + + + + + +
+
+ + +
+ +
+
 
+

+ Inside NYTimes.com

+
+ + +
+ + + + + + + + + + + + + + + + + +
+
+
+ Business » +
+
+ Carriers Warn of Crisis in Mobile Spectrum +
+
Carriers Warn of Crisis in Mobile Spectrum
+
+ 		+
+
+ Dining & Wine » +
+
+ The Pizza Issue +
+
The Pizza Issue
+
+ 		+
+
Opinion »
+

Is Veganism For Everyone?

+

It’s a commitment, so Room for Debate asks, does the regimen really have enough benefits to benefit all body types and mindsets?

+
+ 		+
+
+ World » +
+
+ Shipwreck Cuts Two Ways for Island’s Tourism +
+
Shipwreck Cuts Two Ways for Island’s Tourism
+
+ 		+
+
+ Opinion »
+
+ Ann Patchett: And the Winner Isn’t ... +
+
Ann Patchett: And the Winner Isn’t ...
+
+ 		+
+
+ Dining & Wine » +
+
+ Restaurant Review: Alison Eighteen +
+
Restaurant Review: Alison Eighteen
+
+
+
+ +
+ +
+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/tests/test_sample_articles.py b/src/tests/test_sample_articles.py new file mode 100644 index 00000000..a52f5bdc --- /dev/null +++ b/src/tests/test_sample_articles.py @@ -0,0 +1,31 @@ +"""Process all of the samples and make sure that process without error.""" +import os +import unittest + +from readability_lxml.readability import Document + + +SAMPLES = os.path.join(os.path.dirname(__file__), 'samples') + +sample_list = [ + 'nyt.sample.html', + 'si-game.sample.html', +] + + +def load_sample(filename): + """Helper to get the content out of the sample files""" + return open(os.path.join(SAMPLES, filename)).read() + + +def test_processes(): + for article in sample_list: + yield process_article, article + + +def process_article(article): + sample = load_sample(article) + doc = Document(sample) + res = doc.summary() + failed_msg = "Failed to process the article: " + article + assert '
Date: Wed, 18 Apr 2012 21:01:51 -0400 Subject: [PATCH 15/31] Start some lower level unit tests --- src/readability_lxml/readability.py | 15 +++++++++------ src/tests/test_readability.py | 13 +++++++++++++ 2 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 src/tests/test_readability.py diff --git a/src/readability_lxml/readability.py b/src/readability_lxml/readability.py index aaa8dab9..2c8c6308 100755 --- a/src/readability_lxml/readability.py +++ b/src/readability_lxml/readability.py @@ -82,10 +82,10 @@ class Document: TEXT_LENGTH_THRESHOLD = 25 RETRY_LENGTH = 250 - def __init__(self, input, **options): + def __init__(self, input_doc, **options): """Generate the document - :param input: string of the html content. + :param input_doc: string of the html content. kwargs: - attributes: @@ -95,17 +95,20 @@ def __init__(self, input, **options): - url: will allow adjusting links to be absolute """ - self.input = input + if input_doc is None: + raise ValueError('You must supply a document to process.') + + self.input_doc = input_doc self.options = options self.html = None def _html(self, force=False): if force or self.html is None: - self.html = self._parse(self.input) + self.html = self._parse(self.input_doc) return self.html - def _parse(self, input): - doc = build_doc(input) + def _parse(self, input_doc): + doc = build_doc(input_doc) doc = html_cleaner.clean_html(doc) base_href = self.options.get('url', None) if base_href: diff --git a/src/tests/test_readability.py b/src/tests/test_readability.py new file mode 100644 index 00000000..c55dbaa2 --- /dev/null +++ b/src/tests/test_readability.py @@ -0,0 +1,13 @@ +import unittest + +from readability_lxml.readability import Document + + +class TestReadabilityDocument(unittest.TestCase): + """Test the Document parser.""" + + def test_none_input_raises_exception(self): + """Feeding a None input to the document should blow up.""" + + doc = None + self.assertRaises(ValueError, Document, doc) From 8b0210c4dc2d7276b85776ac5e5a932837d8ccd4 Mon Sep 17 00:00:00 2001 From: Richard Harding Date: Wed, 18 Apr 2012 21:31:11 -0400 Subject: [PATCH 16/31] Add a license file --- LICENSE | 201 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From 58c69651d37af756caf40d30ddf8dd18b4703e93 Mon Sep 17 00:00:00 2001 From: Richard Harding Date: Wed, 18 Apr 2012 21:31:42 -0400 Subject: [PATCH 17/31] Update README to be a rst file and clean up a little bit. --- README => README.rst | 60 ++++++++++++++++++----------- src/readability_lxml/readability.py | 8 +--- 2 files changed, 39 insertions(+), 29 deletions(-) rename README => README.rst (52%) diff --git a/README b/README.rst similarity index 52% rename from README rename to README.rst index 9e0471c0..ee9aa664 100644 --- a/README +++ b/README.rst @@ -1,14 +1,14 @@ -This code is under the Apache License 2.0. http://www.apache.org/licenses/LICENSE-2.0 +readability_lxml +================ -This is a python port of a ruby port of arc90's readability project +This is a python port of a ruby port of `arc90's readability`_ project -http://lab.arc90.com/experiments/readability/ - -In few words, Given a html document, it pulls out the main body text and cleans it up. It also can clean up title based on latest readability.js code. -Based on: + +Inspiration +----------- - Latest readability.js ( https://github.com/MHordecki/readability-redux/blob/master/readability/readability.js ) - Ruby port by starrhorne and iterationlabs - Python port by gfxmonk ( https://github.com/gfxmonk/python-readability , based on BeautifulSoup ) @@ -16,13 +16,29 @@ Based on: - "BR to P" fix from readability.js which improves quality for smaller texts. - Github users contributions. -Installation:: - easy_install readability-lxml - or - pip install readability-lxml +Installation +------------- +:: + + $ easy_install readability-lxml + # or + $ pip install readability-lxml + + +Usage +------ -Usage:: +Command Line Client +~~~~~~~~~~~~~~~~~~~ +:: + + $ readability http://pypi.python.org/pypi/readability-lxml + $ readability /home/rharding/sampledoc.html + +As a Library +~~~~~~~~~~~~ +:: from readability.readability import Document import urllib @@ -30,21 +46,19 @@ Usage:: readable_article = Document(html).summary() readable_title = Document(html).short_title() -Command-line usage:: - - python -m readability.readability -u http://pypi.python.org/pypi/readability-lxml - +Optional `Document` keyword argument: -Document() kwarg options: +- attributes: +- debug: output debug messages +- min_text_length: +- retry_length: +- url: will allow adjusting links to be absolute - - attributes: - - debug: output debug messages - - min_text_length: - - retry_length: - - url: will allow adjusting links to be absolute +History +------- -Updates + - `0.2.5`` Update setup.py for uploading .tar.gz to pypi - - 0.2.5 Update setup.py for uploading .tar.gz to pypi +.. _arc90's readability: http://lab.arc90.com/experiments/readability/ diff --git a/src/readability_lxml/readability.py b/src/readability_lxml/readability.py index 2c8c6308..c83b46fd 100755 --- a/src/readability_lxml/readability.py +++ b/src/readability_lxml/readability.py @@ -102,11 +102,6 @@ def __init__(self, input_doc, **options): self.options = options self.html = None - def _html(self, force=False): - if force or self.html is None: - self.html = self._parse(self.input_doc) - return self.html - def _parse(self, input_doc): doc = build_doc(input_doc) doc = html_cleaner.clean_html(doc) @@ -136,7 +131,8 @@ def summary(self, enclose_with_html_tag=False): try: ruthless = True while True: - self._html(True) + self.html = self._parse(self.input_doc) + for i in self.tags(self.html, 'script', 'style'): i.drop_tree() for i in self.tags(self.html, 'body'): From 8f420bd9500798f62e148f9906090ef2c197baa5 Mon Sep 17 00:00:00 2001 From: Richard Harding Date: Wed, 18 Apr 2012 21:35:58 -0400 Subject: [PATCH 18/31] Fix setup.py to pull the rst readme --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 637813e5..130c05ea 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ author_email="burchik@gmail.com", description="fast python port of arc90's readability tool", keywords='readable read parse html document readability', - long_description=open("README").read(), + long_description=open("README.rst").read(), license="Apache License 2.0", classifiers=[ "Environment :: Web Environment", From 8091a75f0086671d6773b5c2c43c1d839b8b49cc Mon Sep 17 00:00:00 2001 From: Richard Harding Date: Wed, 18 Apr 2012 21:38:38 -0400 Subject: [PATCH 19/31] fix my rst --- README.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index ee9aa664..e7a1e14e 100644 --- a/README.rst +++ b/README.rst @@ -9,12 +9,12 @@ It also can clean up title based on latest readability.js code. Inspiration ----------- - - Latest readability.js ( https://github.com/MHordecki/readability-redux/blob/master/readability/readability.js ) - - Ruby port by starrhorne and iterationlabs - - Python port by gfxmonk ( https://github.com/gfxmonk/python-readability , based on BeautifulSoup ) - - Decruft effort to move to lxml ( http://www.minvolai.com/blog/decruft-arc90s-readability-in-python/ ) - - "BR to P" fix from readability.js which improves quality for smaller texts. - - Github users contributions. +- Latest readability.js ( https://github.com/MHordecki/readability-redux/blob/master/readability/readability.js ) +- Ruby port by starrhorne and iterationlabs +- Python port by gfxmonk ( https://github.com/gfxmonk/python-readability , based on BeautifulSoup ) +- Decruft effort to move to lxml ( http://www.minvolai.com/blog/decruft-arc90s-readability-in-python/ ) +- "BR to P" fix from readability.js which improves quality for smaller texts. +- Github users contributions. Installation @@ -58,7 +58,7 @@ Optional `Document` keyword argument: History ------- - - `0.2.5`` Update setup.py for uploading .tar.gz to pypi +- `0.2.5`` Update setup.py for uploading .tar.gz to pypi .. _arc90's readability: http://lab.arc90.com/experiments/readability/ From b0063ffb3c9476df330ebc882bdf71d46cdafe32 Mon Sep 17 00:00:00 2001 From: Richard Harding Date: Wed, 18 Apr 2012 21:43:14 -0400 Subject: [PATCH 20/31] garden docs --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index e7a1e14e..320bd9d3 100644 --- a/README.rst +++ b/README.rst @@ -58,7 +58,7 @@ Optional `Document` keyword argument: History ------- -- `0.2.5`` Update setup.py for uploading .tar.gz to pypi +- `0.2.5` Update setup.py for uploading .tar.gz to pypi .. _arc90's readability: http://lab.arc90.com/experiments/readability/ From a4b6957be264ff60a04033f5e3c7616f168984a3 Mon Sep 17 00:00:00 2001 From: Richard Harding Date: Wed, 18 Apr 2012 22:50:07 -0400 Subject: [PATCH 21/31] Update html to be a property with a getter --- src/readability_lxml/readability.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/readability_lxml/readability.py b/src/readability_lxml/readability.py index c83b46fd..5a19374e 100755 --- a/src/readability_lxml/readability.py +++ b/src/readability_lxml/readability.py @@ -100,7 +100,15 @@ def __init__(self, input_doc, **options): self.input_doc = input_doc self.options = options - self.html = None + self._html = None + + @property + def html(self): + """The parsed html document from the input""" + if not self._html: + self._html = self._parse(self.input_doc) + + return self._html def _parse(self, input_doc): doc = build_doc(input_doc) @@ -113,13 +121,13 @@ def _parse(self, input_doc): return doc def content(self): - return get_body(self._html(True)) + return get_body(self.html) def title(self): - return get_title(self._html(True)) + return get_title(self.html) def short_title(self): - return shorten_title(self._html(True)) + return shorten_title(self.html) def summary(self, enclose_with_html_tag=False): """Generate the summary of the html docuemnt @@ -255,7 +263,7 @@ def score_paragraphs(self, ): self.TEXT_LENGTH_THRESHOLD) candidates = {} ordered = [] - for elem in self.tags(self._html(), "p", "pre", "td"): + for elem in self.tags(self.html, "p", "pre", "td"): parent_node = elem.getparent() if parent_node is None: continue From aa51283dff7b9231d225bbae168a0e110a3f2ae5 Mon Sep 17 00:00:00 2001 From: Richard Harding Date: Thu, 19 Apr 2012 15:16:49 -0400 Subject: [PATCH 22/31] work on doing some more pep8 work on things --- src/readability_lxml/readability.py | 75 +++++++++++++++-------------- 1 file changed, 40 insertions(+), 35 deletions(-) diff --git a/src/readability_lxml/readability.py b/src/readability_lxml/readability.py index 5a19374e..61a8c86a 100755 --- a/src/readability_lxml/readability.py +++ b/src/readability_lxml/readability.py @@ -22,18 +22,28 @@ REGEXES = { - 'unlikelyCandidatesRe': re.compile('combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter', re.I), - 'okMaybeItsACandidateRe': re.compile('and|article|body|column|main|shadow', re.I), - 'positiveRe': re.compile('article|body|content|entry|hentry|main|page|pagination|post|text|blog|story', re.I), - 'negativeRe': re.compile('combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget', re.I), - 'divToPElementsRe': re.compile('<(a|blockquote|dl|div|img|ol|p|pre|table|ul)', re.I), + 'unlikelyCandidatesRe': re.compile( + ('combx|comment|community|disqus|extra|foot|header|menu|remark|rss|' + 'shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|' + 'tweet|twitter'), re.I), + 'okMaybeItsACandidateRe': re.compile( + 'and|article|body|column|main|shadow', re.I), + 'positiveRe': re.compile( + ('article|body|content|entry|hentry|main|page|pagination|post|text|' + 'blog|story'), re.I), + 'negativeRe': re.compile( + ('combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|' + 'outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|' + 'tool|widget'), re.I), + 'divToPElementsRe': re.compile( + '<(a|blockquote|dl|div|img|ol|p|pre|table|ul)', re.I), #'replaceBrsRe': re.compile('(]*>[ \n\r\t]*){2,}',re.I), #'replaceFontsRe': re.compile('<(\/?)font[^>]*>',re.I), #'trimRe': re.compile('^\s+|\s+$/'), #'normalizeRe': re.compile('\s{2,}/'), #'killBreaksRe': re.compile('((\s| ?)*){1,}/'), #'videoRe': re.compile('http:\/\/(www\.)?(youtube|vimeo)\.com', re.I), - #skipFootnoteLink: /^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i, + #skipFootnoteLink: /^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i, } @@ -132,8 +142,8 @@ def short_title(self): def summary(self, enclose_with_html_tag=False): """Generate the summary of the html docuemnt - :param enclose_with_html_tag: return only the div of the document, don't wrap - in html and body tags. + :param enclose_with_html_tag: return only the div of the document, + don't wrap in html and body tags. """ try: @@ -187,7 +197,8 @@ def summary(self, enclose_with_html_tag=False): log.exception('error getting summary: ') raise Unparseable(str(e)), None, sys.exc_info()[2] - def get_article(self, candidates, best_candidate, enclose_with_html_tag=False): + def get_article(self, candidates, best_candidate, + enclose_with_html_tag=False): # Now that we have the top candidate, look through its siblings for # content that might also be related. # Things like preambles, content split by ads that we removed, etc. @@ -235,7 +246,9 @@ def get_article(self, candidates, best_candidate, enclose_with_html_tag=False): return output def select_best_candidate(self, candidates): - sorted_candidates = sorted(candidates.values(), key=lambda x: x['content_score'], reverse=True) + sorted_candidates = sorted(candidates.values(), + key=lambda x: x['content_score'], + reverse=True) for candidate in sorted_candidates[:5]: elem = candidate['elem'] self.debug("Top 5 : %6.3f %s" % ( @@ -466,7 +479,8 @@ def sanitize(self, node, candidates): reason = "less than 3x

s than s" to_remove = True elif content_length < (MIN_LEN) and (counts["img"] == 0 or counts["img"] > 2): - reason = "too short content length %s without a single image" % content_length + reason = ('too short content length %s without a single' + ' image') % content_length to_remove = True elif weight < 25 and link_density > 0.2: reason = "too many links %.3f for its weight %s" % ( @@ -477,36 +491,26 @@ def sanitize(self, node, candidates): link_density, weight) to_remove = True elif (counts["embed"] == 1 and content_length < 75) or counts["embed"] > 1: - reason = "s with too short content length, or too many s" + reason = ('s with too short content length, or too' + ' many s') to_remove = True -# if el.tag == 'div' and counts['img'] >= 1 and to_remove: -# imgs = el.findall('.//img') -# valid_img = False -# self.debug(tounicode(el)) -# for img in imgs: -# -# height = img.get('height') -# text_length = img.get('text_length') -# self.debug ("height %s text_length %s" %(repr(height), repr(text_length))) -# if to_int(height) >= 100 or to_int(text_length) >= 100: -# valid_img = True -# self.debug("valid image" + tounicode(img)) -# break -# if valid_img: -# to_remove = False -# self.debug("Allowing %s" %el.text_content()) -# for desnode in self.tags(el, "table", "ul", "div"): -# allowed[desnode] = True - - #find x non empty preceding and succeeding siblings + + # don't really understand what this is doing. Originally + # the i/j were =+ which sets the value to 1. I think that + # was supposed to be += which would increment. But then + # it's compared to x which is hard set to 1. So you only + # ever do one loop in each iteration and don't understand + # it. Will have to investigate when we get to testing more + # pages. i, j = 0, 0 x = 1 + siblings = [] for sib in el.itersiblings(): #self.debug(sib.text_content()) sib_content_length = text_length(sib) if sib_content_length: - i =+ 1 + i += 1 siblings.append(sib_content_length) if i == x: break @@ -514,7 +518,7 @@ def sanitize(self, node, candidates): #self.debug(sib.text_content()) sib_content_length = text_length(sib) if sib_content_length: - j =+ 1 + j += 1 siblings.append(sib_content_length) if j == x: break @@ -526,7 +530,8 @@ def sanitize(self, node, candidates): allowed[desnode] = True if to_remove: - self.debug("Cleaned %6.3f %s with weight %s cause it has %s." % + self.debug( + "Cleaned %6.3f %s with weight %s cause it has %s." % (content_score, describe(el), weight, reason)) #print tounicode(el) #self.debug("pname %s pweight %.3f" %(pname, pweight)) From 35792e7a595f52de0acc13405dad1fa4e2d75210 Mon Sep 17 00:00:00 2001 From: Richard Harding Date: Thu, 19 Apr 2012 16:03:07 -0400 Subject: [PATCH 23/31] garden --- src/readability_lxml/readability.py | 2 +- src/tests/test_article_only.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/readability_lxml/readability.py b/src/readability_lxml/readability.py index 61a8c86a..e315c904 100755 --- a/src/readability_lxml/readability.py +++ b/src/readability_lxml/readability.py @@ -159,7 +159,6 @@ def summary(self, enclose_with_html_tag=False): self.remove_unlikely_candidates() self.transform_misused_divs_into_paragraphs() candidates = self.score_paragraphs() - best_candidate = self.select_best_candidate(candidates) if best_candidate: @@ -249,6 +248,7 @@ def select_best_candidate(self, candidates): sorted_candidates = sorted(candidates.values(), key=lambda x: x['content_score'], reverse=True) + for candidate in sorted_candidates[:5]: elem = candidate['elem'] self.debug("Top 5 : %6.3f %s" % ( diff --git a/src/tests/test_article_only.py b/src/tests/test_article_only.py index 0d7ddc37..d8595c95 100644 --- a/src/tests/test_article_only.py +++ b/src/tests/test_article_only.py @@ -36,4 +36,3 @@ def test_si_sample_html_partial(self): doc = Document(sample, url='http://sportsillustrated.cnn.com/baseball/mlb/gameflash/2012/04/16/40630_preview.html') res = doc.summary(enclose_with_html_tag=True) self.assertEqual('

0.33: + class_weight = self.class_weight(header) + link_density = self.get_link_density(header) + if class_weight < 0 or link_density > 0.33: header.drop_tree() for elem in self.tags(node, "form", "iframe", "textarea"): @@ -455,7 +463,8 @@ def sanitize(self, node, candidates): parent_node = el.getparent() if parent_node is not None: if parent_node in candidates: - content_score = candidates[parent_node]['content_score'] + parent = candidates[parent_node] + content_score = parent['content_score'] else: content_score = 0 #if parent_node is not None: From 93ac1111a18bfbda34b5937ee5fc6805cc7fb09a Mon Sep 17 00:00:00 2001 From: Richard Harding Date: Thu, 19 Apr 2012 22:07:17 -0400 Subject: [PATCH 25/31] Add try it out to the readable server --- README.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.rst b/README.rst index 320bd9d3..2b17ee62 100644 --- a/README.rst +++ b/README.rst @@ -17,6 +17,14 @@ Inspiration - Github users contributions. +Try it out! +----------- +You can try out the parser by entering your test urls on the following test +service. + +http://readable.bmark.us + + Installation ------------- :: From 3347f16d939d397c23793490c7c6e91538801f53 Mon Sep 17 00:00:00 2001 From: Richard Harding Date: Fri, 20 Apr 2012 06:33:42 -0400 Subject: [PATCH 26/31] Fix the flipped nature of the wrapping setting --- src/readability_lxml/readability.py | 12 ++++++------ src/tests/test_article_only.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/readability_lxml/readability.py b/src/readability_lxml/readability.py index 4bcf8dfc..255b626f 100755 --- a/src/readability_lxml/readability.py +++ b/src/readability_lxml/readability.py @@ -139,7 +139,7 @@ def title(self): def short_title(self): return shorten_title(self.html) - def summary(self, enclose_with_html_tag=False): + def summary(self, enclose_with_html_tag=True): """Generate the summary of the html docuemnt :param enclose_with_html_tag: return only the div of the document, @@ -197,7 +197,7 @@ def summary(self, enclose_with_html_tag=False): raise Unparseable(str(e)), None, sys.exc_info()[2] def get_article(self, candidates, best_candidate, - enclose_with_html_tag=False): + enclose_with_html_tag=True): # Now that we have the top candidate, look through its siblings for # content that might also be related. # Things like preambles, content split by ads that we removed, etc. @@ -206,9 +206,9 @@ def get_article(self, candidates, best_candidate, best_candidate['content_score'] * 0.2]) # create a new html document with a html->body->div if enclose_with_html_tag: - output = fragment_fromstring('
') - else: output = document_fromstring('
') + else: + output = fragment_fromstring('
') best_elem = best_candidate['elem'] for sibling in best_elem.getparent().getchildren(): # in lxml there no concept of simple text @@ -238,9 +238,9 @@ def get_article(self, candidates, best_candidate, # We don't want to append directly to output, but the div # in html->body->div if enclose_with_html_tag: - output.append(sibling) - else: output.getchildren()[0].getchildren()[0].append(sibling) + else: + output.append(sibling) #if output is not None: # output.append(best_elem) return output diff --git a/src/tests/test_article_only.py b/src/tests/test_article_only.py index d8595c95..858d9c68 100644 --- a/src/tests/test_article_only.py +++ b/src/tests/test_article_only.py @@ -34,5 +34,5 @@ def test_si_sample_html_partial(self): """Using the si sample, make sure we can get the article alone.""" sample = load_sample('si-game.sample.html') doc = Document(sample, url='http://sportsillustrated.cnn.com/baseball/mlb/gameflash/2012/04/16/40630_preview.html') - res = doc.summary(enclose_with_html_tag=True) + res = doc.summary(enclose_with_html_tag=False) self.assertEqual('
document + or just the
html partial. + + """ + summary = self._summary(enclose_with_html_tag=enclose_with_html_tag) + # For this call return the raw Summary object. + return summary + def summary(self, enclose_with_html_tag=True): - """Generate the summary of the html docuemnt + """Generate the summary of the html document + + :param enclose_with_html_tag: Bool do you want a full document + or just the
html partial. + + """ + summary = self._summary(enclose_with_html_tag=enclose_with_html_tag) + # Only return the html to be consistent with the backwards api. + return summary.html + + def _summary(self, enclose_with_html_tag=True): + """Helper used in a few places to generate the summary of the content - :param enclose_with_html_tag: return only the div of the document, - don't wrap in html and body tags. + :param enclose_with_html_tag: Bool do you want a full document + or just the
html partial. """ try: @@ -162,6 +190,7 @@ def summary(self, enclose_with_html_tag=True): best_candidate = self.select_best_candidate(candidates) if best_candidate: + confidence = best_candidate['content_score'] article = self.get_article(candidates, best_candidate, enclose_with_html_tag=enclose_with_html_tag) else: @@ -177,7 +206,8 @@ def summary(self, enclose_with_html_tag=True): log.debug( ("Ruthless and lenient parsing did not work. " "Returning raw html")) - article = self.html.find('body') + article = self.html.find('body') or self.html + confidence = 0 if article is None: article = self.html cleaned_article = self.sanitize(article, candidates) @@ -191,7 +221,7 @@ def summary(self, enclose_with_html_tag=True): # Loop through and try again. continue else: - return cleaned_article + return Summary(confidence=confidence, html=cleaned_article) except StandardError, e: log.exception('error getting summary: ') raise Unparseable(str(e)), None, sys.exc_info()[2] diff --git a/src/tests/test_article_only.py b/src/tests/test_article_only.py index 858d9c68..fc8518fb 100644 --- a/src/tests/test_article_only.py +++ b/src/tests/test_article_only.py @@ -36,3 +36,15 @@ def test_si_sample_html_partial(self): doc = Document(sample, url='http://sportsillustrated.cnn.com/baseball/mlb/gameflash/2012/04/16/40630_preview.html') res = doc.summary(enclose_with_html_tag=False) self.assertEqual('
document diff --git a/src/tests/test_article_only.py b/src/tests/test_article_only.py index fc8518fb..4575f306 100644 --- a/src/tests/test_article_only.py +++ b/src/tests/test_article_only.py @@ -41,7 +41,7 @@ def test_si_sample_full_summary(self): """We should parse the doc and get a full summary with confidence""" sample = load_sample('si-game.sample.html') doc = Document(sample, url='http://sportsillustrated.cnn.com/baseball/mlb/gameflash/2012/04/16/40630_preview.html') - res = doc.get_summary_with_metadata(enclose_with_html_tag=False) + res = doc.summary_with_metadata(enclose_with_html_tag=False) self.assertTrue(hasattr(res, 'html'), 'res should have an html attrib') self.assertTrue(hasattr(res, 'confidence'), 'res should have an html attrib') self.assertEqual('