Your IP : 216.73.216.11
<?php
/**
*
* @package JMAP::FRAMEWORK::administrator::components::com_jmap
* @subpackage framework
* @subpackage aigenerator
* @author Joomla! Extensions Store
* @copyright (C) 2021 - Joomla! Extensions Store
* @license GNU/GPLv2 http://www.gnu.org/licenses/gpl-2.0.html
*/
// no direct access
defined ( '_JEXEC' ) or die ( 'Restricted access' );
class JMapAigeneratorReadability {
public $version = '1.7.1-without-multi-page';
public $convertLinksToFootnotes = false;
public $revertForcedParagraphElements = true;
public $articleTitle;
public $articleContent;
public $dom;
public $url = null; // optional - URL where HTML was retrieved
public $debug = false;
public $lightClean = true; // preserves more content (experimental) added 2012-09-19
protected $body = null; //
protected $bodyCache = null; // Cache the body HTML in case we need to re-use it later
protected $flags = 7; // 1 | 2 | 4; // Start with all flags set.
protected $success = false; // indicates whether we were able to extract or not
protected $logger;
/**
* All of the regular expressions in use within readability.
* Defined up here so we don't instantiate them repeatedly in loops.
**/
public $regexps = array(
'unlikelyCandidates' => '/combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad\-break|agegate|pagination|pager|skip\-to\-text\-link|popup|flash|js|el__featured\-video|kicker|meta|kicker\-label|headline|page\-title/i',
'okMaybeItsACandidate' => '/and|article|body|column|main|continues|postContent|content|post|story|related|shadow|story\-content|story\-body\-supplemental|story\-body|story\-body\-text|story\-continues|story\-content|story\-body\-text|story\-continues\-2|el__leafmedia\-\-speakable\-paragraph|no\-js|story\-content|morning\-briefing\-weather\-module/i',
'positive' => '/article|body|story\-content|content|entry|hentry|main|page|attachment|pagination|post|text|blog|postContent|story|story\-body|story\-body\-supplemental|story\-body\-text|story\-continues|story\-content|story\-body\-text|story\-continues\-2|speakable|el__leafmedia\-\-speakable\-paragraph|articleBody|story\-content|story\-body|story\-body\-supplemental/i',
'negative' => '/combx|comment|skip\-to\-text\-link|com\-|contact|foot|footer|_nav|footnote|masthead|media|meta|outbrain|taboola|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget|el__featured\-video|flash\-state|video__end\-slate\-\-inactive|js|zn\-large\-media|script|fav|unmute|el__video__collection__close\-\-expandable|js__video__collection__close\-\-expandable|kicker|meta|kicker\-label|headline|page\-title/i',
'divToPElements' => '/<(a|blockquote|dl|div|img|ol|p|pre|table|ul)/i',
'replaceBrs' => '/(<br[^>]*>[ \n\r\t]*){2,}/i',
'replaceFonts' => '/<(\/?)font[^>]*>/i',
// 'trimRe' => '/^\s+|\s+$/g', // PHP has trim()
'normalize' => '/\s{2,}/u',
'killBreaks' => '/(<br\s*\/?>(\s| ?)*){1,}/',
'video' => '!//(player\.|www\.)?(youtube|vimeo|viddler)\.com!i',
'skipFootnoteLink' => '/^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i',
);
/* constants */
const FLAG_STRIP_UNLIKELYS = 1;
const FLAG_WEIGHT_CLASSES = 2;
const FLAG_CLEAN_CONDITIONALLY = 4;
/**
* Create instance of Readability
*
* @param string UTF-8 encoded string
* @param string (optional) URL associated with HTML (used for footnotes)
* @param string which parser to use for turning raw HTML into a DOMDocument (either 'libxml' or 'html5lib')
*/
function __construct( $html, $url = null, $parser = 'libxml', $logging_function = false ) {
$this->url = $url;
$this->logger = $logging_function;
/* Turn all double br's into p's */
$html = preg_replace( $this->regexps['replaceBrs'], '</p><p>', $html );
$html = preg_replace( $this->regexps['replaceFonts'], '<$1span>', $html );
$html = mb_encode_numericentity($html, [0x80, 0xFFFF, 0, 0xFFFF], 'UTF-8');
if ( trim( $html ) == '' ) { $html = '<html></html>'; }
if ( $parser == 'html5lib' && ($this->dom = HTML5_Parser::parse( $html )) ) {
// all good
} else {
$this->dom = new DOMDocument('1.0', 'UTF-8');
$this->dom->preserveWhiteSpace = false;
@$this->dom->loadHTML( $html );
}
$this->dom->registerNodeClass( 'DOMElement', 'JMapAigeneratorJslikehtmlelement' );
}
/**
* Get article title element
*
* @return DOMElement
*/
public function getTitle() {
return $this->articleTitle;
}
/**
* Get article content element
*
* @return DOMElement
*/
public function getContent() {
return $this->articleContent;
}
/**
* Runs readability.
*
* Workflow:
* 1. Prep the document by removing script tags, css, etc.
* 2. Build readability's DOM tree.
* 3. Grab the article content from the current dom tree.
* 4. Replace the current DOM tree with the new one.
* 5. Read peacefully.
*
* @return boolean true if we found content, false otherwise
**/
public function init() {
if ( ! isset( $this->dom->documentElement ) ) { return false; }
$this->removeScripts( $this->dom );
// die($this->getInnerHTML($this->dom->documentElement));
// Assume successful outcome
$this->success = true;
$bodyElems = $this->dom->getElementsByTagName( 'body' );
if ( $bodyElems->length > 0 ) {
if ( $this->bodyCache == null ) {
$this->bodyCache = $bodyElems->item( 0 )->innerHTML;
}
if ( $this->body == null ) {
$this->body = $bodyElems->item( 0 );
}
}
$this->prepDocument();
// die($this->dom->documentElement->parentNode->nodeType);
// $this->setInnerHTML($this->dom->documentElement, $this->getInnerHTML($this->dom->documentElement));
// die($this->getInnerHTML($this->dom->documentElement));
/* Build readability's DOM tree */
$overlay = $this->dom->createElement( 'div' );
$innerDiv = $this->dom->createElement( 'div' );
$articleTitle = $this->getArticleTitle();
$articleContent = $this->grabArticle();
if ( ! $articleContent ) {
$this->success = false;
$articleContent = $this->dom->createElement( 'div' );
$articleContent->setAttribute( 'id', 'readability-content' );
$articleContent->innerHTML = '<p>Sorry, Readability was unable to parse this page for content.</p>';
}
$overlay->setAttribute( 'id', 'readOverlay' );
$innerDiv->setAttribute( 'id', 'readInner' );
/* Glue the structure of our document together. */
$innerDiv->appendChild( $articleTitle );
$innerDiv->appendChild( $articleContent );
$overlay->appendChild( $innerDiv );
/* Clear the old HTML, insert the new content. */
$this->body->innerHTML = '';
$this->body->appendChild( $overlay );
// document.body.insertBefore(overlay, document.body.firstChild);
$this->body->removeAttribute( 'style' );
$this->postProcessContent( $articleContent );
// Set title and content instance variables
$this->articleTitle = $articleTitle;
$this->articleContent = $articleContent;
return $this->success;
}
protected function external_logger( $msg ) {
if ( false !== $this->logger && is_callable( $this->logger ) ) {
call_user_func_array( $this->logger, array( $msg ) );
}
}
/**
* Check if the url is an absolute URL in some way
*
* @param string $url
* @return bool
*/
protected function isFullyQualified($url) {
$isFullyQualified = substr($url, 0, 7) == 'http://' || substr($url, 0, 8) == 'https://' || substr($url, 0, 2) == '//';
return $isFullyQualified;
}
/**
* Get the top level host domain for each kind of URL needed to avoid redirects on CURL exec
*
* @access private
* @param string $url
* @return string
*/
protected function getHost($url) {
if (strpos ( $url, "http" ) !== false) {
$httpurl = $url;
} else {
$httpurl = "https://" . $url;
}
$parse = parse_url ( $httpurl );
$scheme = $parse ['scheme'];
$domain = $parse ['host'];
return $scheme . '://' . $domain;
}
/**
* Debug
*/
protected function dbg( $msg ) {
if ( $this->debug ) { echo '* ',$msg, "\n"; }
$this->external_logger( $msg );
}
/**
* Run any post-process modifications to article content as necessary.
*
* @param DOMElement
* @return void
*/
public function postProcessContent( $articleContent ) {
if ( $this->convertLinksToFootnotes && ! preg_match( '/wikipedia\.org/', @$this->url ) ) {
$this->addFootnotes( $articleContent );
}
}
/**
* Get the article title as an H1.
*
* @return DOMElement
*/
protected function getArticleTitle() {
$curTitle = '';
$origTitle = '';
try {
$curTitle = $origTitle = $this->getInnerText( $this->dom->getElementsByTagName( 'title' )->item( 0 ) );
} catch (Exception $e) {}
if ( preg_match( '/ [\|\-] /', $curTitle ) ) {
$curTitle = preg_replace( '/(.*)[\|\-] .*/i', '$1', $origTitle );
if ( count( explode( ' ', $curTitle ) ) < 3 ) {
$curTitle = preg_replace( '/[^\|\-]*[\|\-](.*)/i', '$1', $origTitle );
}
} elseif ( strpos( $curTitle, ': ' ) !== false ) {
$curTitle = preg_replace( '/.*:(.*)/i', '$1', $origTitle );
if ( count( explode( ' ', $curTitle ) ) < 3 ) {
$curTitle = preg_replace( '/[^:]*[:](.*)/i','$1', $origTitle );
}
} elseif ( strlen( $curTitle ) > 150 || strlen( $curTitle ) < 15 ) {
$hOnes = $this->dom->getElementsByTagName( 'h1' );
if ( $hOnes->length == 1 ) {
$curTitle = $this->getInnerText( $hOnes->item( 0 ) );
}
}
$curTitle = trim( $curTitle );
if ( count( explode( ' ', $curTitle ) ) <= 4 ) {
$curTitle = $origTitle;
}
$articleTitle = $this->dom->createElement( 'h1' );
$articleTitle->innerHTML = $curTitle;
return $articleTitle;
}
/**
* Prepare the HTML document for readability to scrape it.
* This includes things like stripping javascript, CSS, and handling terrible markup.
*
* @return void
**/
protected function prepDocument() {
/**
* In some cases a body element can't be found (if the HTML is totally hosed for example)
* so we create a new body node and append it to the document.
*/
if ( $this->body == null ) {
$this->body = $this->dom->createElement( 'body' );
$this->dom->documentElement->appendChild( $this->body );
}
$this->body->setAttribute( 'id', 'readabilityBody' );
/* Remove all style tags in head */
$styleTags = $this->dom->getElementsByTagName( 'style' );
for ( $i = $styleTags->length -1; $i >= 0; $i-- ) {
$styleTags->item( $i )->parentNode->removeChild( $styleTags->item( $i ) );
}
/*
Turn all double br's into p's */
/*
Note, this is pretty costly as far as processing goes. Maybe optimize later. */
// document.body.innerHTML = document.body.innerHTML.replace(readability.regexps.replaceBrs, '</p><p>').replace(readability.regexps.replaceFonts, '<$1span>');
// We do this in the constructor for PHP as that's when we have raw HTML - before parsing it into a DOM tree.
// Manipulating innerHTML as it's done in JS is not possible in PHP.
}
/**
* For easier reading, convert this document to have footnotes at the bottom rather than inline links.
*
* @see http://www.roughtype.com/archives/2010/05/experiments_in.php
*
* @return void
**/
public function addFootnotes( $articleContent ) {
$footnotesWrapper = $this->dom->createElement( 'div' );
$footnotesWrapper->setAttribute( 'id', 'readability-footnotes' );
$footnotesWrapper->innerHTML = '<h3>References</h3>';
$articleFootnotes = $this->dom->createElement( 'ol' );
$articleFootnotes->setAttribute( 'id', 'readability-footnotes-list' );
$footnotesWrapper->appendChild( $articleFootnotes );
$articleLinks = $articleContent->getElementsByTagName( 'a' );
$linkCount = 0;
for ( $i = 0; $i < $articleLinks->length; $i++ ) {
$articleLink = $articleLinks->item( $i );
$footnoteLink = $articleLink->cloneNode( true );
$refLink = $this->dom->createElement( 'a' );
$footnote = $this->dom->createElement( 'li' );
$linkDomain = @parse_url( $footnoteLink->getAttribute( 'href' ), PHP_URL_HOST );
if ( ! $linkDomain && isset( $this->url ) ) { $linkDomain = @parse_url( $this->url, PHP_URL_HOST ); }
// linkDomain = footnoteLink.host ? footnoteLink.host : document.location.host,
$linkText = $this->getInnerText( $articleLink );
if ( (strpos( $articleLink->getAttribute( 'class' ), 'readability-DoNotFootnote' ) !== false) || preg_match( $this->regexps['skipFootnoteLink'], $linkText ) ) {
continue;
}
$linkCount++;
/** Add a superscript reference after the article link */
$refLink->setAttribute( 'href', '#readabilityFootnoteLink-' . $linkCount );
$refLink->innerHTML = '<small><sup>[' . $linkCount . ']</sup></small>';
$refLink->setAttribute( 'class', 'readability-DoNotFootnote' );
$refLink->setAttribute( 'style', 'color: inherit;' );
// does this work or should we use DOMNode.isSameNode()?
if ( $articleLink->parentNode->lastChild == $articleLink ) {
$articleLink->parentNode->appendChild( $refLink );
} else {
$articleLink->parentNode->insertBefore( $refLink, $articleLink->nextSibling );
}
$articleLink->setAttribute( 'style', 'color: inherit; text-decoration: none;' );
$articleLink->setAttribute( 'name', 'readabilityLink-' . $linkCount );
$footnote->innerHTML = '<small><sup><a href="#readabilityLink-' . $linkCount . '" title="Jump to Link in Article">^</a></sup></small> ';
$footnoteLink->innerHTML = ($footnoteLink->getAttribute( 'title' ) != '' ? $footnoteLink->getAttribute( 'title' ) : $linkText);
$footnoteLink->setAttribute( 'name', 'readabilityFootnoteLink-' . $linkCount );
$footnote->appendChild( $footnoteLink );
if ( $linkDomain ) { $footnote->innerHTML = $footnote->innerHTML . '<small> (' . $linkDomain . ')</small>'; }
$articleFootnotes->appendChild( $footnote );
}
if ( $linkCount > 0 ) {
$articleContent->appendChild( $footnotesWrapper );
}
}
/**
* Reverts P elements with class 'readability-styled'
* to text nodes - which is what they were before.
*
* @param DOMElement
* @return void
*/
function revertReadabilityStyledElements( $articleContent ) {
$xpath = new \DOMXPath( $articleContent->ownerDocument );
$elems = $xpath->query( './/p[@class="readability-styled"]', $articleContent );
// $elems = $articleContent->getElementsByTagName('p');
for ( $i = $elems->length -1; $i >= 0; $i-- ) {
$e = $elems->item( $i );
$e->parentNode->replaceChild( $articleContent->ownerDocument->createTextNode( $e->textContent ), $e );
// if ($e->hasAttribute('class') && $e->getAttribute('class') == 'readability-styled') {
// $e->parentNode->replaceChild($this->dom->createTextNode($e->textContent), $e);
// }
}
}
/**
* Prepare the article node for display. Clean out any inline styles,
* iframes, forms, strip extraneous <p> tags, etc.
*
* @param DOMElement
* @return void
*/
function prepArticle( $articleContent ) {
$this->cleanStyles( $articleContent );
$this->killBreaks( $articleContent );
if ( $this->revertForcedParagraphElements ) {
$this->revertReadabilityStyledElements( $articleContent );
}
/* Clean out junk from the article content */
$this->cleanConditionally( $articleContent, 'form' );
$this->clean( $articleContent, 'object' );
$this->clean( $articleContent, 'script' );
$this->clean( $articleContent, 'link' );
$this->clean( $articleContent, 'meta' );
// $this->clean($articleContent, 'h1');
/**
* If there is only one h2, they are probably using it
* as a header and not a subheader, so remove it since we already have a header.
*/
if ( ! $this->lightClean && ($articleContent->getElementsByTagName( 'h2' )->length == 1) ) {
// $this->clean($articleContent, 'h2');
}
$this->clean( $articleContent, 'iframe' );
$this->cleanHeaders( $articleContent );
/* Do these last as the previous stuff may have removed junk that will affect these */
$this->cleanConditionally( $articleContent, 'table' );
$this->cleanConditionally( $articleContent, 'ul' );
$this->cleanConditionally( $articleContent, 'div' );
// Make imgs always absolute URLs, use 2 methods: getElementsByTagName and XPath
$imagesFoundDOM = $articleContent->getElementsByTagName( 'img' );
if( $imagesFoundDOM->length > 0) {
for ( $im = $imagesFoundDOM->length -1; $im >= 0; $im-- ) {
$imgTag = $imagesFoundDOM->item( $im );
$originalSrc = $imgTag->getAttribute('src');
if($originalSrc && !$this->isFullyQualified($originalSrc) && !preg_match('/data:image/i', $originalSrc)) {
// Relative image has been found, go on to append the original src domain
$originalSrc = $this->getHost($this->url) . '/' . ltrim($originalSrc, '/');
$imgTag->setAttributeNode(new \DOMAttr('src', $originalSrc));
}
// Support for custom lazy src attributes
$arrayOfDataSrcAttributes = [
'data-src',
'data-lazyload',
'data-original',
'data-dt-lazy-src',
'data-lazy-src'
];
foreach ($arrayOfDataSrcAttributes as $singleDataSrcAttribute) {
if($imgTag->hasAttribute($singleDataSrcAttribute)) {
$originalDataSrc = $imgTag->getAttribute($singleDataSrcAttribute);
if($originalDataSrc) {
if(!$this->isFullyQualified($originalDataSrc)) {
$imgTag->setAttributeNode(new \DOMAttr($singleDataSrcAttribute, $this->getHost($this->url) . '/' . ltrim($originalDataSrc, '/')));
}
// Nullify the src attribute for the regular expression in the purify contents
$imgTag->setAttributeNode(new \DOMAttr('src', 'data:image/png;'));
}
}
}
}
}
/* Remove extra paragraphs */
$articleParagraphs = $articleContent->getElementsByTagName( 'p' );
for ( $i = $articleParagraphs->length -1; $i >= 0; $i-- ) {
$imgCount = $articleParagraphs->item( $i )->getElementsByTagName( 'img' )->length;
$embedCount = $articleParagraphs->item( $i )->getElementsByTagName( 'embed' )->length;
$objectCount = $articleParagraphs->item( $i )->getElementsByTagName( 'object' )->length;
$iframeCount = $articleParagraphs->item( $i )->getElementsByTagName( 'iframe' )->length;
if ( $imgCount === 0 && $embedCount === 0 && $objectCount === 0 && $iframeCount === 0 && $this->getInnerText( $articleParagraphs->item( $i ), false ) == '' ) {
$articleParagraphs->item( $i )->parentNode->removeChild( $articleParagraphs->item( $i ) );
}
}
try {
$articleContent->innerHTML = preg_replace( '/<br[^>]*>\s*<p/i', '<p', $articleContent->innerHTML );
// articleContent.innerHTML = articleContent.innerHTML.replace(/<br[^>]*>\s*<p/gi, '<p');
} catch (Exception $e) {
$this->dbg( 'Cleaning innerHTML of breaks failed. This is an IE strict-block-elements bug. Ignoring.: ' . $e );
}
}
/**
* Initialize a node with the readability object. Also checks the
* className/id for special names to add to its score.
*
* @param Element
* @return void
**/
protected function initializeNode( $node ) {
$readability = $this->dom->createAttribute( 'readability' );
$readability->value = 0; // this is our contentScore
$node->setAttributeNode( $readability );
switch ( strtoupper( $node->tagName ) ) { // unsure if strtoupper is needed, but using it just in case
case 'DIV':
$readability->value += 5;
break;
case 'PRE':
case 'TD':
case 'BLOCKQUOTE':
$readability->value += 3;
break;
case 'ADDRESS':
case 'OL':
case 'UL':
case 'DL':
case 'DD':
case 'DT':
case 'LI':
case 'FORM':
$readability->value -= 3;
break;
case 'H1':
case 'H2':
case 'H3':
case 'H4':
case 'H5':
case 'H6':
case 'TH':
$readability->value -= 5;
break;
case 'SCRIPT':
$readability->value -= 100;
break;
}
$readability->value += $this->getClassWeight( $node );
}
/**
*
* grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is
* most likely to be the stuff a user wants to read. Then return it wrapped up in a div.
*
* @return DOMElement
**/
protected function grabArticle( $page = null ) {
$stripUnlikelyCandidates = $this->flagIsActive( self::FLAG_STRIP_UNLIKELYS );
if ( ! $page ) { $page = $this->dom; }
$allElements = $page->getElementsByTagName( '*' );
/**
* First, node prepping. Trash nodes that look cruddy (like ones with the class name "comment", etc), and turn divs
* into P tags where they have been used inappropriately (as in, where they contain no other block level elements.)
*
* Note: Assignment from index for performance. See http://www.peachpit.com/articles/article.aspx?p=31567&seqNum=5
* Shouldn't this be a reverse traversal?
*/
$node = null;
$nodesToScore = array();
//var_dump('<pre>',$allElements->item( $nodeIndex )); //die();
for ( $nodeIndex = 0; ($node = $allElements->item( $nodeIndex )); $nodeIndex++ ) {
// for ($nodeIndex=$targetList->length-1; $nodeIndex >= 0; $nodeIndex--) {
// $node = $targetList->item($nodeIndex);
$tagName = strtoupper( $node->tagName );
//var_dump($node);
/* Remove unlikely candidates */
if ( $stripUnlikelyCandidates ) {
$unlikelyMatchString = $node->getAttribute( 'class' ) . $node->getAttribute( 'id' );
if (
preg_match( $this->regexps['unlikelyCandidates'], $unlikelyMatchString ) &&
! preg_match( $this->regexps['okMaybeItsACandidate'], $unlikelyMatchString ) &&
$tagName != 'BODY'
) {
$this->dbg( 'Removing unlikely candidate - ' . $unlikelyMatchString );
// $nodesToRemove[] = $node;
$node->parentNode->removeChild( $node );
$nodeIndex--;
continue;
}
}
if ( $tagName == 'P' || $tagName == 'TD' || $tagName == 'PRE' ) {
$nodesToScore[] = $node;
}
/* Turn all divs that don't have children block level elements into p's */
if ( $tagName == 'DIV' ) {
if ( ! preg_match( $this->regexps['divToPElements'], $node->innerHTML ) ) {
// $this->dbg('Altering div to p');
$newNode = $this->dom->createElement( 'p' );
try {
$newNode->innerHTML = $node->innerHTML;
// $nodesToReplace[] = array('new'=>$newNode, 'old'=>$node);
$node->parentNode->replaceChild( $newNode, $node );
$nodeIndex--;
$nodesToScore[] = $node; // or $newNode?
} catch (Exception $e) {
$this->dbg( 'Could not alter div to p, reverting back to div.: ' . $e );
}
} else {
/*
EXPERIMENTAL */
// change these p elements back to text nodes after processing
for ( $i = 0, $il = $node->childNodes->length; $i < $il; $i++ ) {
$childNode = $node->childNodes->item( $i );
if ( $childNode->nodeType == 3 ) { // XML_TEXT_NODE
// $this->dbg('replacing text node with a p tag with the same content.');
$p = $this->dom->createElement( 'p' );
$p->innerHTML = $childNode->nodeValue;
$p->setAttribute( 'style', 'display: inline;' );
$p->setAttribute( 'class', 'readability-styled' );
$childNode->parentNode->replaceChild( $p, $childNode );
}
}
}
}
}
/**
* Loop through all paragraphs, and assign a score to them based on how content-y they look.
* Then add their score to their parent node.
*
* A score is determined by things like number of commas, class names, etc. Maybe eventually link density.
*/
$candidates = array();
for ( $pt = 0; $pt < count( $nodesToScore ); $pt++ ) {
$parentNode = $nodesToScore[ $pt ]->parentNode;
// $grandParentNode = $parentNode ? $parentNode->parentNode : null;
$grandParentNode = ! $parentNode ? null : (($parentNode->parentNode instanceof DOMElement) ? $parentNode->parentNode : null);
$grandGrandParentNode = ! $grandParentNode ? null : (($grandParentNode->parentNode instanceof DOMElement) ? $grandParentNode->parentNode : null);
$innerText = $this->getInnerText( $nodesToScore[ $pt ] );
if ( ! $parentNode || ! isset( $parentNode->tagName ) ) {
continue;
}
/* If this paragraph is less than 25 characters, don't even count it. */
if ( strlen( $innerText ) < 25 ) {
continue;
}
/* Initialize readability data for the parent. */
if ( ! $parentNode->hasAttribute( 'readability' ) ) {
$this->initializeNode( $parentNode );
$candidates[] = $parentNode;
}
/* Initialize readability data for the grandparent. */
if ( $grandParentNode && ! $grandParentNode->hasAttribute( 'readability' ) && isset( $grandParentNode->tagName ) ) {
$this->initializeNode( $grandParentNode );
$candidates[] = $grandParentNode;
}
/* Initialize readability data for the grandgrandparent. */
if ( $grandGrandParentNode && ! $grandGrandParentNode->hasAttribute( 'readability' ) && isset( $grandGrandParentNode->tagName ) ) {
$this->initializeNode( $grandGrandParentNode );
$candidates[] = $grandGrandParentNode;
}
$contentScore = 0;
/* Add a point for the paragraph itself as a base. */
$contentScore++;
/* Add points for any commas within this paragraph */
$contentScore += count( explode( ',', $innerText ) );
/* For every 100 characters in this paragraph, add another point. Up to 3 points. */
$contentScore += min( floor( strlen( $innerText ) / 100 ), 3 );
/* Add the score to the parent. The grandparent gets half. */
$parentNode->getAttributeNode( 'readability' )->value += $contentScore;
if ( $grandParentNode ) {
$grandParentNode->getAttributeNode( 'readability' )->value += $contentScore / 2;
}
}
/**
* After we've calculated scores, loop through all of the possible candidate nodes we found
* and find the one with the highest score.
*/
$topCandidate = null;
for ( $c = 0, $cl = count( $candidates ); $c < $cl; $c++ ) {
/**
* Scale the final candidates score based on link density. Good content should have a
* relatively small link density (5% or less) and be mostly unaffected by this operation.
*/
$readability = $candidates[ $c ]->getAttributeNode( 'readability' );
$readability->value = $readability->value * (1 -$this->getLinkDensity( $candidates[ $c ] ));
if ( 'article' == $candidates[ $c ]->tagName ) {
$readability->value = $readability->value + 300;
// var_dump($candidates[$c]); die();
}
$this->dbg( 'Candidate: ' . $candidates[ $c ]->tagName . ' (' . $candidates[ $c ]->getAttribute( 'class' ) . ':' . $candidates[ $c ]->getAttribute( 'id' ) . ') with score ' . $readability->value );
if ( ! $topCandidate || $readability->value > (int) $topCandidate->getAttribute( 'readability' ) ) {
$topCandidate = $candidates[ $c ];
}
}
/**
* If we still have no top candidate, just use the body as a last resort.
* We also have to copy the body node so it is something we can modify.
*/
if ( $topCandidate === null || strtoupper( $topCandidate->tagName ) == 'BODY' ) {
$topCandidate = $this->dom->createElement( 'div' );
if ( $page instanceof DOMDocument ) {
if ( ! isset( $page->documentElement ) ) {
// we don't have a body either? what a mess! :)
} else {
$topCandidate->innerHTML = $page->documentElement->innerHTML;
$page->documentElement->innerHTML = '';
$page->documentElement->appendChild( $topCandidate );
}
} else {
$topCandidate->innerHTML = $page->innerHTML;
$page->innerHTML = '';
$page->appendChild( $topCandidate );
}
$this->initializeNode( $topCandidate );
}
/**
* Now that we have the top candidate, look through its siblings for content that might also be related.
* Things like preambles, content split by ads that we removed, etc.
*/
$articleContent = $this->dom->createElement( 'div' );
$articleContent->setAttribute( 'id', 'readability-content' );
$siblingScoreThreshold = max( 10, ((int) $topCandidate->getAttribute( 'readability' )) * 0.2 );
$siblingNodes = $topCandidate->parentNode->childNodes;
if ( ! isset( $siblingNodes ) ) {
$siblingNodes = new stdClass;
$siblingNodes->length = 0;
}
for ( $s = 0, $sl = $siblingNodes->length; $s < $sl; $s++ ) {
$siblingNode = $siblingNodes->item( $s );
$append = false;
$this->dbg( 'Looking at sibling node: ' . $siblingNode->nodeName . (($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute( 'readability' )) ? (' with score ' . $siblingNode->getAttribute( 'readability' )) : '') );
// dbg('Sibling has score ' . ($siblingNode->readability ? siblingNode.readability.contentScore : 'Unknown'));
if ( $siblingNode === $topCandidate ) {
$append = true;
}
$contentBonus = 0;
/* Give a bonus if sibling nodes and top candidates have the example same classname */
if ( $siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->getAttribute( 'class' ) == $topCandidate->getAttribute( 'class' ) && $topCandidate->getAttribute( 'class' ) != '' ) {
$contentBonus += ((int) $topCandidate->getAttribute( 'readability' )) * 0.2;
}
if ( $siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute( 'readability' ) && (((int) $siblingNode->getAttribute( 'readability' )) + $contentBonus) >= $siblingScoreThreshold ) {
$append = true;
}
if ( strtoupper( $siblingNode->nodeName ) == 'P' ) {
$linkDensity = $this->getLinkDensity( $siblingNode );
$nodeContent = $this->getInnerText( $siblingNode );
$nodeLength = strlen( $nodeContent );
if ( $nodeLength > 80 && $linkDensity < 0.25 ) {
$append = true;
} elseif ( $nodeLength < 80 && $linkDensity === 0 && preg_match( '/\.( |$)/', $nodeContent ) ) {
$append = true;
}
}
if ( $append ) {
$this->dbg( 'Appending node: ' . $siblingNode->nodeName );
$nodeToAppend = null;
$sibNodeName = strtoupper( $siblingNode->nodeName );
if ( $sibNodeName != 'DIV' && $sibNodeName != 'P' ) {
/* We have a node that isn't a common block level element, like a form or td tag. Turn it into a div so it doesn't get filtered out later by accident. */
$this->dbg( 'Altering siblingNode of ' . $sibNodeName . ' to div.' );
$nodeToAppend = $this->dom->createElement( 'div' );
try {
$nodeToAppend->setAttribute( 'id', $siblingNode->getAttribute( 'id' ) );
$nodeToAppend->innerHTML = $siblingNode->innerHTML;
} catch (Exception $e) {
$this->dbg( 'Could not alter siblingNode to div, reverting back to original.' );
$nodeToAppend = $siblingNode;
$s--;
$sl--;
}
} else {
$nodeToAppend = $siblingNode;
$s--;
$sl--;
}
/* To ensure a node does not interfere with readability styles, remove its classnames */
$nodeToAppend->removeAttribute( 'class' );
/* Append sibling and subtract from our list because it removes the node when you append to another node */
$articleContent->appendChild( $nodeToAppend );
}
}
/**
* So we have all of the content that we need. Now we clean it up for presentation.
*/
$this->prepArticle( $articleContent );
/**
* Now that we've gone through the full algorithm, check to see if we got any meaningful content.
* If we didn't, we may need to re-run grabArticle with different flags set. This gives us a higher
* likelihood of finding the content, and the sieve approach gives us a higher likelihood of
* finding the -right- content.
*/
if ( strlen( $this->getInnerText( $articleContent, false ) ) < 250 ) {
// find out why element disappears sometimes, e.g. for this URL http://www.businessinsider.com/6-hedge-fund-etfs-for-average-investors-2011-7
// in the meantime, we check and create an empty element if it's not there.
if ( ! isset( $this->body->childNodes ) ) { $this->body = $this->dom->createElement( 'body' ); }
$this->body->innerHTML = $this->bodyCache;
if ( $this->flagIsActive( self::FLAG_STRIP_UNLIKELYS ) ) {
$this->removeFlag( self::FLAG_STRIP_UNLIKELYS );
return $this->grabArticle( $this->body );
} elseif ( $this->flagIsActive( self::FLAG_WEIGHT_CLASSES ) ) {
$this->removeFlag( self::FLAG_WEIGHT_CLASSES );
return $this->grabArticle( $this->body );
} elseif ( $this->flagIsActive( self::FLAG_CLEAN_CONDITIONALLY ) ) {
$this->removeFlag( self::FLAG_CLEAN_CONDITIONALLY );
return $this->grabArticle( $this->body );
} else {
return false;
}
}
return $articleContent;
}
/**
* Remove script tags from document
*
* @param DOMElement
* @return void
*/
public function removeScripts( $doc ) {
$scripts = $doc->getElementsByTagName( 'script' );
for ( $i = $scripts->length -1; $i >= 0; $i-- ) {
$scripts->item( $i )->parentNode->removeChild( $scripts->item( $i ) );
}
}
/**
* Get the inner text of a node.
* This also strips out any excess whitespace to be found.
*
* @param DOMElement $
* @param boolean $normalizeSpaces (default: true)
* @return string
**/
public function getInnerText( $e, $normalizeSpaces = true ) {
$textContent = '';
if ( ! isset( $e->textContent ) || $e->textContent == '' ) {
return '';
}
$textContent = trim( $e->textContent );
if ( $normalizeSpaces ) {
return preg_replace( $this->regexps['normalize'], ' ', $textContent );
} else {
return $textContent;
}
}
/**
* Get the number of times a string $s appears in the node $e.
*
* @param DOMElement $e
* @param string - what to count. Default is ","
* @return number (integer)
**/
public function getCharCount( $e, $s = ',' ) {
return substr_count( $this->getInnerText( $e ), $s );
}
/**
* Remove the style attribute on every $e and under.
*
* @param DOMElement $e
* @return void
*/
public function cleanStyles( $e ) {
if ( ! is_object( $e ) ) { return; }
$elems = $e->getElementsByTagName( '*' );
foreach ( $elems as $elem ) {
$elem->removeAttribute( 'style' );
}
}
/**
* Get the density of links as a percentage of the content
* This is the amount of text that is inside a link divided by the total text in the node.
*
* @param DOMElement $e
* @return number (float)
*/
public function getLinkDensity( $e ) {
$links = $e->getElementsByTagName( 'a' );
$textLength = strlen( $this->getInnerText( $e ) );
$linkLength = 0;
for ( $i = 0, $il = $links->length; $i < $il; $i++ ) {
$linkLength += strlen( $this->getInnerText( $links->item( $i ) ) );
}
if ( $textLength > 0 ) {
return $linkLength / $textLength;
} else {
return 0;
}
}
/**
* Get an elements class/id weight. Uses regular expressions to tell if this
* element looks good or bad.
*
* @param DOMElement $e
* @return number (Integer)
*/
public function getClassWeight( $e ) {
if ( ! $this->flagIsActive( self::FLAG_WEIGHT_CLASSES ) ) {
return 0;
}
$weight = 0;
/* Look for a special classname */
if ( $e->hasAttribute( 'class' ) && $e->getAttribute( 'class' ) != '' ) {
if ( preg_match( $this->regexps['negative'], $e->getAttribute( 'class' ) ) ) {
$weight -= 25;
}
if ( preg_match( $this->regexps['positive'], $e->getAttribute( 'class' ) ) ) {
$weight += 25;
}
}
//var_dump('<pre>', $e); //die();
/* Look for a special ID */
if ( $e->hasAttribute( 'id' ) && $e->getAttribute( 'id' ) != '' ) {
if ( preg_match( $this->regexps['negative'], $e->getAttribute( 'id' ) ) ) {
$weight -= 55;
}
if ( preg_match( $this->regexps['positive'], $e->getAttribute( 'id' ) ) ) {
$weight += 55;
}
if ( 'story' == $e->getAttribute( 'id' ) && 'article' == $e->tagName ){
$weight += 300;
}
if ( 'main' == $e->getAttribute( 'id' ) && 'main' == $e->tagName ){
$weight += 300;
}
}
if ( $e->hasAttribute( 'itemprop' ) && $e->getAttribute( 'itemprop' ) != '' ) {
if ( preg_match( $this->regexps['negative'], $e->getAttribute( 'itemprop' ) ) ) {
$weight -= 25;
}
if ( preg_match( $this->regexps['positive'], $e->getAttribute( 'itemprop' ) ) ) {
$weight += 25;
}
if ( 'articleBody' == $e->getAttribute( 'itemprop' ) ) {
$weight += 400;
}
}
if ( $e->hasAttribute( 'role' ) && $e->getAttribute( 'role' ) != '' ) {
if ( preg_match( $this->regexps['negative'], $e->getAttribute( 'role' ) ) ) {
$weight -= 25;
}
if ( preg_match( $this->regexps['positive'], $e->getAttribute( 'role' ) ) ) {
$weight += 25;
}
if ( 'main' == $e->getAttribute( 'role' ) ) {
$weight += 400;
}
}
if ( $e->hasAttribute( 'data-para-count' ) && $e->getAttribute( 'data-para-count' ) != '' ) {
if ( intval($e->getAttribute( 'data-para-count')) > 0 ) {
$weight += 200;
}
}
return $weight;
}
/**
* Remove extraneous break tags from a node.
*
* @param DOMElement $node
* @return void
*/
public function killBreaks( $node ) {
$html = $node->innerHTML;
$html = preg_replace( $this->regexps['killBreaks'], '<br />', $html );
$node->innerHTML = $html;
}
/**
* Clean a node of all elements of type "tag".
* (Unless it's a youtube/vimeo video. People love movies.)
*
* Updated 2012-09-18 to preserve youtube/vimeo iframes
*
* @param DOMElement $e
* @param string $tag
* @return void
*/
public function clean( $e, $tag ) {
$targetList = $e->getElementsByTagName( $tag );
$isEmbed = ($tag == 'iframe' || $tag == 'object' || $tag == 'embed');
for ( $y = $targetList->length -1; $y >= 0; $y-- ) {
/* Allow youtube and vimeo videos through as people usually want to see those. */
if ( $isEmbed ) {
$attributeValues = '';
for ( $i = 0, $il = $targetList->item( $y )->attributes->length; $i < $il; $i++ ) {
$attributeValues .= $targetList->item( $y )->attributes->item( $i )->value . '|'; // DOMAttr
}
/* First, check the elements attributes to see if any of them contain youtube or vimeo */
if ( preg_match( $this->regexps['video'], $attributeValues ) ) {
continue;
}
/* Then check the elements inside this element for the same. */
if ( preg_match( $this->regexps['video'], $targetList->item( $y )->innerHTML ) ) {
continue;
}
}
$targetList->item( $y )->parentNode->removeChild( $targetList->item( $y ) );
}
}
/**
* Clean an element of all tags of type "tag" if they look fishy.
* "Fishy" is an algorithm based on content length, classnames,
* link density, number of images & embeds, etc.
*
* @param DOMElement $e
* @param string $tag
* @return void
*/
public function cleanConditionally( $e, $tag ) {
if ( ! $this->flagIsActive( self::FLAG_CLEAN_CONDITIONALLY ) ) {
return;
}
$tagsList = $e->getElementsByTagName( $tag );
$curTagsLength = $tagsList->length;
/**
* Gather counts for other typical elements embedded within.
* Traverse backwards so we can remove nodes at the same time without effecting the traversal.
*
* Consider taking into account original contentScore here.
*/
for ( $i = $curTagsLength -1; $i >= 0; $i-- ) {
$weight = $this->getClassWeight( $tagsList->item( $i ) );
$contentScore = ($tagsList->item( $i )->hasAttribute( 'readability' )) ? (int) $tagsList->item( $i )->getAttribute( 'readability' ) : 0;
$this->dbg( 'Cleaning Conditionally ' . $tagsList->item( $i )->tagName . ' (' . $tagsList->item( $i )->getAttribute( 'class' ) . ':' . $tagsList->item( $i )->getAttribute( 'id' ) . ')' . (($tagsList->item( $i )->hasAttribute( 'readability' )) ? (' with score ' . $tagsList->item( $i )->getAttribute( 'readability' )) : '') );
if ( $weight + $contentScore < 0 ) {
$tagsList->item( $i )->parentNode->removeChild( $tagsList->item( $i ) );
} elseif ( $this->getCharCount( $tagsList->item( $i ), ',' ) < 10 ) {
/**
* If there are not very many commas, and the number of
* non-paragraph elements is more than paragraphs or other ominous signs, remove the element.
*/
$p = $tagsList->item( $i )->getElementsByTagName( 'p' )->length;
$img = $tagsList->item( $i )->getElementsByTagName( 'img' )->length;
$li = $tagsList->item( $i )->getElementsByTagName( 'li' )->length -100;
$input = $tagsList->item( $i )->getElementsByTagName( 'input' )->length;
$a = $tagsList->item( $i )->getElementsByTagName( 'a' )->length;
$embedCount = 0;
$embeds = $tagsList->item( $i )->getElementsByTagName( 'embed' );
for ( $ei = 0, $il = $embeds->length; $ei < $il; $ei++ ) {
if ( preg_match( $this->regexps['video'], $embeds->item( $ei )->getAttribute( 'src' ) ) ) {
$embedCount++;
}
}
$embeds = $tagsList->item( $i )->getElementsByTagName( 'iframe' );
for ( $ei = 0, $il = $embeds->length; $ei < $il; $ei++ ) {
if ( preg_match( $this->regexps['video'], $embeds->item( $ei )->getAttribute( 'src' ) ) ) {
$embedCount++;
}
}
$linkDensity = $this->getLinkDensity( $tagsList->item( $i ) );
$contentLength = strlen( $this->getInnerText( $tagsList->item( $i ) ) );
$toRemove = false;
if ( $this->lightClean ) {
$this->dbg( 'Light clean...' );
if ( ($img > $p) && ($img > 4) ) {
$this->dbg( ' more than 4 images and more image elements than paragraph elements' );
$toRemove = true;
} elseif ( $li > $p && $tag != 'ul' && $tag != 'ol' ) {
$this->dbg( ' too many <li> elements, and parent is not <ul> or <ol>' );
$toRemove = true;
} elseif ( $input > floor( $p / 3 ) ) {
$this->dbg( ' too many <input> elements' );
$toRemove = true;
} elseif ( $contentLength < 25 && ($embedCount === 0 && ($img === 0 || $img > 2)) ) {
$this->dbg( ' content length less than 25 chars, 0 embeds and either 0 images or more than 2 images' );
$toRemove = true;
} elseif ( $weight < 25 && $linkDensity > 0.2 ) {
$this->dbg( ' weight smaller than 25 and link density above 0.2' );
$toRemove = true;
} elseif ( $a > 2 && ($weight >= 25 && $linkDensity > 0.5) ) {
$this->dbg( ' more than 2 links and weight above 25 but link density greater than 0.5' );
$toRemove = true;
} elseif ( $embedCount > 3 ) {
$this->dbg( ' more than 3 embeds' );
$toRemove = true;
}
} else {
$this->dbg( 'Standard clean...' );
if ( $img > $p ) {
$this->dbg( ' more image elements than paragraph elements' );
$toRemove = true;
} elseif ( $li > $p && $tag != 'ul' && $tag != 'ol' ) {
$this->dbg( ' too many <li> elements, and parent is not <ul> or <ol>' );
$toRemove = true;
} elseif ( $input > floor( $p / 3 ) ) {
$this->dbg( ' too many <input> elements' );
$toRemove = true;
} elseif ( $contentLength < 25 && ($img === 0 || $img > 2) ) {
$this->dbg( ' content length less than 25 chars and 0 images, or more than 2 images' );
$toRemove = true;
} elseif ( $weight < 25 && $linkDensity > 0.2 ) {
$this->dbg( ' weight smaller than 25 and link density above 0.2' );
$toRemove = true;
} elseif ( $weight >= 25 && $linkDensity > 0.5 ) {
$this->dbg( ' weight above 25 but link density greater than 0.5' );
$toRemove = true;
} elseif ( ($embedCount == 1 && $contentLength < 75) || $embedCount > 1 ) {
$this->dbg( ' 1 embed and content length smaller than 75 chars, or more than one embed' );
$toRemove = true;
}
}
// var_dump($tagsList->item($i-2)); die();
if ( ( false !== stripos( $tagsList->item( $i )->textContent, 'et al' ) ) || ( 1 === preg_match( '/\([1-9]\d{3,}\)/', $tagsList->item( $i )->textContent ) ) ) {
$this->dbg( ' content of element indicates reference.' );
$toRemove = false;
} elseif (
( (false != $tagsList) && method_exists( $tagsList, 'item' ) && ( $tagsList->item( $i )) && method_exists( $tagsList->item( $i ), 'parentNode' ) && ( $tagsList->item( $i )->parentNode) ) &&
(
(
(
'li' === $tagsList->item( $i )->parentNode->tagName ||
(
(isset( $tagsList->item( $i )->parentNode->parentNode )) &&
( 'li' === $tagsList->item( $i )->parentNode->parentNode->tagName )
)
) &&
( false !== stripos( $tagsList->item( $i -1 )->textContent, 'et al' ) )
) ||
(( $tagsList->item( $i -1 )) &&
( 1 === preg_match( '/\([1-9]\d{3,}\)/', $tagsList->item( $i -1 )->textContent ) )
)
)
) {
$this->dbg( ' content of element indicates reference.' );
$toRemove = false;
}
if ( $toRemove ) {
// $this->dbg('Removing: '.$tagsList->item($i)->innerHTML);
$tagsList->item( $i )->parentNode->removeChild( $tagsList->item( $i ) );
}
}
}
}
/**
* Clean out spurious headers from an Element. Checks things like classnames and link density.
*
* @param DOMElement $e
* @return void
*/
public function cleanHeaders( $e ) {
for ( $headerIndex = 1; $headerIndex < 3; $headerIndex++ ) {
$headers = $e->getElementsByTagName( 'h' . $headerIndex );
for ( $i = $headers->length -1; $i >= 0; $i-- ) {
if ( $this->getClassWeight( $headers->item( $i ) ) < 0 || $this->getLinkDensity( $headers->item( $i ) ) > 0.33 ) {
$headers->item( $i )->parentNode->removeChild( $headers->item( $i ) );
}
}
}
}
public function flagIsActive( $flag ) {
return ($this->flags & $flag) > 0;
}
public function addFlag( $flag ) {
$this->flags = $this->flags | $flag;
}
public function removeFlag( $flag ) {
$this->flags = $this->flags & ~$flag;
}
}