The Internet
// ==UserScript==
// @name The internet
// @namespace AggregatorByGabyDeWilde
// @include https://opml.go-here.nl/internet.html
// @include https://opml.go-here.nl/configuration.html
// @version 0.090
// @grant GM_xmlhttpRequest
// @grant GM_getValue
// @grant GM_setValue
// @grant GM_listValues
// @grant GM_deleteValue
// @grant GM_registerMenuCommand
// @updateURL https://opml.go-here.nl/the-internet.meta.js
// @connect *
// ==/UserScript==
/* https://github.com/gaby-de-wilde/salamisushi */
//"use strict"
//window.variables = Object.keys( window ); // keep track of used variables
window.id = function(x){ return document.getElementById( x )} // abstract
window.setValue = function(a,b){return !GM_setValue(a,b.join(',')) } // manage gm storage set
window.getValue = function(a,b){return GM_getValue(a,b).split(',') } // manage gm storage get
window.confLink = "https://opml.go-here.nl/configuration.html"; // configuration url
window.aggregatorLink = "https://opml.go-here.nl/internet.html"; // aggregator url
window.bugTracker = "https://github.com/gaby-de-wilde/salamisushi/issues";
id('install').innerHTML = ''; // remove installation instructions
// load configuration
(window.urlArrays = ['unsubscribe','rss_blacklist','rss_suspended','opml','rss']).forEach( function(x){ window[x] = getValue(x, '')});
if( location.href == window.aggregatorLink ){ // detect aggregation page
if(!((GM_listValues()).includes('configuration'))){ location.href = window.confLink } // detect lack of configuration
window.pref = JSON.parse(GM_getValue('configuration', '')); // parse configuration object
// configuration not included in configuration page
window.pref.refresh_delay = 20;
window.pref.blacklist_feed_with_script = false;
window.pref.maxPending = 30;
window.pref.feed_date = "on";
window.pref.advanced_details = "off";
window.pref.publish_news = "no";
window.pref.publish_news_url = "https://";
window.pref.publish_news_password = "";
window.disabledConsoles = [];
/* 'parse_html', 'suspended', 'rss_request_url', 'rss_response_url', 'no_new_items', 'failure_date_error', 'title_result',
'word_filter', 'duplicate_title', 'considered', 'to_short', 'failure_future_date_error', 'no_link', 'no_title',
'title_similarity', 'blacklist', 'abort', 'aborted', 'timeout', 'failure_request_error','failure_parse_error',
'failure_no_items_in_feed','opml_request_url', 'opml_response_url', 'feed_filter', 'pain', 'feed_origin',
'var_monitor', 'stages', 'feeds', 'performance', 'average_time', 'storage', 'faiure_parse_error'//*/
window.linkTarget = "_blank";
// Initialize global variables
'feedsRequested feedResponses feedsSkipped opmlRequested opmlResponses titlesFiltered duplicateTitles toShortTitles titleCount countInOpml xml_retreaved_from_opml oldestEntry renewTimer mouseMove'.split(' ').forEach(function(x){window[x]=0});
window.titleResult = [];
window.HTMLresultA = [[]];
window.lastParse = Infinity;
window.oldTimeA = Math.floor( Date.now() / 1000 );
window.rss_suspended_length = window.rss_suspended.length;
window.rss_blacklist_length = window.rss_blacklist.length;
window.maxPending = window.pref.maxPending;
// log things to their consoles
window.log = function(){}
window.monitorMode = false;
document.getElementById('consolecontainer').style.visibility = "hidden";
//////// GM MENU ENTRIES /////////////
// we need these first so that things can be deleted if something goes wrong
window.qelm = id('output');
window.g = GM_registerMenuCommand;
window.c = function(x){return confirm(x)};
window.monitor = function(){
window.monitorMode = true;
window.log = function(logConsole, logMessage){
if(window.disabledConsoles.indexOf(logConsole) == -1){
unsafeWindow.console_factory.write( ""+logConsole , ""+logMessage );
}
}
document.getElementById('consolecontainer').style.visibility = "visible";
window.qelm.innerHTML = '';
window.localStorage['monitor'] = 1;
}
g("monitor",window.monitor);
g("news",function(){
window.log = function(){};
window.monitorMode = false;
window.scrollArray(100,true);
document.getElementById('consolecontainer').style.visibility = "hidden";
window.localStorage['monitor'] = 0;
});
g("configuration",function(){ location.href = confLink });
g("# reset blacklist",function(){
if(c("Feeds with errors are blacklisted but these errors might not be permanent. They can be inside a news item or the webmaster can finally fix the bugs after you harass him by email for many years.\n\n Do you want to reset the blacklist?")){
window.rss_blacklist = ['https://example.com/feed/'];
setValue('rss_blacklist', 'https://example.com/feed/');
window.serviceGMstorage()}});
g("# reset unsubscribed",function(){
if(c("Restore subscription to all unsubscribed feeds?")){
/*window.domainUnsubscribe=*/window.unsubscribe=['https://example.com/feed/'];
setValue('unsubscribe','https://example.com/feed/')}});
g("# erase old news",function(){
if(c("Erase old news items?")){
window.HTMLresultA = [[]];
window.HTMLresultX = [];
localStorage.finalarray = [[0,'https://void','https://example.com','dummy title','void' ]];
location.href = window.aggregatorLink}});
g("# reset suspended",function(){
if(c("Erase suspended list?")){
window.rss_suspended = [0,'https://example.com/feed/'];
window.rss_suspended_url = ['https://example.com/feed/'];
window.suspended_regex = window.buildRegex(window.rss_suspended_url);
setValue('rss_suspended', [0,'https://example.com/feed/']);
window.serviceGMstorage()}});
g("# erase autodetect",function(){
if(c("Erase auto detected list?")){
localStorage.autoDetect = []}});
g("display autodetect",function(){
document.getElementsByTagName('body')[0].innerHTML =
'<div style="color:red;font-size:16px;"><ul><li>' + localStorage.autoDetect + '</li></ul></div>'});
if(!!(window.localStorage['monitor']*1)){window.monitor()}
window.gr = function(val){ return '<span style="color:#00FF00">' + val + '</span>'; };
window.br = function(val){ return '<span style="color:#C6AE79">' + val + '</span>'; };
window.red = function(val){ return '<span style="color:red">' + val + '</span>'; };
window.bl = function(val){ return '<span style="color:#00F9E0">' + val + '</span>'; };
window.ora = function(val){ return '<span style="color:#FFA100">' + val + '</span>'; };
// default to http protocol
[window.cleanProtocol,window.getDomain] = (function(){
var cleanProtocol_regex = new RegExp('^(?:\/\/|https?:\/\/|feed:\/\/?)','i');//^(\/\/|http:\/\/|https:\/\/|feed:\/\/)
return [
function(url){return url?url.replace(cleanProtocol_regex,'https://'):"" },
function(url){
if(!url){return "https://example.com"}
var b,c;
url = url.replace(cleanProtocol_regex,''); // strip protocol
url = url.split('/')[0]; // discard folders
if((c = ( b = url.split('.') ).length) > 2){ // discard sub domains
url = ( b[c-2] == 'co' ? b[c-3] + '.co' : b[c-2]) + '.' + b[c-1]; // manage .co.dinosaur domains
}
if(url.length > 4){return url;} // dispose of fakes
}
]
})();
window.titleResult_set = new Set([]);
window.unsubscribe_set = new Set(window.unsubscribe);
window.unsubscribe_fresh = [];
window.unsubscribe_fresh_set = new Set([]);
window.buildRegex = (function(){
var buildREgex_regex = new RegExp(/[-\/\\^$*+[?].()|[\]{}]/g);
return function(x){
x.push('asdfasddasfasdfdasfdasf');
var y = [];
x.forEach(function(x){ y.push(x.trim().replace(buildREgex_regex, '\\$&'));buildREgex_regex.lastIndex = 0; });
x.pop();
return new RegExp( '^(' + y.join('|') + ')$','g' );
}
})();
// create unsubscribe regular expressions
window.buildUnsubscribeRegex = function(){
var unsubscribe_url = [],
unsubscribe_domain = [];
window.unsubscribe.forEach(function(x){
x=window.cleanProtocol(x.trim());
if(x.startsWith('http')){ unsubscribe_url.push(x) }
else{ unsubscribe_domain.push(x) }
});
window.unsubscribe_url_regex = window.buildRegex(unsubscribe_url);
window.unsubscribe_domain_regex = window.buildRegex(unsubscribe_domain);
};
window.buildUnsubscribeRegex();
// create blacklist regular expressions
window.rss_blacklist_url = [];
window.rss_blacklist_domain = [];
window.rss_blacklist.forEach(function(x){
if(x.trim().startsWith('http')){
window.rss_blacklist_url.push(window.cleanProtocol(x.trim()));
}else{
window.rss_blacklist_domain.push(x.trim());
}
});
window.rss_blacklist_url_regex = window.buildRegex(window.rss_blacklist_url);
window.rss_blacklist_domain_regex = window.buildRegex(window.rss_blacklist_domain);
// unsuspending feeds
if(window.rss_suspended.length > 0){
log('stages','checking '+ ora(window.rss_suspended.length)+ ' suspended feed dates');
var theNow = Date.now();
for(x=0;x<window.rss_suspended.length;x=x+2){
if(isNaN(window.rss_suspended[x])){log('suspended', window.rss_suspended[x]+' is not a date');x--;continue}
if(+window.rss_suspended[x]+172800000 < theNow){
if(window.rss_suspended[x+1]){ log('suspended','clearing: '+ ora(window.rss_suspended[x+1])) };
window.rss_suspended.splice(x,2);x=x-2;
}
}
}
setValue('rss_suspended', window.rss_suspended);
// create suspended regular expression
window.rss_suspended_url = [];
window.rss_suspended.forEach(function(x){
if(x.trim().startsWith('http')){
window.rss_suspended_url.push(x);
}
})
window.suspended_regex = window.buildRegex(window.rss_suspended_url);
// create badwords regular expression
var badwords = [];
for(var x=1;x<9;x++){
badwords = badwords.concat(window.pref['badwords'+x].split(','));
delete window.pref['badwords'+x]
}
for(x=0;x<badwords.length;x++){ badwords[x] = badwords[x].trim().toLowerCase().replace(/[^a-z0-9]/g, " ") }
badwords = badwords.filter(function(x){ return x.length > 2});
badwords.push('asdfdasfasfasffas');
window.badwords_regex = new RegExp( '(^|\\b)('+badwords.join('|')+')($|\\b)' );
delete window.badwords;
// define words not to highlight
window.noHighlight = new Set("three four five seven eight nine eleven twelve thirteen fourteen fifteen sixteen seventeen eighteen nineteen twenty january february march april may june july august september october november december 2015 2016 720p 1080p x264 nbsp 8217 your with this comment program national upgrade against president scientist year from body that more will have says what back more about best holiday after years video over most news just series high last first world review down life secret city before announcement week congress deal know online test york almost people photos season america american americans bluray economy make next real report school some believe than time when would build coming facts free government hand girl here home kill leak plan shows ways north east south west white following control crazy dies does food game hits into miracle much only open other stories story take they things thread tips want anxiety attack call cars change changes death early every final found gets good hike holidays lady like looking love made making master message play power price start stay there these under between become today ahead calls case charges been house biggest hard getting though amid cover work happy hard lost should thanks becoming through young around though door great sign boyfriend girlfriend across wants their could crash earth force support being children dead edition list star state storms true beyond family fans federal lost million right sheltered student woman awesome been black business camera cold cookies daily date discussion ever fashionable fighters forces gift happy intelligent kids lead least lunch mobile nick radio record reserve system talks team tech times weather where brings response called worst freedom really century energy general since update where which anti behind better community enough international legal lives look looks marriage mini minute mother movie need official plans plus post return selling spill spirit think wrong awakens bill book brand building card check country days fast feel likely proof learn giving district close everything tree help cause travel full major space spain airport favorite flight live music tree captured close internet intervention model proof security stocks trade campaign court crisis district during issue mandatory matter mission show sold traffic units updates used watch annual armed catastrophe cause central collapse configuration emergency episode event everything feds fight finds friends giving group guide heart http jubilee latest lawsuit learn left likely long mall near party popular mine inspired justice keep".split(' '));
// hide undesired table column (switching it on/off doesn't modify old news)
if(window.pref.feed_origin == "off"){
var style = document.createElement('style');
style.type = 'text/css';
style.innerHTML = "table tr > td:first-child + td + td + td + td{display:none !important}";
document.getElementsByTagName('head')[0].appendChild(style);
}
if(window.pref.feed_date == "off"){
var style = document.createElement('style');
style.type = 'text/css';
style.innerHTML = "table tr > td:first-child {display:none !important}";
document.getElementsByTagName('head')[0].appendChild(style);
}
// unsubscribe (in stead of turning the array into a set and back we maintain both)
window.unsubscribeFeed = function(badFeed){
badFeed = window.cleanProtocol( badFeed.trim() );
if (!badFeed){alert('error: feed url undefined');return;}
if(confirm("Remove (skip) this subscription?\n\n(cancel for domain options)\n\n\n" + badFeed )){
window.unsubscribe_url_regex.lastIndex = 0;
window.unsubscribe_domain_regex.lastIndex = 0;
if(!unsubscribe_url_regex.test(badFeed)){
window.unsubscribe.push(badFeed);
window.unsubscribe_set.add(badFeed);
setValue('unsubscribe', window.unsubscribe );
let unsub_url = [];
window.unsubscribe.forEach(function(x){
x=window.cleanProtocol(x.trim());
if(x.startsWith('http')){ unsub_url.push(x) }
});
window.unsubscribe_url_regex = window.buildRegex(unsub_url);
if(!unsubscribe_url_regex.test(badFeed)){alert('unsubscribe is broken')}
window.renewResults2(true,true);
}else if(window.unsubscribe.indexOf(badFeed) != -1){ alert('Error, regex failed to match feed: \n\n'+badFeed)}
else{ alert('error \n\n'+badFeed + '\n\n was already unsubscribed'); }
}else{
badFeed = getDomain(badFeed);
if (confirm("Remove (skip) all subscriptions for this domain?\n\n\n" + badFeed )){
if(!unsubscribe_domain_regex.test(badFeed)){
window.unsubscribe.push(badFeed);
window.unsubscribe_set.add(badFeed);
setValue('unsubscribe', window.unsubscribe );
let unsub_domain = [];
window.unsubscribe.forEach(function(x){
x=window.cleanProtocol(x.trim());
if(!x.startsWith('http')){ unsub_domain.push(x) }
});
window.unsubscribe_domain_regex = window.buildRegex(unsub_domain);
window.renewResults2(true,true);
}else{ alert('error \n\n'+badFeed + '\n\n was already unsubscribed'); }
}
}
}
// periodically backup the blacklist and suspended feeds list
window.serviceGMstorage = function(){
var c = window.rss_blacklist.length;
var d = window.rss_suspended.length;
var a = window.rss_blacklist_length != c;
var b = window.rss_suspended_length != d;
if(a){
setValue('rss_blacklist', window.rss_blacklist);
window.rss_blacklist_length = c;
}
if(b){
setValue('rss_suspended', window.rss_suspended);
window.rss_suspended_length = d;
}
if(window.rss_suspended_length != window.rss_suspended.length){log('blacklist', 'error')}
if(a||b){ log('blacklist', red(window.rss_blacklist_length) +' blacklisted, '+ora(window.rss_suspended.length/2)+' suspended');}
}
////////////////////////// results to HTML /////////////////////////////////////
// compare title similarity
window.compareFilter = function(a){ return !this.has(a); }
window.compareLength = function(a,b){return Array.from(a).filter(window.compareFilter,b).length; }
window.compareTitles = function(a,b){
var aa,bb;
var wordCount=((aa=new Set(a=a.split(/\W+/g))).size+(bb=new Set(b=b.split(/\W+/g))).size)/2;
var NotMatch=(window.compareLength(aa,bb)+window.compareLength(bb,aa))/2;
return (wordCount-NotMatch)/wordCount;
}
// heavy title similarity check (experimental, should be a web worker)
window.countDifferences = function(a, b) {
var was = a.split(/\W+/g);
var wbs = b.split(/\W+/g);
var aSet = Object.create(null);
var bSet = Object.create(null);
var countA = 0;
var countB = 0;
var totalWords = 0;
var j = 0;
for (var i = 0; i < was.length; ++i) {
var w = was[i];
if (!aSet[w]) {
aSet[w] = true;
++countA;
++totalWords;
// remove duplicates as we go since we need
// to iterate over the unique entries in this
// array at the end.
was[j++] = w;
}
}
was.length = j;
for (var i = 0; i < wbs.length; ++i) {
var w = wbs[i];
if (!bSet[w]) {
bSet[w] = true;
++countB;
++totalWords;
if (aSet[w]) {
--countA;
}
}
}
for (var i = 0; i < was.length; ++i) {
var w = was[i];
if (bSet[w]) {
--countB;
}
}
var wordCount = totalWords * 0.5;
var notMatch = (countA + countB) * 0.5;
return (wordCount - notMatch) / wordCount;
}
/* convert domain names into colors */
function stringToColor(str){
var r = 0,
g = 0,
b = 0,
/*turn the string into an array of numbers */
arr = (' '+str).split('').map(x=>x.charCodeAt(0));
/*extend the array length to be a multiple of 3*/
while(arr.length%3){arr.push(0)}
/* divide the character code over r,g and b */
for(var y=0, x=arr.length;y<x;y+=3){
r = (r += arr.pop())%94;
g = (g += arr.pop())%94;
b = (b += arr.pop())%94;
}
/* make a contrasting background color */
rb = (r + 128)%256;
gb = (g + 128)%256;
bb = (b + 128)%256;
return ['rgb('+rb+', '+gb+', '+bb+')',
'rgb('+r+', '+g+', '+b+')'];
}
window.qelm.style.position = "absolute";
window.qelm.style.top = "14px";
document.getElementsByClassName('foo')[0].innerHTML = "";
window.qelm.style.height = window.innerHeight+'px';
window.last_known_scroll_position = 0;
window.currentOffset = 0;
window.itemArray = window.localStorage.getItem('finalarray');
// old news is not incorruptable
if(window.itemArray){try{
window.itemArray = JSON.parse(window.itemArray);
}catch(e){
window.itemArray = [[0,'https://void','https://example.com','this is a dummy news item because the old news was corrupted and deleted','void' ]];
}}else{
window.itemArray = [[0,'https://void','https://example.com','dummy title','void' ]];
}
var amount = 1000;
window.theLastNinja = 0;
window.clickHistory = {};
window.oldPageNumber = 10000;
window.pageNumber = 0;
window.refreshTime = 0;
window.scrollArray = function( scroll_pos, forced_update ){
if(!scroll_pos){scroll_pos=window.last_known_scroll_position}
if( window.monitorMode || !window.itemArray || window.itemArray.length==0){ window.qelm.innerHTML = '';return }
// are we going up or down?
document.body.style.height = window.itemArray.length*43+"px";
window.currentOffset = (scroll_pos/43)<<0;
window.pageNumber=(window.currentOffset/30)<<0;
console.log('page: '+window.pageNumber);
console.log('offset page top: '+window.pageNumber*30*43);
console.log('offset scroll: '+scroll_pos);
//var relaxation = Date.now();
//if(window.pageNumber == window.oldPageNumber && relaxation-window.refreshTime<window.pref.refresh_delay*1000 && !forced_update){return}
//window.refreshTime = relaxation;
if(window.pageNumber == window.oldPageNumber && !forced_update){return}
window.oldPageNumber = window.pageNumber;
console.log(window.pageNumber);
// number of items before top item displayed
var skiped = 'top';
// number of items beyond bottom item displayed
var togo = window.pageNumber;
// extract array sub set
var subset = window.itemArray.slice(window.pageNumber*30,(window.pageNumber+1)*60);
// make html from array
var outputResult = [];
for(var x=0;x<subset.length;x++){
// define item class
var itemClass = '';
// identify comments
if(subset[x][3].indexOf('Comment on') == 0
|| subset[x][3].indexOf('RE:') == 0
|| subset[x][3].indexOf('Re:') == 0
|| subset[x][3].indexOf('re:') == 0
|| subset[x][2].indexOf('#comment') != -1
|| subset[x][2].indexOf('/comment') != -1){ itemClass += 'comment' }
// class for undefined source
if(subset[x][4] == "not defined"){ itemClass += ' autodetect' }
// try to obtain the host url
if(subset[x][2].indexOf('feedproxy.google.com')>0
&& subset[x][2].indexOf('feedproxy.google.com')<9
&& subset[x][2].split('/')[4] ){ var domainIndicator = subset[x][2].split('/')[4];
}else{ var domainIndicator = getDomain(subset[x][2]); }
if(!subset[x][4]||window.pref.feed_origin == "off"){ subset[x][4] = "" }
var minutesAgo=(Date.now()-subset[x][0])/60000<<0;
var hoursAgo = minutesAgo/60<<0;
var andMinutes = minutesAgo - (hoursAgo * 60);
var bgcolor = (subset[x][0]+0).toString(5).slice(8,14);
var col = stringToColor(domainIndicator);
var visitedClass = '';
// faster visited link coloring
var vdomain = getDomain(subset[x][2]);
if(Array.isArray(window.clickHistory[ vdomain ]) && window.clickHistory[vdomain].indexOf(subset[x][2].split(vdomain)[1]) !=-1 ){
visitedClass = ' visited';
}
outputResult.push([
/*0*/ '<tbody><tr>',
/*1*/ '<td rowspan="',
/*2*/ 1,
/*3*/ '" style="background-color:#' + bgcolor + '">',
/*4*/ (hoursAgo?hoursAgo + ' hours and<br>':'') + andMinutes + ' minutes ago',
/*5*/ '</td>',
/*6*/ '<td><button data-feed="'+
subset[x][1]+
'">X</button></td><td><a class="tr" target="_blank" href="https://translate.google.com/translate?hl=en&langpair=auto|en&tbb=1&ie=UTF-8&u='+
subset[x][2]+
'">語</a> <a href="'+
subset[x][2]+
'" class="'+
itemClass + visitedClass +
'" target="' +
window.linkTarget +
'">' +
subset[x][3]+
'</a></td>',
/*7*/ '<td rowspan="',
/*8*/ 1,
/*9*/ '" style="color:'+col[0]+';background:'+col[1]+'">',
/*10*/ domainIndicator,
/*11*/ '</td>',
/*12*/ '<td>'+
subset[x][4],
/*13*/ '</td></tr></tbody>']);
}
var outputResultHTML = '';
for(var i=0;i<outputResult.length;i++){
var y=1;
var rowspan = 1;
while(outputResult[i+y] && outputResult[i][2] != '' && outputResult[i][4] === outputResult[i+y][4]){
rowspan++;
outputResult[i+y][0]='<tr>';
outputResult[i+y][1]='<td style="display:none"></td>';
outputResult[i+y][2]='';
outputResult[i+y][3]='';
outputResult[i+y][4]='';
outputResult[i+y][5]='';
outputResult[i+y-1][13]='</td></tr>';
y++;
outputResult[i][2] = rowspan;
}
outputResultHTML += outputResult[i].join('');
}
// display html
window.qelm.style.top = (window.pageNumber*30*43)+"px";
window.qelm.innerHTML = skiped + '<table>' + outputResultHTML + "</table>"+togo;
// make unsubscribe buttons
var buttons = document.getElementsByTagName('button');
var buttonslength = buttons.length;
for(x=0; x<buttonslength; x++){
buttons[x].addEventListener("click", function(){
window.unsubscribeFeed(this.dataset.feed)
}, false);
}
}
scrollArray(0, true);
window.addEventListener('scroll', function(e) {
window.last_known_scroll_position = window.scrollY;
window.scrollArray(window.last_known_scroll_position,0);
});
document.addEventListener("click", function(e){
if (e.target.tagName !== 'A' || !e.target.hasAttribute('href')) return;
var vdomain = getDomain(e.target.href);
window.clickHistory[vdomain] = window.clickHistory[vdomain] || [];
window.clickHistory[vdomain].push(e.target.href.split(vdomain)[1]);
window.maxPending = 0;
}, false);
// build the html <table>
window.buildTable = function(s,forced_update){ scrollArray(0,forced_update) }
// sort frequent words by frequency
window.sortByCount = function(a, b){ return b.count - a.count }
// filter out duplicates
window.filterHtmlResult = function(elem, pos){
return this[0].indexOf(elem[2]) == pos && this[1].indexOf(elem[3]) == pos;
}
// filter out unsubscribed
window.removeUnsubscribe = function(elem){
window.unsubscribe_url_regex.lastIndex = 0;
window.unsubscribe_domain_regex.lastIndex = 0;
var url = window.cleanProtocol(elem[1]);
return !window.unsubscribe_url_regex.test(url) && !window.unsubscribe_domain_regex.test(getDomain(url));
}
// update localstorrage with fresh results
window.updateStoredResult = function(s){
// merge old results
if(!window.itemArray){ window.itemArray = [[0,'https://void','https://example.com','dummy title','void' ]] }
log('parse_html', 'merging '+gr(s.length)+' new news with '+gr(window.itemArray.length)+' old news');
var rangeEnd, rangeStart, trial;
for(var x = 0; x < s.length; x++){
rangeEnd = window.itemArray.length;
rangeStart = 0;
while(rangeEnd - rangeStart > 1){
trial = Math.ceil((rangeEnd + rangeStart) / 2);
if(s[x][0] < window.itemArray[trial][0]) rangeStart = trial; else rangeEnd = trial;
}
window.itemArray.splice(s[x][0] > window.itemArray[rangeStart][0] ? rangeStart : rangeEnd, 0, s[x]);
}
// remove unsubscribed items
window.unsubscribe_url_regex.lastIndex = 0;
window.unsubscribe_domain_regex.lastIndex = 0;
window.itemArray = window.itemArray.filter(window.removeUnsubscribe)
log('parse_html',' after removing unsubscribed items '+ora(window.itemArray.length)+' items remain.');
// limit results per feed source
var ma = new Map(),tempz;
for(var r=0;r<window.itemArray.length;r++){
if(tempz = ma.get(window.itemArray[r][1])){
if(tempz > window.pref.items_per_rss_feed){
window.itemArray.splice(r,1);
r--;
}else{
ma.set(window.itemArray[r][1],tempz+1);
}
}else{
ma.set(window.itemArray[r][1],1);
}
}
log('parse_html',' after counting the number of items per feed '+ora(window.itemArray.length)+' items remain.');
// trim results by preference
if(window.itemArray.length > window.pref.items_to_keep){
log('parse_html', 'trimming news item list from '+red(window.itemArray.length)+' to '+gr(window.pref.items_to_keep));
window.itemArray.splice(window.pref.items_to_keep,999999999);
}
// trim results by local storage maximum
var sJSON = JSON.stringify(window.itemArray);
while(sJSON.length > 2636625){
window.itemArray.pop();
sJSON = JSON.stringify(window.itemArray);
log('parse_html', 'trimming news item to fit in storage<br>' + red(window.itemArray.length) + ' items remaining ');
}
// make new title array
window.titleResult = window.itemArray.map(function(d){ return d[3] });
window.titleResult_set = new Set(window.titleResult);
if(window.titleResult_set.size != window.titleResult.length){
var sLength = window.itemArray.length;
var f = window.itemArray.map(function(d){ return d[2] });
var g = window.itemArray.map(function(d){ return d[3] });
window.itemArray = window.itemArray.filter(window.filterHtmlResult,[f,g]);
log('parse_html', (sLength-window.itemArray.length) + ' duplicate titles removed');
sJSON = JSON.stringify(window.itemArray);
}
// store new results
window.localStorage.setItem('finalarray' , sJSON);
// obtain oldest result date
if(window.itemArray.length >= window.pref.items_to_keep){ window.oldestEntry = window.itemArray[window.itemArray.length-1][0] ;
}else{ window.oldestEntry = 0 }
return window.itemArray;
}
// perform big update (rebuilding the table is faster than inserting many rows)
window.bigUpdate = function(x,forced_update){
log('parse_html', 'parse big update');
// remove duplicate titles
var xLength = x.length;
var f = x.map(function(d){ return d[2] });
var g = x.map(function(d){ return d[3] });
x = x.filter(window.filterHtmlResult,[f,g]);
x = x.filter(function(w){ return !window.titleResult_set.has(w[3])});
if(xLength-x.length){log('parse_html', red(xLength-x.length) + ' item'+(xLength-x.length>1?'s':'')+' removed')}
var HTMLresultX = window.updateStoredResult(x);
var HTMLresultXlength = HTMLresultX.length;
if(false && window.pref.highlight_frequent_words != "off"){ // https://jsfiddle.net/5hk6329u/
var words = window.titleResult.join(' ').toLowerCase().split(/\W+/g).sort();
var result = [];
for (var x = 0; x < words.length; x++) {
var word = words[x];
if(word.length > 3 && word.length < 20 && !window.noHighlight.has(word)){
var count = 1;
while(words[x+10] === word){
count+=10;
x+=10;
}
while(words[x+1] === word){
count++;
x++;
}
result.push({word: word, count: count});
if(result.length > 200){
result.sort(function(a, b){ return b.count - a.count });
result.length = 100;
}
}
}
result.sort(window.sortByCount);
result.length = 100;
w = result.map(function(z){ return z.word });
log('frequent_words', '<br>'+w.join(' '));
w = new Set(w);
for(x=0;x<HTMLresultXlength;x++){
var a = HTMLresultX[x][3];
var b = a.split(/[^A-Za-z]/);
var len = 0;
var spanstart = [];
var spanend = [];
for(b_length=b.length,o=0;o<b_length;o++){
if(w.has(b[o])){
spanstart.push(len)
spanend.push(len+b[o].length);
}
len += b[o].length+1;
}
if(spanstart.length>0){
b=a.split('');
for(z=spanstart.length-1;z>=0;z--){
b.splice(spanend[z], 0, '</span>');
b.splice(spanstart[z], 0, '<span style="color:white">');
}
HTMLresultX[x][3]=b.join('');
}
}
}
// combine array into string and write it to the page
window.buildTable(HTMLresultX,forced_update)
}
// parse and write results to the page
window.renewResults2 = function(forceUpdate,forced_update){
if( window.HTMLresultA[0].length > 0||forceUpdate){
log('parse_html', 'renew '+bl(window.HTMLresultA[0].length)+' results');
window.HTMLresultA.unshift([]);
var HTMLresultX = window.HTMLresultA.pop();
window.bigUpdate( HTMLresultX,forced_update )
}
window.renewResults = [window.renewResults1,window.renewResults2]
}
// deal with race conditions, function is called with renewResults.pop()()
window.renewResults = [window.renewResults1,window.renewResults2]
window.renewResults1 = function(forceUpdate){
log('parse_html', 'parser is bussy');
window.renewResults.unshift(window.renewResults1)
setTimeout((function(){renewResults(x)}).bind(0,forceUpdate),100)
}
// load the feeds
window.lastReqest = '';
window.pendingBussy = false;
window.pending_feeds = [];
window.loadFeedsInterval = (function(){
var get_domain = window.getDomain;
var clean_protocol = window.cleanProtocol;
var rssList = [];
getRssLength = function(){
return rssList.length + window.rss.length;
}
return function(){
if(rssList.length === 0 && window.rss.length > 0){
rssList = window.rss.splice(-100,100)
}
if(!window.stopRequest && window.pending_feeds.length < window.maxPending && getRssLength() > 0){
var currentFeed = rssList.pop().trim();
if(!currentFeed){ return }
if( currentFeed.indexOf('#') > -1){
currentFeed = currentFeed.split('#');
var currentOrigin = currentFeed[1].trim();
currentFeed = currentFeed[0].trim();
}else{
var currentOrigin = "not defined";
}
// skip blacklisted, unsubscribed and suspended feed urls
window.rss_blacklist_url_regex.lastIndex = 0;
window.unsubscribe_url_regex.lastIndex = 0;
window.suspended_regex.lastIndex = 0;
var baseProtocol = clean_protocol(currentFeed);
if(window.lastReqest != currentFeed && baseProtocol &&
!window.rss_blacklist_url_regex.test(baseProtocol) &&
!window.unsubscribe_url_regex.test(baseProtocol) &&
!window.suspended_regex.test(baseProtocol)){
// skip blacklisted and unsubscribed domains
var feedDomain = get_domain(currentFeed);
window.rss_blacklist_domain_regex.lastIndex = 0;
window.unsubscribe_domain_regex.lastIndex = 0;
if(feedDomain &&
!window.rss_blacklist_domain_regex.test(feedDomain) &&
!window.unsubscribe_domain_regex.test(feedDomain)){
window.lastReqest = currentFeed;
window.feedsRequested++;
log('rss_request_url', window.feedsRequested + ' ' + ora(currentFeed));
(function (reqestUrl,requestOrigin) {
window.pending_feeds.push(
[Date.now(),reqestUrl,GM_xmlhttpRequest({
method: 'GET',
url: clean_protocol(reqestUrl),
onload: function(response){
window.parseFeed[0]( response, reqestUrl, requestOrigin )
},
timeout: window.pref.wait_for_rss*1000,
onerror: function(){
window.rss_blacklist.push( reqestUrl );
window.removePending(reqestUrl)
window.feedResponses++;
log('rss_request_url', window.feedsRequested + ' ' + red(reqestUrl));
log('failure_request_error', red( reqestUrl ) );
return;
},
ontimeout: function(){
window.rss_blacklist.push( reqestUrl );
window.removePending(reqestUrl)
window.feedResponses++;
log('timeout',red( reqestUrl ))
},
onabort: function(){
window.rss_blacklist.push( reqestUrl );
window.removePending(reqestUrl)
window.feedResponses++;
log('aborted',red( reqestUrl ))
}
})]);
})(currentFeed,currentOrigin);
}else{ window.feedsSkipped++; loadFeedsInterval() }
}else{ window.feedsSkipped++; loadFeedsInterval() }
}
if(window.pending_feeds.length < window.maxPending && getRssLength() > 0){
setTimeout(window.loadFeedsInterval, window.pref.rss_loading_delay*1)
}
}
})()
// remove response from pending feed list
window.removePending = function(x){
for(var i=0;i < window.pending_feeds.length;i++){
if(window.pending_feeds[i][1] == x){
window.pending_feeds.splice(i,1);
break;
}
}
}
//////////////////////// parse xml responses //////////////////////////////////
buildParser = function(){
var regexParser = {},
lastDateError = 0,
noNewItems = 0,
consideredFeeds = 0;
// encode html entities
var htmlEncode = function( htmlToEncode ) {
var virtualDom = document.createElement( 'div' );
virtualDom.appendChild( document.createTextNode( htmlToEncode ) );
return virtualDom.innerHTML;
}
// test attributes
var testAtr = function(xml , testThis , atr, defaultVal){
if( xml.contains( xml.getElementsByTagName(testThis)[0] ) &&
xml.getElementsByTagName(testThis)[0].hasAttribute(atr) ){
return htmlEncode( xml.getElementsByTagName(testThis)[0].getAttribute(atr) )
}
}
// test attributes with regex (if dom parser fails)
var testAtrRegex = function(xml , testThis , atr){
regexParser[testThis+atr] =
regexParser[testThis+atr] || new RegExp("<"+testThis+"[\\s\\S]+?"+atr+"=['\"]([^'\">]+)","g");
var val = regexParser[testThis+atr].exec(xml);
regexParser[testThis+atr].lastIndex = 0;
if(val != null) return htmlEncode( val[1] );
}
// test elements
var testElm = function(xml , testThis){
var elm, val;
for(var x=0;x<testThis.length;x++){
if( xml.contains( elm=xml.getElementsByTagName(testThis[x])[0] ) ){
if(xml.contains( elm.childNodes[0] ) ){
val = htmlEncode( elm.childNodes[0].nodeValue );
if(val.trim() !="") return val;
}
return htmlEncode( elm.textContent );
}
}
}
// test elements with regex (if dom parser fails)
var testElmRegex = function(xml , testThis){
for(var x=0;x<testThis.length;x++){
regexParser[testThis[x]] = regexParser[testThis[x]] || new RegExp("<" + testThis[x] + "[^>]*?>([^<]+)","gm");
var val = regexParser[testThis[x]].exec(xml);
regexParser[testThis[x]].lastIndex = 0;
if(val != null){
return htmlEncode( val[1] );
}
}
}
// get elements by tag name
var getTags = function(a,b){ return a.getElementsByTagName(b) }
// get elements by tag name with "regex" (if dom parser fails)
var getTagsRegex = function(a,b){ a = a.split('<'+b); a.shift(); return a }
var pendingParser = [];
var itemPubDate_regex = new RegExp("<item>[\\s\\S]+?<pubDate>([^<]+)","g");
return [
function( response, reqestedUrl, requestOrigin ){
pendingParser.push({
response : response,
reqestedUrl : reqestedUrl,
requestOrigin : requestOrigin
})},
function(){
if(pendingParser.length==0){return}
var pain = Math.floor(document.getElementsByTagName('meter')[0].value);
log('pending_parsing', 'browser lag is '+pain+' (lower is better)');
log('pending_parsing', ora(pendingParser.length) +' responses awaiting parsing' );
do{
var task = pendingParser.pop();
lastParse = Date.now();
window.feedResponses++;
setTimeout(log,0,'rss_response_url', window.feedResponses + ' ' + gr(task.response.finalUrl));
setTimeout((function(x){
window.removePending(x);
}).bind(undefined, task.reqestedUrl),0);
}while(pendingParser.length > 50)
// get rid of everything with a <script tag in it
if(window.pref.blacklist_feed_with_script && task.response.responseText.indexOf('<script') !=-1){
log('error_found_script', 'found script tag in '+ora(task.reqestedUrl) );
window.rss_blacklist.push( task.reqestedUrl );
return;
}
// date 1 - quickly check the first 6 dates before parsing the xml
var i=0;
var pub;
var pubD;
itemPubDate_regex.lastIndex = 0;
pub = itemPubDate_regex.exec(task.response.responseText);
if(pub!=null){ // no date was found, skipping rest of test
pubD = Date.parse(pub[1]);
if(!isNaN(pubD) && pubD < window.oldestEntry){
do{ // the first date was valid but not newer than the oldest entry
pub = itemPubDate_regex.exec(task.response.responseText);
if(pub==null){ // no new news found
log('no_new_items', (++noNewItems) + ' ' +ora(task.reqestedUrl));
return;
}
pubD = Date.parse(pub[1]);
if(isNaN(pubD)){ continue } // skip broken dates
if(pubD > window.oldestEntry){ break } // new item found!
if(i>=6){ // no new news found
log('no_new_items', (++noNewItems) + ' ' +ora(task.reqestedUrl));
return;
}
}while(++i)
}
}
// deal with blogspot bullshit link attributes
if(task.response.finalUrl.indexOf('.blogspot.') != -1 || task.reqestedUrl.indexOf('.blogspot.') != -1||task.response.finalUrl.indexOf('.blogger.') != -1 || task.reqestedUrl.indexOf('.blogger.') != -1 ){
task.response.responseText = task.response.responseText.split(/<link.+application\/atom[^>]+>/).join('');
}
// try to use the dom parser
var xml = new DOMParser();
xml = xml.parseFromString(task.response.responseText.trim(), "text/xml");
if(xml.documentElement.nodeName == "parsererror"){
// if the dom parser doesn't work use regex
xml = task.response.responseText;
var parser = {
testAtri : testAtrRegex,
testEleme : testElmRegex,
getTagNames : getTagsRegex,
type : 'regex'
}
}else{
var parser = {
testAtri : testAtr,
testEleme : testElm,
getTagNames : getTags,
type : 'dom'
}
}
// gather items
var feedItems = parser.getTagNames(xml,"item")
if(feedItems.length === 0 || !feedItems ){
feedItems = parser.getTagNames(xml,"entry");
}
if(feedItems.length === 0){
log('failure_no_items_in_feed', window.feedResponses + ' ' + ora( task.reqestedUrl ));
window.rss_suspended.push(Date.now());
window.rss_suspended.push(task.reqestedUrl);
return;
}
var lastNoNew = "";
var logNoNew = false;
var newItem = 0;
var maxLength = feedItems.length;
for(var itemNr = 0; itemNr < maxLength; itemNr++ ){
var feedItemsNode = feedItems[itemNr];
// date 2
var itemPubDate = parser.testEleme(feedItemsNode,["pubDate"]);
if(itemPubDate && Date.parse( itemPubDate ) < window.oldestEntry){
if( lastNoNew != task.reqestedUrl){
lastNoNew = task.reqestedUrl;
logNoNew = true;
}
continue;
}
// link
var itemLink = parser.testEleme(feedItemsNode , ["link","guid"]) || parser.testAtri(feedItemsNode , "link" , "href") || parser.testAtri(feedItemsNode , "enclosure" , "url")
if(!itemLink && parser.type==='dom'){
var serializer = new XMLSerializer();
var feedItemTextNode = serializer.serializeToString(feedItemsNode)
itemLink = testElmRegex(feedItemTextNode,["link","guid"]) || testAtrRegex(feedItemTextNode,"link","href") || testAtrRegex(feedItemTextNode,"enclosure","url");
if(!itemLink){log('no_link', red(task.response.finalUrl));continue}
}
if(!itemLink || !window.cleanProtocol(itemLink.trim()).startsWith('http')){continue}
itemLink = window.cleanProtocol(itemLink.trim());
// title
var itemTitle = parser.testEleme(feedItemsNode,["title"]);
if( !itemTitle || itemTitle.length<3){
itemTitle = parser.testEleme(feedItemsNode,["description"])
if( !itemTitle || itemTitle.length<3){
itemTitle = parser.testEleme(feedItemsNode,["summary"])
if( !itemTitle ){
log('no_title', red(task.response.finalUrl));
var h = itemLink.split('/');
h = h.reduce(function(a,b){ return (a.length>b.length)?a:b});
h = h.replace(/([-_]|\.html|\.php)/g,' ');
itemTitle = h.charAt(0).toUpperCase() + h.slice(1);
}
}
}
itemTitle = itemTitle.trim();
if(itemTitle.length > 200){ itemTitle = itemTitle.substring(0,200)+"..." }
// date 3
itemPubDate = itemPubDate || parser.testEleme(feedItemsNode,["published","created","dc:date","updated","modified","atom:updated"]);
// try find date in url like /2015/12/26/ ==> 2015-12-28T00:00:00+00:00
if( ! itemPubDate ){
var lDArray = itemLink.match(/\/?(\d{1,4})\/(\d{1,2})(?:\/(\d{0,2})|)/);
var curDate = new Date();
var da = curDate.getFullYear();
var db = "01";
var dc = "01";
if(lDArray && lDArray[1] && lDArray[2]){// looks like a date?
if(!lDArray[3]){ // when only 2 numbers in url, first might be the month or the year
if((1*lDArray[1]) <= 12){ // smaller than 13 must be a month (at least up to 2100)
lDArray.unshift('');
// if the month is or comes before this one assume publication this year
if((1*lDArray[2]) <= curDate.getMonth()+1){ lDArray[1]=da }
// if the month comes after this month publication must have been last year
else{ lDArray[1]=da-1 }
}
}
// extend 2 digit year
if((lDArray[1]+'').length == 2){ lDArray[1]= "20"+lDArray[1] }
if((lDArray[1]+'').length == 4 && 1*lDArray[1] <= da){
da = lDArray[1];
if( lDArray[2] && (lDArray[2]+'').length == 1){ lDArray[2]= "0"+lDArray[2] }
if( lDArray[2] && (1*lDArray[2]) <= curDate.getMonth()+1){
db = lDArray[2];
if((lDArray[3]+'').length == 1){ lDArray[3]= "0"+lDArray[3] }
if(lDArray[3] && 1*lDArray[3] <= curDate.getDate()){
dc = lDArray[3];
}
}
itemPubDate = da+"-"+db+"-"+dc+"T00:00:00+00:00"
}
}
}
// date 3 (any date in xml)
itemPubDate = itemPubDate || parser.testEleme( xml , ["modified","updated","atom:updated","published","created","dc:date","lastBuildDate","pubDate"] )
if( itemPubDate ){
itemPubDate = itemPubDate.trim();
if( isNaN( Date.parse( itemPubDate ))){
var chunks = itemPubDate.split(' '); // try correct 2 digit years
if(chunks[3] && chunks[3].length == 2){
chunks[3] = '20'+chunks[3]; itemPubDate = chunks.join(' ');
}
//itemPubDate = itemPubDate.split(' PM ').join(' '); // try correct PM
}
}
// fix further date errors
if(isNaN( Date.parse( itemPubDate ))){
if(lastDateError != task.reqestedUrl){
log('failure_date_error', red( itemPubDate ) + ':<br>' + ora( task.reqestedUrl ));
lastDateError = task.reqestedUrl;
}
itemPubDate = "Mon, 18 Mar 1974 00:00:00 +0000";
}else if( Date.now() < Date.parse( itemPubDate ) ){
log('failure_future_date_error', red( itemPubDate ) + ' = ' + red( new Date( itemPubDate ).toLocaleTimeString() ) + '<br>' + ora( task.reqestedUrl ));
continue;
}
if(!(Date.parse( itemPubDate ) > window.oldestEntry)){
if( lastNoNew != task.reqestedUrl){
lastNoNew = task.reqestedUrl;
logNoNew = true;
}
continue;
}
// did we have this exact title already?
if(!window.titleResult_set.has(itemTitle) ){
log("title_result", (++window.titleCount) + ' ' + ora(itemTitle));
// filter out titles with badwords
var stripTitle = itemTitle.slice(0).toLowerCase().replace(/[^a-z0-9]/g, " ");
var titleArray = stripTitle.split(' ');
var longEnough = titleArray.length >= window.pref.minimum_number_of_words_in_a_title;
window.badwords_regex.lastIndex = 0;
if( longEnough ){
if( !window.badwords_regex.test(stripTitle) ){
window.titleResult.push( itemTitle );
window['HTMLresultA'][0].unshift([
/*[0]*/ Date.parse( itemPubDate ),
/*[1]*/ task.reqestedUrl,
/*[2]*/ itemLink,
/*[3]*/ itemTitle,
/*[4]*/ task.requestOrigin ]);
itemPubDate = false;
itemLink = false;
itemTitle = false;
logNoNew = false;
lastNoNew = task.reqestedUrl;
newItem++;
if(newItem >= window.pref.items_per_rss_feed){break}
// describe errors and filtered items
}else{ log('word_filter', red(++window.titlesFiltered) + ' titles discarded') }
}else{ log('to_short',(++window.toShortTitles)+' '+red(itemTitle))}
}else{ log('duplicate_title', (++window.duplicateTitles) + ' ' + red(itemTitle)) }
}
if(logNoNew){ log('no_new_items', (++noNewItems) + ' ' + bl(task.reqestedUrl) ) }
if(newItem){ log('considered', (++consideredFeeds) + ' ' + ora(task.reqestedUrl)) }
}]
}
window.parseFeed = buildParser();
// extend the rss list
window.rssPushFeeds = [];
window.rssPush = function(newFeedUrl){
if(window.rss == undefined){ window.rss = [newFeedUrl]; return; }
if(window.rss.indexOf( newFeedUrl ) === -1 && window.rssPushFeeds.indexOf( newFeedUrl ) === -1){
window.rssPushFeeds.push( newFeedUrl );
setTimeout(function(){
if(window.rssPushFeeds.length > 0){
window.rss = window.rss.concat(window.rssPushFeeds);
window.rssPushFeeds = [];
}
},7000)
}
}
// request opml files
window.opmlReadingIntervalFunction = function(){
window.opmlRequested++;
var currentOPML = window.opml.pop();
log('opml_request_url', window.opmlRequested + ' ' + ora(currentOPML));
(function (reqestUrl) {
GM_xmlhttpRequest({
method : 'GET',
url : reqestUrl.trim(),
timeout: window.pref.wait_for_opml*1000,
onerror: function(){ log('opml_request_error', red(reqestUrl))},
onload : function(response){
window.opmlResponses++;
log('opml_response_url', window.opmlResponses + ' ' + gr( response.finalUrl.split('<')[0] ));
// manage flat feed lists and comma separated feed lists
var openTag = response.responseText.indexOf('<');
if(openTag == -1){
var temp_list = response.responseText;
temp_list = temp_list.split(',').join('\n').split('\n');
var result_list = [];
for(var e=0;e<temp_list.length;e++){
var temp_val = temp_list[e].trim()
if( temp_val != ""){
result_list.push(temp_val+"#"+reqestUrl);
}
}
var temp_listLength = result_list.length;
result_list = Array.from(new Set(result_list));
window.rss = window.rss.concat(result_list);
countInOpml += temp_listLength;
log('stages','receaved rss list ' + gr(window.opmlResponses) + ' with ' + gr(temp_listLength) + ' feeds for a total of ' + bl(countInOpml) + ' feeds');
return 0;
}
// use dom parser
var xml = new DOMParser();
xml = xml.parseFromString(response.responseText, "text/xml");
if(xml.documentElement.nodeName == "parsererror"){ log('opml_failure','parse error ' + red( reqestUrl ) ); return; }
var outline = xml.getElementsByTagName("outline");
var outlineLength =outline.length;
countInOpml += outlineLength;
log('stages','receaved opml ' + gr(window.opmlResponses) + ' with ' + gr(outlineLength) + ' outlines for a total of ' + bl(countInOpml) + ' feeds');
if(outlineLength == 0){ log('opml_failure','no outlines found ' + red( currentOPML ) ); return;
}
for ( var k = 0; k < outlineLength; k++ ) {
if(outline[k].hasAttribute('xmlUrl')){
var xmlUrl = outline[k].getAttribute('xmlUrl');
if(typeof window.rss === 'undefined'){
window.rss = [xmlUrl.trim()+"#"+reqestUrl];
}else if(window.rss.indexOf(xmlUrl.trim()+"#"+reqestUrl) == -1){
window.rssPush( xmlUrl.trim()+"#"+reqestUrl );
window.xml_retreaved_from_opml++;
}
}
}
}
})
})(currentOPML)
}
// delay parsing html if mouse is moving
/*
window.mouseUpdate = function(){
window.mouseMove = 15;
document.getElementsByTagName('title')[0].innerHTML = "mouse moving";
}
document.addEventListener('mousemove', mouseUpdate, false);
*/
/////////////// STAGES //////////////////////
window.publish_time = Date.now();
// 1 - display old news
log('stages','display old news');
window.renewResults.pop()(true);
// keep blacklist up to date
window.serviceGMstorage();
window['serviceGMstorageTimer'] = setInterval(function(){ window.serviceGMstorage() },5000);
// stage tracking object
window.ApplicationStages = {
"waiting" : 0,
"aquire_feeds_from_config" : 1,
"aquire_feeds_from_local_storage" : [1],
"aquire_feeds_from_opml_files" : window.opml.slice().fill(1),// window.opml.length,
"finish" : [1]
}
// load the feed list from configuration
countInOpml += window.rss.length;
log('stages','loading ' + gr(window.rss.length) + ' user defined feeds for a total of ' + bl(countInOpml) + ' feeds');
for(x = window.rss.length; x>=0 ; x--){ window.rss[x] = window.rss[x] + "#user defined"; }
window.rss = Array.from(new Set(window.rss));
window.progressInterval = function(){
// show progress every second
window.oldProgressSeconds = window.progressSeconds;
window.progressSeconds = Math.floor( Date.now() / 1000 )-window.oldTimeA;
if(window.progressSeconds != window.oldProgressSeconds){
log('feeds', ora(
window.progressSeconds) +' sec ' +
gr(window.feedResponses) +' done '+
red(window.feedsSkipped) +' skipped ' +
bl(window.rss.length) +' in task ' +
bl(window.pending_feeds.length) + ' pending'
);
log('average_time', 'averaging '+ gr((window.progressSeconds/window.feedResponses).toFixed(4))+' sec per feed '+gr((window.progressSeconds/window.titleCount).toFixed(4))+' sec per title considered');
// renew html results if mouse is not moving
if(window.mouseMove != 0){ window.mouseMove-- }
else{
document.getElementsByTagName('title')[0].innerHTML = "internet";
window.oldRenewTimer = window.renewTimer;
window.renewTimer = Math.floor(window.progressSeconds / window.pref.html_parsing_delay );
if(window.renewTimer != window.oldRenewTimer){ setTimeout(window.renewResults.pop()(),5) }
}
var expired = 0;
var y = Date.now();
for(var i=0;i < window.pending_feeds.length;i++){
if(y-window.pending_feeds[i][0]>window.pref.wait_for_rss*1000){
window.pending_feeds[i][2].abort();
window.rss_blacklist.push(window.pending_feeds[i][1]);
window.pending_feeds.splice(i,1);
i--;
expired++;
}
}
if(expired){
log('abort', ora(expired)+' requests expired, '+bl(window.pending_feeds.length)+' requests pending expiration')
}
}
if(window.publish_time+300000 < Date.now()){
window.publish_time = Date.now();
// export the news results
if(window.pref.publish_news === "yes"){
log('publishing','publishing....' + bl(window.publish_time));
var subset = JSON.parse(localStorage.getItem('finalarray', false));
if(!subset){return}
// make html from array
var outputResult = '',domainIndicator;
for(var x=0;x<subset.length&&x<3000;x++){
// define item class
var itemClass = 'class="';
// identify comments
if(subset[x][3].indexOf('Comment on') == 0
|| subset[x][3].indexOf('RE:') == 0
|| subset[x][3].indexOf('Re:') == 0
|| subset[x][3].indexOf('re:') == 0
|| subset[x][2].indexOf('#comment') != -1
|| subset[x][2].indexOf('/comment') != -1){ itemClass += 'comment' }
// class for undefined source
if(subset[x][4] == "not defined"){ itemClass += ' autodetect' }
itemClass += '"';
// try to obtain the host url
if(subset[x][2].indexOf('feedproxy.google.com')>0
&& subset[x][2].indexOf('feedproxy.google.com')<9
&& subset[x][2].split('/')[4] ){
domainIndicator = subset[x][2].split('/')[4];
}else{
domainIndicator = getDomain(subset[x][2]);
}
outputResult +=
'<tr><td>' +
(new Date(subset[x][0])+'') +
'</td><td>'+
domainIndicator+
'</td><td><a href="'+
subset[x][2]+
'" '+
itemClass +
' target="_blank">'+
subset[x][3]+
'</a></td></tr>';
}
var finalData = '<table>'+outputResult+'</table>';
GM_xmlhttpRequest({
method: 'POST',
url: window.pref.publish_news_url,
onload: function(response){},
data: "password="+window.pref.publish_news_password+"&news="+encodeURIComponent(finalData),
headers: { "Content-Type": "application/x-www-form-urlencoded" }
})
}
}
if(window.rss == null){ window.rss = []; }
if(window.rss.length > 200 ){ return }
if(0 < window.ApplicationStages.waiting--){ return }
// 3 - load the feeds from localStorage
if( window.ApplicationStages.aquire_feeds_from_local_storage.pop() ){
window.ApplicationStages.waiting = 30;
var rssL = localStorage.getItem('autoDetect',false);
if (rssL){
rssL = rssL.split(',');
log('stages','imported ' + gr(rssL.length) + ' feed urls from localStorage');
for(x = rssL.length; x>=0 ; x--){ rssL[x] += "#local storage"; }
window.rss = window.rss.concat(rssL);
}else{
log('stages','no feeds in local storage');
}
// 4 - load opml files
}else if( window.opml.length ){
window.ApplicationStages.waiting = 100;
log('stages','loading ' + gr(window.opml.length) + ' opml files');
opmlReadingIntervalFunction();
}else if( window.pending_feeds.length === 0 && window.ApplicationStages.finish.pop() ){
window.ApplicationStages.waiting = 1000;
window.renewResults.pop()();
clearInterval( window.antiFreezeTimer );
log('stages','<b>finished</b>');
setTimeout(function(){
clearTimeout(window.antiFreezeTimer);
clearInterval(window.ParseTimer);
clearInterval(window.serviceGMstorageTimer);
unsafeWindow.console_factory.stop;
for(o=0;o<100;o++){clearTimeout(o);clearInterval(o)};
setTimeout(function(){
document.location.reload(false);
},3600000);
},10000);
}
}
window.dat = window.oldDat = window.localHostTest = Date.now();
window.oldPain = 10000;
window.port = 65535;
window.pingArray = [50,50];
window.pingAverage = 0;
window.pingActive = false;
window.stopRequest = false;
window.antiFreeze = function(){
window.stopRequest = false;
if(window.maxPending < 0){window.maxPending = 0}
if(Date.now() - window.localHostTest > 5000 && !window.pingActive){
window.pingActive = true;
window.localHostTest = Date.now();
GM_xmlhttpRequest({
method: "GET",
url: "https://localhost:" +window.port,
timeout: ((( window.pingArray.reduce(function(a,b){return a+b},0) )/ window.pingArray.length)+200),
ontimeout: function() {
window.pingActive = false;
log('localhost',red('connection timed out: 5 sec pause...'));
window.stopRequest = true;
if(window.maxPending>5){
if(window.maxPending>20){ log('localhost', red('reducing max request to: '+(window.maxPending-=20)))
}else{ log('localhost', red('reducing max request to: '+(window.maxPending=0))) }
}
clearTimeout(window.antiFreezeTimer);
window.antiFreezeTimer = setTimeout( window.antiFreeze, 5000 );
window.oldDat = Date.now();
window.progressInterval();
window.parseFeed[1](); window.parseFeed[1](); window.parseFeed[1]();
window.parseFeed[1](); window.parseFeed[1]();
},
onload: function(response){ window.pingActive = false; log('localhost',ora('acidentally hit website: changing port number to '+ window.port--)) },
onerror: function(response) {
window.pingActive = false;
var lastPing = Date.now() - window.localHostTest;
window.pingArray.push(lastPing);
if(window.pingArray.length>60){ window.pingArray.shift() };;
window.pingAverage = ( window.pingArray.reduce(function(a,b){return a+b},0) ) / window.pingArray.length;
log('localhost','ping: '+lastPing+' average over last '+window.pingArray.length+' : '+window.pingAverage);
if(lastPing>(window.pingAverage+30)){
if(window.maxPending>10){ log('localhost',ora('prematurely decreasing max request to'+': '+(window.maxPending-=2))) }
} else if(window.maxPending<100){ log('localhost', bl('increasing max request from '+ora(window.maxPending)+' to '+ora((window.maxPending+=20))));
}else{ log('localhost',gr('local host is available, max pending: '+ gr(window.maxPending))) }
window.progressInterval();
window.parseFeed[1](); window.parseFeed[1](); window.parseFeed[1]();
window.parseFeed[1](); window.parseFeed[1](); window.parseFeed[1]();
window.parseFeed[1](); window.parseFeed[1]();
}
});
}else if(Date.now() - window.localHostTest > 50000 && window.pingActive){ window.pingActive = false }
dat = Date.now();
if(dat-oldDat > 50000){ log('stages', red('<b>Aborting: browser froze for '+((dat-oldDat)/1000)+' seconds.<br> Please try restarting the browser</b>')) }
oldDat = dat;
var pain = Math.floor(document.getElementsByTagName('meter')[0].value);
var delay = (pain < 2020)?500:(pain > 10000)?10000:pain-2000;
window.antiFreezeTimer = setTimeout( window.antiFreeze, delay );
window.progressInterval();
window.loadFeedsInterval();
if(pain > 5000){ log('localhost',red('rendering lag exceeds 5000 units, decreasing max request to '+(window.maxPending-=10)))
}else{ window.parseFeed[1]() }
window.oldPain = pain;
}
window.antiFreezeTimer = setTimeout(window.antiFreeze,500)
window.onpagehide = function (e) { window.maxPendingOld = window.maxPending || 1; window.maxPending = 0; }
window.onpageshow = function (e) { window.maxPending = window.maxPendingOld || 1 }
}else{
// load old configuration into the form
id('configuration').value = GM_getValue('configuration', '');
window.urlArrays.forEach(function(x){ if(window[x][0]){ id(x).value = window[x].join('\n') } });
// update configuration if form is submitted
setInterval(function(){
if(id('submitCheck').value=="checked"){
id('submitCheck').value = "unchecked";
GM_setValue( 'configuration', id('configuration').value );
window.urlArrays.forEach(function(x){setValue(x, id(x).value.split('\n'))});
if(confirm("\n\n Settings saved! \n\nProceed to aggregator?\n\n")){ location.href = window.aggregatorLink }
}
},50);
GM_registerMenuCommand("feature requests / bug report", function(){ location.href = window.pref.bugTracker });
}