// Copyright 2012-2017 (c) Peter Širka <petersirka@gmail.com>

//

// Permission is hereby granted, free of charge, to any person obtaining a

// copy of this software and associated documentation files (the

// "Software"), to deal in the Software without restriction, including

// without limitation the rights to use, copy, modify, merge, publish,

// distribute, sublicense, and/or sell copies of the Software, and to permit

// persons to whom the Software is furnished to do so, subject to the

// following conditions:

//

// The above copyright notice and this permission notice shall be included

// in all copies or substantial portions of the Software.

//

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS

// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF

// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN

// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,

// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR

// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE

// USE OR OTHER DEALINGS IN THE SOFTWARE.

/**

* @module FrameworkUtils

* @version 2.5.0

*/

'use strict';

const Dns = require('dns');

const Url = require('url');

const Qs = require('querystring');

const Http = require('http');

const Https = require('https');

const Path = require('path');

const Fs = require('fs');

const Events = require('events');

const Crypto = require('crypto');

const CONCAT = [null, null];

if (!global.framework_utils)

global.framework_utils = exports;

var regexpSTATIC = /\.\w{2,8}($|\?)+/;

const regexpTRIM = /^[\s]+|[\s]+$/g;

const regexpDATE = /(\d{1,2}\.\d{1,2}\.\d{4})|(\d{4}\-\d{1,2}\-\d{1,2})|(\d{1,2}\:\d{1,2}(\:\d{1,2})?)/g;

const regexpDATEFORMAT = /yyyy|yy|M+|d+|HH|H|hh|h|mm|m|ss|s|a|ww|w/g;

const regexpSTRINGFORMAT = /\{\d+\}/g;

const regexpPATH = /\\/g;

const regexpTags = /<\/?[^>]+(>|$)/g;

const regexpDiacritics = /[^\u0000-\u007e]/g;

const regexpXML = /\w+\=\".*?\"/g;

const regexpDECODE = /&#?[a-z0-9]+;/g;

const regexpPARAM = /\{{2}[^}\n]*\}{2}/g;

const regexpINTEGER = /(^\-|\s-)?[0-9]+/g;

const regexpFLOAT = /(^\-|\s-)?[0-9\.\,]+/g;

const regexpALPHA = /^[A-Za-z0-9]+$/;

const regexpSEARCH = /[^a-zA-Zá-žÁ-Ž\d\s:]/g;

const regexpDECRYPT = /\-|\_/g;

const regexpENCRYPT = /\/|\+/g;

const regexpUNICODE = /\\u([\d\w]{4})/gi;

const regexpTERMINAL = /[\w\S]+/g;

const SOUNDEX = { a: '', e: '', i: '', o: '', u: '', b: 1, f: 1, p: 1, v: 1, c: 2, g: 2, j: 2, k: 2, q: 2, s: 2, x: 2, z: 2, d: 3, t: 3, l: 4, m: 5, n: 5, r: 6 };

const ENCODING = 'utf8';

const NEWLINE = '\r\n';

const isWindows = require('os').platform().substring(0, 3).toLowerCase() === 'win';

const DIACRITICSMAP = {};

const STREAM_READONLY = { flags: 'r' };

const STREAM_END = { end: false };

const ALPHA_INDEX = { '&lt': '<', '&gt': '>', '&quot': '"', '&apos': '\'', '&amp': '&', '&lt;': '<', '&gt;': '>', '&quot;': '"', '&apos;': '\'', '&amp;': '&' };

const EMPTYARRAY = [];

const EMPTYOBJECT = [];

const NODEVERSION = parseFloat(process.version.toString().replace('v', '').replace(/\./g, ''));

const STREAMPIPE = { end: false };

exports.MONTHS = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'];

exports.DAYS = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday'];

Object.freeze(EMPTYARRAY);

Object.freeze(EMPTYOBJECT);

var DIACRITICS=[{b:' ',c:'\u00a0'},{b:'0',c:'\u07c0'},{b:'A',c:'\u24b6\uff21\u00c0\u00c1\u00c2\u1ea6\u1ea4\u1eaa\u1ea8\u00c3\u0100\u0102\u1eb0\u1eae\u1eb4\u1eb2\u0226\u01e0\u00c4\u01de\u1ea2\u00c5\u01fa\u01cd\u0200\u0202\u1ea0\u1eac\u1eb6\u1e00\u0104\u023a\u2c6f'},{b:'AA',c:'\ua732'},{b:'AE',c:'\u00c6\u01fc\u01e2'},{b:'AO',c:'\ua734'},{b:'AU',c:'\ua736'},{b:'AV',c:'\ua738\ua73a'},{b:'AY',c:'\ua73c'},{b:'B',c:'\u24b7\uff22\u1e02\u1e04\u1e06\u0243\u0181'},{b:'C',c:'\u24b8\uff23\ua73e\u1e08\u0106C\u0108\u010a\u010c\u00c7\u0187\u023b'},{b:'D',c:'\u24b9\uff24\u1e0a\u010e\u1e0c\u1e10\u1e12\u1e0e\u0110\u018a\u0189\u1d05\ua779'},{b:'Dh',c:'\u00d0'},{b:'DZ',c:'\u01f1\u01c4'},{b:'Dz',c:'\u01f2\u01c5'},{b:'E',c:'\u025b\u24ba\uff25\u00c8\u00c9\u00ca\u1ec0\u1ebe\u1ec4\u1ec2\u1ebc\u0112\u1e14\u1e16\u0114\u0116\u00cb\u1eba\u011a\u0204\u0206\u1eb8\u1ec6\u0228\u1e1c\u0118\u1e18\u1e1a\u0190\u018e\u1d07'},{b:'F',c:'\ua77c\u24bb\uff26\u1e1e\u0191\ua77b'}, {b:'G',c:'\u24bc\uff27\u01f4\u011c\u1e20\u011e\u0120\u01e6\u0122\u01e4\u0193\ua7a0\ua77d\ua77e\u0262'},{b:'H',c:'\u24bd\uff28\u0124\u1e22\u1e26\u021e\u1e24\u1e28\u1e2a\u0126\u2c67\u2c75\ua78d'},{b:'I',c:'\u24be\uff29\u00cc\u00cd\u00ce\u0128\u012a\u012c\u0130\u00cf\u1e2e\u1ec8\u01cf\u0208\u020a\u1eca\u012e\u1e2c\u0197'},{b:'J',c:'\u24bf\uff2a\u0134\u0248\u0237'},{b:'K',c:'\u24c0\uff2b\u1e30\u01e8\u1e32\u0136\u1e34\u0198\u2c69\ua740\ua742\ua744\ua7a2'},{b:'L',c:'\u24c1\uff2c\u013f\u0139\u013d\u1e36\u1e38\u013b\u1e3c\u1e3a\u0141\u023d\u2c62\u2c60\ua748\ua746\ua780'}, {b:'LJ',c:'\u01c7'},{b:'Lj',c:'\u01c8'},{b:'M',c:'\u24c2\uff2d\u1e3e\u1e40\u1e42\u2c6e\u019c\u03fb'},{b:'N',c:'\ua7a4\u0220\u24c3\uff2e\u01f8\u0143\u00d1\u1e44\u0147\u1e46\u0145\u1e4a\u1e48\u019d\ua790\u1d0e'},{b:'NJ',c:'\u01ca'},{b:'Nj',c:'\u01cb'},{b:'O',c:'\u24c4\uff2f\u00d2\u00d3\u00d4\u1ed2\u1ed0\u1ed6\u1ed4\u00d5\u1e4c\u022c\u1e4e\u014c\u1e50\u1e52\u014e\u022e\u0230\u00d6\u022a\u1ece\u0150\u01d1\u020c\u020e\u01a0\u1edc\u1eda\u1ee0\u1ede\u1ee2\u1ecc\u1ed8\u01ea\u01ec\u00d8\u01fe\u0186\u019f\ua74a\ua74c'}, {b:'OE',c:'\u0152'},{b:'OI',c:'\u01a2'},{b:'OO',c:'\ua74e'},{b:'OU',c:'\u0222'},{b:'P',c:'\u24c5\uff30\u1e54\u1e56\u01a4\u2c63\ua750\ua752\ua754'},{b:'Q',c:'\u24c6\uff31\ua756\ua758\u024a'},{b:'R',c:'\u24c7\uff32\u0154\u1e58\u0158\u0210\u0212\u1e5a\u1e5c\u0156\u1e5e\u024c\u2c64\ua75a\ua7a6\ua782'},{b:'S',c:'\u24c8\uff33\u1e9e\u015a\u1e64\u015c\u1e60\u0160\u1e66\u1e62\u1e68\u0218\u015e\u2c7e\ua7a8\ua784'},{b:'T',c:'\u24c9\uff34\u1e6a\u0164\u1e6c\u021a\u0162\u1e70\u1e6e\u0166\u01ac\u01ae\u023e\ua786'}, {b:'Th',c:'\u00de'},{b:'TZ',c:'\ua728'},{b:'U',c:'\u24ca\uff35\u00d9\u00da\u00db\u0168\u1e78\u016a\u1e7a\u016c\u00dc\u01db\u01d7\u01d5\u01d9\u1ee6\u016e\u0170\u01d3\u0214\u0216\u01af\u1eea\u1ee8\u1eee\u1eec\u1ef0\u1ee4\u1e72\u0172\u1e76\u1e74\u0244'},{b:'V',c:'\u24cb\uff36\u1e7c\u1e7e\u01b2\ua75e\u0245'},{b:'VY',c:'\ua760'},{b:'W',c:'\u24cc\uff37\u1e80\u1e82\u0174\u1e86\u1e84\u1e88\u2c72'},{b:'X',c:'\u24cd\uff38\u1e8a\u1e8c'},{b:'Y',c:'\u24ce\uff39\u1ef2\u00dd\u0176\u1ef8\u0232\u1e8e\u0178\u1ef6\u1ef4\u01b3\u024e\u1efe'}, {b:'Z',c:'\u24cf\uff3a\u0179\u1e90\u017b\u017d\u1e92\u1e94\u01b5\u0224\u2c7f\u2c6b\ua762'},{b:'a',c:'\u24d0\uff41\u1e9a\u00e0\u00e1\u00e2\u1ea7\u1ea5\u1eab\u1ea9\u00e3\u0101\u0103\u1eb1\u1eaf\u1eb5\u1eb3\u0227\u01e1\u00e4\u01df\u1ea3\u00e5\u01fb\u01ce\u0201\u0203\u1ea1\u1ead\u1eb7\u1e01\u0105\u2c65\u0250\u0251'},{b:'aa',c:'\ua733'},{b:'ae',c:'\u00e6\u01fd\u01e3'},{b:'ao',c:'\ua735'},{b:'au',c:'\ua737'},{b:'av',c:'\ua739\ua73b'},{b:'ay',c:'\ua73d'}, {b:'b',c:'\u24d1\uff42\u1e03\u1e05\u1e07\u0180\u0183\u0253\u0182'},{b:'c',c:'\uff43\u24d2\u0107\u0109\u010b\u010d\u00e7\u1e09\u0188\u023c\ua73f\u2184'},{b:'d',c:'\u24d3\uff44\u1e0b\u010f\u1e0d\u1e11\u1e13\u1e0f\u0111\u018c\u0256\u0257\u018b\u13e7\u0501\ua7aa'},{b:'dh',c:'\u00f0'},{b:'dz',c:'\u01f3\u01c6'},{b:'e',c:'\u24d4\uff45\u00e8\u00e9\u00ea\u1ec1\u1ebf\u1ec5\u1ec3\u1ebd\u0113\u1e15\u1e17\u0115\u0117\u00eb\u1ebb\u011b\u0205\u0207\u1eb9\u1ec7\u0229\u1e1d\u0119\u1e19\u1e1b\u0247\u01dd'}, {b:'f',c:'\u24d5\uff46\u1e1f\u0192'},{b:'ff',c:'\ufb00'},{b:'fi',c:'\ufb01'},{b:'fl',c:'\ufb02'},{b:'ffi',c:'\ufb03'},{b:'ffl',c:'\ufb04'},{b:'g',c:'\u24d6\uff47\u01f5\u011d\u1e21\u011f\u0121\u01e7\u0123\u01e5\u0260\ua7a1\ua77f\u1d79'},{b:'h',c:'\u24d7\uff48\u0125\u1e23\u1e27\u021f\u1e25\u1e29\u1e2b\u1e96\u0127\u2c68\u2c76\u0265'},{b:'hv',c:'\u0195'},{b:'i',c:'\u24d8\uff49\u00ec\u00ed\u00ee\u0129\u012b\u012d\u00ef\u1e2f\u1ec9\u01d0\u0209\u020b\u1ecb\u012f\u1e2d\u0268\u0131'}, {b:'j',c:'\u24d9\uff4a\u0135\u01f0\u0249'},{b:'k',c:'\u24da\uff4b\u1e31\u01e9\u1e33\u0137\u1e35\u0199\u2c6a\ua741\ua743\ua745\ua7a3'},{b:'l',c:'\u24db\uff4c\u0140\u013a\u013e\u1e37\u1e39\u013c\u1e3d\u1e3b\u017f\u0142\u019a\u026b\u2c61\ua749\ua781\ua747\u026d'},{b:'lj',c:'\u01c9'},{b:'m',c:'\u24dc\uff4d\u1e3f\u1e41\u1e43\u0271\u026f'},{b:'n',c:'\u24dd\uff4e\u01f9\u0144\u00f1\u1e45\u0148\u1e47\u0146\u1e4b\u1e49\u019e\u0272\u0149\ua791\ua7a5\u043b\u0509'},{b:'nj', c:'\u01cc'},{b:'o',c:'\u24de\uff4f\u00f2\u00f3\u00f4\u1ed3\u1ed1\u1ed7\u1ed5\u00f5\u1e4d\u022d\u1e4f\u014d\u1e51\u1e53\u014f\u022f\u0231\u00f6\u022b\u1ecf\u0151\u01d2\u020d\u020f\u01a1\u1edd\u1edb\u1ee1\u1edf\u1ee3\u1ecd\u1ed9\u01eb\u01ed\u00f8\u01ff\ua74b\ua74d\u0275\u0254\u1d11'},{b:'oe',c:'\u0153'},{b:'oi',c:'\u01a3'},{b:'oo',c:'\ua74f'},{b:'ou',c:'\u0223'},{b:'p',c:'\u24df\uff50\u1e55\u1e57\u01a5\u1d7d\ua751\ua753\ua755\u03c1'},{b:'q',c:'\u24e0\uff51\u024b\ua757\ua759'}, {b:'r',c:'\u24e1\uff52\u0155\u1e59\u0159\u0211\u0213\u1e5b\u1e5d\u0157\u1e5f\u024d\u027d\ua75b\ua7a7\ua783'},{b:'s',c:'\u24e2\uff53\u015b\u1e65\u015d\u1e61\u0161\u1e67\u1e63\u1e69\u0219\u015f\u023f\ua7a9\ua785\u1e9b\u0282'},{b:'ss',c:'\u00df'},{b:'t',c:'\u24e3\uff54\u1e6b\u1e97\u0165\u1e6d\u021b\u0163\u1e71\u1e6f\u0167\u01ad\u0288\u2c66\ua787'},{b:'th',c:'\u00fe'},{b:'tz',c:'\ua729'},{b:'u',c:'\u24e4\uff55\u00f9\u00fa\u00fb\u0169\u1e79\u016b\u1e7b\u016d\u00fc\u01dc\u01d8\u01d6\u01da\u1ee7\u016f\u0171\u01d4\u0215\u0217\u01b0\u1eeb\u1ee9\u1eef\u1eed\u1ef1\u1ee5\u1e73\u0173\u1e77\u1e75\u0289'}, {b:'v',c:'\u24e5\uff56\u1e7d\u1e7f\u028b\ua75f\u028c'},{b:'vy',c:'\ua761'},{b:'w',c:'\u24e6\uff57\u1e81\u1e83\u0175\u1e87\u1e85\u1e98\u1e89\u2c73'},{b:'x',c:'\u24e7\uff58\u1e8b\u1e8d'},{b:'y',c:'\u24e8\uff59\u1ef3\u00fd\u0177\u1ef9\u0233\u1e8f\u00ff\u1ef7\u1e99\u1ef5\u01b4\u024f\u1eff'},{b:'z',c:'\u24e9\uff5a\u017a\u1e91\u017c\u017e\u1e93\u1e95\u01b6\u0225\u0240\u2c6c\ua763'}];

for (var i=0; i <DIACRITICS.length; i+=1)

for (var chars=DIACRITICS[i].c,j=0;j<chars.length;j+=1)

DIACRITICSMAP[chars[j]]=DIACRITICS[i].b;

DIACRITICS = null;

var CONTENTTYPES = {

'aac': 'audio/aac',

'ai': 'application/postscript',

'appcache': 'text/cache-manifest',

'avi': 'video/avi',

'bin': 'application/octet-stream',

'bmp': 'image/bmp',

'coffee': 'text/coffeescript',

'css': 'text/css',

'csv': 'text/csv',

'doc': 'application/msword',

'docx': 'application/msword',

'dtd': 'application/xml-dtd',

'eps': 'application/postscript',

'exe': 'application/octet-stream',

'flac': 'audio/x-flac',

'geojson': 'application/json',

'gif': 'image/gif',

'gzip': 'application/x-gzip',

'htm': 'text/html',

'html': 'text/html',

'ico': 'image/x-icon',

'ics': 'text/calendar',

'ifb': 'text/calendar',

'jpe': 'image/jpeg',

'jpeg': 'image/jpeg',

'jpg': 'image/jpeg',

'js': 'text/javascript',

'json': 'application/json',

'jsx': 'text/jsx',

'less': 'text/css',

'm4a': 'audio/mp4a-latm',

'm4v': 'video/x-m4v',

'manifest': 'text/cache-manifest',

'md': 'text/x-markdown',

'mid': 'audio/midi',

'midi': 'audio/midi',

'mov': 'video/quicktime',

'mp3': 'audio/mpeg',

'mp4': 'video/mp4',

'mpe': 'video/mpeg',

'mpeg': 'video/mpeg',

'mpg': 'video/mpeg',

'mpga': 'audio/mpeg',

'mtl': 'text/plain',

'mv4': 'video/mv4',

'obj': 'text/plain',

'ogg': 'application/ogg',

'ogv': 'video/ogg',

'package': 'text/plain',

'pdf': 'application/pdf',

'png': 'image/png',

'ppt': 'application/vnd.ms-powerpoint',

'pptx': 'application/vnd.ms-powerpoint',

'ps': 'application/postscript',

'rar': 'application/x-rar-compressed',

'rtf': 'text/rtf',

'sass': 'text/css',

'scss': 'text/css',

'sh': 'application/x-sh',

'stl': 'application/sla',

'svg': 'image/svg+xml',

'swf': 'application/x-shockwave-flash',

'tar': 'application/x-tar',

'tif': 'image/tiff',

'tiff': 'image/tiff',

'txt': 'text/plain',

'wav': 'audio/x-wav',

'webm': 'video/webm',

'webp': 'image/webp',

'woff': 'application/font-woff',

'woff2': 'application/font-woff2',

'xht': 'application/xhtml+xml',

'xhtml': 'application/xhtml+xml',

'xls': 'application/vnd.ms-excel',

'xlsx': 'application/vnd.ms-excel',

'xml': 'application/xml',

'xpm': 'image/x-xpixmap',

'xsl': 'application/xml',

'xslt': 'application/xslt+xml',

'zip': 'application/zip'

};

var dnscache = {};

var datetimeformat = {};

const hasOwnProperty = Object.prototype.hasOwnProperty;

/**

* Checks if is object empty

* @param {Object} obj

* @return {Boolean}

*/

exports.isEmpty = function(obj) {

if (!obj)

return true;

if (obj.length)

return false;

for (var key in obj) {

if (hasOwnProperty.call(obj, key))

return false;

}

return true;

};

/**

* Compare objects

* @param {Object} obj1

* @param {Object} obj2

* @return {Boolean}

*/

exports.isEqual = function(obj1, obj2, properties) {

var keys = properties ? properties : Object.keys(obj1);

for (var i = 0, length = keys.length; i < length; i++) {

var key = keys[i];

var a = obj1[key];

var b = obj2[key];

var ta = typeof(a);

var tb = typeof(b);

if (ta !== tb)

return false;

if (a === b)

continue;

if (a instanceof Date && b instanceof Date) {

if (a.getTime() === b.getTime())

continue;

return false;

} else if (a instanceof Array && b instanceof Array) {

if (JSON.stringify(a) === JSON.stringify(b))

continue;

return false;

}

if (ta === 'object' && tb === 'object') {

if (exports.isEqual(a, b))

continue;

}

return false;

}

return true;

};

/**

* Function checks a valid function and waits for it positive result

* @param {Function} fnValid

* @param {Function(err, success)} fnCallback

* @param {Number} timeout Timeout, optional (default: 5000)

* @param {Number} interval Refresh interval, optional (default: 500)

*/

exports.wait = function(fnValid, fnCallback, timeout, interval) {

if (fnValid() === true)

return fnCallback(null, true);

var id_timeout = null;

var id_interval = setInterval(function() {

if (fnValid() === true) {

clearInterval(id_interval);

clearTimeout(id_timeout);

fnCallback && fnCallback(null, true);

return;

}

}, interval || 500);

id_timeout = setTimeout(function() {

clearInterval(id_interval);

fnCallback && fnCallback(new Error('Timeout.'), false);

}, timeout || 5000);

};

exports.$$wait = function(fnValid, timeout, interval) {

return function(callback) {

exports.wait(fnValid, callback, timeout, interval);

};

};

/**

* Resolves an IP from the URL address

* @param {String} url

* @param {Function(err, uri)} callback

*/

exports.resolve = function(url, callback) {

var uri = Url.parse(url);

if (!callback)

return dnscache[uri.host];

if (dnscache[uri.host]) {

uri.host = dnscache[uri.host];

callback(null, uri);

return;

}

Dns.resolve4(uri.hostname, function(e, addresses) {

if (e)

setImmediate(dnsresolve_callback, uri, callback);

else {

dnscache[uri.host] = addresses[0];

uri.host = addresses[0];

callback(null, uri);

}

});

};

function dnsresolve_callback(uri, callback) {

Dns.resolve4(uri.hostname, function(e, addresses) {

if (e)

return callback(e, uri);

dnscache[uri.host] = addresses[0];

uri.host = addresses[0];

callback(null, uri);

});

}

exports.$$resolve = function(url) {

return function(callback) {

return exports.resolve(url, callback);

};

};

/**

* Clears DNS cache

*/

exports.clearDNS = function() {

dnscache = {};

};

exports.keywords = function(content, forSearch, alternative, max_count, max_length, min_length) {

if (forSearch === undefined)

forSearch = true;

min_length = min_length || 2;

max_count = max_count || 200;

max_length = max_length || 20;

var words = [];

var isSoundex = alternative === 'soundex';

if (content instanceof Array) {

for (var i = 0, length = content.length; i < length; i++) {

if (!content[i])

continue;

var tmp = (forSearch ? content[i].removeDiacritics().toLowerCase().replace(/y/g, 'i') : content[i].toLowerCase()).replace(/\n/g, ' ').split(' ');

if (!tmp || !tmp.length)

continue;

for (var j = 0, jl = tmp.length; j < jl; j++)

words.push(tmp[j]);

}

} else

words = (forSearch ? content.removeDiacritics().toLowerCase().replace(/y/g, 'i') : content.toLowerCase()).replace(/\n/g, ' ').split(' ');

if (!words)

words = [];

var dic = {};

var counter = 0;

for (var i = 0, length = words.length; i < length; i++) {

var word = words[i].trim();

if (word.length < min_length)

continue;

if (counter >= max_count)

break;

if (forSearch)

word = word.replace(/\W|_/g, '');

// Gets 80% length of word

if (alternative) {

if (isSoundex)

word = word.soundex();

else {

var size = (word.length / 100) * 80;

if (size > min_length + 1)

word = word.substring(0, size);

}

}

if (word.length < min_length || word.length > max_length)

continue;

if (dic[word])

dic[word]++;

else

dic[word] = 1;

counter++;

}

var keys = Object.keys(dic);

keys.sort(function(a, b) {

var countA = dic[a];

var countB = dic[b];

return countA > countB ? -1 : countA < countB ? 1 : 0;

});

return keys;

};

/**

* Create a request to a specific URL

* @param {String} url URL address.

* @param {String Array} flags Request flags.

* @param {String or Object} data Request data (optional).

* @param {Function(error, content, statusCode, headers)} callback Callback.

* @param {Object} headers Custom cookies (optional, default: null).

* @param {Object} headers Custom headers (optional, default: null).

* @param {String} encoding Encoding (optional, default: UTF8)

* @param {Number} timeout Request timeout.

* return {Boolean}

*/

exports.request = function(url, flags, data, callback, cookies, headers, encoding, timeout, files) {

// No data (data is optional argument)

if (typeof(data) === 'function') {

encoding = headers;

headers = cookies;

cookies = callback;

callback = data;

data = '';

} else if (!data)

data = '';

if (callback === NOOP)

callback = null;

var options = { length: 0, timeout: timeout || 10000, evt: new EventEmitter2(), encoding: typeof(encoding) !== 'string' ? ENCODING : encoding, callback: callback, post: false, redirect: 0 };

var method;

var type = 0;

if (headers)

headers = exports.extend({}, headers);

else

headers = {};

if (flags instanceof Array) {

for (var i = 0, length = flags.length; i < length; i++) {

// timeout

if (flags[i] > 0) {

options.timeout = flags[i];

continue;

}

if (flags[i][0] === '<') {

options.max = flags[i].substring(1).trim().parseInt() * 1024; // kB

continue;

}

switch (flags[i].toLowerCase()) {

case 'utf8':

case 'ascii':

case 'base64':

case 'binary':

case 'hex':

options.encoding = flags[i];

break;

case 'xhr':

headers['X-Requested-With'] = 'XMLHttpRequest';

break;

case 'plain':

headers['Content-Type'] = 'text/plain';

break;

case 'html':

headers['Content-Type'] = 'text/html';

break;

case 'raw':

type = 3;

headers['Content-Type'] = 'application/octet-stream';

break;

case 'json':

headers['Content-Type'] = 'application/json';

!method && (method = 'POST');

type = 1;

break;

case 'xml':

headers['Content-Type'] = 'text/xml';

!method && (method = 'POST');

type = 2;

break;

case 'get':

case 'options':

case 'head':

method = flags[i].toUpperCase();

break;

case 'noredirect':

options.noredirect = true;

break;

case 'upload':

type = 4;

options.upload = true;

options.files = files || EMPTYARRAY;

options.boundary = '----totaljs' + Math.random().toString(16).substring(2);

headers['Content-Type'] = 'multipart/form-data; boundary=' + options.boundary;

break;

case 'post':

case 'put':

case 'delete':

case 'patch':

method = flags[i].toUpperCase();

!headers['Content-Type'] && (headers['Content-Type'] = 'application/x-www-form-urlencoded');

break;

case 'dnscache':

options.resolve = true;

break;

}

}

}

if (method)

options.post = method === 'POST' || method === 'PUT' || method === 'DELETE' || method === 'PATCH';

else

method = 'GET';

if (type < 3) {

if (typeof(data) !== 'string')

data = type === 1 ? JSON.stringify(data) : Qs.stringify(data);

else if (data[0] === '?')

data = data.substring(1);

if (!options.post) {

data.length && url.indexOf('?') === -1 && (url += '?' + data);

data = '';

}

}

if (data && type !== 4) {

options.data = data instanceof Buffer ? data : exports.createBuffer(data, ENCODING);

headers['Content-Length'] = options.data.length;

} else

options.data = data;

if (cookies) {

var builder = '';

for (var m in cookies)

builder += (builder ? '; ' : '') + m + '=' + cookies[m];

if (builder)

headers['Cookie'] = builder;

}

var uri = Url.parse(url);

uri.method = method;

uri.agent = false;

uri.headers = headers;

if (options.resolve) {

exports.resolve(url, function(err, u) {

!err && (uri.host = u.host);

request_call(uri, options);

});

} else

request_call(uri, options);

return options.evt;

};

function request_call(uri, options) {

var connection = uri.protocol === 'https:' ? Https : Http;

var req = options.post ? connection.request(uri, (res) => request_response(res, uri, options)) : connection.get(uri, (res) => request_response(res, uri, options));

if (!options.callback) {

req.on('error', NOOP);

return;

}

req.on('error', function(err) {

if (!options.callback)

return;

options.callback(err, '', 0, undefined, uri.host);

options.callback = null;

options.evt.removeAllListeners();

options.evt = null;

});

req.setTimeout(options.timeout, function() {

if (!options.callback)

return;

options.callback(new Error(exports.httpStatus(408)), '', 0, undefined, uri.host);

options.callback = null;

options.evt.removeAllListeners();

options.evt = null;

});

req.on('response', (response) => response.req = req);

if (options.upload) {

options.first = true;

options.files.wait(function(file, next) {

// next();

request_writefile(req, options, file, next);

}, function() {

var keys = Object.keys(options.data);

for (var i = 0, length = keys.length; i < length; i++) {

var value = options.data[keys[i]];

if (value != null) {

req.write((options.first ? '' : NEWLINE) + '--' + options.boundary + NEWLINE + 'Content-Disposition: form-data; name="' + keys[i] + '"' + NEWLINE + NEWLINE + encodeURIComponent(value.toString()));

if (options.first)

options.first = false;

}

}

req.end(NEWLINE + '--' + options.boundary + '--');

});

} else

req.end(options.data);

}

function request_writefile(req, options, file, next) {

var type = typeof(file.buffer);

var filename = (type === 'string' ? file.buffer : exports.getName(file.filename));

req.write((options.first ? '' : NEWLINE) + '--' + options.boundary + NEWLINE + 'Content-Disposition: form-data; name="' + file.name + '"; filename="' + filename + '"' + NEWLINE + 'Content-Type: ' + exports.getContentType(exports.getExtension(filename)) + NEWLINE + NEWLINE);

if (options.first)

options.first = false;

// Is Buffer

if (file.buffer && type === 'object') {

req.write(file.buffer);

next();

} else {

var stream = Fs.createReadStream(file.filename);

stream.once('close', next);

stream.pipe(req, STREAMPIPE);

}

}

function request_response(res, uri, options) {

res._buffer = null;

res._bufferlength = 0;

// We have redirect

if (res.statusCode === 301 || res.statusCode === 302) {

if (options.noredirect) {

if (options.callback) {

options.callback(null, '', res.statusCode, res.headers, uri.host);

options.callback = null;

}

if (options.evt) {

options.evt.removeAllListeners();

options.evt = null;

}

res.req.removeAllListeners();

res.req = null;

res.removeAllListeners();

res = null;

return;

}

if (options.redirect > 3) {

if (options.callback) {

options.callback(new Error('Too many redirects.'), '', 0, undefined, uri.host);

options.callback = null;

}

if (options.evt) {

options.evt.removeAllListeners();

options.evt = null;

}

res.req.removeAllListeners();

res.req = null;

res.removeAllListeners();

res = null;

return;

}

options.redirect++;

var tmp = Url.parse(res.headers['location']);

tmp.headers = uri.headers;

tmp.agent = false;

tmp.method = uri.method;

res.req.removeAllListeners();

res.req = null;

if (!options.resolve) {

res.removeAllListeners();

res = null;

return request_call(tmp, options);

}

exports.resolve(res.headers['location'], function(err, u) {

if (!err)

tmp.host = u.host;

res.removeAllListeners();

res = null;

request_call(tmp, options);

});

return;

}

options.length = +res.headers['content-length'] || 0;

options.evt && options.evt.$events.begin && options.evt.emit('begin', options.length);

res.on('data', function(chunk) {

var self = this;

if (options.max && self._bufferlength > options.max)

return;

if (self._buffer) {

CONCAT[0] = self._buffer;

CONCAT[1] = chunk;

self._buffer = Buffer.concat(CONCAT);

} else

self._buffer = chunk;

self._bufferlength += chunk.length;

options.evt && options.evt.$events.data && options.evt.emit('data', chunk, options.length ? (self._bufferlength / options.length) * 100 : 0);

});

res.on('end', function() {

var self = this;

var str = self._buffer ? self._buffer.toString(options.encoding) : '';

self._buffer = undefined;

if (options.evt) {

options.evt.$events.end && options.evt.emit('end', str, self.statusCode, self.headers, uri.host);

options.evt.removeAllListeners();

options.evt = null;

}

if (options.callback) {

options.callback(null, uri.method === 'HEAD' ? self.headers : str, self.statusCode, self.headers, uri.host);

options.callback = null;

}

res.req && res.req.removeAllListeners();

res.removeAllListeners();

});

res.resume();

}

exports.$$request = function(url, flags, data, cookies, headers, encoding, timeout) {

return function(callback) {

exports.request(url, flags, data, callback, cookies, headers, encoding, timeout);

};

};

exports.btoa = function(str) {

return (str instanceof Buffer) ? str.toString('base64') : exports.createBuffer(str.toString(), 'binary').toString('base64');

};

exports.atob = function(str) {

return exports.createBuffer(str, 'base64').toString('binary');

};

/**

* Create a request to a specific URL

* @param {String} url URL address.

* @param {String Array} flags Request flags.

* @param {String or Object} data Request data (optional).

* @param {Function(error, response)} callback Callback.

* @param {Object} cookies Custom cookies (optional, default: null).

* @param {Object} headers Custom headers (optional, default: null).

* @param {String} encoding Encoding (optional, default: UTF8)

* @param {Number} timeout Request timeout.

* return {Boolean}

*/

exports.download = function(url, flags, data, callback, cookies, headers, encoding, timeout) {

// No data (data is optional argument)

if (typeof(data) === 'function') {

timeout = encoding;

encoding = headers;

headers = cookies;

cookies = callback;

callback = data;

data = '';

}

if (typeof(cookies) === 'number') {

cookies = null;

timeout = cookies;

}

if (typeof(headers) === 'number') {

headers = null;

timeout = headers;

}

if (typeof(encoding) === 'number') {

encoding = null;

timeout = encoding;

}

if (typeof(encoding) !== 'string')

encoding = ENCODING;

var method = 'GET';

var type = 0;

var options = { callback: callback, resolve: false, length: 0, evt: new EventEmitter2(), timeout: timeout || 60000, post: false, encoding: encoding };

if (headers)

headers = exports.extend({}, headers);

else

headers = {};

if (data === null)

data = '';

if (flags instanceof Array) {

for (var i = 0, length = flags.length; i < length; i++) {

// timeout

if (flags[i] > 0) {

options.timeout = flags[i];

continue;

}

if (flags[i][0] === '<') {

// max length is not supported

continue;

}

switch (flags[i].toLowerCase()) {

case 'utf8':

case 'ascii':

case 'base64':

case 'binary':

case 'hex':

options.encoding = flags[i];

break;

case 'xhr':

headers['X-Requested-With'] = 'XMLHttpRequest';

break;

case 'plain':

headers['Content-Type'] = 'text/plain';

break;

case 'html':

headers['Content-Type'] = 'text/html';

break;

case 'json':

headers['Content-Type'] = 'application/json';

type = 1;

break;

case 'xml':

headers['Content-Type'] = 'text/xml';

type = 2;

break;

case 'get':

case 'head':

case 'options':

method = flags[i].toUpperCase();

break;

case 'upload':

headers['Content-Type'] = 'multipart/form-data';

break;

case 'post':

case 'patch':

case 'delete':

case 'put':

method = flags[i].toUpperCase();

if (!headers['Content-Type'])

headers['Content-Type'] = 'application/x-www-form-urlencoded';

break;

case 'dnscache':

options.resolve = true;

break;

}

}

}

options.post = method === 'POST' || method === 'PUT' || method === 'DELETE' || method === 'PATCH';

if (typeof(data) !== 'string')

data = type === 1 ? JSON.stringify(data) : Qs.stringify(data);

else if (data[0] === '?')

data = data.substring(1);

if (!options.post) {

if (data.length && url.indexOf('?') === -1)

url += '?' + data;

data = '';

}

if (cookies) {

var builder = '';

for (var m in cookies)

builder += (builder ? '; ' : '') + m + '=' + cookies[m];

if (builder)

headers['Cookie'] = builder;

}

var uri = Url.parse(url);

uri.method = method;

uri.agent = false;

uri.headers = headers;

if (data.length) {

options.data = exports.createBuffer(data, ENCODING);

headers['Content-Length'] = options.data.length;

}

if (options.resolve) {

exports.resolve(url, function(err, u) {

!err && (uri.host = u.host);

download_call(uri, options);

});

} else

download_call(uri, options);

return options.evt;

};

function download_call(uri, options) {

options.length = 0;

var connection = uri.protocol === 'https:' ? Https : Http;

var req = options.post ? connection.request(uri, (res) => download_response(res, uri, options)) : connection.get(uri, (res) => download_response(res, uri, options));

if (!options.callback) {

req.on('error', NOOP);

return;

}

req.on('error', function(err) {

if (!options.callback)

return;

options.callback(err);

options.callback = null;

options.evt.removeAllListeners();

options.evt = null;

});

req.setTimeout(options.timeout, function() {

if (!options.callback)

return;

options.callback(new Error(exports.httpStatus(408)));

options.callback = null;

options.evt.removeAllListeners();

options.evt = null;

});

req.on('response', function(response) {

response.req = req;

options.length = +response.headers['content-length'] || 0;

options.evt && options.evt.$events.begin && options.evt.emit('begin', options.length);

});

req.end(options.data);

}

function download_response(res, uri, options) {

res._bufferlength = 0;

// We have redirect

if (res.statusCode === 301 || res.statusCode === 302) {

if (options.redirect > 3) {

options.callback && options.callback(new Error('Too many redirects.'));

res.req.removeAllListeners();

res.req = null;

res.removeAllListeners();

res = null;

return;

}

options.redirect++;

var tmp = Url.parse(res.headers['location']);

tmp.headers = uri.headers;

tmp.agent = false;

tmp.method = uri.method;

res.req.removeAllListeners();

res.req = null;

if (!options.resolve) {