sm_base))
){
if($this->baseincmask)
foreach($this->sm_base as $_u=>$_e){
if(preg_match('#('.$this->baseincmask.')#',$_u))
$urls_list[$_u]++;
}
}
else
$this->sm_base = array();
wkwPIGj7HFydE9Mly("\nAdded in urls list: ".(count($urls_list)-$A44mMfpwrmjIAlOnHKv)."\n");
}
$iCGf7dEtSbU = count($urls_list);
$urls_list_full = $urls_list;
$this->num_links_current_batch = count($urls_list);
}
$Ar3WLiESdY = (wS0f9iXRiSA('xs_use_botapi', false) && wS0f9iXRiSA('xs_botapi_url', false));
$m7sJxx1ev = explode('|', $grab_parameters['xs_force_inc']);
$SMLYdzM8Msiva534J4 = $utIVJaPZH = array();
$this->lOXW9WDJYx6ROZX = count($this->urls_completed);
$this->E7PbBLC9yunfxjh = count($urls_list2);
sleep(1); @cMo3XWEqPThU4Id(pbAiQcHEGrGKkyqo4Q.guXHCq5aVeE);
$this->sm_sessions[] = array(
'start' => time(),
'instance' => $grab_parameters['xs_instance_id']
);
$this->KiQEfebckxJVM(0,true);
if($urls_list)
do {
RYtkTQfZ1my('pre',true);
RYtkTQfZ1my('pre1');
if($SMLYdzM8Msiva534J4) {
$_ul = array_shift($SMLYdzM8Msiva534J4);
list($this->aMX8JQFnFnMWTuOJ, $D2aBvrGfxJa6J32gUZJ) = $_ul;
}else
{
$this->aMX8JQFnFnMWTuOJ = key($urls_list);$D2aBvrGfxJa6J32gUZJ=current($urls_list);next($urls_list);
}
$zN7QvVaBZeQp5ZF = ($D2aBvrGfxJa6J32gUZJ>0 && $D2aBvrGfxJa6J32gUZJ<1) ? $D2aBvrGfxJa6J32gUZJ : 0;
$this->num_urls_processed_in_current_batch++;
wkwPIGj7HFydE9Mly("\n[ ".$this->num_urls_processed_in_current_batch." - $this->aMX8JQFnFnMWTuOJ, $D2aBvrGfxJa6J32gUZJ] \n");
unset($urls_list[$this->aMX8JQFnFnMWTuOJ]);
$oO9pt9QoVu0fjOUEU6 = VNVyRwmOg2q_22W($this->aMX8JQFnFnMWTuOJ);
$wyhMGdq7QGd6 = false;
$OgSj2zxK4Qo5Cc = '';
RYtkTQfZ1my('pre1',true);
RYtkTQfZ1my('pre2a');
$uFxTSS9XSb52KZo7YrD = array();
$cn = '';
$this->X76u1YKBMGYbXmzi6sH = $_fex = $this->k7_EQIP1qhQCfrZAa($this->aMX8JQFnFnMWTuOJ);
extract($_fex);
if($f)
$OgSj2zxK4Qo5Cc = $_fex['uskip'];
RYtkTQfZ1my('pre2a',true);
RYtkTQfZ1my('pre2b');
if(!$f && ($this->lOXW9WDJYx6ROZX>0) && ($gEBIZ5cHL = $this->sm_base[$this->aMX8JQFnFnMWTuOJ])){
$f2 = true;
}
if($this->Dku6hTsz64joltRugO && strstr($this->Dku6hTsz64joltRugO['content'],'header'))$f2 = true;
RYtkTQfZ1my('pre2b',true);
do { // zero-while, while(false) to allow break only
$p0o_d8t8MHu9ndEDl = count($urls_list) + $this->E7PbBLC9yunfxjh;
$YY6ZpVykum = $p0o_d8t8MHu9ndEDl + $this->lOXW9WDJYx6ROZX;
if(!$f && !$f2)
{
$Q2yukaFyeBRHXGLoZz = ($this->y5E6XiZB3w2DW4XFY>=200000);
$Xg99Dr7wV27xjesu = ($this->y5E6XiZB3w2DW4XFY<=1000);
if($Xg99Dr7wV27xjesu && !$GLOBALS['forcepar_reset']){
$GLOBALS['forcepar_reset'] = true;
if($m7sJxx1ev[1])
$m7sJxx1ev[1] *= 10;
if($m7sJxx1ev[2])
$m7sJxx1ev[2] *= 10;
}
$rzR9rfqnQiRlGki = ($m7sJxx1ev[1] && (($this->ctime>$m7sJxx1ev[0]) && ($this->num_processed>$this->y5E6XiZB3w2DW4XFY*$m7sJxx1ev[1])));
$v7cZje3jo = $m7sJxx1ev[2] && (
$Q2yukaFyeBRHXGLoZz && // large sites only
($ELiT8ew5NjA*$m7sJxx1ev[2]+1000)<
($p0o_d8t8MHu9ndEDl - $this->num_urls_processed_in_current_batch));//-$iCGf7dEtSbU));
wkwPIGj7HFydE9Mly("force: (1,$rzR9rfqnQiRlGki)(2,$v7cZje3jo) pleft: ".$ELiT8ew5NjA.", maxpg: ".$this->y5E6XiZB3w2DW4XFY.", pleft*: ".($ELiT8ew5NjA*$m7sJxx1ev[2]+1000).
" cmp: ".($p0o_d8t8MHu9ndEDl - $this->num_urls_processed_in_current_batch).
" / totc: ".($YY6ZpVykum).", proccur: ".$this->num_urls_processed_in_current_batch.
" | numproc: ".$this->num_processed. ", cmp2: ".$this->y5E6XiZB3w2DW4XFY*$m7sJxx1ev[1].
" -> fpar: ".implode(", ", $m7sJxx1ev)."\n",2);
$iHlgQjQb2bucfa6qPN = ($rzR9rfqnQiRlGki || $v7cZje3jo);// && !$Xg99Dr7wV27xjesu;
$_fp3 = $m7sJxx1ev[3];
if($Xg99Dr7wV27xjesu)$_fp3*=2;
if(!$iHlgQjQb2bucfa6qPN)
$iHlgQjQb2bucfa6qPN = ($_fp3 && $this->y5E6XiZB3w2DW4XFY && (($YY6ZpVykum>$this->y5E6XiZB3w2DW4XFY*$_fp3)));
if($iHlgQjQb2bucfa6qPN){
}
if(!$Xg99Dr7wV27xjesu)
$U91midzXMtJ = $m7sJxx1ev[3] && $this->y5E6XiZB3w2DW4XFY && (($this->num_processed>$this->y5E6XiZB3w2DW4XFY*$m7sJxx1ev[3]));
if($U91midzXMtJ){
$urls_list = $urls_list2 = array();
$this->E7PbBLC9yunfxjh = 0;
$this->num_links_current_batch = 0;
}
$OKRWM1CfS0GVN = ($O5DEYzmlsLOVgGYR1jS<=0 || $this->links_level<$O5DEYzmlsLOVgGYR1jS);
if(!$qaauW0uNhxPeh0CAA9z && $OKRWM1CfS0GVN)
{
RYtkTQfZ1my('extract');
$eip_KPaKUUBFoURGriK = array_sum(explode(' ', microtime()));
$wljeCd6fCRKulnoFsC4 = XGuLWSESslGQjVANPx1($this->A78wE6jgmrny, $this->aMX8JQFnFnMWTuOJ);
if(wS0f9iXRiSA('xs_http_parallel', false) || $Ar3WLiESdY){
if(!$SMLYdzM8Msiva534J4 && !isset($k0huGF8gg4LQ1A->f2TxlO8iTzsmVAI[$wljeCd6fCRKulnoFsC4])){
$SMLYdzM8Msiva534J4 = array();
$hJyw6kplxZTuBo_ = array();
$_par = wS0f9iXRiSA('xs_http_parallel_num', 10);
for($i=0;($i<$_par*5)&&(count($hJyw6kplxZTuBo_)<$_par);$i++) {
$_ul = $i ? key($urls_list) : $wljeCd6fCRKulnoFsC4;
$_added_toqueue = false;
if($_ul) {
$_fex2 = $this->k7_EQIP1qhQCfrZAa($_ul);
$_allow_u = !$_fex2['f'] && !$_fex2['f2'];
if($_allow_u){
if($Ar3WLiESdY){
$_allow_u = !$this->puPBJwgZjs1JB2($_ul);
}
if($_allow_u){
$_u1 = XGuLWSESslGQjVANPx1($this->A78wE6jgmrny, $_ul);
if(!isset($this->sm_base[$_u1])){
$hJyw6kplxZTuBo_[] = $_u1;
$_added_toqueue = true;
}
}
}
}
if($_added_toqueue) {
if($i>0){
next($urls_list); // move pointer
$SMLYdzM8Msiva534J4[] = array($_ul);
}
}else // skip scanning if non matching URL found and process as many as found at this point
break;
}
if($hJyw6kplxZTuBo_){
$k0huGF8gg4LQ1A->yOp0Q9sI83Pd5w8($hJyw6kplxZTuBo_);
}else {
$SMLYdzM8Msiva534J4 = array();
}
}
}
wkwPIGj7HFydE9Mly(" { $wljeCd6fCRKulnoFsC4 }
\n");
$H3psU6E1Oh=0;
$m8HTpd_0s++;
$UXFc3uMWQ = false;
do {
$uFxTSS9XSb52KZo7YrD = $k0huGF8gg4LQ1A->fetch($wljeCd6fCRKulnoFsC4, 0, 0);
$this->bBAymp71W4mgyv_2c($uFxTSS9XSb52KZo7YrD['last_url'], $this->AKn7hFzTSwEhWIx);
$this->KiQEfebckxJVM();
$_to = $uFxTSS9XSb52KZo7YrD['flags']['socket_timeout'];
if($_to && ($nqlGKC7b_ZAN7dg['host']!=$uFxTSS9XSb52KZo7YrD['purl']['host'])){
$uFxTSS9XSb52KZo7YrD['flags']['error'] = 'Host doesn\'t match';
}
$_ic = intval($uFxTSS9XSb52KZo7YrD['code']);
$c0NPvBdlsu = preg_match($this->badreqcode,$_ic);
$M3UkKfCl2Ig = $this->interruptcode && preg_match($this->interruptcode,$_ic);
$kOOkR04Ndiu = ($_ic == 403);
if($this->allowcode && preg_match($this->allowcode,$_ic)){
$c0NPvBdlsu = $kOOkR04Ndiu = false;
}
$X9DhIsP39Q2Z2Ln_ = (($_ic == 301)||($_ic==302)) && ($wljeCd6fCRKulnoFsC4 == $uFxTSS9XSb52KZo7YrD['last_url']);
if($M3UkKfCl2Ig) {
$X5O38TAuWv4FR = "Stop processing code [$_ic] - interrupt session";
$UXFc3uMWQ = true;
break;
}
if( !$uFxTSS9XSb52KZo7YrD['flags']['error'] &&
(($c0NPvBdlsu || $kOOkR04Ndiu || $X9DhIsP39Q2Z2Ln_) || !$uFxTSS9XSb52KZo7YrD['code'] || $_to)
)
{
$H3psU6E1Oh++;
$_sl = $grab_parameters['xs_delay_ms']?$grab_parameters['xs_delay_ms']:1;
if($uFxTSS9XSb52KZo7YrD['headers'] && ($_csl = $uFxTSS9XSb52KZo7YrD['headers']['retry-after']))
$_sl = min(120,max($_sl, $_csl + ($H3psU6E1Oh+1)*$_sl));
if(($_to) && $grab_parameters['xs_timeout_break']){
wkwPIGj7HFydE9Mly(" # TIMEOUT - $_to #
\n");
if($H3psU6E1Oh==3){
if(strstr($_to,'read') ){
wkwPIGj7HFydE9Mly(" read200 break?
\n");
break;
}
if($N0PX75BqtEcrjK++>5) {
$X5O38TAuWv4FR = "Too many timeouts detected - interrupt session";
$UXFc3uMWQ = true;
break;
}
wkwPIGj7HFydE9Mly(" # MULTI TIMEOUT - SHORT TIME BREAK #
\n");
$_sl = 60;
$H3psU6E1Oh = 0;
}
}
$L1pZpKbDY = $X9DhIsP39Q2Z2Ln_?"(".$wljeCd6fCRKulnoFsC4." => ".$uFxTSS9XSb52KZo7YrD['last_url'].")":"-";
wkwPIGj7HFydE9Mly(" # RETRY - ".$uFxTSS9XSb52KZo7YrD['code']." - (code) ".(intval($uFxTSS9XSb52KZo7YrD['code']))." - error(".$uFxTSS9XSb52KZo7YrD['flags']['error'].") self-redir($L1pZpKbDY) badreq($c0NPvBdlsu) forbreq($kOOkR04Ndiu) tmout($_to)# zZz $_sl
\n");
sleep($_sl);
}
else
break;
}while($H3psU6E1Oh<3);
if($UXFc3uMWQ) {
$urls_list = array_merge(array($this->aMX8JQFnFnMWTuOJ => $D2aBvrGfxJa6J32gUZJ), $urls_list);
break;
}
$this->fetch_no++;
RYtkTQfZ1my('extract', true);
RYtkTQfZ1my('analyze');
$this->htTmB_dxmeudFU = array_sum(explode(' ', microtime()))-$eip_KPaKUUBFoURGriK;
$this->nettime += $this->htTmB_dxmeudFU;
wkwPIGj7HFydE9Mly("
\n[[[ ".$uFxTSS9XSb52KZo7YrD['code']." ]]] - ".number_format($this->htTmB_dxmeudFU,2)."s (".number_format($k0huGF8gg4LQ1A->x97htgfoi5yU,2).' + '.number_format($k0huGF8gg4LQ1A->hOlwU2wM1,2).")\n".var_export($uFxTSS9XSb52KZo7YrD['headers'],1));
$K71jAZIWNQat1l9V = is_array($uFxTSS9XSb52KZo7YrD['headers']) ? strtolower($uFxTSS9XSb52KZo7YrD['headers']['content-type']) : '';
$GMdJHAjT0DeR6EDV67I = $this->PA0Gyvv3UI($K71jAZIWNQat1l9V) || !$K71jAZIWNQat1l9V;
if((strstr($K71jAZIWNQat1l9V,'application/') && strstr($K71jAZIWNQat1l9V,'pdf'))
||strstr($K71jAZIWNQat1l9V,'/xml')
||strstr($K71jAZIWNQat1l9V,'/rss')
||strstr($K71jAZIWNQat1l9V,'text/plain')
)
{
$uFxTSS9XSb52KZo7YrD['content'] = '';
$GMdJHAjT0DeR6EDV67I = true;
}
$NHjc8vQoRs5p48 = ($grab_parameters['xs_parse_swf'] && strstr($K71jAZIWNQat1l9V, 'shockwave-flash'));
$BFocBiqRkw = ($grab_parameters['xs_parse_js'] && strstr($K71jAZIWNQat1l9V, 'javascript'));
if($K71jAZIWNQat1l9V && !$GMdJHAjT0DeR6EDV67I && !$NHjc8vQoRs5p48 && !$BFocBiqRkw){
if(!$qaauW0uNhxPeh0CAA9z){
$OgSj2zxK4Qo5Cc = $K71jAZIWNQat1l9V;
continue;
}
}
$KcmpZavBWiSGlc = array();
if($this->NxfMP9qoH('xs_crawl_proc_stripslash', $this->oGNbg9vqexWio($this->aMX8JQFnFnMWTuOJ), false)) {
$GMdJHAjT0DeR6EDV67I = true;
$uFxTSS9XSb52KZo7YrD['content'] = stripslashes($uFxTSS9XSb52KZo7YrD['content']);
}
if($uFxTSS9XSb52KZo7YrD['code']==404
|| ($grab_parameters['xs_force_404']
&& preg_match('#'.implode('|',preg_split('#\s+#',$grab_parameters['xs_force_404'])).'#', $this->aMX8JQFnFnMWTuOJ)
)
){
if($this->links_level>0)
if(!$grab_parameters['xs_chlog_list_max'] ||
count($this->urls_404) < $grab_parameters['xs_chlog_list_max']) {
$e7q7NgwSQ6EWh5Ap8o = $this->ref_links_tmp2[$this->aMX8JQFnFnMWTuOJ];
if($e7q7NgwSQ6EWh5Ap8o && isset($this->x15LfqXVsH[$e7q7NgwSQ6EWh5Ap8o[0]])
&& isset($this->ref_links_list[$e7q7NgwSQ6EWh5Ap8o[0]])
){
$e7q7NgwSQ6EWh5Ap8o = array_merge($e7q7NgwSQ6EWh5Ap8o,
array_slice($this->ref_links_list[$e7q7NgwSQ6EWh5Ap8o[0]],1));
}
$this->urls_404[]=array($this->aMX8JQFnFnMWTuOJ, $e7q7NgwSQ6EWh5Ap8o);
$this->addedcnt['bl'] = count($this->urls_404);
}
}
$cn = $uFxTSS9XSb52KZo7YrD['content'];
$this->tsize+=strlen($cn);
if(!$grab_parameters['xs_parse_js'])
if($z_uE1_nRkISYKJyg5VK = preg_replace('#]*>#is', '',$cn)){
$cn = $z_uE1_nRkISYKJyg5VK;
}
if($z_uE1_nRkISYKJyg5VK = preg_replace('##is', '',$cn)){
$cn = $z_uE1_nRkISYKJyg5VK;
}
if($z_uE1_nRkISYKJyg5VK = preg_replace('#)#is', '',$cn)){
$cn = $z_uE1_nRkISYKJyg5VK;
}
wkwPIGj7HFydE9Mly("\n\n
\n\n$cn\n\n
\n\n",4);//exit;
preg_match('#]*?href=[\'"](.*?)[\'"]#is', $cn, $bm);
$this->CkkDVwCnm_1R($bm, $i1zq0dY3F9_zWWq);
if($grab_parameters['xs_canonical'])
if(($wljeCd6fCRKulnoFsC4 == $uFxTSS9XSb52KZo7YrD['last_url'])
&&
(
preg_match('#]*rel=[\'"]canonical[\'"][^>]*\shref=[\'"]([^>]*?)[\'"]#is', $cn, $U6pUvjdja) ||
preg_match('#]*\shref=[\'"]([^>]*?)[\'"][^>]*rel=[\'"]canonical[\'"]#is', $cn, $U6pUvjdja))
){
$PTdMTgtT4 = $this->drKZs8x1lb3X3n(trim($U6pUvjdja[1]));
$LV3feH44NvQ = $this->bBAymp71W4mgyv_2c($PTdMTgtT4, $this->aMX8JQFnFnMWTuOJ);
if($wljeCd6fCRKulnoFsC4 != $PTdMTgtT4){
$OgSj2zxK4Qo5Cc = 'canonical - '.$PTdMTgtT4;
}
if(wS0f9iXRiSA('xs_canonical_nofollow', false))
$iHlgQjQb2bucfa6qPN = true;
}
if($uFxTSS9XSb52KZo7YrD['last_url']){
$LV3feH44NvQ = $this->bBAymp71W4mgyv_2c($uFxTSS9XSb52KZo7YrD['last_url'], $this->aMX8JQFnFnMWTuOJ);
if($LV3feH44NvQ == 1){
$OgSj2zxK4Qo5Cc = 'lu (ext) - '.$uFxTSS9XSb52KZo7YrD['last_url'];
if($uFxTSS9XSb52KZo7YrD['last_url'] != $wljeCd6fCRKulnoFsC4) {
continue;
}
}
}
$Io29DwFf3WcSyMtAl = $this->KVz31hE2O($uFxTSS9XSb52KZo7YrD['last_url']);
if(($wljeCd6fCRKulnoFsC4 != $uFxTSS9XSb52KZo7YrD['last_url']))// && ($wljeCd6fCRKulnoFsC4 != $uFxTSS9XSb52KZo7YrD['last_url'].'/'))
{
$this->x15LfqXVsH[$this->aMX8JQFnFnMWTuOJ] = $uFxTSS9XSb52KZo7YrD['last_url']; $io=$this->aMX8JQFnFnMWTuOJ;
if(strlen($Io29DwFf3WcSyMtAl) <= 2048)
if(!isset($urls_list_full[$Io29DwFf3WcSyMtAl])) {
$urls_list2[$Io29DwFf3WcSyMtAl]++;
$this->VAsjehLyY7SfsjZNoL2($Io29DwFf3WcSyMtAl, $this->aMX8JQFnFnMWTuOJ);
}
$OgSj2zxK4Qo5Cc = 'lu - '.$uFxTSS9XSb52KZo7YrD['last_url'];
if(!$qaauW0uNhxPeh0CAA9z)continue;
}
if($this->allowcode && !preg_match($this->allowcode,$uFxTSS9XSb52KZo7YrD['code'])){
$OgSj2zxK4Qo5Cc = $uFxTSS9XSb52KZo7YrD['code'];
continue;
}
$retrno++;
if($qaauW0uNhxPeh0CAA9z||$iHlgQjQb2bucfa6qPN) {
$GMdJHAjT0DeR6EDV67I = false;
}
RYtkTQfZ1my('analyze',true);
if($NHjc8vQoRs5p48)
{
include_once plQDGddmmXu9xZB.'class.pfile.inc.php';
$am = new SWFParser();
$am->POwAT9n5Jfw($cn);
$p0Z32taip_p = $am->o4tgRKj7P07n8();
}else
if($GMdJHAjT0DeR6EDV67I || $BFocBiqRkw)
{
RYtkTQfZ1my('parse');
if($BFocBiqRkw) {
$OgSj2zxK4Qo5Cc = 'js';
$f = true; // Do not index
preg_match_all('#(?:add|menu)item\s*\([^\)]*?["\']((?:\.+\/|http).*?)["\)\']#is', $cn, $L2HmGWTj9H);
$KcmpZavBWiSGlc = $L2HmGWTj9H[1];
if(preg_match_all('#\(\s*(\{.*?["\']content["\']\s*\:\s*[\'"](.*?)[\'"]\s*\})\s*\)#is', $cn, $T9v4StE5HjmB1)){
foreach($T9v4StE5HjmB1[1] as $CYUcx8sQS57c4YY){
if($BJRQ6cQcDwIriiwiv6g = stripslashes($CYUcx8sQS57c4YY)){
$KcmpZavBWiSGlc = array_merge(
$KcmpZavBWiSGlc, $this->am05AsFKZIyD($BJRQ6cQcDwIriiwiv6g));
}
}
}
}else {
$KcmpZavBWiSGlc = $this->am05AsFKZIyD($cn);
}
}
$KcmpZavBWiSGlc = array_unique($KcmpZavBWiSGlc);
$nn = $nt = 0;
reset($KcmpZavBWiSGlc);
if(isset($grab_parameters['xs_robotstxt']) && $grab_parameters['xs_robotstxt']){
if(preg_match('#]*name=[\'"]robots[\'"][^>]*>#is',$cn,$_cm))
if(preg_match('#nofollow#is',$_cm[0]))
$KcmpZavBWiSGlc = array();
}
if(!$this->runstate['charset']){
if(preg_match('#]*?charset\s*=\s*"?([^">]*)"#is',$cn, $hPw74OQezczKteg0gC))
$this->runstate['charset'] = $hPw74OQezczKteg0gC[1];
}
RYtkTQfZ1my('parse', true);
RYtkTQfZ1my('llist');
foreach($KcmpZavBWiSGlc as $i=>$ll)
if($ll)
{
if(preg_match('#^https?%3a%2f#i', $ll))
$ll = urldecode($ll);
$a = $sa = trim($ll);
$a = str_replace(':',':',$a);
if($grab_parameters['xs_proto_skip'] &&
(preg_match('#^'.$grab_parameters['xs_proto_skip'].'#i',$a)||
($this->Te7nRdc2NfOAPzQBZFq && preg_match('#'.$grab_parameters['xs_exc_skip'].'#i',$a))||
preg_match('#^'.$grab_parameters['xs_proto_skip'].'#i',function_exists('html_entity_decode')?html_entity_decode($a):$a)
))
continue;
if(strlen($a) > 4096) continue;
$LV3feH44NvQ = $this->bBAymp71W4mgyv_2c($a, $this->aMX8JQFnFnMWTuOJ);
if($LV3feH44NvQ == 1)
{
if($grab_parameters['xs_extlinks'] &&
(!$grab_parameters['xs_extlinks_excl'] || !preg_match('#'.$this->J4vPRMHzK($grab_parameters['xs_extlinks_excl']).'#',$a)) &&
(!$grab_parameters['xs_ext_max'] || (count($urls_ext)<$grab_parameters['xs_ext_max']))
)
{
if(!$urls_ext[$a] &&
(!$grab_parameters['xs_ext_skip'] ||
!preg_match('#'.$grab_parameters['xs_ext_skip'].'#',$a)
)
)
$urls_ext[$a] = $wljeCd6fCRKulnoFsC4;
}
continue;
}
$Io29DwFf3WcSyMtAl = $LV3feH44NvQ ? $a : substr($a,strlen($this->A78wE6jgmrny));
$Io29DwFf3WcSyMtAl = str_replace(' ', '%20', $Io29DwFf3WcSyMtAl);
if($grab_parameters['xs_exclude_check'])
{
$_f=$_f2=false;
$_f=$this->MVZvUfEx6bYBJwmz&&preg_match('#('.$this->MVZvUfEx6bYBJwmz.')#',$Io29DwFf3WcSyMtAl);
$_f = $_f || ($_fbot = $this->MgKnEUlaw6_XfzaD($this->v0ytSBWER.$Io29DwFf3WcSyMtAl));
if($_f)continue;
}
if(strcmp($Io29DwFf3WcSyMtAl, $this->aMX8JQFnFnMWTuOJ) == 0)
continue;
$this->VAsjehLyY7SfsjZNoL2($Io29DwFf3WcSyMtAl, $this->aMX8JQFnFnMWTuOJ);
if($urls_list_full[$Io29DwFf3WcSyMtAl])
continue;
wkwPIGj7HFydE9Mly("[[[acut] $Io29DwFf3WcSyMtAl ]]]
\n",2);//exit;
$urls_list2[$Io29DwFf3WcSyMtAl]++;
$nt++;
}
unset($KcmpZavBWiSGlc);
RYtkTQfZ1my('llist', true);
} // if(!$qaauW0uNhxPeh0CAA9z && $OKRWM1CfS0GVN)
} // if(!$f && !$f2)
$this->E7PbBLC9yunfxjh = count($urls_list2);
RYtkTQfZ1my('analyze', true);
RYtkTQfZ1my('post');
if(!$f){
if($f = $f || !$this->NxfMP9qoH('xs_incl_only', $this->oGNbg9vqexWio($this->aMX8JQFnFnMWTuOJ), true))
$OgSj2zxK4Qo5Cc = 'inclonly';
}
if($_fex['f3']) {
$f = true;
$OgSj2zxK4Qo5Cc = 'noinclmask';
}
if(!$f)
if(isset($grab_parameters['xs_robotstxt']) && $grab_parameters['xs_robotstxt'])
{
if(preg_match('#]*name=[\'"]robots[\'"][^>]*>#is',$cn,$_cm))
$f = preg_match('#noindex#is',$_cm[0]);
if($f){
if(strstr($cn, 'Incapsula'))
$OgSj2zxK4Qo5Cc = 'incapsula';
else
$OgSj2zxK4Qo5Cc = 'mrob';
}
}
if(!$f && !$OgSj2zxK4Qo5Cc)
{
if(!$gEBIZ5cHL) {
$gEBIZ5cHL = array(
'link' => preg_replace('#//+$#','/',
preg_replace('#^([^/\:\?]/)/+#','\\1',
(preg_match('#^\w+://#',$this->aMX8JQFnFnMWTuOJ) ? $this->aMX8JQFnFnMWTuOJ : $this->A78wE6jgmrny . $this->aMX8JQFnFnMWTuOJ)
))
);
$j0dFHpRFFgv = 1024;
if($grab_parameters['xs_makehtml']||$grab_parameters['xs_makeror']||$grab_parameters['xs_rssinfo']||$grab_parameters['xs_newsinfo'])
{
preg_match('#]*?>([^<]*?)#is', $uFxTSS9XSb52KZo7YrD['content'], $h7F8_WaY6lWU5mvC);
$gEBIZ5cHL['t'] = substr(trim(strip_tags($h7F8_WaY6lWU5mvC[1])),0,$j0dFHpRFFgv);
}
if($grab_parameters['xs_metadesc']||$grab_parameters['xs_videoinfo'])
{
if(!preg_match('#]*(?:http-equiv|name)\s*=\s*"?description[^>]*content\s*=\s*["]?([^>\"]*)#is', $cn, $MdWOO751erxbvs))
preg_match('#]*content\s*=\s*["]?([^>\"]*)[^>]*?(?:http-equiv|name)\s*=\s*"?description#is', $cn, $MdWOO751erxbvs);
if($MdWOO751erxbvs[1])
$gEBIZ5cHL['d'] = substr(trim($MdWOO751erxbvs[1]),0,$j0dFHpRFFgv);
}
if($grab_parameters['xs_makeror']||$grab_parameters['xs_autopriority'])
$gEBIZ5cHL['o'] = max(0,$this->links_level);
if($zN7QvVaBZeQp5ZF)
$gEBIZ5cHL['p'] = $zN7QvVaBZeQp5ZF;
if(preg_match('#]*(?:http-equiv|name)\s*=\s*"?last-modified[^>]*content\s*=\s*["]?([^>\"]*)#is', $cn, $MdWOO751erxbvs)){
$gEBIZ5cHL['clm'] = str_replace('@',' ',$MdWOO751erxbvs[1]);
}
if(preg_match('#]*(?:http-equiv|name)\s*=\s*"?priority[^>]*content\s*=\s*["]?([\d\.]+)#is', $cn, $MdWOO751erxbvs)){
$gEBIZ5cHL['p'] = $MdWOO751erxbvs[1];
}
if(preg_match('#]*(?:http-equiv|name)\s*=\s*"?changefreq[^>]*content\s*=\s*["]?([^>\"]*)#is', $cn, $MdWOO751erxbvs)){
$gEBIZ5cHL['f'] = $MdWOO751erxbvs[1];
}else
if(preg_match('#]*(?:http-equiv|name)\s*=\s*"?revisit-after[^>]*content\s*=\s*["]?([^>\"]*)#is', $cn, $MdWOO751erxbvs)){
if(preg_match('#(\d+)\s*hour#',$MdWOO751erxbvs[1])){
$gEBIZ5cHL['f'] = 'hourly';
}
if(preg_match('#(\d+)\s*month#',$MdWOO751erxbvs[1])){
$gEBIZ5cHL['f'] = 'monthly';
}
if(preg_match('#(\d+)\s*day#',$MdWOO751erxbvs[1], $MdWOO751erxbvs)){
$d = $MdWOO751erxbvs[1]+0;
if($d<4)$gEBIZ5cHL['f'] = 'daily';
else
if($d<22)$gEBIZ5cHL['f'] = 'weekly';
else
$gEBIZ5cHL['f'] = 'monthly';
}
}
if(preg_match('#'.$Xmfk5aHUVgA7WxET.'#',$this->oGNbg9vqexWio($this->aMX8JQFnFnMWTuOJ),$eADRrzkKihKqzHhKTq7))
{
for($_i=0;$_iy5E6XiZB3w2DW4XFY*100;
if(!$_maxi_tot || ($this->addedcnt['i']<$_maxi_tot))
if($grab_parameters['xs_imginfo'])
{
$_imgext = '(?:jpg|png|gif|jpeg|bmp)';
preg_match_all('#<(?:(?:amp-)?img|div)[^>]*?(?:(?:src|data-(?:image|thumbnail|original))(?:set)?\s*=\s*|image\:\s*url\s*\(\s*)("([^">]+)|\'([^\'>]+)|([^\s\"\\\\>]+))(.*?>)#is', $cn, $dbZYiSonNrlDO42usc, PREG_SET_ORDER);
preg_match_all('#]*\shref\s*=\s*[\'"]?([^>\'" ]*?\.'.$_imgext.')([\'" ][^>]*?>|>)#is', $cn, $L2HmGWTj9H, PREG_SET_ORDER);
$T9v4StE5HjmB1 = array();
$ai = $ae = array();
$dbZYiSonNrlDO42usc = array_merge($dbZYiSonNrlDO42usc, $L2HmGWTj9H, $T9v4StE5HjmB1);
$_maxi = $grab_parameters['xs_img_maxperpage'];
if(!$_maxi)$_maxi = 1000;
if($uFxTSS9XSb52KZo7YrD['inpdata'] && ($_dimg = $uFxTSS9XSb52KZo7YrD['inpdata']['img'])){
foreach($_dimg as $im)
{
$dbZYiSonNrlDO42usc[] = array('', $im['url']);
}
}
foreach($dbZYiSonNrlDO42usc as $im)
{
$_il = array($im[1]);
if(strstr($im[0],'srcset')){
$_il = preg_split('#(\s*,\s*|\s+[^\./]+\s*)#', $im[1]);
}
foreach($_il as $_ili)
if($_ili)
{
$_im1 = trim(preg_replace('#^[\'\"]+#','',$_ili));
$LV3feH44NvQ = $this->bBAymp71W4mgyv_2c($_im1, $this->aMX8JQFnFnMWTuOJ,
$grab_parameters['xs_img_allow_domains']);
if($LV3feH44NvQ == 1)
continue;
if($grab_parameters['xs_imgincmask'])
if(!$this->NxfMP9qoH('xs_imgincmask', $_im1, true))
continue;
$XegIxMo9HSXuEr = $this->v0ytSBWER.substr($_im1,strlen($this->A78wE6jgmrny));
if($LV3feH44NvQ != 2)
{
if($this->MgKnEUlaw6_XfzaD($XegIxMo9HSXuEr))
continue;
}
if(!$this->MVZvUfEx6bYBJwmz||!@preg_match('#('.$this->MVZvUfEx6bYBJwmz.')#',$XegIxMo9HSXuEr))
{
preg_match('#(?:title|alt)\s*=\s*"(.*?)"#i', $im[0], $tm);
$t = trim($tm[1]);
if(!$imlist[md5($_im1.($grab_parameters['xs_img_hashtitle'] ? '-'.$t : ''))]++)
{
$ai[] = array($_im1, $t);
}
}
if(count($ai) >= $_maxi)break;
}
}
if($ai){
$gEBIZ5cHL['i'] = $ai;
$this->addedcnt['i'] += count($ai);
}
wkwPIGj7HFydE9Mly("{ images added: ".count($ai)." }
\n",2);
}
$xz = '/img';
$xz = 'video';
$_maxv_tot = $this->y5E6XiZB3w2DW4XFY * 10;
if(!$_maxv_tot || ($this->addedcnt['v']<$_maxv_tot))
if($grab_parameters['xs_videoinfo'])
{
$BJS7s_fNyAAZU26 = array(
'(?:https?\:)?//(?:www\.)?youtube(?:-nocookie)?.com/(?:v/|embed/|watch\?v=)',
'(?:https?\:)?//youtu\.be/',
'(?:https?\:)?//video.google.com/googleplayer\.swf\?docid=',
'(?:https?\:)?//(?:player\.)?vimeo.com/(?:video/|moogaloop\.swf\?clip_id=)',
'(?:https?\:)?//(?:www\.)?dailymotion.com/(?:embed|swf)/video/',
'(?:https?\:)?//(?:media.mtvnservices.com|www.mtv.com/player/)[^>]*?vid(?:=|%3D)',
'(?:https?\:)?//blip.tv/play/',
'(?:https?\:)?//network.coull.com/(?:embediframe/|embed)\?v='
);
$BJS7s_fNyAAZU26 = '(?:'. implode('|', $BJS7s_fNyAAZU26).')';
$IQXRrpl7X = '[a-z0-9\-\_]';
$BJS7s_fNyAAZU26 = '('.$BJS7s_fNyAAZU26.'('.$IQXRrpl7X.'+))';
preg_match_all('#]*name=[\'"\s]*?(?:movie|src)[^>]*value=[\'"\s]*?'.$BJS7s_fNyAAZU26.'([^>\'"]*)#is', $cn, $dbZYiSonNrlDO42usc, PREG_SET_ORDER);
preg_match_all('#flashvars\s*=\s*\{[^\}]*?file:\s*[\'"]'.$BJS7s_fNyAAZU26.'#is', $cn, $s2DT0A9dOu, PREG_SET_ORDER);
preg_match_all('#<(?:embed|script)[^>]*src=[\'"\s]*?'.$BJS7s_fNyAAZU26.'([^>\'"]*)#is', $cn, $p0Z32taip_p, PREG_SET_ORDER);
preg_match_all('#<(?:iframe|video)[^>]*src=[^>]*?'.$BJS7s_fNyAAZU26.'([^>\'"]*)#is', $cn, $m07f5HVKi, PREG_SET_ORDER);
preg_match_all('#<[^>]*(?:href|data-path)=[^>]*?'.$BJS7s_fNyAAZU26.'([^>\'"]*)#is', $cn, $yUuYWZ7cq, PREG_SET_ORDER);
preg_match_all('#file\"\s*\:\s*\[\'"]'.$BJS7s_fNyAAZU26.'#is', $cn, $vieMCCQX4, PREG_SET_ORDER);
preg_match_all('#\.addVariable\([\'\"]file[\'\"],[\'\"]'.$BJS7s_fNyAAZU26.'#is', $cn, $t, PREG_SET_ORDER);
preg_match_all('#var\s+youtube\s+=\s+\'('.$IQXRrpl7X.'+)#is', $cn, $_ym1, PREG_SET_ORDER);
foreach($_ym1 as $_yv){
$dbZYiSonNrlDO42usc[] = array('', 'http://youtu.be/'.$_yv[1],$_yv[1]);
}
$ai = $ae = $UTnnUtzvzSy0XAP = array();
preg_match_all('#jwplayer\([^\)]*?\)\.setup\(\{(.*?)\}\)#is', $cn, $ps);
if($ps && $ps[1]) {
foreach($ps[1] as $lodni7zmzvSYqgs){
if(
preg_match('#\"provider\"\s*\:\s*\"youtube\"#is', $lodni7zmzvSYqgs, $m1)
&& preg_match('#\"file\"\s*\:\s*\"[^\"]*?(?:v=)?('.$IQXRrpl7X.'+)#is', $lodni7zmzvSYqgs, $m2)
){
preg_match('#\"image\"\s*\:\s*\"([^\"]+)#is', $lodni7zmzvSYqgs, $m3);
$_th = $m3 ? $m3[1] : '';
if($_th)
$this->bBAymp71W4mgyv_2c($_th, $this->aMX8JQFnFnMWTuOJ);
$UTnnUtzvzSy0XAP[] = array(
'',
'http://www.youtube.com/watch?v='.$m2[1],
$m2[1],
'',
$gEBIZ5cHL['t'],
$gEBIZ5cHL['d'],
$_th
);
}
}
}
preg_match_all('#]*class="youtube[^>]*data-id="(.*?)"#is', $cn, $ps);
if($ps && $ps[1]) {
foreach($ps[1] as $m2){
$UTnnUtzvzSy0XAP[] = array(
'',
'http://www.youtube.com/watch?v='.$m2,
$m2
);
}
}
$dbZYiSonNrlDO42usc = array_merge($dbZYiSonNrlDO42usc,$p0Z32taip_p,$m07f5HVKi,$s2DT0A9dOu, $yUuYWZ7cq, $vieMCCQX4, $UTnnUtzvzSy0XAP, $t);
preg_match_all('#<[^>]*id="video".*?>(.*?)#is', $cn, $ps);
if($ps && $ps[1]) {
foreach($ps[1] as $lodni7zmzvSYqgs){
if(
preg_match('#<[^>]*href="(.*?)"[^>]*data-type="video"#is', $lodni7zmzvSYqgs, $m_l)
&& preg_match('#]*src="(.*?)"#is', $lodni7zmzvSYqgs, $m_i)
){
$_th = $m_i[1];
$this->bBAymp71W4mgyv_2c($_th, $this->aMX8JQFnFnMWTuOJ);
$this->bBAymp71W4mgyv_2c($m_l[1], $this->aMX8JQFnFnMWTuOJ);
$ai[] = array(
'title' => strip_tags($gEBIZ5cHL['t']),
'playerloc' => $m_l[1],
'thumb' => $_th,
'desc' => strip_tags($gEBIZ5cHL['d']),
);
}
}
}
$ps = preg_split('#<[^>]*itemtype=\"http\:\/\/schema\.org\/VideoObject\"#is', $cn);
if(count($ps)>1) {
$Agor9azeTRg65OdCMf = array();
foreach($ps as $pp){
if(
(preg_match('#<[^>]*itemprop=\"name\"[^>]*content="([^>]*?)"#is', $pp, $m1)
||preg_match('#<[^>]*itemprop=\"name\"[^>]*?>(.*?)<#is', $pp, $m1)
)
&& preg_match('#<[^>]*itemprop=\"(?:(?:content|embed)(?:url)?)\"[^>]*content="([^>]*?)"#is', $pp, $m2)
&& preg_match('#<[^>]*itemprop=\"thumbnail(?:url)?\"[^>]*content="([^>]*?)"#is', $pp, $m3)
){
preg_match('#<[^>]*itemprop=\"duration\"[^>]*content="([^>]*?)"#is', $pp, $m11);
if(!preg_match('#<[^>]*itemprop=\"description\"[^>]*?content="([^>]*?)"#is', $pp, $m4))
preg_match('#<[^>]*itemprop=\"description\"[^>]*?>(.*?)bBAymp71W4mgyv_2c($m2[1], $this->aMX8JQFnFnMWTuOJ, true);
$this->bBAymp71W4mgyv_2c($m3[1], $this->aMX8JQFnFnMWTuOJ, true);
$_anx = array(
'title' => strip_tags($m1[1]),
'playerloc' => $m2[1],
'thumb' => $m3[1],
'desc' => strip_tags(preg_replace('##', "\n", ($m4 ? $m4[1] : ($gEBIZ5cHL['d'] ? $gEBIZ5cHL['d'] : $m1[1])))),
'dur' => $dfUGNFzE8,
);
if(preg_match('#'.$BJS7s_fNyAAZU26.'#is', $m2[1], $_lm))
$_anx['vid'] = trim($_lm[2]);
$Agor9azeTRg65OdCMf[] = $_anx;
}
}
if($Agor9azeTRg65OdCMf){
$ai = $Agor9azeTRg65OdCMf;
}
}
preg_match_all('#]*application/ld\+json[^>]*>(.+?)#is', $cn, $ps);
if(count($ps)>1) {
$Agor9azeTRg65OdCMf = array();
foreach($ps[1] as $pp){
$Z4oarkWIP004duZRulZ = json_decode($pp, true);
if($Z4oarkWIP004duZRulZ)
foreach($Z4oarkWIP004duZRulZ as $k=>$v)$Z4oarkWIP004duZRulZ[strtolower($k)] = $v;
if(
($Z4oarkWIP004duZRulZ['@type'] == 'VideoObject')
&& $Z4oarkWIP004duZRulZ['embedurl']
&& $Z4oarkWIP004duZRulZ['thumbnailurl']
){
preg_match('#\"duration\"\s*\:\s*\"(.*?)\"#is', $pp, $m11);
preg_match('#\"description\"\s*\:\s*\"(.*?)\"#is', $pp, $m4);
$dfUGNFzE8 = $Z4oarkWIP004duZRulZ['duration'];
if(preg_match('#T(?:([\d\.]+)H)?(?:([\d\.]+)M)?([\d\.]+)S#i', $dfUGNFzE8, $dm)) {
$dfUGNFzE8 = $dm[3] + $dm[2]*60 + $dm[1]*3600;;
}
$this->bBAymp71W4mgyv_2c($m2[1], $this->aMX8JQFnFnMWTuOJ, true);
$this->bBAymp71W4mgyv_2c($m3[1], $this->aMX8JQFnFnMWTuOJ, true);
$_anx = array(
'title' => strip_tags($Z4oarkWIP004duZRulZ['name']),
'playerloc' => $Z4oarkWIP004duZRulZ['embedurl'],
'thumb' => $Z4oarkWIP004duZRulZ['thumbnailurl'],
'desc' => strip_tags($Z4oarkWIP004duZRulZ['description']),
'dur' => $dfUGNFzE8,
);
if(preg_match('#'.$BJS7s_fNyAAZU26.'#is', $Z4oarkWIP004duZRulZ['embedurl'], $_lm))
$_anx['vid'] = trim($_lm[2]);
$Agor9azeTRg65OdCMf[] = $_anx;
}
}
if($Agor9azeTRg65OdCMf){
$dbZYiSonNrlDO42usc = array();
$ai = $Agor9azeTRg65OdCMf;
}
}
if(preg_match('#]*property="og:type"[^>]*content="video#is', $cn)) {
preg_match('#]*property="og:duration"[^>]*content="(.*?)"#is', $cn, $m1 );
preg_match('#]*property="og:url"[^>]*content="(.*?)"#is', $cn, $m2 );
preg_match('#]*property="og:image"[^>]*content="(.*?)"#is', $cn, $m3 );
preg_match('#]*property="og:title"[^>]*content="(.*?)"#is', $cn, $m4 );
preg_match('#]*property="og:description"[^>]*content="(.*?)"#is', $cn, $m5 );
if($m1&& $m2&& $m3&& $m4) {
$cAdokkPjZ4JrKCaE6J0 = explode(':', $m1[1]);
$dfUGNFzE8 = $cAdokkPjZ4JrKCaE6J0[0]*60+$cAdokkPjZ4JrKCaE6J0[1];
if(count($cAdokkPjZ4JrKCaE6J0)==3)
$dfUGNFzE8 = $dfUGNFzE8*60+$dfUGNFzE8[2];
$dbZYiSonNrlDO42usc = $ai = array();
$_anx = array(
'thumb' => $m3[1],
'title' => $m4[1],
'desc' => $m5?$m5[1]:$m4[1],
'dur' => $dfUGNFzE8,
'playerloc' => $m2[1],
);
if(preg_match('#'.$BJS7s_fNyAAZU26.'#is', $m2[1], $_lm))
$_anx['vid'] = trim($_lm[2]);
$ai[] = $_anx;
}
}
foreach($dbZYiSonNrlDO42usc as $im)
{
array_shift($im);
$ai[] = $im;
}
if($ai){
$gEBIZ5cHL['v'] = $ai;
}
if(function_exists('xml_extra_video_step1'))
xml_extra_video_step1($gEBIZ5cHL, $cn, $imlist);
}
if($_ev = $gEBIZ5cHL['v']){
$ai = array();
foreach($_ev as $im)
{
if($_vid = $im['vid'] ?$im['vid'] : ($im['playerloc']?$im['playerloc']:$im[1])){
$_i5 = md5(trim($_vid));
$imlist[$_i5]++;
if(!$imlist2[$_i5]++)
$ai[] = $im;
}
}
$gEBIZ5cHL['v'] = $ai;
$this->addedcnt['v'] += count($ai);
wkwPIGj7HFydE9Mly("{ videos added: ".count($ai)." }
\n",2);
}
$xz = '/video';
if($grab_parameters['xs_hreflang']){
if(
preg_match_all('#]*rel\s*=\s*"alternate"[^>]*>#is', $cn, $G9M1u0W69Tn7MIsQ4, PREG_SET_ORDER)
){
$_la = array();
foreach($G9M1u0W69Tn7MIsQ4 as $_alt1){
if(preg_match('#\s(hreflang|media)\s*=\s*"([^">]*?)"[^>]*>#is', $_alt1[0], $_alt)
&& preg_match('#\s*href\s*=\s*[\'"]([^>]*?)[\'"]#is', $_alt1[0], $_hm)
) {
$this->bBAymp71W4mgyv_2c($_hm[1], $this->aMX8JQFnFnMWTuOJ);
$_la[] = array('t' => $_alt[1], 'l' => $_alt[2], 'u' => $_hm[1]);
}
}
$gEBIZ5cHL['hl'] = $_la;
}
}
if($grab_parameters['xs_lastmod_notparsed'] && $f2)
{
$uFxTSS9XSb52KZo7YrD = $k0huGF8gg4LQ1A->fetch($wljeCd6fCRKulnoFsC4, 0, 1, false, "", array('req'=>'HEAD'));
}
if(!$gEBIZ5cHL['lm'] && isset($uFxTSS9XSb52KZo7YrD['headers']['last-modified']))
$gEBIZ5cHL['lm'] = $uFxTSS9XSb52KZo7YrD['headers']['last-modified'];
}
if(!$gEBIZ5cHL['lm'] &&
preg_match('#]*?name\s*=\s*\"last-modified\" content\s*=\s*"?([^">]*)"#is',$uFxTSS9XSb52KZo7YrD['content'], $hPw74OQezczKteg0gC))
$gEBIZ5cHL['lm'] = $hPw74OQezczKteg0gC[1];
RYtkTQfZ1my('post', true);
RYtkTQfZ1my('post-save1');
wkwPIGj7HFydE9Mly("\n((include ".$gEBIZ5cHL['link']."))
\n");
$wyhMGdq7QGd6 = true;
if($grab_parameters['xs_memsave'])
{
DSdHxXD2UR($oO9pt9QoVu0fjOUEU6, $gEBIZ5cHL);
$this->urls_completed[] = $oO9pt9QoVu0fjOUEU6;
}
else $this->urls_completed[] = $this->dtRHNsRCOCS7gtKcKk3($gEBIZ5cHL);
$this->lOXW9WDJYx6ROZX++;
RYtkTQfZ1my('post-save1',true);
RYtkTQfZ1my('post-save2');
if($grab_parameters['xs_prev_sm_base']
&& $this->basecachemask &&
preg_match('#('.$this->basecachemask.')#',$this->aMX8JQFnFnMWTuOJ)){
$this->sm_base[$this->aMX8JQFnFnMWTuOJ] = $gEBIZ5cHL;
}
$ELiT8ew5NjA = $this->y5E6XiZB3w2DW4XFY - $this->lOXW9WDJYx6ROZX;
RYtkTQfZ1my('post-save2',true);
}
}while(false);// zero-while
RYtkTQfZ1my('post-progress1');
if($this->num_urls_processed_in_current_batch>=$this->num_links_current_batch)
{
$this->links_level++;
wkwPIGj7HFydE9Mly("\n
NEXT LEVEL:$this->links_level ($this->num_urls_processed_in_current_batch>=$this->num_links_current_batch)
\n");
unset($urls_list);
$this->num_urls_processed_in_current_batch = 0;
$urls_list = $urls_list2;
reset($urls_list);
$urls_list_full += $urls_list;
$this->num_links_current_batch = count($urls_list);
unset($this->ref_links_tmp2);
$this->ref_links_tmp2 = $this->ref_links_tmp;
unset($this->ref_links_tmp); unset($urls_list2);
$this->ref_links_tmp = array();
$urls_list2 = array();
}
if(!$wyhMGdq7QGd6){
wkwPIGj7HFydE9Mly("\n({skipped ".$this->aMX8JQFnFnMWTuOJ." - $OgSj2zxK4Qo5Cc})
\n");
if(!$grab_parameters['xs_chlog_list_max'] ||
count($urls_list_skipped) < $grab_parameters['xs_chlog_list_max']) {
$urls_list_skipped[$this->aMX8JQFnFnMWTuOJ] = $OgSj2zxK4Qo5Cc;
}
}
RYtkTQfZ1my('post-progress1',true);
RYtkTQfZ1my('post-progress2');
$this->num_processed++;
R1r_jX8NavH();
$this->pl=min($this->num_links_current_batch - $this->num_urls_processed_in_current_batch,$ELiT8ew5NjA);
$eG1Pm653b = ($this->lOXW9WDJYx6ROZX>=$this->y5E6XiZB3w2DW4XFY) || ($this->num_urls_processed_in_current_batch >= $this->num_links_current_batch);
if(!$X5O38TAuWv4FR) {
if($X5O38TAuWv4FR = VzipJxoohDvbtWM()){
if(!@cMo3XWEqPThU4Id(pbAiQcHEGrGKkyqo4Q.guXHCq5aVeE))
$X5O38TAuWv4FR=0;
}
}
RYtkTQfZ1my('post-progress2',true);
RYtkTQfZ1my('post-progress3');
$progpar = $this->KiQEfebckxJVM(false, $eG1Pm653b||$X5O38TAuWv4FR);
RYtkTQfZ1my('post-progress3',true);
RYtkTQfZ1my('post-progress4');
if($grab_parameters['xs_exec_time'] &&
((time()-$dgHgts6lFbG) > $grab_parameters['xs_exec_time']) ){
$X5O38TAuWv4FR = 'Time limit exceeded - '.($grab_parameters['xs_exec_time']).' - '.(time()-$dgHgts6lFbG);
}
if($grab_parameters['xs_savestate_time']>0 &&
(
($this->ctime-$epcQYfCl1>$grab_parameters['xs_savestate_time'])
|| $eG1Pm653b
|| $X5O38TAuWv4FR
)
)
{
$epcQYfCl1 = $this->ctime;
wkwPIGj7HFydE9Mly("(saving dump)
\n");
$mFUcZxJGA4itwRbJ = array();
$gHdDA3umqlMQEf47qO = array(
'num_urls_processed_in_current_batch',
'urls_list','urls_list2','num_links_current_batch',
'ref_links_tmp','ref_links_tmp2','ref_links_list',
'urls_list_full','urls_completed',
'urls_404',
'nt','tsize','num_processed','links_level','ctime', 'urls_ext','fetch_no',
'starttime', 'retrno', 'nettime', 'urls_list_skipped',
'imlist', 'imlist2', 'progpar', 'runstate', 'sm_sessions',
'addedcnt'
);
foreach($gHdDA3umqlMQEf47qO as $k){
$mFUcZxJGA4itwRbJ[$k] = isset($this->$k) ? $this->$k : $$k;
}
$mFUcZxJGA4itwRbJ['time']=time();
$y5GpEuKtv9=Ie0qgN_fQ8HA($mFUcZxJGA4itwRbJ);
Ndm7I4IRr(TYR4q027D0OWvh,$y5GpEuKtv9,pbAiQcHEGrGKkyqo4Q,true);
unset($mFUcZxJGA4itwRbJ);
unset($y5GpEuKtv9);
}
if($grab_parameters['xs_delay_req'] && $grab_parameters['xs_delay_ms'] &&
(($m8HTpd_0s-$WCC6wCSTlQ4MjKyNm3)==$grab_parameters['xs_delay_req']))
{
$WCC6wCSTlQ4MjKyNm3 = $m8HTpd_0s;
sleep(intval($grab_parameters['xs_delay_ms']));
}
RYtkTQfZ1my('post-progress4', true);
}while(!$eG1Pm653b && !$X5O38TAuWv4FR);
wkwPIGj7HFydE9Mly("\n\n
Crawling completed
\n");
if($_GET['ddbgexit']){
echo '
Dbg exit
';
echo $k0huGF8gg4LQ1A->K7DRsNrsbdnzu9hi.' / '.$k0huGF8gg4LQ1A->nettime.'
';
echo DfjNmPlYlDtVYeKoRsL().'
';
exit;
}
return array(
'u404'=>$this->urls_404,
'ref_links_list'=>$this->ref_links_list,
'starttime'=>$starttime,
'topmu' => $K5qP9_ZsHx,
'ctime'=>$this->ctime,
'tsize'=>$this->tsize,
'retrno' => $retrno,
'nettime' => $this->nettime,
'errmsg'=>'',
'initurl'=>$this->wOd8ZRUj4KujS,
'initdir'=>$this->A78wE6jgmrny,
'ucount'=>$this->lOXW9WDJYx6ROZX,
'crcount'=>$this->num_processed,
'fetch_no'=>$this->fetch_no,
'time'=>time(),
'params'=>$this->SVH4BzIyh56a_HEGqt,
'sm_sessions'=>$this->sm_sessions,
'interrupt'=>$X5O38TAuWv4FR,
'runstate' => $this->runstate,
'urls_ext'=>$urls_ext,
'urls_list_skipped' => $urls_list_skipped,
'max_reached' => $this->lOXW9WDJYx6ROZX>=$this->y5E6XiZB3w2DW4XFY
);
}
}
$AIqYzQK_mhY = new SiteCrawler();
function xvQcTIF8B113J(){
@cMo3XWEqPThU4Id(pbAiQcHEGrGKkyqo4Q.crI8G26xJzM);
if(@file_exists(pbAiQcHEGrGKkyqo4Q.m093Bbc4Eg))
@rename(pbAiQcHEGrGKkyqo4Q.m093Bbc4Eg,pbAiQcHEGrGKkyqo4Q.crI8G26xJzM);
}