/* Copyright (C) 2000, 2001  SWsoft, Singapore
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

/*  $Id: sqldbi.cpp,v 1.9 2001/11/10 15:38:32 al Exp $
    Author : Alexander F. Avdonkin, Igor Sukhih
*/

#include <string.h>
#include "aspseek-cfg.h"
#include "sqldbi.h"
#include "parse.h"
#include "index.h"
#include "rank.h"
#include "logger.h"
#include "lastmod.h"
#include "timer.h"
#include "geo.h"

#include "deltas.h"

extern hash_set<CURLName> Sites;

typedef struct
{
	ULONG urlID, siteID;
	int time;
} URLN;

int CSQLDatabaseI::AddUrls(CSQLQuery *query, double& time, int& maxtime, CIntSet* set)
{
	CSQLAnswer *answ;
	int numr = 0;
	{
		CTimerAdd timer(time);
		answ = sql_query(query);
	}
	if (answ)
	{
		vector<URLN> urls;
		while (answ->FetchRow())
		{
			URLN url;
			answ->GetColumn(0, url.urlID);
			answ->GetColumn(1, url.siteID);
			answ->GetColumn(2, url.time);
			urls.push_back(url);
		}
		delete answ;
		int sz = set ? set->size() : 0;
		for (URLN* purl = urls.begin(); purl != urls.end(); purl++)
		{
			if ((sz == 0) || (set == NULL) || (set->find(purl->urlID) == set->end()))
			{
				m_squeue.AddURL(purl->siteID, purl->urlID, this);
				numr++;
			}
			if (sz == 0)
			{
				if (purl->time != maxtime)
				{
					maxtime = purl->time;
					if (set) set->clear();
				}
				if (set) set->insert(purl->urlID);
			}
		}
	}
	return numr;
}

ULONG CSQLDatabaseI::GetNextLink(int numthreads)
{
	while (m_squeue.m_inactiveSize < (numthreads << 2))
	{
		int numr = 0;
		double sel = 0;
		CIntSet set;
		int mintime = m_maxtime;
/* "SELECT url_id,site_id,next_index_time FROM urlword WHERE deleted=0 \
AND next_index_time<=%d AND next_index_time > %i %s ORDER BY next_index_time LIMIT %lu",
(int)now(), m_maxtime, Limit.c_str(), NextDocLimit);
*/
		CSQLParam p;
		int tm = now();
		p.AddParam(&tm);
		p.AddParam(&m_maxtime);
		p.AddParam(Limit.c_str());
		CSQLQuery *sqlquery = m_sqlquery->GetNextLinkS(&p);
		sqlquery->AddLimit(NextDocLimit);

		numr = AddUrls(sqlquery, sel, m_maxtime, &set);
		if (numr > 0)
		{
/* "SELECT url_id,site_id,next_index_time FROM urlword \
WHERE deleted=0 AND next_index_time=%d %s", m_maxtime, Limit.c_str());
*/
			CSQLParam p;
			p.AddParam(&m_maxtime);
			p.AddParam(Limit.c_str());
			CSQLQuery *sqlquery = m_sqlquery->GetNextLinkS1(&p);
			numr += AddUrls(sqlquery, sel, m_maxtime, &set);
		}
		if (numr && m_logFile)
		{
			fprintf(m_logFile, "Got next %6i URLs for: %7.3f seconds. Queued docs: %5lu.Time %i-%i.%s\n",
numr, sel, m_squeue.m_qDocs, mintime, m_maxtime,m_available ? "" : "Not available!");
			fflush(m_logFile);
			m_got = 1;
		}
		if (numr == 0)
		{
			m_available = 0;
			break;
		}
	}
	return m_squeue.ExtractURL();
}

ULONG CSQLDatabaseI::GetNextLinkH(int numthreads)
{
	while ((m_maxhops < 0x10000) && (m_squeue.m_inactiveSize < (numthreads << 2)))
	{
		int numr = 0;
//		CIntSet set;
		double sel = 0;
		int& maxtime = m_maxtimes[m_maxhops];
		int nw = (int)now();
//		if (maxtime >= nw) break;
		if (maxtime < nw)
		{
/* "SELECT url_id,site_id,next_index_time FROM urlword
WHERE deleted=0 AND next_index_time<=%d AND hops=%i AND next_index_time > %i %s
ORDER BY next_index_time", nw, m_maxhops, maxtime, Limit.c_str());
*/
			CSQLParam p;
			p.AddParam(&nw);
			p.AddParam(&m_maxhops);
			p.AddParam(&maxtime);
			p.AddParam(Limit.c_str());
			CSQLQuery *sqlquery = m_sqlquery->GetNextLinkHS(&p);

			numr = AddUrls(sqlquery, sel, maxtime, NULL);
/*			if (numr > 0)
			{
//"SELECT url_id,site_id,next_index_time
//FROM urlword WHERE deleted=0 AND hops=%i AND next_index_time=%d %s",m_maxhops, maxtime, Limit.c_str());
				CSQLParam p;
				p.AddParam(&m_maxhops);
				p.AddParam(&maxtime);
				p.AddParam(Limit.c_str());
				CSQLQuery *sqlquery = m_sqlquery->GetNextLinkHS1(&p);

				numr += AddUrls(sqlquery, sel, maxtime, set);
			}
			else*/
			{
				maxtime = nw;
			}
			if (numr && m_logFile)
			{
				fprintf(m_logFile, "Got next %6i URLs for: %7.3f seconds.Queued docs: %5lu. Hops: %i.%s\n",
numr, sel, m_squeue.m_qDocs, m_maxhops,m_available ? "" : "Not available!");
				fflush(m_logFile);
				m_got = 1;
			}
		}
		if (/*(numr || (m_squeue.m_qDocs == 0)) && */(m_maxhops < m_maxhopsa))
		{
			m_maxhops++;
		}
		else
		{
			break;
		}
/*
                else
                {
                        if (m_maxhops < m_maxhopsa)
                        {
                                m_maxhops++;
                        }
                        else
                        {
                                break;
                        }
                }*/
	}
	return m_squeue.ExtractURL();
}


void CSQLDatabaseI::InsertWordSite(const char* word, ULONG* sites, int size)
{
// "INSERT INTO wordsite(word, sites) VALUES('%s', '", word);
//	pquery = memcpyq(pquery, (char*)sites, size << 2);
	CSQLParam p;
        p.AddParam(word);
        p.AddParamEsc((char*)sites, size << 2);
        CSQLQuery *sqlquery = m_sqlquery->InsertWordSiteI(&p);
 	sql_real_query(sqlquery);
}

void CSQLDatabaseI::UpdateWordSite(char* hostname, ULONG site_id)
{
	hash_map<CWord, ULONG> hmap;
	char* host = hostname;
	Tolower((unsigned char*)host, CHARSET_USASCII);
	ULONG bit = 1;
	ULONG bits = 32;
	while (host)
	{
		char* dot = strchr(host, '.');
		if (dot) *dot = 0;
		if ((dot ? dot - host : strlen(host)) <= MAX_WORD_LEN) hmap[host] |= bit;
		host = dot ? dot + 1 : NULL;
		bit <<= 1;
		bits--;
	}
	for (hash_map<CWord, ULONG>::iterator it = hmap.begin(); it != hmap.end(); it++)
	{
// "SELECT sites FROM wordsite WHERE word='%s'", it->first.Word());
		CSQLParam p;
		p.AddParam(it->first.Word());
		CSQLQuery *sqlquery = m_sqlquery->UpdateWordSiteS(&p);
		CSQLAnswer *answ = sql_query(sqlquery);
		ULONG* newSites;
		ULONG* pNew;
		if (answ && answ->FetchRow())
		{
			ULONG len = answ->GetLength(0) >> 2;
			newSites = new ULONG[len + 1];
			pNew = newSites;
			ULONG* pold = (ULONG*)answ->GetColumn(0);
			ULONG* eold = pold + len;
			while ((pold < eold) && ((*pold & 0xFFFFFF) < site_id))
			{
				*pNew++ = *pold++;
			}
			*pNew++ = site_id | ((it->second << bits) & 0xFF000000);
			if ((pold < eold) && ((*pold & 0xFFFFFF)== site_id))
			{
				pold++;
			}
			memcpy(pNew, pold, (eold - pold) << 2);
			pNew += (eold - pold);
// "DELETE FROM wordsite WHERE word = '%s'", it->first.Word());
			CSQLParam p;
	                p.AddParam(it->first.Word());
		        sqlquery = m_sqlquery->UpdateWordSiteD(&p);
			sql_query2(sqlquery);
		}
		else
		{
			newSites = new ULONG[1];
			newSites[0] = site_id | ((it->second << bits) & 0xFF000000);
			pNew = newSites + 1;
		}
		InsertWordSite(it->first.Word(), newSites, pNew - newSites);
		delete newSites;
		if (answ) delete answ;
	}
}

void CSQLDatabaseI::GenerateWordSite()
{
	logger.log(CAT_ALL, L_INFO, "Generating word site .");
//  "SELECT site, site_id FROM sites order by site_id");
	CSQLQuery *sqlquery = m_sqlquery->GenerateWordSiteS();
        CSQLAnswer *answ = sql_query(sqlquery);
	if(!answ)
		return;
	hash_map<CWord, vector<ULONG> > words;
	hash_map<CWord, vector<ULONG> >::iterator wit;
	while (answ && answ->FetchRow())
	{
		CUrl url;
		hash_map<CWord, ULONG> hmap;
		ULONG site_id;
		answ->GetColumn(1, site_id);
		url.ParseURL(answ->GetColumn(0));
		char* host = url.m_hostinfo;
		Tolower((unsigned char*)host, CHARSET_USASCII);
		ULONG bit = 1;
		ULONG bits = 32;
		while (host)
		{
			char* dot = strchr(host, '.');
			if (dot) *dot = 0;
			if ((dot ? dot - host : strlen(host)) <= MAX_WORD_LEN) hmap[host] |= bit;
			host = dot ? dot + 1 : NULL;
			bit <<= 1;
			bits--;
		}
		for (hash_map<CWord, ULONG>::iterator it = hmap.begin(); it != hmap.end(); it++)
		{
			words[it->first].push_back(site_id | ((it->second << bits) & 0xFF000000));
		}
	}
	if (answ) delete answ;
	logger.log(CAT_ALL, L_INFO, ".");
// "SELECT word FROM wordsite");
        sqlquery = m_sqlquery->GenerateWordSiteS1();
        answ = sql_query(sqlquery);
	while (answ && answ->FetchRow())
	{
		if (strlen(answ->GetColumn(0)) <= MAX_WORD_LEN)
		{
			wit = words.find(answ->GetColumn(0));
// "DELETE FROM wordsite WHERE word = '%s'", row[0]);
			CSQLParam p;
			p.AddParam(answ->GetColumn(0));
		        sqlquery = m_sqlquery->GenerateWordSiteD(&p);
			sql_query2(sqlquery);
			if (wit != words.end())
			{
				vector<ULONG>& sites = wit->second;
				InsertWordSite(answ->GetColumn(0), sites.begin(), sites.size());
				words.erase(wit);
			}
		}
	}
	for (wit = words.begin(); wit != words.end(); wit++)
	{
		vector<ULONG>& sites = wit->second;
		InsertWordSite(wit->first.Word(), sites.begin(), sites.size());
	}
	if (answ) delete answ;
	logger.log(CAT_ALL, L_INFO, ". done\n");
}

void CSQLDatabaseI::GetTmpUrls(CTmpUrlSource& tmpus)
{
// "SELECT url_id FROM tmpurl");
	CSQLQuery *sqlquery = m_sqlquery->GetTmpUrlsS();
        CSQLAnswer *answ = sql_query(sqlquery);
//Fixme:
//	tmpus.m_urls.reserve(answ->GetNumRows());
	while (answ && answ->FetchRow())
	{
		ULONG url_id;
		answ->GetColumn(0, url_id);
		tmpus.m_urls.push_back(url_id);
	}
	if (answ) delete answ;

}

int CSQLDatabaseI::GetUrlID(const char* url, CHref& href, int nit, ULONG& urlID, ULONG& lost)
{
	int ret = 0;
//	"SELECT url_id, deleted FROM urlword WHERE url='");
//	pq = strcpyq(pq, url);
//	strcpy(pq, "'");
	CSQLParam p;
	p.AddParamEsc(url);
        CSQLQuery *sqlquery = m_sqlquery->GetUrlIDS2(&p);
        CSQLAnswer *answ = sql_query(sqlquery);
	if (answ && answ->FetchRow())
	{
		int del;
		answ->GetColumn(0, urlID);
		answ->GetColumn(1, del);
		lost++;
		if (del != 0)
		{
// "UPDATE urlword SET hops=%d,referrer=%d,last_index_time=%ld,next_index_time=%d,status=0,deleted=0 WHERE url_id=%ld",
// href.m_hops, href.m_referrer, now(), nit, urlID);
    			CSQLParam p;
			p.AddParam(&href.m_hops);
			p.AddParam(&href.m_referrer);
			int tm = now();
			p.AddParam(&tm);
			p.AddParam(&nit);
			p.AddParam(&urlID);
			sqlquery = m_sqlquery->GetUrlIDU(&p);
			sql_query2(sqlquery);
			ret = 1;
		}
	}
	else
	{
		urlID = 0;
	}
	if (answ) delete answ;
	return ret;
}

ULONG CSQLDatabaseI::StoreHref2(const char* url, CHref& href, const char* server, int hopsord, ULONG& lost)
{
	ULONG site_id;
// "SELECT site_id FROM sites WHERE site='%s'", server);
	CSQLParam p;
	p.AddParam(server);
	CSQLQuery *sqlquery = m_sqlquery->GetSiteS(&p);
	CSQLAnswer *answ = sql_query(sqlquery);
	if (answ && answ->FetchRow())
	{
		answ->GetColumn(0, site_id);
	}
	else
	{
// "INSERT INTO sites(site) VALUES('%s')", server);
		CSQLParam p;
		p.AddParam(server);
		sqlquery = m_sqlquery->StoreHref2I(&p);
		site_id = sql_query1(sqlquery);
	}
	if (answ) delete answ;
	int nit = now() - href.m_delta;
// "INSERT INTO urlword(site_id, hops, referrer, last_index_time, next_index_time, status, url) VALUES(%ld,%d,%d,%ld,%d,0,'",
// site_id, href.m_hops, href.m_referrer, now(), nit);
//	char* pq = strcpyq(qIns + strlen(qIns), url);
	int err;
	p.Clear();
	p.AddParam(&site_id);
	p.AddParam(&href.m_hops);
	p.AddParam(&href.m_referrer);
	int tm = now();
	p.AddParam(&tm);
	p.AddParam(&nit);
	p.AddParamEsc(url);
	sqlquery = m_sqlquery->StoreHref2I1(&p);

	ULONG urlID = sql_query1(sqlquery, &err);
	if ((err != DB_ER_DUP_ENTRY) || GetUrlID(url, href, nit, urlID, lost))
	{
		int* maxtime = hopsord ? &m_maxtimes[href.m_hops] : &m_maxtime;
		if (href.m_hops > m_maxhopsa)
		{
			m_maxhopsa = href.m_hops;
		}
		if (maxtime)
		{
			if ((*maxtime >= nit) || (hopsord && (href.m_hops < m_maxhops)))
			{
				CLocker lock(&m_squeue.m_mutex);
				m_squeue.AddURL(site_id, urlID, url);
			}
			else
			{
				m_available = 1;
			}
		}
	}
	return urlID;
}

ULONG CSQLDatabaseI::StoreHref1(const char* url, CHref& href, const char* server, int hopsord, ULONG& lost)
{
	ULONG site_id;
// "SELECT site_id FROM sites WHERE site='%s'", server);
	CSQLParam p;
	p.AddParam(server);
	CSQLQuery *sqlquery = m_sqlquery->GetSiteS(&p);
	CSQLAnswer *answ = sql_query(sqlquery);
	
	if (answ && answ->FetchRow())
	{
		answ->GetColumn(0, site_id);
	}
	else
	{
// "INSERT INTO sites(site) VALUES('%s')", server);
    		CSQLParam p;
		p.AddParam(server);
		sqlquery = m_sqlquery->StoreHref1I(&p);
		site_id = sql_query1(sqlquery);
	}
	if (answ) delete answ;
	int nit = now() - href.m_delta;
// "INSERT INTO urlword(site_id, hops, referrer, last_index_time, next_index_time, status, url) VALUES(%ld,%d,%d,%ld,%d,0,'", 
//site_id, href.m_hops, href.m_referrer, now(), nit);
//	char* pq = strcpyq(qIns + strlen(qIns), url);
	int err;
	p.Clear();
	p.AddParam(&site_id);
	p.AddParam(&href.m_hops);
	p.AddParam(&href.m_referrer);
	int tm = now();
	p.AddParam(&tm);
	p.AddParam(&nit);
	p.AddParamEsc(url);
	sqlquery = m_sqlquery->StoreHref1I1(&p);
	ULONG urlID = sql_query1(sqlquery, &err);
	int b = (err != DB_ER_DUP_ENTRY) || GetUrlID(url, href, nit, urlID, lost);
	StoredHrefs.AddHref(url, err == DB_ER_DUP_ENTRY ? 1 : 0, urlID);
	if (b)
	{
		int* maxtime = hopsord ? &m_maxtimes[href.m_hops] : &m_maxtime;
		if (maxtime)
		{
			if (*maxtime >= nit)
			{
				if (m_logFile) fprintf(m_logFile, "Added URL: %lu\n", urlID);
				m_squeue.AddURL(site_id, urlID, url);
			}
			else
			{
				m_available = 1;
			}
			if (href.m_hops < m_maxhops) m_maxhops = href.m_hops;
		}
		return urlID;
	}
	return 0;
}

void CSQLDatabaseI::StoreHref(const char* url, CHref& href, const char* server, int* maxtime)
{
	ULONG site_id;
// "SELECT site_id FROM sites WHERE site='%s'", server);
        CSQLParam p;
	p.AddParam(server);
	CSQLQuery *sqlquery = m_sqlquery->GetSiteS(&p);
        CSQLAnswer *answ = sql_query(sqlquery);

	if (answ && answ->FetchRow())
	{
		answ->GetColumn(0, site_id);
	}
	else
	{
// "INSERT INTO sites(site) VALUES('%s')", server);
		CSQLParam p;
		p.AddParam(server);
		sqlquery = m_sqlquery->StoreHrefI(&p);
		site_id = sql_query1(sqlquery);
	}
	if (answ) delete answ;
	int nit = now() - href.m_delta;
//"INSERT INTO urlword(site_id, hops, referrer, last_index_time, next_index_time, status, url) VALUES(%ld,%d,%d,%ld,%d,0,'",
//site_id, href.m_hops, href.m_referrer, now(), nit);
//	char* pq = strcpyq(qIns + strlen(qIns), url);
	int err;
	p.Clear();
	p.AddParam(&site_id);
	p.AddParam(&href.m_hops);
	p.AddParam(&href.m_referrer);
	int tm = now();
	p.AddParam(&tm);
	p.AddParam(&nit);
	p.AddParamEsc(url);
	sqlquery = m_sqlquery->StoreHrefI1(&p);
	ULONG urlID = sql_query1(sqlquery, &err);
	StoredHrefs.AddHref(url, err == DB_ER_DUP_ENTRY ? 1 : 0, urlID);
	if (err != DB_ER_DUP_ENTRY)
	{
		if (maxtime)
		{
			if (*maxtime >= nit)
			{
				if (m_logFile) fprintf(m_logFile, "Added URL: %lu\n", urlID);
				m_squeue.AddURL(site_id, urlID, url);
			}
			else
			{
				m_available = 1;
			}
			if (href.m_hops < m_maxhops) m_maxhops = href.m_hops;
		}
	}
}

void CSQLDatabaseI::StoreHrefs(int hopsord)
{
	CStringToHrefMap::iterator it;
	for (it = Hrefs.begin(); it != Hrefs.end(); it++)
	{
		CHref& href = it->second;
		ULONG lost = 0;
		ULONG urlID = StoreHref1(it->first.c_str(), href, href.m_server.c_str(), hopsord, lost);
		if (urlID) href.m_urlID = urlID;
	}
	Hrefs.clear();
}

void CSQLDatabaseI::SaveLink(const CUrlRank& rank)
{
// "DELETE FROM citation WHERE url_id = %lu", rank.m_urlID);
	CSQLParam p;
	p.AddParam(&rank.m_urlID);
	CSQLQuery *sqlquery = m_sqlquery->SaveLinkD(&p);
	sql_query2(sqlquery);
	ULONG refcount = rank.m_refend - rank.m_referrers;
	if (refcount > 0)
	{
		CUrlRankPtrS* rankp = new CUrlRankPtrS[refcount];
		CUrlRankPtrS* prank = rankp;
		CUrlRankPtrS* erank;
		for (CUrlRank** r = rank.m_referrers; r < rank.m_refend; r++, prank++)
		{
			prank->m_rank = *r;
		}
		erank = prank;

		sort<CUrlRankPtrS*>(rankp, erank);
		ULONG* sitebuf = new ULONG[(refcount << 1) + 4];
		ULONG* urlbuf = new ULONG[refcount];
		ULONG* ps = sitebuf;
		ULONG* pu = urlbuf;
		prank = rankp;
		while (prank < erank)
		{
			ULONG site_id = prank->m_rank->m_siteID;
			*ps++ = (BYTE*)pu - (BYTE*)urlbuf;
			*ps++ = site_id;
			while ((prank < erank) && (site_id == prank->m_rank->m_siteID))
			{
				*pu++ = prank->m_rank->m_urlID;
				prank++;
			}
		}
		*ps++ = (BYTE*)pu - (BYTE*)urlbuf;
		ULONG* es = ps;
		ULONG d = (BYTE*)ps - (BYTE*)sitebuf;
		for (ps = sitebuf; ps < es; ps += 2)
		{
			*ps += d;
		}
//  "INSERT INTO citation(url_id, referrers) VALUES(%lu, '", rank.m_urlID);
// pquery = memcpyq(pquery, (char*)sitebuf, (BYTE*)es - (BYTE*)sitebuf);
// pquery = memcpyq(pquery, (char*)urlbuf, (BYTE*)pu - (BYTE*)urlbuf);
		p.Clear();
		p.AddParam(&rank.m_urlID);
		char *buf = new char[((BYTE*)es - (BYTE*)sitebuf) + ((BYTE*)pu - (BYTE*)urlbuf)];
		memcpy(buf, (char*)sitebuf, (BYTE*)es - (BYTE*)sitebuf);
		memcpy(buf + ((BYTE*)es - (BYTE*)sitebuf), (char*)urlbuf, (BYTE*)pu - (BYTE*)urlbuf);

		p.AddParamEsc((char*)buf, ((BYTE*)es - (BYTE*)sitebuf) + ((BYTE*)pu - (BYTE*)urlbuf));

//                p.AddParamEsc((char*)sitebuf, (BYTE*)es - (BYTE*)sitebuf);
//                p.AddParamEsc((char*)urlbuf, (BYTE*)pu - (BYTE*)urlbuf, 1);
		sqlquery = m_sqlquery->SaveLinkI(&p);
 		sql_real_query(sqlquery);
		delete buf;
		delete sitebuf;
		delete urlbuf;
		delete rankp;
	}
}

void CSQLDatabaseI::SaveLinks(CUrlRanks& ranks)
{
	logger.log(CAT_ALL, L_INFO, "Saving citation    [");
	size_t s = ranks.size();
	int n = s / 50 + 1; // we want about 50 ....s
	int i = 0;
	for (CUrlRanks::iterator it = ranks.begin(); it != ranks.end(); it++)
	{
		SaveLink(*it);
		if (!(i++ % n))
			logger.log(CAT_ALL, L_INFO, ".");
	}
	logger.log(CAT_ALL, L_INFO, "] done.\n");
}

void CSQLDatabaseI::CalcTotal()
{
	ULONG total = 0;
	for (int i = 0; i < 16; i++)
	{
		char buf[3];
// "SELECT count(*) FROM urlwords%02i", i);
		sprintf(buf, "%02i", i);
	        CSQLParam p;
		p.AddParam(buf);
		CSQLQuery *sqlquery = m_sqlquery->CalcTotalS(&p);
	        CSQLAnswer *answ = sql_query(sqlquery);
;
		if (answ && answ->FetchRow())
		{
			ULONG total_t;
			answ->GetColumn(0, total_t);
			total += total_t;
		}
		if (answ) delete answ;
	}

	string fntotal = DBWordDir + "/total";
	FILE* f = fopen(fntotal.c_str(), "w");
	if (f)
	{
		fprintf(f, "%lu", total);
		fclose(f);
	}
	else
	{
		logger.log(CAT_ALL, L_ERR, "Couldn't open file: %s\n", fntotal.c_str());
	}
}

void CSQLDatabaseI::LoadRanks(CUrlRanks& ranks)
{
	logger.log(CAT_ALL, L_INFO, "Loading ranks      [");
	ranks.Load();
//	ULONG j = 0;
	ULONG maxUrlId = 0;
// ("SELECT MAX(url_id) FROM urlword");
	CSQLQuery *sqlquery = m_sqlquery->LoadRanksS();
	CSQLAnswer *answ = sql_query(sqlquery);

	if (answ && answ->FetchRow())
	{
		answ->GetColumn(0, maxUrlId);
//		logger.log(CAT_ALL, L_ERR, "\nmaxUrlId = %lu\n", maxUrlId);
	}
	if (answ) delete answ;

// "SELECT url_id, redir, site_id, last_modified FROM urlword WHERE status!=0");
	sqlquery = m_sqlquery->LoadRanksS1();
	answ = sql_query(sqlquery);
	if (answ)
	{
//		ULONG num_res = answ->GetNumRows();
//		int n = num_res / 18 + 1; // we want 18 ...s
		CMapULONGToULONG& redir = ranks.m_redir;
		time_t* lmbuf = new time_t[maxUrlId + 1];
		memset((void *)lmbuf, 0, (maxUrlId + 1) * sizeof(time_t));
		time_t t, maxt = 0;
		time_t t_limit = now() + 60 * 60 * 24; // now + 1 day
		while (answ->FetchRow())
		{
			ULONG from, to;
			answ->GetColumn(0, from);
			if (!answ->GetColumn(1, to))
			{
				to = 0;
			}
			t = httpDate2time_t(answ->GetColumn(3));
			if (t != BAD_DATE)
			{
				if (t > t_limit)
					t = t_limit;
				lmbuf[from] = t;
				if (t > maxt)
					maxt = t;
			}
			ULONG f_siteID;
			if (!answ->GetColumn(2, f_siteID))
			{
				f_siteID = 0;
			}
			ranks[from].m_siteID = f_siteID;
//			if (!(j++ % n))
//				logger.log(CAT_ALL, L_INFO, ".");
			if (to != 0)
			{
				ULONG t = to;
				int r = 1;
				CULONGSet set;
				while (true)
				{
					CMapULONGToULONG::iterator it = redir.find(t);
					if (it == redir.end()) break;
					ULONG t1 = it->second;
					if ((t1 == from) || (set.find(t1) != set.end()))
					{
						r = 0;
						break;
					}
					set.insert(t1);
					t = t1;
				}
				if (r) redir[from] = to;
			}
		}
		// Save lastmod file
		CLastMods lm;
		lm.Init(DBWordDir.c_str());
		lm.reserve(maxUrlId + 1);
		lm.CalcOurEpoch(maxt);
		lm.ReadTimesFromBuf(lmbuf, maxUrlId);
		lm.Save();
		delete lmbuf;
		if (answ) delete answ;
	}
	for (int i = 0; i < 16; i++)
	{
		logger.log(CAT_ALL, L_INFO, "."); // more 16 ...s
		char buf[3];
// "SELECT url_id, hrefs FROM urlwords%02i WHERE length(hrefs) > 0", i);
		sprintf(buf, "%02i", i);
		CSQLParam p;
		p.AddParam(buf);
	        CSQLQuery *sqlquery = m_sqlquery->LoadRanksS2(&p);
		answ = sql_query(sqlquery);
		while (answ && answ->FetchRow())
		{
			ULONG len = answ->GetLength(1);
			if (len > 0)
			{
//				ULONG from = atoi(row[0]);
//				CUrlRank& rank = ranks[from];
				ULONG* to = (ULONG*)answ->GetColumn(1);
				for (int j = len >> 2; j > 0; j--)
				{
					ULONG r = ranks.Redir(*to++);
					if (r == 0)
					{
						logger.log(CAT_ALL, L_ERR, "ERROR in %s:%d: Refferring to 0\n", __FILE__, __LINE__);
					}
					ranks.IncRefCount(r);
				}
			}

		}
		if (answ) delete answ;
	}

	
	ranks.Allocate();
	for (int i = 0; i < 16; i++)
	{
		char buf[3];
		logger.log(CAT_ALL, L_INFO, "."); // more 16 ...s
// "SELECT url_id, hrefs FROM urlwords%02i WHERE length(hrefs) > 0", i);
		sprintf(buf, "%02i", i);
	        CSQLParam p;
		p.AddParam(buf);
		sqlquery = m_sqlquery->LoadRanksS2(&p);
		answ = sql_query(sqlquery);

		if (!answ)
			return;
//		answ->SeekRow(0);
		while (answ && answ->FetchRow())
		{
			ULONG len = answ->GetLength(1);
			if (len > 0)
			{
				ULONG from;
				answ->GetColumn(0, from);
				ULONG* to = (ULONG*)answ->GetColumn(1);
				CUrlRanks::iterator ifr = ranks.find(from);
				if (ifr != ranks.end())
				{
					CULONGSet set;
					for (int j = len >> 2; j > 0; j--)
					{
						ULONG r = ranks.Redir(*to++);
						set.insert(r);
					}
					for (CULONGSet::iterator it = set.begin(); it != set.end(); it++)
					{
						CUrlRanks::iterator ito = ranks.find(*it);
						if (ito != ranks.end())
						{
							ranks.AddLinkFromTo(const_cast<CUrlRank&>(*ifr), const_cast<CUrlRank&>(*ito));
						}
					}
				}
			}
		}
		if (answ) delete answ;
	}
	logger.log(CAT_ALL, L_INFO, "] done.\n");
}

void CSQLDatabaseI::LoadSites()
{
// "SELECT site from sites");
	CSQLQuery *sqlquery = m_sqlquery->LoadSitesS();
	CSQLAnswer *answ = sql_query(sqlquery);
	while (answ && answ->FetchRow())
	{
		Sites.insert(answ->GetColumn(0));
	}
	if (answ) delete answ;
}

void CSQLDatabaseI::FindMaxHops()
{
//  "SELECT max(hops) from urlword where 1=1 %s", Limit.c_str());
        CSQLParam p;
//	p.AddParam(Limit.c_str());
	p.AddParam("");
	CSQLQuery *sqlquery = m_sqlquery->FindMaxHopsS(&p);
	CSQLAnswer *answ = sql_query(sqlquery);
	if (answ && answ->FetchRow() && answ->GetColumn(0))
	{
		answ->GetColumn(0, m_maxhopsa);
	}
	if (answ) delete answ;
}

int CSQLDatabaseI::Startup()
{
	LoadRobots();
	LoadSites();
	FindMaxHops();
//	SaveCountriesIfNotEmpty(this);
//	LoadCountriesIfEmpty(this);
	return 1;
}

void CSQLDatabaseI::LoadRobots()
{
	Robots.clear();
// "SELECT hostinfo,path FROM robots");
	CSQLQuery *sqlquery = m_sqlquery->LoadRobotsS();
	CSQLAnswer *answ = sql_query(sqlquery);
	while (answ && answ->FetchRow())
	{
		Robots[answ->GetColumn(0)].push_back(answ->GetColumn(1));
	}
	if (answ) delete answ;
}

void CSQLDatabaseI::LoadRobotsOfHost(char* hostinfo)
{
	if ((strlen(hostinfo) <= MAX_URL_LEN))
	{
//  "SELECT path FROM robots WHERE hostinfo = '");
//		pquery = strcpyq(pquery, hostinfo);

		CStringVector& robots = Robots[hostinfo];
    		CSQLParam p;
		p.AddParamEsc(hostinfo);
		CSQLQuery *sqlquery = m_sqlquery->LoadRobotsOfHostS(&p);
	        CSQLAnswer *answ = sql_query(sqlquery);
		robots.clear();
		while (answ && answ->FetchRow())
		{
			robots.push_back(answ->GetColumn(0));
		}
		if (answ) delete answ;
	}
}

void CSQLDatabaseI::DeleteRobotsFromHost(char *host)
{
	if (strlen(host) <= MAX_URL_LEN)
	{
//  "DELETE FROM robots WHERE hostinfo='");
//		pquery = strcpyq(pquery, host);
		CSQLParam p;
		p.AddParamEsc(host);
		CSQLQuery *sqlquery = m_sqlquery->DeleteRobotsFromHostD(&p);
		sql_query2(sqlquery);
	}
}

void CSQLDatabaseI::AddRobotsToHost(char *host, char *s)
{
	if ((strlen(host) <= MAX_URL_LEN) && (strlen(s) <= MAX_URL_LEN))
	{
// "INSERT INTO robots (hostinfo,path) VALUES ('");
//		pquery = strcpyq(pquery, host);
//		pquery += sprintf(pquery, "', '");
//		pquery = strcpyq(pquery, s);

	        CSQLParam p;
		p.AddParamEsc(host);
		p.AddParamEsc(s);
		CSQLQuery *sqlquery = m_sqlquery->AddRobotsToHostI(&p);
		sql_query2(sqlquery);
	}
}

void CSQLDatabaseI::DeleteUrlFromSpace(const char* curl, ULONG space)
{
	CUrl url;
	if ((strlen(curl) <= MAX_URL_LEN) && (url.ParseURL(curl) == 0))
	{
//"lock tables spaces write, sites read");
		CSQLQuery *sqlquery = m_sqlquery->DeleteUrlFromSpaceL();
		sql_query2(sqlquery);
// "SELECT site_id FROM sites WHERE site='%s://", url.m_schema);
//		pq = strcpyq(pq, url.m_hostinfo);
//		strcpy(pq, "/'");
		char urlp[(MAX_URL_LEN + 1) << 1];
		sprintf(urlp, "%s://%s/", url.m_schema, url.m_hostinfo);
	        CSQLParam p;
		p.AddParamEsc(urlp);
		sqlquery = m_sqlquery->DeleteUrlFromSpaceS(&p);
                CSQLAnswer *answ = sql_query(sqlquery);

		if (answ && answ->FetchRow())
		{
// "SELECT site_id FROM spaces WHERE site_id=%s and space_id=%lu", row[0], space);
		        CSQLParam p;
			p.AddParam(answ->GetColumn(0));
			p.AddParam(&space);
			sqlquery = m_sqlquery->DeleteUrlFromSpaceS1(&p);
			
			CSQLAnswer *answ1 = sql_query(sqlquery);
			if (answ1 && answ1->FetchRow())
			{
//  "DELETE FROM spaces WHERE site_id=%s AND space_id=%lu", row[0], space);
			        CSQLParam p;
	                        p.AddParam(answ->GetColumn(0));
		                p.AddParam(&space);
				sqlquery = m_sqlquery->DeleteUrlFromSpaceD(&p);
				sql_query2(sqlquery);

				char fname[1000];
				sprintf(fname, "%s/subsets/space%lu", DBWordDir.c_str(), space);
				FILE* f = fopen(fname, "r+");
				if (f)
				{
					ULONG site_id;
					answ->GetColumn(0, site_id);
					ULONG nmins = site_id >> 5;
					ULONG mins;
					fread(&mins, 1, sizeof(mins), f);
					if (nmins >= mins)
					{
						ULONG off = (nmins - mins + 1) << 2;
						fseek(f, off, SEEK_SET);
						long noff = ftell(f);
						if ((ULONG)noff == off)
						{
							ULONG elem;
							fread(&elem, 1, sizeof(elem), f);
							elem &= ~(1 << (site_id & 0x1F));
							fseek(f, off, SEEK_SET);
							fwrite(&elem, 1, sizeof(elem), f);
						}
					}
					fclose(f);
					printf("Status: OK\n");
				}
			}
			else
			{
				printf("Status: Warning. Site doesn't belongs to space\n");
			}
			if (answ1) delete answ1;
		}
		else
		{
			printf("Status: Error. Site not found\n");
		}
		sqlquery = m_sqlquery->UnlockTables();
		sql_query2(sqlquery);
		if (answ) delete answ;
	}
}

void CSQLDatabaseI::AddUrlToSpace(const char* curl, ULONG space)
{
	CUrl url;
	if ((strlen(curl) <= MAX_URL_LEN) && (url.ParseURL(curl) == 0))
	{
// "lock tables spaces write, sites read");
		CSQLQuery *sqlquery = m_sqlquery->AddUrlToSpaceL();
		sql_query2(sqlquery);
/* "SELECT site_id FROM sites WHERE site='%s://", url.m_schema);
		pq = strcpyq(pq, url.m_hostinfo);
		strcpy(pq, "/'");
*/
		char urlp[(MAX_URL_LEN + 1)  << 1];
		sprintf(urlp, "%s://%s/", url.m_schema, url.m_hostinfo);
	        CSQLParam p;
		p.AddParamEsc(urlp);
		sqlquery = m_sqlquery->AddUrlToSpaceS(&p);
                CSQLAnswer *answ = sql_query(sqlquery);

		if (answ && answ->FetchRow())
		{
// "SELECT site_id FROM spaces WHERE site_id=%s and space_id=%lu", row[0], space);
		        CSQLParam p;
			p.AddParam(answ->GetColumn(0));
			p.AddParam(&space);
			sqlquery = m_sqlquery->AddUrlToSpaceS1(&p);
            		CSQLAnswer *answ1 = sql_query(sqlquery);
			if (answ1 && answ1->FetchRow())
			{
				logger.log(CAT_ALL, L_ERR, "Status: Warning. Site already belongs to space\n");
			}
			else
			{
				ULONG site_id;
				answ->GetColumn(0, site_id);
// "INSERT INTO spaces(site_id,space_id) VALUES (%s, %lu)", row[0], space);
			        CSQLParam p;
				p.AddParam(answ->GetColumn(0));
				p.AddParam(&space);
				sqlquery = m_sqlquery->AddUrlToSpaceI(&p);
				sql_query2(sqlquery);
				char fname1[1000], fname[1000];
				sprintf(fname1, "%s/subsets/space%lu.1", DBWordDir.c_str(), space);
				sprintf(fname, "%s/subsets/space%lu", DBWordDir.c_str(), space);
				FILE* f = fopen(fname, "r");
				FILE* fo = fopen(fname1, "w");
				if (fo)
				{
					ULONG mins;
					ULONG nmins = site_id >> 5;
					ULONG elem;
					if (f)
					{
						fread(&mins, 1, sizeof(mins), f);
						if (nmins < mins)
						{
							ULONG z = 0;
							fwrite(&nmins, 1, sizeof(nmins), fo);
							elem = 1 << (site_id & 0x1F);
							fwrite(&elem, 1, sizeof(elem), fo);
							nmins++;
							while (nmins < mins)
							{
								fwrite(&z, 1, sizeof(z), fo);
								nmins++;
							}
						}
						else
						{
							fwrite(&mins, 1, sizeof(mins), fo);
							while (mins < nmins)
							{
								elem = 0;
								fread(&elem, 1, sizeof(elem), f);
								fwrite(&elem, 1, sizeof(elem), fo);
								mins++;
							}
							elem = 0;
							fread(&elem, 1, sizeof(elem), f);
							elem |= (1 << (site_id & 0x1F));
							fwrite(&elem, 1, sizeof(elem), fo);
						}
						while (fread(&elem, 1, sizeof(elem), f) > 0)
						{
							fwrite(&elem, 1, sizeof(elem), fo);
						}
						fclose(f);
					}
					else
					{
						fwrite(&nmins, 1, sizeof(nmins), fo);
						elem = 1 << (site_id & 0x1F);
						fwrite(&elem, 1, sizeof(elem), fo);
					}
					fclose(fo);
					unlink(fname);
					rename(fname1, fname);
					printf("Status: OK\n");
				}
			}
			if (answ1) delete answ;
		}
		else
		{
			printf("Status: Error. Site not found\n");
		}
		sqlquery = m_sqlquery->UnlockTables();
		sql_query2(sqlquery);
		if (answ) delete answ;
	}
}

void CSQLDatabaseI::AddCountry(const char* country, ULONG addr, ULONG mask)
{
//  "INSERT INTO countries(country, addr, mask, expires) VALUES('%s', %lu, %lu, %li)",
// country, addr, mask, now() + 7 * 64800);
        CSQLParam p;
	p.AddParam(country);
	p.AddParam(&addr);
	p.AddParam(&mask);
	int tm = (int)now() + 7 * 64800;
	p.AddParam(&tm);
	CSQLQuery *sqlquery = m_sqlquery->AddCountryI(&p);
	sql_query2(sqlquery);
}

void CSQLDatabaseI::LoadCountries(CIPTree* tree)
{
// "SELECT country, addr, mask, expires FROM countries");
	CSQLQuery *sqlquery = m_sqlquery->LoadCountriesS();
	CSQLAnswer *answ = sql_query(sqlquery);
	while (answ && answ->FetchRow())
	{
		ULONG addr, mask, expires;
		sscanf(answ->GetColumn(1), "%lu", &addr);
		sscanf(answ->GetColumn(2), "%lu", &mask);
                sscanf(answ->GetColumn(3), "%lu", &expires);
		tree->AddCountry(addr, mask, answ->GetColumn(0), expires);
	}
	if (answ) delete answ;

}

void CSQLDatabaseI::ReplaceCountry(ULONG istart, const char* country)
{
//        sprintf(query, "UPDATE countries SET country = '%s', expires = %li WHERE addr = %lu", country, now() + COUNTRY_EXPIRATION, istart);
	CSQLParam p;
	p.AddParam(country);
	int tm = now() + COUNTRY_EXPIRATION;	
	p.AddParam(&tm);
	p.AddParam(&istart);
	CSQLQuery *sqlquery = m_sqlquery->ReplaceCountryU(&p);
	sql_query2(sqlquery);
}
void CSQLDatabaseI::NarrowCountry(ULONG istart, ULONG mask, int off)
{
//        sprintf(query, "UPDATE countries SET addr=%lu, mask=%lu WHERE addr=%lu", istart + (off ? mask : 0), ~((mask | (mask - 1)) >> 1), istart);
	CSQLParam p;
	ULONG istart1 = istart + (off ? mask : 0);
	p.AddParam(&istart1);
	ULONG istart2 = ~((mask | (mask - 1)) >> 1);
	p.AddParam(&istart2);
	p.AddParam(&istart);
	CSQLQuery *sqlquery = m_sqlquery->NarrowCountryU(&p);
	sql_query2(sqlquery);
}

void CheckHref(ULONG url, FILE* citations, ULONG prevOffset, ULONG endOffset, ULONG clen, char* refs)
{
	ULONG size = endOffset - prevOffset;
	int eq = 0;
	if (size == clen)
	{
		char* hr = (char*)alloca(size);
		fseek(citations, prevOffset, SEEK_SET);
		fread(hr, 1, size, citations);
		eq = memcmp(hr, refs, size) == 0;
	}
	if ((size != clen) || (eq == 0))
	{
		printf("Wrong citations for URL: %lu\n", url);
	}
}
void CSQLDatabaseI::CheckHrefDeltas()
{
	CSQLQuery* query = m_sqlquery->SetQuery("SELECT url_id, referrers FROM citation ORDER by mod(url_id, 4), url_id");
	CSQLAnswer *answ = sql_query(query);
	if (answ)
	{
		printf("Checking citations...\n");
		int availc = answ->FetchRow();
		while (availc)
		{
			char fname[1000], fnamec[1000];
			ULONG urlID;
			answ->GetColumn(0, urlID);
			ULONG f = urlID % NUM_HREF_DELTAS;
			CCitationFile file(f, 0, fname, fnamec);
			ULONG url, endOffset, prevOffset = 0;
			int avail = file.GetNextUrl(url, endOffset);
			while (availc && (f == (urlID % NUM_HREF_DELTAS)))
			{
				if ((avail == 0) || (url > urlID))
				{
					printf("URL %lu is absent\n", urlID);
					availc = answ->FetchRow();
					if (availc) answ->GetColumn(0, urlID);
				}
				else if (url < urlID)
				{
					if (endOffset > prevOffset)
					{
						printf("Extra URL %lu\n", url);
					}
					prevOffset = endOffset;
					avail = file.GetNextUrl(url, endOffset);
				}
				else
				{
					CheckHref(url, file.m_citations, prevOffset, endOffset, answ->GetLength(1), answ->GetColumn(1));
					availc = answ->FetchRow();
					if (availc) answ->GetColumn(0, urlID);
					prevOffset = endOffset;
					avail = file.GetNextUrl(url, endOffset);
				}
			}
		}
		delete answ;
	}
}

ULONG CSQLDatabaseI::DeleteUrls(CWordCache* wcache, const char *limit)
{
// SELECT url_id, site_id, url, crc, origin FROM urlword WHERE url LIKE '%s';
#ifdef OPTIMIZE_CLONE
	ULONG ucnt = 0;
        CSQLParam p;
	p.AddParam(limit);
	CSQLQuery* sqlquery = m_sqlquery->DeleteUrlsS(&p);
	CSQLAnswer *answ = sql_query(sqlquery);
	while (answ && answ->FetchRow())
	{
		ucnt += 1;
		ULONG urlID, siteID, originID;
		char *crc, *url;
		CUrl curl;

		answ->GetColumn(0, urlID);
		answ->GetColumn(1, siteID);
		url = answ->GetColumn(2);
		crc = answ->GetColumn(3);
		answ->GetColumn(4, originID);

		curl.ParseURL(url);
		CServer* CurSrv = curl.FindServer();

		if (originID == 1)
		{
			CUrlContent ucontent;
			ucontent.m_cache = wcache;
			ucontent.m_url.m_urlID = urlID;
			ucontent.m_url.m_siteID = siteID;
			ucontent.m_url.m_crc = crc;
			
			curl.SetNewOrigin(crc, &ucontent, CurSrv, wcache);
		}

		wcache->DeleteWordsFromURL(urlID, siteID, CurSrv);

		MarkDeleted(urlID, 2);

		logger.log(CAT_ALL, L_ERR, "Delete url: %s\n", url);
	}
	if (answ) delete answ;
	return ucnt;
#else
	logger.log(CAT_ALL, L_ERR, "Deletion urls limit work only with --enable-fast-clones\n");
	return 0;
#endif
}

void CSQLDatabaseI::UpdateOrigin(ULONG urlID)
{
	CSQLParam p;
	p.AddParam(&urlID);
	CSQLQuery* query = m_sqlquery->UpdateOrigin(&p);
	sql_query2(query);
}

void CSQLDatabaseI::CheckCharsetGuesser()
{
#ifdef USE_CHARSET_GUESSER
	ULONG nexturl = 0, fl = 1;
	ULONG urlcnt = 0, totalurl = 0, bad = 0, totalbad = 0;
	struct timeval tm, lastSave, firstSave;
	double dif;

	gettimeofday(&tm, NULL);
	lastSave = firstSave = tm;
	while (fl){
		CSQLParam p;
		p.AddParam(&nexturl);
		CSQLQuery* query = m_sqlquery->SetQuery("SELECT url_id, url FROM urlword WHERE status=200 AND deleted!=1 AND url_id>:1", &p);
		query->AddLimit(1000);
		CSQLAnswer *answ = sql_query(query);
		fl = 0;
		while (answ && answ->FetchRow())
		{
			BYTE* content;
			ULONG urlid, wordCount, uLen;
		        char content_type[50];
		        string charset_db;
			int size, charset_gid;
			char *url;
			
			answ->GetColumn(0, urlid);
			url = answ->GetColumn(1);

			nexturl = urlid;
			size = GetContents(urlid, (BYTE**)& content, wordCount, NULL, NULL, content_type, charset_db);
			BYTE* uncompr = Uncompress(content, size, uLen);
			if (uncompr)
			{
				charset_gid = langs.GetLang((char*)uncompr);
#ifdef UNICODE
				if (charset_gid != CHARSET_USASCII && GetUCharset(charset_db.c_str()) != GetUCharset(charset_gid))
#else
				if (charset_gid != CHARSET_USASCII && charset_gid != GetCharset(charset_db.c_str()))
#endif
				{
					logger.log(CAT_ALL, L_ERR, "Url: %s charset %s - %s\n", url, charset_db.c_str(),
#ifdef UNICODE
GetCharsetName(charset_gid)
#else
charsetMap.CharsetsStr(charset_gid)
#endif
);
					bad++;
					totalbad++;
				}
				delete uncompr;
			}
			urlcnt++;
			totalurl++;
			gettimeofday(&tm, NULL);
			if ((dif = timedif(tm, lastSave)) > 60)
			{
				logger.log(CAT_ALL, L_ERR, "Sec: %7.3f Urls: %d Bad: %d \n", dif, urlcnt, bad);
				lastSave = tm;
				urlcnt = 0;
				bad = 0;
			}
			if (content) delete content;
			fl = 1;
		}
		if (answ) delete answ;

	} 
	gettimeofday(&tm, NULL);
	dif = timedif(tm, lastSave);
	logger.log(CAT_ALL, L_ERR, "Sec: %7.3f Urls: %d Bad: %d \n", dif, urlcnt, bad);
	dif = timedif(tm, firstSave);
	logger.log(CAT_ALL, L_ERR, "Total sec: %7.3f Total urls: %d Total bad: %d Last url_id: %d\n", dif, totalurl, totalbad, nexturl);

#else
	logger.log(CAT_ALL, L_ERR, "Charset guesser not enabled\n");
#endif
}
