From 3f21c0c6c17c0767ea3d5a460650e00f9f8c8898 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Sun, 20 Sep 2026 12:30:49 +0200 Subject: [PATCH] Bound web structure growth while preserving active crawl roots and connected paths --- help/Crawler_p.md | 2 + help/WatchWebStructure_p.md | 6 + source/net/yacy/crawler/CrawlStacker.java | 2 + source/net/yacy/crawler/CrawlSwitchboard.java | 38 ++- .../net/yacy/crawler/data/CrawlProfile.java | 46 +++ source/net/yacy/htroot/Crawler_p.java | 2 +- .../peers/graphics/WebStructureGraph.java | 154 ++++++++-- source/net/yacy/search/Switchboard.java | 7 +- .../yacy/crawler/CrawlSwitchboardTest.java | 272 ++++++++++++++++++ .../yacy/crawler/data/CrawlProfileTest.java | 78 +++++ .../peers/graphics/WebStructureGraphTest.java | 267 +++++++++++++++++ 11 files changed, 836 insertions(+), 38 deletions(-) create mode 100644 test/java/net/yacy/crawler/CrawlSwitchboardTest.java create mode 100644 test/java/net/yacy/crawler/data/CrawlProfileTest.java diff --git a/help/Crawler_p.md b/help/Crawler_p.md index d8b421e8d..46d97179c 100644 --- a/help/Crawler_p.md +++ b/help/Crawler_p.md @@ -40,6 +40,8 @@ Crawler pages separate three decisions: the source to load, the rules that decid Begin with the smallest crawl that proves the idea. Use exact start URLs, prefer restrictive boundaries, set limits for unfamiliar sites, and watch the queue after submission. A crawler is not a magic search box: it creates search results only after documents have been loaded, parsed, and indexed. +Crawl profiles retain local start-host metadata to prioritize their connected web-structure graph during pruning. URL starts are recorded at submission; sitemap and file-import starts are recorded when accepted local depth-zero requests reach the crawl stacker. Descendant pages do not become additional roots. Up to 10,000 start hosts are recorded per profile. This metadata is not included in peer crawl-news announcements and requires no additional request parameter. See [Web Structure](WatchWebStructure_p.md) for retention limits. + ## Access And Safety Administrator access is required. YaCy protects `_p` pages as administration pages. diff --git a/help/WatchWebStructure_p.md b/help/WatchWebStructure_p.md index c559b9263..e57b95e45 100644 --- a/help/WatchWebStructure_p.md +++ b/help/WatchWebStructure_p.md @@ -67,6 +67,12 @@ GET /WatchWebStructure_p.html?host=...&depth=...&nodes=...&time=...&width=... ## What To Expect +The web-structure graph retains up to 10,000 host entries across its older and recent data. When the limit is exceeded, both at startup and during crawling, pruning prioritizes active crawl start hosts and follows their outgoing links breadth-first. This preserves intermediate hosts along retained paths, rather than leaving a root surrounded by missing host entries. Within the normal 9,000-host retention budget, nearer hosts take priority over farther ones. Remaining space is filled by recent unrelated hosts. If the roots alone exceed 9,000 entries, the budget expands up to the 10,000-host limit; no bounded graph can retain every connection of an arbitrarily large crawl. + +Within the protected neighbourhood, HTTP, HTTPS, and port variants of a hostname are kept together, matching the picture's aggregation of their outgoing links. A group that does not fit the remaining retention budget is not partially protected. + +URL starts persist their start hosts in the crawl profile. Sitemap and file imports also record accepted local depth-zero hosts as their requests pass through the crawl stacker, up to 10,000 distinct start hosts per profile. This metadata is local and excluded from peer crawl-news announcements. Older profiles use their comma-separated host names as a fallback; abbreviated or custom legacy names may not identify the original roots until those hosts are processed again. Stopping a crawl removes its priority at the next pruning pass. A host entry present in both maps counts once. This limits graph history; it does not delete indexed documents or reconstruct previously pruned graph entries. + Expect observations: counts, logs, queues, timing, network rows, thread states, or resource values. Monitoring does not fix the issue by itself; it points to the next page or setting to change. ## Related Pages diff --git a/source/net/yacy/crawler/CrawlStacker.java b/source/net/yacy/crawler/CrawlStacker.java index c27185883..9dc07e98c 100644 --- a/source/net/yacy/crawler/CrawlStacker.java +++ b/source/net/yacy/crawler/CrawlStacker.java @@ -396,6 +396,8 @@ public final class CrawlStacker implements WorkflowTask{ return error; } + if (local) this.crawler.recordStartURL(profile, entry); + if (global) { // it may be possible that global == true and local == true, so do not check an error case against it if (proxy) CrawlStacker.log.warn("URL '" + entry.url().toString() + "' has conflicting initiator properties: global = true, proxy = true, initiator = proxy" + ", profile.handle = " + profile.handle()); diff --git a/source/net/yacy/crawler/CrawlSwitchboard.java b/source/net/yacy/crawler/CrawlSwitchboard.java index 8ad8373c2..20a996add 100644 --- a/source/net/yacy/crawler/CrawlSwitchboard.java +++ b/source/net/yacy/crawler/CrawlSwitchboard.java @@ -59,6 +59,7 @@ import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.kelondroException; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; +import net.yacy.server.serverSwitch; public final class CrawlSwitchboard { @@ -119,12 +120,18 @@ public final class CrawlSwitchboard { private final Map defaultPushProfiles; // for each collection one profile private final File queuesRoot; private final Switchboard switchboard; + private final serverSwitch config; public CrawlSwitchboard(Switchboard switchboard) { + this(switchboard, switchboard.queuesRoot, switchboard); + } + /** Initialize profile storage independently of the server's background services. */ + CrawlSwitchboard(final Switchboard switchboard, final File queuesRoot, final serverSwitch config) { this.switchboard = switchboard; - this.log = this.switchboard.log; - this.queuesRoot = this.switchboard.queuesRoot; + this.config = config; + this.log = config.log; + this.queuesRoot = queuesRoot; this.defaultPushProfiles = new ConcurrentHashMap<>(); this.profilesActiveCrawlsCache = Collections.synchronizedMap(new TreeMap(Base64Order.enhancedCoder)); this.profilesActiveCrawlsCounter = new ConcurrentHashMap<>(); @@ -247,7 +254,7 @@ public final class CrawlSwitchboard { return this.profilesPassiveCrawls.keySet(); } - public void removeActive(final byte[] profileKey) { + public synchronized void removeActive(final byte[] profileKey) { if ( profileKey == null ) { return; } @@ -268,6 +275,17 @@ public final class CrawlSwitchboard { this.removePassive(profileKey); } + /** Persist streamed sitemap/file roots without reviving a profile stopped by the user. */ + public void recordStartURL(final CrawlProfile profile, final Request entry) { + if (entry.depth() != 0 || DEFAULT_PROFILES.contains(profile.name())) return; + synchronized (this) { + final byte[] handle = UTF8.getBytes(profile.handle()); + if (getActive(handle) == profile && profile.recordStartURL(entry.url(), entry.depth())) { + this.profilesActiveCrawls.put(handle, profile); + } + } + } + public void putPassive(final byte[] profileKey, final CrawlProfile profile) { this.profilesPassiveCrawls.put(profileKey, profile); this.removeActive(profileKey); @@ -278,7 +296,7 @@ public final class CrawlSwitchboard { } private void initActiveCrawlProfiles() { - final Switchboard sb = Switchboard.getSwitchboard(); + final serverSwitch sb = this.config; // generate new default entry for deep auto crawl this.defaultAutocrawlDeepProfile = @@ -705,6 +723,18 @@ public final class CrawlSwitchboard { return hasDoneSomething; } + /** Start hosts whose surrounding web graph should survive pruning. */ + public Set getActiveStartHosts() { + final Set hosts = new HashSet<>(); + for (final byte[] handle : getActive()) { + final CrawlProfile profile = getActive(handle); + if (profile != null && !DEFAULT_PROFILES.contains(profile.name())) { + hosts.addAll(profile.startHosts()); + } + } + return hosts; + } + public Set getActiveProfiles() { // find all profiles that are candidates for deletion final Set profileKeys = new HashSet<>(); diff --git a/source/net/yacy/crawler/data/CrawlProfile.java b/source/net/yacy/crawler/data/CrawlProfile.java index af6c60926..ba22d9a60 100644 --- a/source/net/yacy/crawler/data/CrawlProfile.java +++ b/source/net/yacy/crawler/data/CrawlProfile.java @@ -482,6 +482,52 @@ public class CrawlProfile extends ConcurrentHashMap implements M return r; } + private static final String START_HOSTS = "startHosts"; + private Set recordedStartHosts; + + /** Persist the actual start hosts, including profiles whose display name is abbreviated. */ + public synchronized void setStartURLs(final Collection urls) { + this.recordedStartHosts = new LinkedHashSet<>(); + for (final MultiProtocolURL url : urls) { + if (this.recordedStartHosts.size() >= net.yacy.peers.graphics.WebStructureGraph.maxhosts) break; + if (url.getHost() != null) this.recordedStartHosts.add(url.getHost().toLowerCase(Locale.ROOT)); + } + this.put(START_HOSTS, String.join(",", this.recordedStartHosts)); + } + + /** Record streamed depth-zero starts once per host, with the same bound as the graph. */ + public synchronized boolean recordStartURL(final MultiProtocolURL url, final int depth) { + if (depth != 0 || url.getHost() == null) return false; + if (this.recordedStartHosts == null) { + this.recordedStartHosts = new LinkedHashSet<>(); + final String stored = this.get(START_HOSTS); + if (stored != null && !stored.isEmpty()) { + for (final String host : stored.split(",")) this.recordedStartHosts.add(host); + } + } + if (this.recordedStartHosts.size() >= net.yacy.peers.graphics.WebStructureGraph.maxhosts + || !this.recordedStartHosts.add(url.getHost().toLowerCase(Locale.ROOT))) return false; + this.put(START_HOSTS, String.join(",", this.recordedStartHosts)); + return true; + } + + /** Local retention metadata must not be included in peer crawl announcements. */ + public Map copyForCrawlNews() { + final Map attributes = new HashMap<>(this); + attributes.remove(START_HOSTS); + return attributes; + } + + /** Older profiles used comma-separated start hosts as their name. */ + public Set startHosts() { + final Set hosts = new HashSet<>(); + final String stored = this.get(START_HOSTS); + for (final String host : (stored == null ? name() : stored).split(",")) { + if (!host.trim().isEmpty()) hosts.add(host.trim().toLowerCase(Locale.ROOT)); + } + return hosts; + } + /** * create a name that takes the collection as name if this is not "user". * @return the name of the collection if that is not "user" or the name() otherwise; diff --git a/source/net/yacy/htroot/Crawler_p.java b/source/net/yacy/htroot/Crawler_p.java index 79934c786..c5583185e 100644 --- a/source/net/yacy/htroot/Crawler_p.java +++ b/source/net/yacy/htroot/Crawler_p.java @@ -672,7 +672,7 @@ public class Crawler_p { // generate a YaCyNews if the global flag was set if (!sb.isRobinsonMode() && crawlOrder) { - final Map m = new HashMap<>(profile); // must be cloned + final Map m = profile.copyForCrawlNews(); m.remove("specificDepth"); m.remove("indexText"); m.remove("indexMedia"); diff --git a/source/net/yacy/peers/graphics/WebStructureGraph.java b/source/net/yacy/peers/graphics/WebStructureGraph.java index e1b544a6f..0ed7241de 100644 --- a/source/net/yacy/peers/graphics/WebStructureGraph.java +++ b/source/net/yacy/peers/graphics/WebStructureGraph.java @@ -30,10 +30,15 @@ package net.yacy.peers.graphics; import java.io.File; import java.io.Serializable; import java.text.ParseException; +import java.util.ArrayDeque; +import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedMap; @@ -41,6 +46,7 @@ import java.util.TreeMap; import java.util.TreeSet; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; +import java.util.function.Supplier; import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.date.MicroDate; @@ -83,6 +89,9 @@ public class WebStructureGraph { /** Eventual backup file */ private final File structureFile; + + /** Queried only when pruning, including initial loading, so profile changes take effect. */ + private final Supplier> activeRootHosts; /** *

Older structure entries (notably loaded from the backup file).

@@ -97,6 +106,9 @@ public class WebStructureGraph { * "'b64hash(6)','hostname" to 'date-yyyymmdd(8)'{'target-b64hash(6)''target-count-hex(4)'}*

* */ private final TreeMap structure_new; + + /** Number of distinct keys across both maps; guarded by structure_new then structure_old. */ + private int hostCount; /** Queue used to receive new entries to store */ private final BlockingQueue publicRefDNSResolvingQueue; @@ -131,6 +143,11 @@ public class WebStructureGraph { * backup file */ public WebStructureGraph(final File structureFile) { + this(structureFile, Collections::emptySet); + } + + public WebStructureGraph(final File structureFile, final Supplier> activeRootHosts) { + this.activeRootHosts = activeRootHosts; this.structure_old = new TreeMap(); this.structure_new = new TreeMap(); this.structureFile = structureFile; @@ -150,26 +167,8 @@ public class WebStructureGraph { } this.structure_old.putAll(loadedStructureB); - // delete out-dated entries in case the structure is too big - if ( this.structure_old.size() > maxhosts ) { - // fill a set with last-modified - dates of the structure - final TreeSet delset = new TreeSet(); - String key; - byte[] value; - for ( final Map.Entry entry : this.structure_old.entrySet() ) { - key = entry.getKey(); - value = entry.getValue(); - if ( value != null && value.length >= 8 ) { - delset.add(UTF8.String(value).substring(0, 8) + key); - } - } - int delcount = this.structure_old.size() - (maxhosts * 9 / 10); - final Iterator j = delset.iterator(); - while ( (delcount > 0) && (j.hasNext()) ) { - this.structure_old.remove(j.next().substring(8)); - delcount--; - } - } + this.hostCount = this.structure_old.size(); + trimHosts(); this.publicRefDNSResolvingWorker = new PublicRefDNSResolvingProcess(); this.publicRefDNSResolvingWorker.start(); @@ -199,8 +198,13 @@ public class WebStructureGraph { * Clear the complete web structure. */ public void clear() { - this.structure_old.clear(); - this.structure_new.clear(); + synchronized (this.structure_new) { + synchronized (this.structure_old) { + this.structure_old.clear(); + this.structure_new.clear(); + this.hostCount = 0; + } + } } public void generateCitationReference(final DigestURL url, final Document document) { @@ -751,9 +755,7 @@ public class WebStructureGraph { if (Switchboard.getSwitchboard() != null && Switchboard.getSwitchboard().shallTerminate()) break; if (!exists(domain)) { // this must be recorded as an host with no references - synchronized ( this.structure_new ) { - this.structure_new.put(domain + "," + u.getHost(), UTF8.getBytes(none2refstr())); - } + putNewHost(domain + "," + u.getHost(), UTF8.getBytes(none2refstr())); } c = 0; Integer existingCount = refs.get(domain); @@ -789,11 +791,95 @@ public class WebStructureGraph { } // store the map back to the structure - synchronized ( this.structure_new ) { - this.structure_new.put(sourceHosthash + "," + url.getHost(), UTF8.getBytes(map2refstr(refs))); + putNewHost(sourceHosthash + "," + url.getHost(), UTF8.getBytes(map2refstr(refs))); + synchronized (this.structure_new) { + synchronized (this.structure_old) { + // Record the source's links before pruning their newly discovered target hosts. + trimHosts(); + } } } + private void putNewHost(final String key, final byte[] value) { + synchronized (this.structure_new) { + synchronized (this.structure_old) { + if (!this.structure_new.containsKey(key) && !this.structure_old.containsKey(key)) { + this.hostCount++; + } + this.structure_new.put(key, value); + } + } + } + + /** Called before the worker starts, or while holding both map locks. */ + private void trimHosts() { + if (this.hostCount <= maxhosts) return; + + // Recent entries override older dates for hosts present in both maps. + final Map hosts = new TreeMap<>(this.structure_old); + hosts.putAll(this.structure_new); + final Set retained = connectedHosts(hosts, this.activeRootHosts.get()); + final int targetSize = Math.max(maxhosts * 9 / 10, retained.size()); + final TreeSet oldest = new TreeSet<>(); + for (final Map.Entry entry : hosts.entrySet()) { + final byte[] value = entry.getValue(); + final String date = value == null || value.length < 8 ? "00000000" : UTF8.String(value).substring(0, 8); + oldest.add(date + entry.getKey()); + } + // Leave headroom so sorting is amortized over many newly learned hosts. + final Iterator i = oldest.iterator(); + while (this.hostCount > targetSize && i.hasNext()) { + final String key = i.next().substring(8); + if (retained.contains(key)) continue; + this.structure_old.remove(key); + this.structure_new.remove(key); + this.hostCount--; + } + } + + /** Keep roots and a breadth-first neighbourhood, so retained paths have their intermediate hosts. */ + private Set connectedHosts(final Map hosts, final Set roots) { + final Set retained = new LinkedHashSet<>(); + if (roots.isEmpty()) return retained; + final Map namesByHash = new HashMap<>(); + final Map> keysByName = new TreeMap<>(); + for (final String key : hosts.keySet()) { + if (key.length() < 8) continue; + final String name = key.substring(7); + namesByHash.putIfAbsent(key.substring(0, 6), name); + keysByName.computeIfAbsent(name, ignored -> new ArrayList<>()).add(key); + } + final ArrayDeque queue = new ArrayDeque<>(); + for (final Map.Entry> group : keysByName.entrySet()) { + final String name = group.getKey(); + if (retained.size() + group.getValue().size() <= maxhosts && (roots.contains(name) + || roots.contains("www." + name) + || (name.startsWith("www.") && roots.contains(name.substring(4))))) { + retained.addAll(group.getValue()); + queue.add(name); + } + } + final int budget = Math.max(maxhosts * 9 / 10, retained.size()); + while (!queue.isEmpty() && retained.size() < budget) { + // The picture aggregates outgoing links over all protocol/port variants of a hostname. + for (final String key : keysByName.get(queue.remove())) { + for (final byte[] refs : new byte[][] {this.structure_new.get(key), this.structure_old.get(key)}) { + if (refs == null) continue; + for (int p = 8; p + 10 <= refs.length && retained.size() < budget; p += 10) { + final String targetName = namesByHash.get(ASCII.String(refs, p, 6)); + final List targetKeys = targetName == null ? null : keysByName.get(targetName); + if (targetKeys != null && !retained.contains(targetKeys.get(0)) + && retained.size() + targetKeys.size() <= budget) { + retained.addAll(targetKeys); + queue.add(targetName); + } + } + } + } + } + return retained; + } + private static void joinStructure(final TreeMap into, final TreeMap from) { for ( final Map.Entry e : from.entrySet() ) { if ( into.containsKey(e.getKey()) ) { @@ -818,8 +904,10 @@ public class WebStructureGraph { */ public void joinOldNew() { synchronized ( this.structure_new ) { - joinStructure(this.structure_old, this.structure_new); - this.structure_new.clear(); + synchronized (this.structure_old) { + joinStructure(this.structure_old, this.structure_new); + this.structure_new.clear(); + } } } @@ -910,7 +998,11 @@ public class WebStructureGraph { * */ private StructureIterator(final boolean latest) { - this.i = ((latest) ? WebStructureGraph.this.structure_new : WebStructureGraph.this.structure_old).entrySet().iterator(); + final TreeMap structure = latest ? WebStructureGraph.this.structure_new : WebStructureGraph.this.structure_old; + synchronized (structure) { + // Runtime eviction must not invalidate an iterator already serving a reader. + this.i = new TreeMap<>(structure).entrySet().iterator(); + } } /** @@ -1035,8 +1127,8 @@ public class WebStructureGraph { + (this.structure_old.size() * 1000 / t) + " entries/second"); } - this.structure_old.clear(); } + clear(); } } } diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index f88ba737a..cf0aee93f 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -703,7 +703,8 @@ public final class Switchboard extends serverSwitch { this.remoteSearchLastAccess = System.currentTimeMillis() - 10000; this.adminAuthenticationLastAccess = 0; // timestamp last admin authentication (as not autenticated here, stamp with 0) this.optimizeLastRun = System.currentTimeMillis(); - this.webStructure = new WebStructureGraph(new File(this.queuesRoot, "webStructure.map")); + this.webStructure = new WebStructureGraph(new File(this.queuesRoot, "webStructure.map"), + () -> this.crawler.getActiveStartHosts()); // configuring list path if ( !(this.listsPath.exists()) ) { @@ -1522,7 +1523,8 @@ public final class Switchboard extends serverSwitch { this.dhtDispatcher = (this.peers.sizeConnected() == 0) ? null : new Dispatcher(this, true, 10000); // create new web structure - this.webStructure = new WebStructureGraph(new File(this.queuesRoot, "webStructure.map")); + this.webStructure = new WebStructureGraph(new File(this.queuesRoot, "webStructure.map"), + () -> this.crawler.getActiveStartHosts()); // load domainList try { @@ -3574,6 +3576,7 @@ public final class Switchboard extends serverSwitch { public void stackURLs(final Collection rootURLs, final CrawlProfile profile, final Set successurls, final Map failurls) { if (rootURLs == null || rootURLs.size() == 0) return; + profile.setStartURLs(rootURLs); if (rootURLs.size() == 1) { // for single stack requests, do not use the multithreading overhead; final DigestURL url = rootURLs.iterator().next(); diff --git a/test/java/net/yacy/crawler/CrawlSwitchboardTest.java b/test/java/net/yacy/crawler/CrawlSwitchboardTest.java new file mode 100644 index 000000000..b10428e46 --- /dev/null +++ b/test/java/net/yacy/crawler/CrawlSwitchboardTest.java @@ -0,0 +1,272 @@ +package net.yacy.crawler; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.util.ConcurrentLog; +import net.yacy.crawler.data.CrawlProfile; +import net.yacy.crawler.data.CrawlProfile.CrawlAttribute; +import net.yacy.crawler.retrieval.Request; +import net.yacy.kelondro.util.FileUtils; +import net.yacy.peers.graphics.WebStructureGraph; +import net.yacy.peers.graphics.WebStructureGraph.StructureEntry; +import net.yacy.server.serverSwitch; + +/** Integration of persistent crawl profiles with graph retention, without network services. */ +public class CrawlSwitchboardTest { + + @Rule + public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private serverSwitch config; + private File queues; + private CrawlSwitchboard crawler; + + @Before + public void openProfiles() throws Exception { + final File root = this.temporaryFolder.getRoot(); + this.config = new serverSwitch(root, root, "absent.init", "settings/yacy.conf"); + this.config.log = new ConcurrentLog("CrawlSwitchboardTest"); + this.queues = this.temporaryFolder.newFolder("queues"); + this.crawler = new CrawlSwitchboard(null, this.queues, this.config); + } + + @After + public void closeProfiles() { + if (this.crawler != null) this.crawler.close(); + } + + private void restartProfiles() { + this.crawler.close(); + this.crawler = null; + this.crawler = new CrawlSwitchboard(null, this.queues, this.config); + } + + private CrawlProfile profile(final String handle, final String name) { + final Map values = new HashMap<>(this.crawler.defaultProxyProfile); + values.put(CrawlAttribute.HANDLE.key, handle); + values.put(CrawlAttribute.NAME.key, name); + return new CrawlProfile(values); + } + + private void activate(final CrawlProfile profile) { + this.crawler.putActive(UTF8.getBytes(profile.handle()), profile); + } + + private static Request request(final CrawlProfile profile, final String host, final int depth) throws Exception { + return new Request(UTF8.getBytes("localpeer000"), new DigestURL("https://" + host + "/"), + null, "start", new Date(), profile.handle(), depth, 0); + } + + @Test + public void submittedAndStreamedRootsSurviveDatabaseRestart() throws Exception { + final CrawlProfile urls = profile("urlprofile00", "abbreviated URL crawl"); + urls.setStartURLs(Arrays.asList(new DigestURL("https://url-root.test/"), + new DigestURL("http://url-root.test/other"))); + activate(urls); + + // Sitemap and file importers feed local depth-zero requests into this persistence boundary. + final CrawlProfile sitemap = profile("sitemapprof0", "sitemap loader for https://source.test/sitemap.xml"); + final CrawlProfile file = profile("fileprofile0", "starts.html"); + for (final CrawlProfile imported : Arrays.asList(sitemap, file)) { + activate(imported); + final String host = imported == sitemap ? "sitemap-root.test" : "file-root.test"; + this.crawler.recordStartURL(imported, request(imported, host, 0)); + this.crawler.recordStartURL(imported, request(imported, host, 0)); + this.crawler.recordStartURL(imported, request(imported, "descendant.test", 1)); + } + final Set expected = new HashSet<>(Arrays.asList( + "url-root.test", "sitemap-root.test", "file-root.test")); + assertEquals(expected, this.crawler.getActiveStartHosts()); + restartProfiles(); + assertEquals(expected, this.crawler.getActiveStartHosts()); + + final CrawlProfile reloaded = this.crawler.getActive(UTF8.getBytes(sitemap.handle())); + this.crawler.recordStartURL(reloaded, request(reloaded, "later-root.test", 0)); + expected.add("later-root.test"); + restartProfiles(); + assertEquals(expected, this.crawler.getActiveStartHosts()); + } + + @Test + public void stoppedAndReplacedProfilesIgnoreLateStartRequests() throws Exception { + final CrawlProfile stopped = profile("stopprofile0", "stopped import"); + activate(stopped); + this.crawler.recordStartURL(stopped, request(stopped, "stopped.test", 0)); + this.crawler.putPassive(UTF8.getBytes(stopped.handle()), stopped); + this.crawler.recordStartURL(stopped, request(stopped, "late.test", 0)); + assertNull(this.crawler.getActive(UTF8.getBytes(stopped.handle()))); + assertEquals(Collections.singleton("stopped.test"), + this.crawler.getPassive(UTF8.getBytes(stopped.handle())).startHosts()); + assertTrue(this.crawler.getActiveStartHosts().isEmpty()); + restartProfiles(); + assertTrue(this.crawler.getActiveStartHosts().isEmpty()); + + final CrawlProfile replacement = profile(stopped.handle(), "replacement import"); + activate(replacement); + this.crawler.recordStartURL(stopped, request(stopped, "obsolete.test", 0)); + this.crawler.recordStartURL(replacement, request(replacement, "replacement.test", 0)); + restartProfiles(); + assertEquals(Collections.singleton("replacement.test"), this.crawler.getActiveStartHosts()); + } + + @Test + public void defaultAndPassiveProfilesDoNotProtectHosts() throws Exception { + for (final byte[] handle : this.crawler.getActive()) { + final CrawlProfile profile = this.crawler.getActive(handle); + assertTrue(CrawlSwitchboard.DEFAULT_PROFILES.contains(profile.name())); + this.crawler.recordStartURL(profile, request(profile, "default.test", 0)); + assertFalse(profile.startHosts().contains("default.test")); + // Even existing metadata on a default profile must not protect its hosts. + profile.setStartURLs(Collections.singletonList(new DigestURL("https://default.test/"))); + activate(profile); + } + final CrawlProfile passive = profile("passiveprof0", "passive.test"); + this.crawler.putPassive(UTF8.getBytes(passive.handle()), passive); + final CrawlProfile legacy = profile("legacyprof00", "Legacy.test, www.second.test"); + activate(legacy); + assertEquals(new HashSet<>(Arrays.asList("legacy.test", "www.second.test")), + this.crawler.getActiveStartHosts()); + restartProfiles(); + assertEquals(new HashSet<>(Arrays.asList("legacy.test", "www.second.test")), + this.crawler.getActiveStartHosts()); + } + + @Test + public void concurrentStreamedRootsPersistWithoutLoss() throws Exception { + final CrawlProfile profile = profile("parallelprof", "parallel import"); + activate(profile); + final ExecutorService executor = Executors.newFixedThreadPool(4); + final CountDownLatch start = new CountDownLatch(1); + final Future[] tasks = new Future[4]; + final Set expected = new HashSet<>(); + try { + for (int worker = 0; worker < tasks.length; worker++) { + final int number = worker; + for (int i = 0; i < 25; i++) expected.add("root-" + worker + "-" + i + ".test"); + tasks[worker] = executor.submit(() -> { + start.await(); + for (int i = 0; i < 25; i++) { + this.crawler.recordStartURL(profile, request(profile, "root-" + number + "-" + i + ".test", 0)); + } + return null; + }); + } + start.countDown(); + for (final Future task : tasks) task.get(20, TimeUnit.SECONDS); + } finally { + executor.shutdownNow(); + assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS)); + } + restartProfiles(); + assertEquals(expected, this.crawler.getActiveStartHosts()); + } + + @Test + public void persistedActiveRootsProtectGraphUntilCrawlStops() throws Exception { + final DigestURL root = new DigestURL("https://root.test/"); + final DigestURL bridge = new DigestURL("http://bridge.test/"); + final DigestURL leaf = new DigestURL("https://leaf.test/"); + final CrawlProfile profile = profile("graphprofile", "abbreviated graph crawl"); + profile.setStartURLs(Collections.singletonList(root)); + activate(profile); + restartProfiles(); + + final Map entries = new TreeMap<>(); + storeHost(entries, root, "19900101" + bridge.hosthash() + "0001"); + storeHost(entries, bridge, "19900101" + leaf.hosthash() + "0001"); + storeHost(entries, leaf, "19900101"); + for (int i = 0; i < WebStructureGraph.maxhosts - 2; i++) { + storeHost(entries, new DigestURL("http://unrelated-" + i + ".test/"), "20990101"); + } + final File backup = this.temporaryFolder.newFile("webStructure.map"); + FileUtils.saveMapB(backup, entries, "active root and newer unrelated hosts"); + WebStructureGraph graph = new WebStructureGraph(backup, () -> this.crawler.getActiveStartHosts()); + try { + assertEquals(9000, hostCount(graph)); + assertPath(graph, root, bridge, leaf); + } finally { + graph.close(); + } + restartProfiles(); + graph = new WebStructureGraph(backup, () -> this.crawler.getActiveStartHosts()); + try { + assertPath(graph, root, bridge, leaf); + // This is the same profile transition used by crawl termination/cleanup. + this.crawler.cleanProfiles(Collections.singleton(profile.handle())); + assertTrue(this.crawler.getActiveStartHosts().isEmpty()); + final DigestURL unrelated = new DigestURL("http://unrelated-0.test/"); + for (int i = 0; i <= 1000; i++) { + graph.generateCitationReference(new DigestURL("https://new-" + i + ".test/"), unrelated); + } + } finally { + // Drain the real learning worker before inspecting the persisted graph. + graph.close(); + } + assertEquals("Runtime pruning must happen before the next startup", 9000, FileUtils.loadMapB(backup).size()); + restartProfiles(); + assertTrue(this.crawler.getActiveStartHosts().isEmpty()); + graph = new WebStructureGraph(backup, () -> this.crawler.getActiveStartHosts()); + try { + assertEquals(9000, hostCount(graph)); + for (final DigestURL url : Arrays.asList(root, bridge, leaf)) assertFalse(graph.exists(url.hosthash())); + } finally { + graph.close(); + } + } + + private static void storeHost(final Map entries, final DigestURL url, final String refs) { + entries.put(url.hosthash() + "," + url.getHost(), UTF8.getBytes(refs)); + } + + private static int hostCount(final WebStructureGraph graph) { + final Set keys = new HashSet<>(); + for (final boolean latest : new boolean[] {false, true}) { + final Iterator entries = graph.structureEntryIterator(latest); + while (entries.hasNext()) { + final StructureEntry entry = entries.next(); + keys.add(entry.hosthash + "," + entry.hostname); + } + } + return keys.size(); + } + + private static void assertPath(final WebStructureGraph graph, final DigestURL... path) { + for (int i = 0; i < path.length; i++) { + assertTrue(graph.exists(path[i].hosthash())); + assertEquals(path[i].getHost(), graph.hostHash2hostName(path[i].hosthash())); + if (i > 0) { + final StructureEntry source = graph.outgoingReferences(path[i - 1].hosthash()); + assertNotNull(source); + assertTrue(source.references.containsKey(path[i].hosthash())); + } + } + } +} diff --git a/test/java/net/yacy/crawler/data/CrawlProfileTest.java b/test/java/net/yacy/crawler/data/CrawlProfileTest.java new file mode 100644 index 000000000..39a973c47 --- /dev/null +++ b/test/java/net/yacy/crawler/data/CrawlProfileTest.java @@ -0,0 +1,78 @@ +package net.yacy.crawler.data; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.List; + +import org.junit.Test; + +import net.yacy.cora.document.id.DigestURL; +import net.yacy.peers.graphics.WebStructureGraph; + +public class CrawlProfileTest { + + @Test + public void startHostsSurviveProfileRoundTripWithAnAbbreviatedName() throws Exception { + final Map values = new HashMap<>(); + values.put(CrawlProfile.CrawlAttribute.NAME.key, "crawl_for_3_start_points_abbreviated"); + final CrawlProfile profile = new CrawlProfile(values); + profile.setStartURLs(Arrays.asList(new DigestURL("https://first.net/"), + new DigestURL("https://second.net/path"), new DigestURL("http://first.net/other"))); + final CrawlProfile reloaded = new CrawlProfile(new HashMap<>(profile)); + assertEquals(new HashSet<>(Arrays.asList("first.net", "second.net")), reloaded.startHosts()); + } + + @Test + public void legacyProfilesUseTheirHostNames() { + final Map values = new HashMap<>(); + values.put(CrawlProfile.CrawlAttribute.NAME.key, "KLG.de, www.example.net"); + assertEquals(new HashSet<>(Arrays.asList("klg.de", "www.example.net")), + new CrawlProfile(values).startHosts()); + } + + @Test + public void streamedSitemapAndFileRootsPersistButDescendantsDoNotBecomeRoots() throws Exception { + for (final String name : Arrays.asList("sitemap loader for https://source.net/sitemap.xml", "starts.html")) { + final Map values = new HashMap<>(); + values.put(CrawlProfile.CrawlAttribute.NAME.key, name); + final CrawlProfile profile = new CrawlProfile(values); + // Both importers submit their start URLs at depth zero to the crawl stacker. + assertTrue(profile.recordStartURL(new DigestURL("https://first.net/a"), 0)); + assertFalse(profile.recordStartURL(new DigestURL("https://first.net/b"), 0)); + assertFalse(profile.recordStartURL(new DigestURL("https://descendant.net/"), 1)); + final CrawlProfile reloaded = new CrawlProfile(new HashMap<>(profile)); + assertTrue(reloaded.recordStartURL(new DigestURL("https://second.net/"), 0)); + assertEquals(new HashSet<>(Arrays.asList("first.net", "second.net")), reloaded.startHosts()); + } + } + + @Test + public void localStartHostsAreOmittedFromCrawlNews() throws Exception { + final Map values = new HashMap<>(); + values.put(CrawlProfile.CrawlAttribute.NAME.key, "crawl_for_2_start_points"); + final CrawlProfile profile = new CrawlProfile(values); + profile.setStartURLs(Arrays.asList(new DigestURL("https://first.net/"), new DigestURL("https://second.net/"))); + final Map news = profile.copyForCrawlNews(); + assertFalse(news.containsKey("startHosts")); + assertEquals(profile.name(), news.get(CrawlProfile.CrawlAttribute.NAME.key)); + assertEquals(2, profile.startHosts().size()); + } + + @Test + public void startHostMetadataIsBoundedForLargeImports() throws Exception { + final CrawlProfile profile = new CrawlProfile(new HashMap<>()); + final List starts = new ArrayList<>(); + for (int i = 0; i <= WebStructureGraph.maxhosts; i++) starts.add(new DigestURL("https://host" + i + ".net/")); + profile.setStartURLs(starts); + assertEquals(WebStructureGraph.maxhosts, profile.startHosts().size()); + assertFalse(profile.recordStartURL(new DigestURL("https://overflow.net/"), 0)); + assertEquals(WebStructureGraph.maxhosts, new CrawlProfile(new HashMap<>(profile)).startHosts().size()); + } +} diff --git a/test/java/net/yacy/peers/graphics/WebStructureGraphTest.java b/test/java/net/yacy/peers/graphics/WebStructureGraphTest.java index d1ffb1003..c7907af04 100644 --- a/test/java/net/yacy/peers/graphics/WebStructureGraphTest.java +++ b/test/java/net/yacy/peers/graphics/WebStructureGraphTest.java @@ -21,16 +21,24 @@ package net.yacy.peers.graphics; +import java.io.File; import java.net.MalformedURLException; +import java.util.Collections; import java.util.HashSet; +import java.util.Iterator; import java.util.Map; import java.util.Set; +import java.util.TreeMap; import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.util.ConcurrentLog; +import net.yacy.kelondro.util.FileUtils; import net.yacy.peers.graphics.WebStructureGraph.LearnObject; import net.yacy.peers.graphics.WebStructureGraph.StructureEntry; @@ -42,6 +50,265 @@ import net.yacy.peers.graphics.WebStructureGraph.StructureEntry; */ public class WebStructureGraphTest { + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private static DigestURL host(final int number) throws MalformedURLException { + return new DigestURL("http://host" + number + ".net/"); + } + + private static Set hostKeys(final WebStructureGraph graph, final boolean latest) { + final Set keys = new HashSet<>(); + final Iterator entries = graph.structureEntryIterator(latest); + while (entries.hasNext()) { + final StructureEntry entry = entries.next(); + keys.add(entry.hosthash + "," + entry.hostname); + } + return keys; + } + + private static int hostCount(final WebStructureGraph graph) { + final Set keys = hostKeys(graph, false); + keys.addAll(hostKeys(graph, true)); + return keys.size(); + } + + @Test + public void testPruningPreservesProtocolAndPortVariantsAlongDisplayedPaths() throws Exception { + for (final boolean atStartup : new boolean[] {false, true}) { + final DigestURL root = new DigestURL("https://root.net/"); + final DigestURL bridgeHttp = new DigestURL("http://bridge.net/"); + final DigestURL bridgeHttps = new DigestURL("https://bridge.net/"); + final DigestURL bridgePort = new DigestURL("http://bridge.net:8080/"); + final DigestURL leaf = new DigestURL("https://leaf.net/"); + final Map entries = new TreeMap<>(); + entries.put(root.hosthash() + ",root.net", UTF8.getBytes("19900101" + bridgeHttp.hosthash() + "0001")); + entries.put(bridgeHttp.hosthash() + ",bridge.net", UTF8.getBytes("19900101")); + entries.put(bridgeHttps.hosthash() + ",bridge.net", UTF8.getBytes("19900101" + leaf.hosthash() + "0001")); + entries.put(bridgePort.hosthash() + ",bridge.net", UTF8.getBytes("19900101" + leaf.hosthash() + "0001")); + entries.put(leaf.hosthash() + ",leaf.net", UTF8.getBytes("19900101")); + for (int i = 0; i < WebStructureGraph.maxhosts - 5; i++) storeHost(entries, i, "20990101"); + if (atStartup) storeHost(entries, WebStructureGraph.maxhosts, "20990101"); + final File backup = this.temporaryFolder.newFile(); + FileUtils.saveMapB(backup, entries, "hostname graph spanning protocols and ports"); + final WebStructureGraph graph = new WebStructureGraph(backup, + () -> Collections.singleton(root.getHost())); + try { + if (!atStartup) graph.learnrefs(new LearnObject(host(WebStructureGraph.maxhosts), Collections.emptySet())); + Assert.assertEquals(9000, hostCount(graph)); + Assert.assertTrue(graph.outgoingReferencesByHostName("root.net").containsKey(bridgeHttp.hosthash())); + for (final DigestURL variant : new DigestURL[] {bridgeHttp, bridgeHttps, bridgePort}) { + Assert.assertTrue("Missing bridge variant " + variant, graph.exists(variant.hosthash())); + } + Assert.assertTrue(graph.outgoingReferencesByHostName("bridge.net").containsKey(leaf.hosthash())); + Assert.assertEquals("leaf.net", graph.hostHash2hostName(leaf.hosthash())); + } finally { + graph.close(); + } + } + } + + private static void storeHost(final Map entries, final int source, + final String date, final int... targets) throws Exception { + final DigestURL url = host(source); + final StringBuilder refs = new StringBuilder(date); + for (final int target : targets) refs.append(host(target).hosthash()).append("0001"); + entries.put(url.hosthash() + "," + url.getHost(), UTF8.getBytes(refs.toString())); + } + + private File graphWithOldRootPaths(final int count) throws Exception { + final Map entries = new TreeMap<>(); + for (int i = 0; i < count; i++) storeHost(entries, i, "20990101"); + storeHost(entries, 0, "19900101", 1); + storeHost(entries, 1, "19900101", 2); + storeHost(entries, 2, "19900101", 3); + storeHost(entries, 3, "19900101", 0); // cycle must not defeat the bound + storeHost(entries, 4, "19900101", 5); + storeHost(entries, 5, "19900101", 6); + storeHost(entries, 6, "19900101", 4); + final File backup = this.temporaryFolder.newFile(); + FileUtils.saveMapB(backup, entries, "old roots and newer unrelated hosts"); + return backup; + } + + private static void assertPath(final WebStructureGraph graph, final int... path) throws Exception { + for (int i = 0; i < path.length; i++) { + Assert.assertTrue("Missing path host " + path[i], graph.exists(host(path[i]).hosthash())); + Assert.assertEquals(host(path[i]).getHost(), graph.hostHash2hostName(host(path[i]).hosthash())); + if (i > 0) { + Assert.assertTrue(graph.outgoingReferences(host(path[i - 1]).hosthash()) + .references.containsKey(host(path[i]).hosthash())); + } + } + } + + @Test + public void testStartupPreservesMultipleOldRootPathsAndRestarts() throws Exception { + final File backup = graphWithOldRootPaths(WebStructureGraph.maxhosts + 1); + final Set roots = new HashSet<>(); + roots.add(host(0).getHost()); + roots.add("www." + host(4).getHost()); + for (int restart = 0; restart < 2; restart++) { + final WebStructureGraph graph = new WebStructureGraph(backup, () -> roots); + try { + Assert.assertEquals(9000, hostCount(graph)); + assertPath(graph, 0, 1, 2, 3, 0); + assertPath(graph, 4, 5, 6, 4); + } finally { + graph.close(); + } + } + } + + @Test + public void testRuntimePreservesRootPathsAndNewTargetsUntilCrawlStops() throws Exception { + final Set roots = new HashSet<>(); + final WebStructureGraph graph = new WebStructureGraph( + graphWithOldRootPaths(WebStructureGraph.maxhosts), () -> roots); + try { + // Starting a crawl after graph construction must affect the next pruning pass. + roots.add(host(0).getHost()); + final DigestURL newTarget = host(WebStructureGraph.maxhosts + 1); + graph.learnrefs(new LearnObject(host(0), Collections.singleton(newTarget))); + assertPath(graph, 0, 1, 2, 3, 0); + assertPath(graph, 0, WebStructureGraph.maxhosts + 1); + Assert.assertEquals(9000, hostCount(graph)); + Assert.assertFalse(graph.exists(host(4).hosthash())); // unrelated old component + graph.joinOldNew(); + roots.clear(); + for (int i = 0; i <= 1000; i++) { + graph.learnrefs(new LearnObject(host(20000 + i), Collections.emptySet())); + } + Assert.assertEquals(9000, hostCount(graph)); + Assert.assertFalse("Stopped crawl's old path can be pruned", graph.exists(host(1).hosthash())); + } finally { + graph.close(); + } + } + + @Test + public void testOversizedConnectedGraphKeepsContinuousPathsWithinBudget() throws Exception { + final File backup = this.temporaryFolder.newFile(); + final Map entries = new TreeMap<>(); + for (int i = 0; i <= WebStructureGraph.maxhosts; i++) { + storeHost(entries, i, "19900101", (i + 1) % (WebStructureGraph.maxhosts + 1)); + } + storeHost(entries, 0, "19900101", 1, WebStructureGraph.maxhosts); + FileUtils.saveMapB(backup, entries, "oversized connected path"); + final WebStructureGraph graph = new WebStructureGraph(backup, + () -> Collections.singleton("host0.net")); + try { + Assert.assertEquals(9000, hostCount(graph)); + for (int i = 0; i < 8999; i++) Assert.assertTrue(graph.exists(host(i).hosthash())); + Assert.assertFalse(graph.exists(host(8999).hosthash())); + assertPath(graph, 0, WebStructureGraph.maxhosts); // nearby branch precedes the long path's tail + assertPath(graph, 0, 1, 2, 3); + } finally { + graph.close(); + } + } + + @Test + public void testRuntimeHostLimitAcrossOldAndNewAndAfterClear() throws Exception { + final WebStructureGraph graph = new WebStructureGraph(null); + try { + for (int i = 0; i < WebStructureGraph.maxhosts; i++) { + graph.learnrefs(new LearnObject(host(i), Collections.emptySet())); + if (i == WebStructureGraph.maxhosts / 2) graph.joinOldNew(); + } + // Updating every old host must not count it again or drain the recent map. + for (int i = 0; i <= WebStructureGraph.maxhosts / 2; i++) { + graph.learnrefs(new LearnObject(host(i), Collections.emptySet())); + } + Assert.assertEquals(WebStructureGraph.maxhosts, hostCount(graph)); + Assert.assertEquals(WebStructureGraph.maxhosts, hostKeys(graph, true).size()); + final Iterator oldSnapshot = graph.structureEntryIterator(false); + final Iterator recentSnapshot = graph.structureEntryIterator(true); + graph.learnrefs(new LearnObject(host(WebStructureGraph.maxhosts), Collections.emptySet())); + Assert.assertEquals(WebStructureGraph.maxhosts * 9 / 10, hostCount(graph)); + Assert.assertEquals(WebStructureGraph.maxhosts / 2 + 1, countEntries(oldSnapshot)); + Assert.assertEquals(WebStructureGraph.maxhosts, countEntries(recentSnapshot)); + graph.joinOldNew(); + // The limit must continue working over successive eviction/merge cycles. + for (int i = 1; i <= 1100; i++) { + graph.learnrefs(new LearnObject(host(WebStructureGraph.maxhosts + i), Collections.emptySet())); + } + Assert.assertEquals(9099, hostCount(graph)); + graph.clear(); + for (int i = 0; i < WebStructureGraph.maxhosts; i++) { + graph.learnrefs(new LearnObject(host(i), Collections.emptySet())); + } + Assert.assertEquals(WebStructureGraph.maxhosts, hostCount(graph)); + } finally { + graph.close(); + } + } + + private static int countEntries(final Iterator entries) { + int count = 0; + while (entries.hasNext()) { + Assert.assertNotNull(entries.next()); + count++; + } + return count; + } + + @Test + public void testTargetHostsAlsoTriggerRuntimeLimit() throws Exception { + final WebStructureGraph graph = new WebStructureGraph(null); + try { + for (int batch = 0; batch < 11; batch++) { + final Set targets = new HashSet<>(); + for (int i = 0; i < 1000; i++) targets.add(host(batch * 1000 + i)); + graph.learnrefs(new LearnObject(host(-1), targets)); + Assert.assertTrue(hostCount(graph) <= WebStructureGraph.maxhosts); + } + } finally { + graph.close(); + } + } + + @Test + public void testRuntimeEvictionUsesRecentDateAndPersistsLimit() throws Exception { + final File backup = this.temporaryFolder.newFile("structure.map"); + final Map stored = new TreeMap<>(); + for (int i = 0; i < WebStructureGraph.maxhosts; i++) { + final DigestURL url = host(i); + stored.put(url.hosthash() + "," + url.getHost(), UTF8.getBytes(i < 2 ? "19900101" : "20000101")); + } + FileUtils.saveMapB(backup, stored, "test graph"); + final WebStructureGraph graph = new WebStructureGraph(backup); + try { + graph.learnrefs(new LearnObject(host(0), Collections.emptySet())); + graph.learnrefs(new LearnObject(host(WebStructureGraph.maxhosts), Collections.emptySet())); + Assert.assertEquals(WebStructureGraph.maxhosts * 9 / 10, hostCount(graph)); + Assert.assertTrue(graph.exists(host(0).hosthash())); + Assert.assertFalse(graph.exists(host(1).hosthash())); + Assert.assertTrue(hostKeys(graph, true).contains(host(0).hosthash() + "," + host(0).getHost())); + } finally { + graph.close(); + } + Assert.assertEquals(WebStructureGraph.maxhosts * 9 / 10, FileUtils.loadMapB(backup).size()); + } + + @Test + public void testStartupStillTrimsOldestHosts() throws Exception { + final File backup = this.temporaryFolder.newFile("oversized.map"); + final Map stored = new TreeMap<>(); + for (int i = 0; i <= WebStructureGraph.maxhosts; i++) { + final DigestURL url = host(i); + stored.put(url.hosthash() + "," + url.getHost(), UTF8.getBytes(i == 0 ? "19900101" : "20000101")); + } + FileUtils.saveMapB(backup, stored, "test graph"); + final WebStructureGraph graph = new WebStructureGraph(backup); + try { + Assert.assertEquals(WebStructureGraph.maxhosts * 9 / 10, hostCount(graph)); + Assert.assertFalse(graph.exists(host(0).hosthash())); + } finally { + graph.close(); + } + } + /** * Most basic out going references unit test */