jody/yacy_search_server
1
0 Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2026-10-25 05:54:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

added collection attribute setting for zim and warc imports

Browse Source 
Having the collection attribute for those imports enables to
use those import methods for YaCy Pack generation which requires
the collection name to produce a pack name.
Fixed also file upload option through browser for both methods.
For large zim/warc files you require large memory settings
to be able to cache the uploaded file.
You also need large memory requirements in your browser to upload
gigabytes of files through multipart/form-data uploads.
This commit is contained in:
Michael Peter Christen committed 2025-08-15 10:26:21 -06:00
1 parent a803c6063b
commit 4c953ee8fd
9 files changed
+340 -268
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

htroot/IndexImportWarc_p.html
+3 -1
View File
@@ -14,7 +14,7 @@
#(import)#
<p>No import thread is running, you can start a new thread here</p>
<form action="IndexImportWarc_p.html" method="get" accept-charset="UTF-8">
<form action="IndexImportWarc_p.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8">
<!-- no post method here, we don't want to transmit the whole file, only the path-->
<fieldset>
<legend>Warc File Selection: select an warc file (which may be gz compressed)</legend>
@@ -30,6 +30,8 @@
<dt class="TableCellDark"><label for="url">Url:</label></dt>
<dd><input name="url" id="url" value="" size="75"/></dd>
<dt></dt>
<dt class="TableCellDark"><label for="collection">Collection:</label></dt>
<dd><input name="collection" id="collection" type="text" value="user" size="75" /></dd>
<dd><input name="submit" class="btn btn-primary" type="submit" value="Import Warc File" /></dd>
</dl>
</fieldset>
htroot/IndexImportZim_p.html
+2
View File
@@ -25,6 +25,8 @@
<dl>
<dt class="TableCellDark"><label for="file">File:</label></dt>
<dd><input name="file" id="file" type="file" value="" size="75" /></dd>
<dt class="TableCellDark"><label for="collection">Collection:</label></dt>
<dd><input name="collection" id="collection" type="text" value="user" size="75" /></dd>
<dt></dt>
<dd><input name="submit" class="btn btn-primary" type="submit" value="Import ZIM File" /></dd>
</dl>
source/net/yacy/crawler/data/CrawlProfile.java
+141 -118
View File
@@ -64,7 +64,7 @@ import net.yacy.server.serverObjects;
/**
* this is a simple record structure that hold all properties of a single crawl start
*/
public class CrawlProfile extends ConcurrentHashMap<String, String> implements Map<String, String> {
public class CrawlProfile extends ConcurrentHashMap<String, String> implements Map<String, String>, Cloneable {
private static final long serialVersionUID = 5527325718810703504L;
@@ -151,7 +151,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
public String toString() {
return this.key;
}
}
}
private Pattern crawlerurlmustmatch = null, crawlerurlmustnotmatch = null;
@@ -166,7 +166,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
private Pattern indexurlmustmatch = null, indexurlmustnotmatch = null;
private Pattern indexcontentmustmatch = null, indexcontentmustnotmatch = null;
/** Pattern on the media type documents must match before being indexed
/** Pattern on the media type documents must match before being indexed
* @see CollectionSchema#content_type */
private Pattern indexMediaTypeMustMatch = null;
@@ -251,58 +251,58 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
throw new NullPointerException("name must not be null or empty");
}
if (name.length() > 256) name = name.substring(256);
this.doms = new ConcurrentHashMap<String, AtomicInteger>();
this.doms = new ConcurrentHashMap<>();
final String handle = Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(name + crawlerUrlMustMatch + depth + crawlerUrlMustNotMatch + domMaxPages + collections)).substring(0, Word.commonHashLength);
put(CrawlAttribute.HANDLE.key, handle);
put(CrawlAttribute.NAME.key, name);
put(CrawlAttribute.AGENT_NAME.key, userAgentName);
put(CrawlAttribute.CRAWLER_ALWAYS_CHECK_MEDIA_TYPE.key, true);
put(CrawlAttribute.CRAWLER_URL_MUSTMATCH.key, (crawlerUrlMustMatch == null) ? CrawlProfile.MATCH_ALL_STRING : crawlerUrlMustMatch);
put(CrawlAttribute.CRAWLER_URL_MUSTNOTMATCH.key, (crawlerUrlMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerUrlMustNotMatch);
put(CrawlAttribute.CRAWLER_ORIGIN_URL_MUSTMATCH.key, (crawlerUrlMustMatch == null) ? CrawlProfile.MATCH_ALL_STRING : crawlerUrlMustMatch);
put(CrawlAttribute.CRAWLER_URL_MUSTNOTMATCH.key, (crawlerUrlMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerUrlMustNotMatch);
put(CrawlAttribute.CRAWLER_IP_MUSTMATCH.key, (crawlerIpMustMatch == null) ? CrawlProfile.MATCH_ALL_STRING : crawlerIpMustMatch);
put(CrawlAttribute.CRAWLER_IP_MUSTNOTMATCH.key, (crawlerIpMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerIpMustNotMatch);
put(CrawlAttribute.CRAWLER_COUNTRY_MUSTMATCH.key, (crawlerCountryMustMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerCountryMustMatch);
put(CrawlAttribute.CRAWLER_URL_NODEPTHLIMITMATCH.key, (crawlerNoDepthLimitMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerNoDepthLimitMatch);
put(CrawlAttribute.INDEXING_URL_MUSTMATCH.key, (indexUrlMustMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : indexUrlMustMatch);
put(CrawlAttribute.INDEXING_URL_MUSTNOTMATCH.key, (indexUrlMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : indexUrlMustNotMatch);
put(CrawlAttribute.INDEXING_CONTENT_MUSTMATCH.key, (indexContentMustMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : indexContentMustMatch);
put(CrawlAttribute.INDEXING_CONTENT_MUSTNOTMATCH.key, (indexContentMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : indexContentMustNotMatch);
put(CrawlAttribute.DEPTH.key, depth);
put(CrawlAttribute.DIRECT_DOC_BY_URL.key, directDocByURL);
put(CrawlAttribute.RECRAWL_IF_OLDER.key, recrawlIfOlder == null ? Long.MAX_VALUE : recrawlIfOlder.getTime());
put(CrawlAttribute.DOM_MAX_PAGES.key, domMaxPages);
put(CrawlAttribute.CRAWLING_Q.key, crawlingQ); // crawling of urls with '?'
put(CrawlAttribute.FOLLOW_FRAMES.key, followFrames); // load pages contained in frames or ifames
put(CrawlAttribute.OBEY_HTML_ROBOTS_NOINDEX.key, obeyHtmlRobotsNoindex); // if false, then a meta robots tag containing 'noindex' is ignored
put(CrawlAttribute.OBEY_HTML_ROBOTS_NOFOLLOW.key, obeyHtmlRobotsNofollow);
put(CrawlAttribute.INDEX_TEXT.key, indexText);
put(CrawlAttribute.INDEX_MEDIA.key, indexMedia);
put(CrawlAttribute.STORE_HTCACHE.key, storeHTCache);
put(CrawlAttribute.REMOTE_INDEXING.key, remoteIndexing);
put(CrawlAttribute.SNAPSHOTS_MAXDEPTH.key, snapshotsMaxDepth);
put(CrawlAttribute.SNAPSHOTS_LOADIMAGE.key, snapshotsLoadImage);
put(CrawlAttribute.SNAPSHOTS_REPLACEOLD.key, snapshotsReplaceOld);
put(CrawlAttribute.SNAPSHOTS_MUSTNOTMATCH.key, snapshotsMustnotmatch);
put(CrawlAttribute.CACHE_STRAGEGY.key, cacheStrategy.toString());
put(CrawlAttribute.COLLECTIONS.key, CommonPattern.SPACE.matcher(collections.trim()).replaceAll(""));
this.put(CrawlAttribute.HANDLE.key, handle);
this.put(CrawlAttribute.NAME.key, name);
this.put(CrawlAttribute.AGENT_NAME.key, userAgentName);
this.put(CrawlAttribute.CRAWLER_ALWAYS_CHECK_MEDIA_TYPE.key, true);
this.put(CrawlAttribute.CRAWLER_URL_MUSTMATCH.key, (crawlerUrlMustMatch == null) ? CrawlProfile.MATCH_ALL_STRING : crawlerUrlMustMatch);
this.put(CrawlAttribute.CRAWLER_URL_MUSTNOTMATCH.key, (crawlerUrlMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerUrlMustNotMatch);
this.put(CrawlAttribute.CRAWLER_ORIGIN_URL_MUSTMATCH.key, (crawlerUrlMustMatch == null) ? CrawlProfile.MATCH_ALL_STRING : crawlerUrlMustMatch);
this.put(CrawlAttribute.CRAWLER_URL_MUSTNOTMATCH.key, (crawlerUrlMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerUrlMustNotMatch);
this.put(CrawlAttribute.CRAWLER_IP_MUSTMATCH.key, (crawlerIpMustMatch == null) ? CrawlProfile.MATCH_ALL_STRING : crawlerIpMustMatch);
this.put(CrawlAttribute.CRAWLER_IP_MUSTNOTMATCH.key, (crawlerIpMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerIpMustNotMatch);
this.put(CrawlAttribute.CRAWLER_COUNTRY_MUSTMATCH.key, (crawlerCountryMustMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerCountryMustMatch);
this.put(CrawlAttribute.CRAWLER_URL_NODEPTHLIMITMATCH.key, (crawlerNoDepthLimitMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerNoDepthLimitMatch);
this.put(CrawlAttribute.INDEXING_URL_MUSTMATCH.key, (indexUrlMustMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : indexUrlMustMatch);
this.put(CrawlAttribute.INDEXING_URL_MUSTNOTMATCH.key, (indexUrlMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : indexUrlMustNotMatch);
this.put(CrawlAttribute.INDEXING_CONTENT_MUSTMATCH.key, (indexContentMustMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : indexContentMustMatch);
this.put(CrawlAttribute.INDEXING_CONTENT_MUSTNOTMATCH.key, (indexContentMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : indexContentMustNotMatch);
this.put(CrawlAttribute.DEPTH.key, depth);
this.put(CrawlAttribute.DIRECT_DOC_BY_URL.key, directDocByURL);
this.put(CrawlAttribute.RECRAWL_IF_OLDER.key, recrawlIfOlder == null ? Long.MAX_VALUE : recrawlIfOlder.getTime());
this.put(CrawlAttribute.DOM_MAX_PAGES.key, domMaxPages);
this.put(CrawlAttribute.CRAWLING_Q.key, crawlingQ); // crawling of urls with '?'
this.put(CrawlAttribute.FOLLOW_FRAMES.key, followFrames); // load pages contained in frames or ifames
this.put(CrawlAttribute.OBEY_HTML_ROBOTS_NOINDEX.key, obeyHtmlRobotsNoindex); // if false, then a meta robots tag containing 'noindex' is ignored
this.put(CrawlAttribute.OBEY_HTML_ROBOTS_NOFOLLOW.key, obeyHtmlRobotsNofollow);
this.put(CrawlAttribute.INDEX_TEXT.key, indexText);
this.put(CrawlAttribute.INDEX_MEDIA.key, indexMedia);
this.put(CrawlAttribute.STORE_HTCACHE.key, storeHTCache);
this.put(CrawlAttribute.REMOTE_INDEXING.key, remoteIndexing);
this.put(CrawlAttribute.SNAPSHOTS_MAXDEPTH.key, snapshotsMaxDepth);
this.put(CrawlAttribute.SNAPSHOTS_LOADIMAGE.key, snapshotsLoadImage);
this.put(CrawlAttribute.SNAPSHOTS_REPLACEOLD.key, snapshotsReplaceOld);
this.put(CrawlAttribute.SNAPSHOTS_MUSTNOTMATCH.key, snapshotsMustnotmatch);
this.put(CrawlAttribute.CACHE_STRAGEGY.key, cacheStrategy.toString());
this.put(CrawlAttribute.COLLECTIONS.key, CommonPattern.SPACE.matcher(collections.trim()).replaceAll(""));
// we transform the ignore_class_name and scraper information into a JSON Array
this.defaultValency = defaultValency;
this.valencySwitchTagNames = valencySwitchTagNames == null ? new HashSet<String>() : valencySwitchTagNames;
this.valencySwitchTagNames = valencySwitchTagNames == null ? new HashSet<>() : valencySwitchTagNames;
String jsonString = new JSONArray(valencySwitchTagNames).toString();
put(CrawlAttribute.DEFAULT_VALENCY.key, defaultValency.name());
put(CrawlAttribute.VALENCY_SWITCH_TAG_NAMES.key, jsonString);
this.put(CrawlAttribute.DEFAULT_VALENCY.key, defaultValency.name());
this.put(CrawlAttribute.VALENCY_SWITCH_TAG_NAMES.key, jsonString);
this.scraper = scraper == null ? new VocabularyScraper() : scraper;
jsonString = this.scraper.toString();
assert jsonString != null && jsonString.length() > 0 && jsonString.charAt(0) == '{' : "jsonString = " + jsonString;
put(CrawlAttribute.SCRAPER.key, jsonString);
put(CrawlAttribute.TIMEZONEOFFSET.key, timezoneOffset);
put(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTMATCH.key, CrawlProfile.MATCH_ALL_STRING);
put(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTNOTMATCH.key, CrawlProfile.MATCH_NEVER_STRING);
put(CrawlAttribute.INDEXING_SOLR_QUERY_MUSTMATCH.key, CrawlProfile.SOLR_MATCH_ALL_QUERY);
put(CrawlAttribute.INDEXING_SOLR_QUERY_MUSTNOTMATCH.key, CrawlProfile.SOLR_EMPTY_QUERY);
put(CrawlAttribute.NOINDEX_WHEN_CANONICAL_UNEQUAL_URL.key, noindexWhenCanonicalUnequalURL);
this.put(CrawlAttribute.SCRAPER.key, jsonString);
this.put(CrawlAttribute.TIMEZONEOFFSET.key, timezoneOffset);
this.put(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTMATCH.key, CrawlProfile.MATCH_ALL_STRING);
this.put(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTNOTMATCH.key, CrawlProfile.MATCH_NEVER_STRING);
this.put(CrawlAttribute.INDEXING_SOLR_QUERY_MUSTMATCH.key, CrawlProfile.SOLR_MATCH_ALL_QUERY);
this.put(CrawlAttribute.INDEXING_SOLR_QUERY_MUSTNOTMATCH.key, CrawlProfile.SOLR_EMPTY_QUERY);
this.put(CrawlAttribute.NOINDEX_WHEN_CANONICAL_UNEQUAL_URL.key, noindexWhenCanonicalUnequalURL);
}
/**
@@ -311,9 +311,9 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
*/
public CrawlProfile(final Map<String, String> ext) {
super(ext == null ? 1 : ext.size());
if (ext != null) putAll(ext);
this.doms = new ConcurrentHashMap<String, AtomicInteger>();
String defaultValency = ext.get(CrawlAttribute.DEFAULT_VALENCY.key);
if (ext != null) this.putAll(ext);
this.doms = new ConcurrentHashMap<>();
final String defaultValency = ext.get(CrawlAttribute.DEFAULT_VALENCY.key);
this.defaultValency = defaultValency == null || defaultValency.length() == 0 ? TagValency.EVAL : TagValency.valueOf(defaultValency);
String jsonString = ext.get(CrawlAttribute.VALENCY_SWITCH_TAG_NAMES.key);
JSONArray a;
@@ -327,10 +327,10 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
a = new JSONArray();
}
}
this.valencySwitchTagNames = new HashSet<String>();
this.valencySwitchTagNames = new HashSet<>();
for (int i = 0; i < a.length(); i++) try {
this.valencySwitchTagNames.add(a.getString(i));
} catch (JSONException e) {}
} catch (final JSONException e) {}
jsonString = ext.get(CrawlAttribute.SCRAPER.key);
if (jsonString == null || jsonString.length() == 0) {
this.scraper = new VocabularyScraper();
@@ -340,12 +340,19 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
loadedScraper = new VocabularyScraper(jsonString);
} catch(final JSONException e) {
ConcurrentLog.logException(e);
loadedScraper = new VocabularyScraper();
loadedScraper = new VocabularyScraper();
}
this.scraper = loadedScraper;
}
}
@Override
public Object clone() {
final ConcurrentHashMap<String, String> m = new ConcurrentHashMap<>(this);
final CrawlProfile cp = new CrawlProfile(m);
return cp;
}
public TagValency defaultValency() {
return this.defaultValency;
}
@@ -389,10 +396,10 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
}
public ClientIdentification.Agent getAgent() {
String agentName = this.get(CrawlAttribute.AGENT_NAME.key);
final String agentName = this.get(CrawlAttribute.AGENT_NAME.key);
return ClientIdentification.getAgent(agentName);
}
public AtomicInteger getCount(final String domain) {
if (domain == null) return new AtomicInteger(0); // in case of file indexing this is required
AtomicInteger dp = this.doms.get(domain);
@@ -436,12 +443,24 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
* @return handle of the profile
*/
public String handle() {
final String r = get(CrawlAttribute.HANDLE.key);
final String r = this.get(CrawlAttribute.HANDLE.key);
assert r != null;
//if (r == null) return null;
return r;
}
public void setHandle() {
final String handle = Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(
this.get(CrawlAttribute.NAME.key) +
this.get(CrawlAttribute.CRAWLER_ORIGIN_URL_MUSTMATCH.key) +
this.get(CrawlAttribute.DEPTH.key) +
this.get(CrawlAttribute.CRAWLER_ORIGIN_URL_MUSTNOTMATCH.key) +
this.domMaxPages() +
this.get(CrawlAttribute.COLLECTIONS.key))).substring(0, Word.commonHashLength);
assert handle != null && handle.length() == Word.commonHashLength : handle + " != " + Word.commonHashLength;
this.put(CrawlAttribute.HANDLE.key, handle);
}
private Map<String, Pattern> cmap = null;
/**
@@ -449,29 +468,33 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
* @return a list of collection names
*/
public Map<String, Pattern> collections() {
if (cmap != null) return cmap;
final String r = get(CrawlAttribute.COLLECTIONS.key);
if (this.cmap != null) return this.cmap;
final String r = this.get(CrawlAttribute.COLLECTIONS.key);
this.cmap = collectionParser(r);
return this.cmap;
}
public static Map<String, Pattern> collectionParser(String collectionString) {
if (collectionString == null || collectionString.length() == 0) return new HashMap<String, Pattern>();
String[] cs = CommonPattern.COMMA.split(collectionString);
final Map<String, Pattern> cm = new LinkedHashMap<String, Pattern>();
for (String c: cs) {
int p = c.indexOf(':');
if (collectionString == null || collectionString.length() == 0) return new HashMap<>();
final String[] cs = CommonPattern.COMMA.split(collectionString);
final Map<String, Pattern> cm = new LinkedHashMap<>();
for (final String c: cs) {
final int p = c.indexOf(':');
if (p < 0) cm.put(c, QueryParams.catchall_pattern); else cm.put(c.substring(0, p), Pattern.compile(c.substring(p + 1)));
}
return cm;
}
public void setCollections(String collectionsList) {
this.put(CrawlAttribute.COLLECTIONS.key, CommonPattern.SPACE.matcher(collectionsList.trim()).replaceAll(""));
}
/**
* Gets the name of the CrawlProfile.
* @return name of the profile
*/
public String name() {
final String r = get(CrawlAttribute.NAME.key);
final String r = this.get(CrawlAttribute.NAME.key);
if (r == null) return "";
return r;
}
@@ -481,8 +504,8 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
* @return the name of the collection if that is not "user" or the name() otherwise;
*/
public String collectionName() {
final String r = get(CrawlAttribute.COLLECTIONS.key);
return r == null || r.length() == 0 || "user".equals(r) ? name() : r;
final String r = this.get(CrawlAttribute.COLLECTIONS.key);
return r == null || r.length() == 0 || "user".equals(r) ? this.name() : r;
}
/**
@@ -491,7 +514,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
*/
public Pattern urlMustMatchPattern() {
if (this.crawlerurlmustmatch == null) {
final String r = get(CrawlAttribute.CRAWLER_URL_MUSTMATCH.key);
final String r = this.get(CrawlAttribute.CRAWLER_URL_MUSTMATCH.key);
try {
this.crawlerurlmustmatch = (r == null || r.equals(CrawlProfile.MATCH_ALL_STRING)) ? CrawlProfile.MATCH_ALL_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
} catch (final PatternSyntaxException e) { this.crawlerurlmustmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -505,11 +528,11 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
* and to prevent exceeding the field size limit for
* CollectionSchema.failreason_s (32k) when the pattern is present in a fail doc
* added to the Solr index.
*
*
* @return the urlMustMatchPattern formatted as a String of limited size
*/
public String formattedUrlMustMatchPattern() {
String patternStr = urlMustMatchPattern().toString();
String patternStr = this.urlMustMatchPattern().toString();
if(patternStr.length() > 1000) {
/* The pattern may be quite large when using the 'From Link-List of URL' crawl start point. */
patternStr = patternStr.substring(0, Math.min(patternStr.length(), 1000)) + "...";
@@ -523,7 +546,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
*/
public Pattern urlMustNotMatchPattern() {
if (this.crawlerurlmustnotmatch == null) {
final String r = get(CrawlAttribute.CRAWLER_URL_MUSTNOTMATCH.key);
final String r = this.get(CrawlAttribute.CRAWLER_URL_MUSTNOTMATCH.key);
try {
this.crawlerurlmustnotmatch = (r == null || r.equals(CrawlProfile.MATCH_NEVER_STRING)) ? CrawlProfile.MATCH_NEVER_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
} catch (final PatternSyntaxException e) { this.crawlerurlmustnotmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -533,7 +556,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
/**
* Get the pattern on the URL a document must match to allow adding its embedded links to the crawl stack
*
*
* @return a {@link Pattern} instance, defaulting to
* {@link CrawlProfile#MATCH_ALL_PATTERN} when the regular expression
* string is not set or its syntax is incorrect
@@ -541,7 +564,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
public Pattern getCrawlerOriginUrlMustMatchPattern() {
if (this.crawlerOriginUrlMustMatch == null) {
/* Cache the compiled pattern for faster next calls */
final String patternStr = get(CrawlAttribute.CRAWLER_ORIGIN_URL_MUSTMATCH.key);
final String patternStr = this.get(CrawlAttribute.CRAWLER_ORIGIN_URL_MUSTMATCH.key);
try {
this.crawlerOriginUrlMustMatch = (patternStr == null
|| patternStr.equals(CrawlProfile.MATCH_ALL_STRING)) ? CrawlProfile.MATCH_ALL_PATTERN
@@ -555,7 +578,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
/**
* Get the pattern on the URL a document must not match to allow adding its embedded links to the crawl stack
*
*
* @return a {@link Pattern} instance, defaulting to
* {@link CrawlProfile#MATCH_NEVER_PATTERN} when the regular expression
* string is not set or its syntax is incorrect
@@ -563,7 +586,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
public Pattern getCrawlerOriginUrlMustNotMatchPattern() {
if (this.crawlerOriginUrlMustNotMatch == null) {
/* Cache the compiled pattern for faster next calls */
final String patternStr = get(CrawlAttribute.CRAWLER_ORIGIN_URL_MUSTNOTMATCH.key);
final String patternStr = this.get(CrawlAttribute.CRAWLER_ORIGIN_URL_MUSTNOTMATCH.key);
try {
this.crawlerOriginUrlMustNotMatch = (patternStr == null
|| patternStr.equals(CrawlProfile.MATCH_NEVER_STRING)) ? CrawlProfile.MATCH_NEVER_PATTERN
@@ -581,7 +604,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
*/
public Pattern ipMustMatchPattern() {
if (this.crawleripmustmatch == null) {
final String r = get(CrawlAttribute.CRAWLER_IP_MUSTMATCH.key);
final String r = this.get(CrawlAttribute.CRAWLER_IP_MUSTMATCH.key);
try {
this.crawleripmustmatch = (r == null || r.equals(CrawlProfile.MATCH_ALL_STRING)) ? CrawlProfile.MATCH_ALL_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
} catch (final PatternSyntaxException e) { this.crawleripmustmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -595,7 +618,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
*/
public Pattern ipMustNotMatchPattern() {
if (this.crawleripmustnotmatch == null) {
final String r = get(CrawlAttribute.CRAWLER_IP_MUSTNOTMATCH.key);
final String r = this.get(CrawlAttribute.CRAWLER_IP_MUSTNOTMATCH.key);
try {
this.crawleripmustnotmatch = (r == null || r.equals(CrawlProfile.MATCH_NEVER_STRING)) ? CrawlProfile.MATCH_NEVER_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
} catch (final PatternSyntaxException e) { this.crawleripmustnotmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -608,7 +631,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
* @return a list of country codes
*/
public String[] countryMustMatchList() {
String countryMustMatch = get(CrawlAttribute.CRAWLER_COUNTRY_MUSTMATCH.key);
String countryMustMatch = this.get(CrawlAttribute.CRAWLER_COUNTRY_MUSTMATCH.key);
if (countryMustMatch == null) countryMustMatch = CrawlProfile.MATCH_NEVER_STRING;
if (countryMustMatch.isEmpty()) return new String[0];
String[] list = CommonPattern.COMMA.split(countryMustMatch);
@@ -622,7 +645,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
*/
public Pattern crawlerNoDepthLimitMatchPattern() {
if (this.crawlernodepthlimitmatch == null) {
final String r = get(CrawlAttribute.CRAWLER_URL_NODEPTHLIMITMATCH.key);
final String r = this.get(CrawlAttribute.CRAWLER_URL_NODEPTHLIMITMATCH.key);
try {
this.crawlernodepthlimitmatch = (r == null || r.equals(CrawlProfile.MATCH_NEVER_STRING)) ? CrawlProfile.MATCH_NEVER_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
} catch (final PatternSyntaxException e) { this.crawlernodepthlimitmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -636,7 +659,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
*/
public Pattern indexUrlMustMatchPattern() {
if (this.indexurlmustmatch == null) {
final String r = get(CrawlAttribute.INDEXING_URL_MUSTMATCH.key);
final String r = this.get(CrawlAttribute.INDEXING_URL_MUSTMATCH.key);
try {
this.indexurlmustmatch = (r == null || r.equals(CrawlProfile.MATCH_ALL_STRING)) ? CrawlProfile.MATCH_ALL_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
} catch (final PatternSyntaxException e) { this.indexurlmustmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -650,7 +673,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
*/
public Pattern indexUrlMustNotMatchPattern() {
if (this.indexurlmustnotmatch == null) {
final String r = get(CrawlAttribute.INDEXING_URL_MUSTNOTMATCH.key);
final String r = this.get(CrawlAttribute.INDEXING_URL_MUSTNOTMATCH.key);
try {
this.indexurlmustnotmatch = (r == null || r.equals(CrawlProfile.MATCH_NEVER_STRING)) ? CrawlProfile.MATCH_NEVER_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
} catch (final PatternSyntaxException e) { this.indexurlmustnotmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -664,7 +687,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
*/
public Pattern indexContentMustMatchPattern() {
if (this.indexcontentmustmatch == null) {
final String r = get(CrawlAttribute.INDEXING_CONTENT_MUSTMATCH.key);
final String r = this.get(CrawlAttribute.INDEXING_CONTENT_MUSTMATCH.key);
try {
this.indexcontentmustmatch = (r == null || r.equals(CrawlProfile.MATCH_ALL_STRING)) ? CrawlProfile.MATCH_ALL_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
} catch (final PatternSyntaxException e) { this.indexcontentmustmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -678,7 +701,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
*/
public Pattern indexContentMustNotMatchPattern() {
if (this.indexcontentmustnotmatch == null) {
final String r = get(CrawlAttribute.INDEXING_CONTENT_MUSTNOTMATCH.key);
final String r = this.get(CrawlAttribute.INDEXING_CONTENT_MUSTNOTMATCH.key);
try {
this.indexcontentmustnotmatch = (r == null || r.equals(CrawlProfile.MATCH_NEVER_STRING)) ? CrawlProfile.MATCH_NEVER_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
} catch (final PatternSyntaxException e) { this.indexcontentmustnotmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -688,7 +711,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
/**
* Get the Pattern on media type that documents must match in order to be indexed
*
*
* @return a {@link Pattern} instance, defaulting to
* {@link CrawlProfile#MATCH_ALL_PATTERN} when the regular expression
* string is not set or its syntax is incorrect
@@ -696,7 +719,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
public Pattern getIndexMediaTypeMustMatchPattern() {
if (this.indexMediaTypeMustMatch == null) {
/* Cache the compiled pattern for faster next calls */
final String patternStr = get(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTMATCH.key);
final String patternStr = this.get(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTMATCH.key);
try {
this.indexMediaTypeMustMatch = (patternStr == null
|| patternStr.equals(CrawlProfile.MATCH_ALL_STRING)) ? CrawlProfile.MATCH_ALL_PATTERN
@@ -710,7 +733,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
/**
* Get the Pattern on media type that documents must not match in order to be indexed
*
*
* @return a {@link Pattern} instance, defaulting to
* {@link CrawlProfile#MATCH_NEVER_PATTERN} when the regular expression
* string is not set or its syntax is incorrect
@@ -718,7 +741,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
public Pattern getIndexMediaTypeMustNotMatchPattern() {
if (this.indexMediaTypeMustNotMatch == null) {
/* Cache the compiled pattern for faster next calls */
final String patternStr = get(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTNOTMATCH.key);
final String patternStr = this.get(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTNOTMATCH.key);
try {
this.indexMediaTypeMustNotMatch = (patternStr == null
|| patternStr.equals(CrawlProfile.MATCH_NEVER_STRING)) ? CrawlProfile.MATCH_NEVER_PATTERN
@@ -736,7 +759,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
* @return depth of crawl job
*/
public int depth() {
final String r = get(CrawlAttribute.DEPTH.key);
final String r = this.get(CrawlAttribute.DEPTH.key);
if (r == null) return 0;
try {
return Integer.parseInt(r);
@@ -751,7 +774,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
* be indexed as links (with metadata only on URL and not on content).
*/
public boolean isIndexNonParseableUrls() {
final String r = get(CrawlAttribute.DIRECT_DOC_BY_URL.key);
final String r = this.get(CrawlAttribute.DIRECT_DOC_BY_URL.key);
if (r == null) return false;
return (r.equals(Boolean.TRUE.toString()));
}
@@ -763,7 +786,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
* with an unknown or unsupported file extension.
*/
public boolean isCrawlerAlwaysCheckMediaType() {
final String r = get(CrawlAttribute.CRAWLER_ALWAYS_CHECK_MEDIA_TYPE.key);
final String r = this.get(CrawlAttribute.CRAWLER_ALWAYS_CHECK_MEDIA_TYPE.key);
if (r == null) {
return false;
}
@@ -771,7 +794,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
}
public CacheStrategy cacheStrategy() {
final String r = get(CrawlAttribute.CACHE_STRAGEGY.key);
final String r = this.get(CrawlAttribute.CACHE_STRAGEGY.key);
if (r == null) return CacheStrategy.IFEXIST;
try {
return CacheStrategy.decode(Integer.parseInt(r));
@@ -782,7 +805,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
}
public void setCacheStrategy(final CacheStrategy newStrategy) {
put(CrawlAttribute.CACHE_STRAGEGY.key, newStrategy.toString());
this.put(CrawlAttribute.CACHE_STRAGEGY.key, newStrategy.toString());
}
/**
@@ -792,7 +815,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
public long recrawlIfOlder() {
// returns a long (millis) that is the minimum age that
// an entry must have to be re-crawled
final String r = get(CrawlAttribute.RECRAWL_IF_OLDER.key);
final String r = this.get(CrawlAttribute.RECRAWL_IF_OLDER.key);
if (r == null) return 0L;
try {
final long l = Long.parseLong(r);
@@ -806,7 +829,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
public int domMaxPages() {
// this is the maximum number of pages that are crawled for a single domain
// if -1, this means no limit
final String r = get(CrawlAttribute.DOM_MAX_PAGES.key);
final String r = this.get(CrawlAttribute.DOM_MAX_PAGES.key);
if (r == null) return Integer.MAX_VALUE;
try {
final int i = Integer.parseInt(r);
@@ -819,61 +842,61 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
}
public boolean crawlingQ() {
final String r = get(CrawlAttribute.CRAWLING_Q.key);
final String r = this.get(CrawlAttribute.CRAWLING_Q.key);
if (r == null) return false;
return (r.equals(Boolean.TRUE.toString()));
}
public boolean followFrames() {
final String r = get(CrawlAttribute.FOLLOW_FRAMES.key);
final String r = this.get(CrawlAttribute.FOLLOW_FRAMES.key);
if (r == null) return false;
return (r.equals(Boolean.TRUE.toString()));
}
public boolean obeyHtmlRobotsNoindex() {
final String r = get(CrawlAttribute.OBEY_HTML_ROBOTS_NOINDEX.key);
final String r = this.get(CrawlAttribute.OBEY_HTML_ROBOTS_NOINDEX.key);
if (r == null) return false;
return (r.equals(Boolean.TRUE.toString()));
}
public boolean obeyHtmlRobotsNofollow() {
final String r = get(CrawlAttribute.OBEY_HTML_ROBOTS_NOFOLLOW.key);
final String r = this.get(CrawlAttribute.OBEY_HTML_ROBOTS_NOFOLLOW.key);
if (r == null) return false;
return (r.equals(Boolean.TRUE.toString()));
}
public boolean indexText() {
final String r = get(CrawlAttribute.INDEX_TEXT.key);
final String r = this.get(CrawlAttribute.INDEX_TEXT.key);
if (r == null) return true;
return (r.equals(Boolean.TRUE.toString()));
}
public boolean indexMedia() {
final String r = get(CrawlAttribute.INDEX_MEDIA.key);
final String r = this.get(CrawlAttribute.INDEX_MEDIA.key);
if (r == null) return true;
return (r.equals(Boolean.TRUE.toString()));
}
public boolean noindexWhenCanonicalUnequalURL() {
final String r = get(CrawlAttribute.NOINDEX_WHEN_CANONICAL_UNEQUAL_URL.key);
final String r = this.get(CrawlAttribute.NOINDEX_WHEN_CANONICAL_UNEQUAL_URL.key);
if (r == null) return true;
return (r.equals(Boolean.TRUE.toString()));
}
public boolean storeHTCache() {
final String r = get(CrawlAttribute.STORE_HTCACHE.key);
final String r = this.get(CrawlAttribute.STORE_HTCACHE.key);
if (r == null) return false;
return (r.equals(Boolean.TRUE.toString()));
}
public boolean remoteIndexing() {
final String r = get(CrawlAttribute.REMOTE_INDEXING.key);
final String r = this.get(CrawlAttribute.REMOTE_INDEXING.key);
if (r == null) return false;
return (r.equals(Boolean.TRUE.toString()));
}
public int snapshotMaxdepth() {
final String r = get(CrawlAttribute.SNAPSHOTS_MAXDEPTH.key);
final String r = this.get(CrawlAttribute.SNAPSHOTS_MAXDEPTH.key);
if (r == null) return -1;
try {
final int i = Integer.parseInt(r);
@@ -886,20 +909,20 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
}
public boolean snapshotLoadImage() {
final String r = get(CrawlAttribute.SNAPSHOTS_LOADIMAGE.key);
final String r = this.get(CrawlAttribute.SNAPSHOTS_LOADIMAGE.key);
if (r == null) return false;
return (r.equals(Boolean.TRUE.toString()));
}
public boolean snapshotReplaceold() {
final String r = get(CrawlAttribute.SNAPSHOTS_REPLACEOLD.key);
final String r = this.get(CrawlAttribute.SNAPSHOTS_REPLACEOLD.key);
if (r == null) return false;
return (r.equals(Boolean.TRUE.toString()));
}
public Pattern snapshotsMustnotmatch() {
if (this.snapshotsMustnotmatch == null) {
final String r = get(CrawlAttribute.SNAPSHOTS_MUSTNOTMATCH.key);
final String r = this.get(CrawlAttribute.SNAPSHOTS_MUSTNOTMATCH.key);
try {
this.snapshotsMustnotmatch = (r == null || r.equals(CrawlProfile.MATCH_ALL_STRING)) ? CrawlProfile.MATCH_ALL_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
} catch (final PatternSyntaxException e) { this.snapshotsMustnotmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -908,11 +931,11 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
}
public int timezoneOffset() {
final String timezoneOffset = get(CrawlAttribute.TIMEZONEOFFSET.key);
final String timezoneOffset = this.get(CrawlAttribute.TIMEZONEOFFSET.key);
if (timezoneOffset == null) return 0;
try {
return Integer.parseInt(timezoneOffset);
} catch (NumberFormatException e) {
} catch (final NumberFormatException e) {
return 0;
}
}
@@ -949,7 +972,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
}
public static String subpathFilter(final Collection<? extends MultiProtocolURL> urls) {
LinkedHashSet<String> filters = new LinkedHashSet<String>(); // first collect in a set to eliminate doubles
final LinkedHashSet<String> filters = new LinkedHashSet<>(); // first collect in a set to eliminate doubles
for (final MultiProtocolURL url: urls) filters.add(mustMatchSubpath(url));
final StringBuilder filter = new StringBuilder();
for (final String urlfilter: filters) filter.append('|').append(urlfilter);
@@ -976,8 +999,8 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
final boolean dark,
final int count,
final int domlistlength) {
boolean terminateButton = active && !CrawlSwitchboard.DEFAULT_PROFILES.contains(this.name());
boolean deleteButton = !active;
final boolean terminateButton = active && !CrawlSwitchboard.DEFAULT_PROFILES.contains(this.name());
final boolean deleteButton = !active;
prop.put(CRAWL_PROFILE_PREFIX + count + "_status", terminateButton ? 1 : deleteButton ? 0 : 2);
prop.put(CRAWL_PROFILE_PREFIX + count + "_terminateButton", terminateButton);
prop.put(CRAWL_PROFILE_PREFIX + count + "_terminateButton_handle", this.handle());
@@ -1006,7 +1029,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
prop.putXML(CRAWL_PROFILE_PREFIX + count + "_indexURLMustNotMatch", this.get(CrawlAttribute.INDEXING_URL_MUSTNOTMATCH.key));
prop.putXML(CRAWL_PROFILE_PREFIX + count + "_indexContentMustMatch", this.get(CrawlAttribute.INDEXING_CONTENT_MUSTMATCH.key));
prop.putXML(CRAWL_PROFILE_PREFIX + count + "_indexContentMustNotMatch", this.get(CrawlAttribute.INDEXING_CONTENT_MUSTNOTMATCH.key));
prop.put(CRAWL_PROFILE_PREFIX + count + "_" + CrawlAttribute.NOINDEX_WHEN_CANONICAL_UNEQUAL_URL, noindexWhenCanonicalUnequalURL() ? 1 : 0);
prop.put(CRAWL_PROFILE_PREFIX + count + "_" + CrawlAttribute.NOINDEX_WHEN_CANONICAL_UNEQUAL_URL, this.noindexWhenCanonicalUnequalURL() ? 1 : 0);
prop.putXML(CRAWL_PROFILE_PREFIX + count + "_" + CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTMATCH.key, this.get(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTMATCH.key));
prop.putXML(CRAWL_PROFILE_PREFIX + count + "_" + CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTNOTMATCH.key, this.get(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTNOTMATCH.key));
prop.putXML(CRAWL_PROFILE_PREFIX + count + "_" + CrawlAttribute.INDEXING_SOLR_QUERY_MUSTMATCH.key, this.get(CrawlAttribute.INDEXING_SOLR_QUERY_MUSTMATCH.key));
@@ -1042,19 +1065,19 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
public static void main(String[] args) {
// test to convert the key set from set to string and back
Set<String> a = new HashSet<>();
final Set<String> a = new HashSet<>();
a.add("eins"); a.add("zwei"); a.add("drei");
JSONArray j = new JSONArray(a);
String s = j.toString();
final String s = j.toString();
System.out.println(s);
JSONTokener o = new JSONTokener(s);
final JSONTokener o = new JSONTokener(s);
try {
j = new JSONArray(o);
System.out.println(j);
Set<String> h = new HashSet<String>();
final Set<String> h = new HashSet<>();
for (int i = 0; i < j.length(); i++) h.add(j.getString(i));
System.out.println(h);
} catch (JSONException e) {
} catch (final JSONException e) {
e.printStackTrace();
}
}
source/net/yacy/document/importer/WarcImporter.java
+23 -9
View File
@@ -22,6 +22,7 @@
*/
package net.yacy.document.importer;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
@@ -44,6 +45,7 @@ import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.ResponseHeader;
import net.yacy.cora.util.ByteBuffer;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.crawler.data.CrawlProfile;
import net.yacy.crawler.retrieval.Request;
import net.yacy.crawler.retrieval.Response;
import net.yacy.document.TextParser;
@@ -83,23 +85,31 @@ public class WarcImporter extends Thread implements Importer {
private final long sourceSize; // length of the input source (for statistic)
private long consumed; // bytes consumed from input source (for statistic)
private boolean abort = false; // flag to signal stop of import
private String collection;
public WarcImporter(MultiProtocolURL url) throws IOException {
public WarcImporter(MultiProtocolURL url, String collection) throws IOException {
super("WarcImporter - from InputStream");
this.recordCnt = 0;
this.sourceSize = -1;
this.name = url.toNormalform(true);
this.source = url.getInputStream(ClientIdentification.yacyInternetCrawlerAgent);
if (this.name.endsWith(".gz")) this.source = new GZIPInputStream(this.source);
this.collection = collection;
}
public WarcImporter(File f) throws IOException {
public WarcImporter(File f, byte[] data, String collection) throws IOException {
super("WarcImporter - from file " + f.getName());
this.name = f.getName();
this.sourceSize = f.length();
this.source = new FileInputStream(f);
if (this.name.endsWith(".gz")) this.source = new GZIPInputStream(this.source);
}
if (!f.exists() && data != null) {
this.sourceSize = data.length;
this.source = new ByteArrayInputStream(data);
} else {
this.sourceSize = f.length();
this.source = new FileInputStream(f);
if (this.name.endsWith(".gz")) this.source = new GZIPInputStream(this.source);
}
this.collection = collection;
}
/**
* Reads a Warc file and adds all contained responses to the index.
@@ -115,6 +125,10 @@ public class WarcImporter extends Thread implements Importer {
job = this;
this.startTime = System.currentTimeMillis();
final CrawlProfile warcProfile = (CrawlProfile) Switchboard.getSwitchboard().crawler.defaultPackProfile.clone();
warcProfile.setCollections(this.collection);
warcProfile.setHandle();
WarcReader localwarcReader = WarcReaderFactory.getReader(f);
WarcRecord wrec = localwarcReader.getNextRecord();
while (wrec != null && !this.abort) {
@@ -165,15 +179,15 @@ public class WarcImporter extends Thread implements Importer {
requestHeader.referer() == null ? null : requestHeader.referer().hash(),
"warc",
responseHeader.lastModified(),
Switchboard.getSwitchboard().crawler.defaultPackProfile.handle(),
warcProfile.handle(),
0,
Switchboard.getSwitchboard().crawler.defaultPackProfile.timezoneOffset());
warcProfile.timezoneOffset());
final Response response = new Response(
request,
requestHeader,
responseHeader,
Switchboard.getSwitchboard().crawler.defaultPackProfile,
warcProfile,
false,
content
);
source/net/yacy/document/importer/ZimImporter.java
+93 -75
View File
@@ -37,6 +37,12 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import org.apache.commons.io.FileUtils;
import org.openzim.ZIMFile;
import org.openzim.ZIMReader;
import org.openzim.ZIMReader.ArticleEntry;
import org.openzim.ZIMReader.DirectoryEntry;
import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.protocol.ClientIdentification;
@@ -44,16 +50,12 @@ import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.ResponseHeader;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.crawler.data.CrawlProfile;
import net.yacy.crawler.retrieval.Request;
import net.yacy.crawler.retrieval.Response;
import net.yacy.document.TextParser;
import net.yacy.search.Switchboard;
import org.openzim.ZIMFile;
import org.openzim.ZIMReader;
import org.openzim.ZIMReader.ArticleEntry;
import org.openzim.ZIMReader.DirectoryEntry;
/**
* ZIM importer
* can import ZIM file i.e. from https://download.kiwix.org/zim/ or mirrors like https://ftp.fau.de/kiwix/zim/
@@ -69,10 +71,12 @@ public class ZimImporter extends Thread implements Importer {
static public ZimImporter job;
private ZIMFile file;
private final ZIMFile file;
private ZIMReader reader;
private String path;
private final String path;
private String guessedSource;
private final byte[] data;
private final String collection;
private int recordCnt;
private long startTime;
@@ -80,62 +84,75 @@ public class ZimImporter extends Thread implements Importer {
private long consumed;
private boolean abort = false;
public ZimImporter(String path) throws IOException {
public ZimImporter(String path, byte[] data, String collection) throws IOException {
super("ZimImporter - from file " + path);
this.path = path;
this.file = new ZIMFile(this.path); // this will read already some of the metadata and could consume some time
this.sourceSize = this.file.length();
this.data = data;
File zimFilePath = new File(path);
if (!zimFilePath.exists() && this.data != null) {
File tempFile = File.createTempFile(zimFilePath.getName().substring(0, zimFilePath.getName().length() - 4), "zim");
FileUtils.writeByteArrayToFile(tempFile, data);
this.file = new ZIMFile(tempFile.getPath());
} else {
this.file = new ZIMFile(this.path); // this will read already some of the metadata and could consume some time
}
this.sourceSize = data == null ? this.file.length() : data.length;
this.collection = collection;
}
@Override
public void run() {
job = this;
this.startTime = System.currentTimeMillis();
Switchboard sb = Switchboard.getSwitchboard();
final Switchboard sb = Switchboard.getSwitchboard();
try {
this.reader = new ZIMReader(this.file);
this.guessedSource = getSource(this.reader);
Date guessedDate = getDate(this.reader);
String dates = HeaderFramework.newRfc1123Format().format(guessedDate);
this.guessedSource = getSource(this.reader, this.file.getName());
final Date guessedDate = getDate(this.reader);
final String dates = HeaderFramework.newRfc1123Format().format(guessedDate);
// verify the source
DirectoryEntry mainEntry = this.reader.getMainDirectoryEntry();
DigestURL mainURL = guessURL(this.guessedSource, mainEntry);
if (!mainURL.exists(ClientIdentification.browserAgent)) {
sb.log.info("zim importer: file " + this.file.getName() + " failed main url existence test: " + mainURL);
return;
}
final DirectoryEntry mainEntry = this.reader.getMainDirectoryEntry();
final DigestURL mainURL = guessURL(this.guessedSource, mainEntry);
//if (!mainURL.exists(ClientIdentification.browserAgent)) {
// sb.log.info("zim importer: file " + this.file.getName() + " failed main url existence test: " + mainURL);
// return;
//}
final CrawlProfile zimProfile = (CrawlProfile) Switchboard.getSwitchboard().crawler.defaultPackProfile.clone();
zimProfile.setCollections(this.collection);
zimProfile.setHandle();
// read all documents
for (int i = 0; i < this.file.header_entryCount; i++) {
try {
if (this.abort) break;
DirectoryEntry de = this.reader.getDirectoryInfo(i);
final DirectoryEntry de = this.reader.getDirectoryInfo(i);
if (!(de instanceof ZIMReader.ArticleEntry)) continue;
ArticleEntry ae = (ArticleEntry) de;
final ArticleEntry ae = (ArticleEntry) de;
if (ae.namespace != 'C' && ae.namespace != 'A') continue;
// check url
DigestURL guessedUrl = guessURL(this.guessedSource, de);
if (recordCnt < 10) {
final DigestURL guessedUrl = guessURL(this.guessedSource, de);
//if (this.recordCnt < 10) {
// critical test for the first 10 urls
if (!guessedUrl.exists(ClientIdentification.browserAgent)) {
sb.log.info("zim importer: file " + this.file.getName() + " failed url " + recordCnt + " existence test: " + guessedUrl);
return;
}
}
// if (!guessedUrl.exists(ClientIdentification.browserAgent)) {
// sb.log.info("zim importer: file " + this.file.getName() + " failed url " + this.recordCnt + " existence test: " + guessedUrl);
// return;
// }
//}
// check availability of text parser
String mimeType = ae.getMimeType();
final String mimeType = ae.getMimeType();
if (!mimeType.startsWith("text/") && !mimeType.equals("application/epub+zip")) continue; // in this import we want only text, not everything that is possible
if (TextParser.supportsMime(mimeType) != null) continue;
// read the content
byte[] b = this.reader.getArticleData(ae);
final byte[] b = this.reader.getArticleData(ae);
// create artificial request and response headers for the indexer
RequestHeader requestHeader = new RequestHeader();
ResponseHeader responseHeader = new ResponseHeader(200);
final RequestHeader requestHeader = new RequestHeader();
final ResponseHeader responseHeader = new ResponseHeader(200);
responseHeader.put(HeaderFramework.CONTENT_TYPE, de.getMimeType()); // very important to tell parser which kind of content
responseHeader.put(HeaderFramework.LAST_MODIFIED, dates); // put in the guessd date to have something that is not the current date
final Request request = new Request(
@@ -144,29 +161,29 @@ public class ZimImporter extends Thread implements Importer {
null, // referrerhash the hash of the referrer URL
de.title, // name the name of the document to crawl
null, // appdate the time when the url was first time appeared
sb.crawler.defaultPackProfile.handle(), // profileHandle the name of the prefetch profile. This must not be null!
zimProfile.handle(), // profileHandle the name of the prefetch profile. This must not be null!
0, // depth the crawling depth of the entry
sb.crawler.defaultPackProfile.timezoneOffset() // timezone offset
zimProfile.timezoneOffset() // timezone offset
);
final Response response = new Response(
request,
requestHeader,
responseHeader,
Switchboard.getSwitchboard().crawler.defaultPackProfile,
zimProfile,
false,
b
);
// throw this to the indexer
String error = sb.toIndexer(response);
final String error = sb.toIndexer(response);
if (error != null) ConcurrentLog.info("ZimImporter", "error parsing: " + error);
this.recordCnt++;
} catch (Exception e) {
} catch (final Exception e) {
// catch any error that could stop the importer
ConcurrentLog.info("ZimImporter", "error loading: " + e.getMessage());
}
}
} catch (IOException e) {
} catch (final IOException e) {
ConcurrentLog.info("ZimImporter", "error reading: " + e.getMessage());
}
ConcurrentLog.info("ZimImporter", "Indexed " + this.recordCnt + " documents");
@@ -190,7 +207,7 @@ public class ZimImporter extends Thread implements Importer {
@Override
public int speed() {
if (this.recordCnt == 0) return 0;
return (int) (this.recordCnt / Math.max(0L, runningTime() ));
return (int) (this.recordCnt / Math.max(0L, this.runningTime() ));
}
@Override
@@ -203,7 +220,7 @@ public class ZimImporter extends Thread implements Importer {
if (this.consumed == 0) {
return 0;
}
long speed = this.consumed / runningTime();
final long speed = this.consumed / this.runningTime();
return (this.sourceSize - this.consumed) / speed;
}
@@ -217,11 +234,11 @@ public class ZimImporter extends Thread implements Importer {
return null; // Handle null or empty input
}
String[] parts = fileName.split("_");
final String[] parts = fileName.split("_");
if (parts.length == 0) {
return null;
}
String firstPart = parts[0];
final String firstPart = parts[0];
// Handling special cases where the domain name might not be obvious
// These are based on your provided list and can be expanded as needed
@@ -293,8 +310,8 @@ public class ZimImporter extends Thread implements Importer {
// Handling domain patterns
if (firstPart.contains(".stackexchange.com")) {
return firstPart;
} else if (firstPart.endsWith(".com") || firstPart.endsWith(".org") || firstPart.endsWith(".de") ||
firstPart.endsWith(".fr") || firstPart.endsWith(".pt") || firstPart.endsWith(".it") ||
} else if (firstPart.endsWith(".com") || firstPart.endsWith(".org") || firstPart.endsWith(".de") ||
firstPart.endsWith(".fr") || firstPart.endsWith(".pt") || firstPart.endsWith(".it") ||
firstPart.endsWith(".ja") || firstPart.endsWith(".es") || firstPart.endsWith(".eo")) {
return firstPart;
} else if (firstPart.contains("-")) {
@@ -303,7 +320,7 @@ public class ZimImporter extends Thread implements Importer {
// Additional general domain extraction logic
if (firstPart.contains(".")) {
int lastDotIndex = firstPart.lastIndexOf('.');
final int lastDotIndex = firstPart.lastIndexOf('.');
if (lastDotIndex > 0 && lastDotIndex < firstPart.length() - 1) {
// Extract up to the next character beyond the TLD, to support TLDs of variable length
int endIndex = firstPart.indexOf('.', lastDotIndex + 1);
@@ -318,19 +335,20 @@ public class ZimImporter extends Thread implements Importer {
return null;
}
public static String getSource(ZIMReader r) throws IOException {
public static String getSource(ZIMReader r, String fileName) throws IOException {
String source = r.getMetadata("Source");
if (source != null) return source;
source = "https://" + guessDomainName(r.getZIMFile().getName()) + "/";
if (fileName == null || fileName.length() == 0) fileName = r.getZIMFile().getName();
source = "https://" + guessDomainName(fileName) + "/";
return source;
}
public static Date getDate(ZIMReader r) throws IOException {
String date = r.getMetadata("Date");
final String date = r.getMetadata("Date");
if (date != null) try {
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd", Locale.US);
final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd", Locale.US);
return format.parse(date);
} catch (ParseException e) {}
} catch (final ParseException e) {}
// failover situation: use file date
return new Date(r.getZIMFile().lastModified());
}
@@ -491,47 +509,47 @@ public class ZimImporter extends Thread implements Importer {
};
public static void main(String[] args) {
Set<String> skip = new HashSet<>();
for (String s: skip_files) skip.add(s);
final Set<String> skip = new HashSet<>();
for (final String s: skip_files) skip.add(s);
// zim file import test
// will test mostly if domain names are included in zim file urls
String zimFilesPath = args[0];
File zimFiles = new File(zimFilesPath);
final String zimFilesPath = args[0];
final File zimFiles = new File(zimFilesPath);
// make ordered file list; order by file size (start with smallest)
String[] filelist = zimFiles.list();
Map<Long, File> orderedFileMap = new TreeMap<>();
final String[] filelist = zimFiles.list();
final Map<Long, File> orderedFileMap = new TreeMap<>();
for (int i = 0; i < filelist.length; i++) {
if (!filelist[i].endsWith(".zim")) continue;
File f = new File(zimFiles, filelist[i]);
final File f = new File(zimFiles, filelist[i]);
orderedFileMap.put(f.length() * 1000 + i, f);
}
Collection<File> orderedFiles = orderedFileMap.values();
Set<String> files_ok = new LinkedHashSet<>();
Set<String> files_nok = new LinkedHashSet<>();
for (File f: orderedFiles) {
final Collection<File> orderedFiles = orderedFileMap.values();
final Set<String> files_ok = new LinkedHashSet<>();
final Set<String> files_nok = new LinkedHashSet<>();
for (final File f: orderedFiles) {
if (skip.contains(f.getName())) continue;
try {
ZIMFile z = new ZIMFile(f.getAbsolutePath());
ZIMReader r = new ZIMReader(z);
DirectoryEntry de = r.getMainDirectoryEntry();
final ZIMFile z = new ZIMFile(f.getAbsolutePath());
final ZIMReader r = new ZIMReader(z);
final DirectoryEntry de = r.getMainDirectoryEntry();
System.out.println("ZIM file: " + f.getAbsolutePath());
for (String key: ZIMReader.METADATA_KEYS) {String s = r.getMetadata(key); if (s != null) System.out.println("Metadata " + key + ": " + s);};
for (final String key: ZIMReader.METADATA_KEYS) {final String s = r.getMetadata(key); if (s != null) System.out.println("Metadata " + key + ": " + s);}
System.out.println("Namespace: " + de.namespace);
System.out.println("Title: " + de.title);
System.out.println("URL: " + de.url);
System.out.println("Mime Type " + de.getMimeType());
System.out.println("guessed domain: " + guessDomainName(f.getName())); // uses a table and rules that deduces a source from the file name
String source = getSource(r);
final String source = getSource(r, null);
System.out.println("guessed Source: " + source); // this uses metadata stored in the zim file
DigestURL mainURL = guessURL(source, de);
final DigestURL mainURL = guessURL(source, de);
System.out.println("guessed main article: " + mainURL);
boolean ok = mainURL.exists(ClientIdentification.browserAgent);
final boolean ok = mainURL.exists(ClientIdentification.browserAgent);
System.out.println("main article exists: " + ok);
if (ok) files_ok.add(f.getName()); else files_nok.add(f.getName());
System.out.println();
} catch (IOException e) {
} catch (final IOException e) {
e.printStackTrace();
}
}
source/net/yacy/htroot/IndexImportWarc_p.java
+7 -3
View File
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.net.MalformedURLException;
import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.order.Base64Order;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.document.importer.WarcImporter;
import net.yacy.server.serverObjects;
@@ -52,11 +53,14 @@ public class IndexImportWarc_p {
if (post != null) {
if (post.containsKey("file") || post.containsKey("url")) {
final String filename = post.get("file");
final String collection = post.get("collection", "user");
final String data64 = post.get("file$file", null); // file uploads are all base64-encoded in YaCyDefaultServlet.parseMultipart
final byte[] data = data64 == null ? null : Base64Order.standardCoder.decode(data64);
if (filename != null && filename.length() > 0) {
final File sourcefile = new File(filename);
if (sourcefile.exists()) {
if (sourcefile.exists() || data != null) {
try {
final WarcImporter wi = new WarcImporter(sourcefile);
final WarcImporter wi = new WarcImporter(sourcefile, data, collection);
wi.start();
prop.put("import_thread", "started");
} catch (final IOException ex) {
@@ -73,7 +77,7 @@ public class IndexImportWarc_p {
if (urlstr != null && urlstr.length() > 0) {
try {
final MultiProtocolURL url = new MultiProtocolURL(urlstr);
final WarcImporter wi = new WarcImporter(url);
final WarcImporter wi = new WarcImporter(url, collection);
wi.start();
prop.put("import_thread", "started");
} catch (final MalformedURLException ex) {
source/net/yacy/htroot/IndexImportZim_p.java
+12 -7
View File
@@ -21,6 +21,7 @@ package net.yacy.htroot;
import java.io.File;
import java.io.IOException;
import net.yacy.cora.order.Base64Order;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.document.importer.ZimImporter;
import net.yacy.server.serverObjects;
@@ -29,16 +30,16 @@ import net.yacy.server.serverSwitch;
public class IndexImportZim_p {
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader request, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
// read multipart data from post request
final serverObjects prop = new serverObjects();
if (ZimImporter.job != null && ZimImporter.job.isAlive()) {
// one import is running, no option to insert anything
prop.put("import", 1);
prop.put("import_thread", "running");
prop.put("import_warcfile", ZimImporter.job.source());
prop.put("import_zimfile", ZimImporter.job.source());
prop.put("import_count", ZimImporter.job.count());
prop.put("import_speed", ZimImporter.job.speed());
prop.put("import_runningHours", (ZimImporter.job.runningTime() / 60) / 60);
@@ -51,13 +52,17 @@ public class IndexImportZim_p {
} else {
prop.put("import", 0);
if (post != null) {
//for (final String s: post.keySet()) System.out.println("key: " + s); // print all post key attributes
if (post.containsKey("file")) {
final String filename = post.get("file");
final String collection = post.get("collection", "user");
final String data64 = post.get("file$file", null); // file uploads are all base64-encoded in YaCyDefaultServlet.parseMultipart
final byte[] data = data64 == null ? null : Base64Order.standardCoder.decode(data64);
if (filename != null && filename.length() > 0) {
final File sourcefile = new File(filename);
if (sourcefile.exists()) {
if (data != null || sourcefile.exists()) {
try {
final ZimImporter zi = new ZimImporter(sourcefile.getAbsolutePath());
final ZimImporter zi = new ZimImporter(sourcefile.getAbsolutePath(), data, collection);
zi.start();
prop.put("import_thread", "started");
} catch (final IOException ex) {
@@ -70,7 +75,7 @@ public class IndexImportZim_p {
prop.put("import_thread", "Error: file not found [" + filename + "]");
}
}
prop.put("import_count", 0);
prop.put("import_speed", 0);
prop.put("import_runningHours", 0);
source/net/yacy/http/servlets/YaCyDefaultServlet.java
+36 -36
View File
@@ -147,7 +147,7 @@ public class YaCyDefaultServlet extends HttpServlet {
this._htDocsPath = sb.htDocsPath;
this._htLocalePath = sb.getDataPath("locale.translated_html", "DATA/LOCALE/htroot");
this._servletContext = getServletContext();
this._servletContext = this.getServletContext();
this._mimeTypes = new MimeTypes();
final String tmpstr = this.getServletContext().getInitParameter("welcomeFile");
@@ -156,12 +156,12 @@ public class YaCyDefaultServlet extends HttpServlet {
} else {
this._welcomes = new String[]{tmpstr,"index.html"};
}
this._acceptRanges = getInitBoolean("acceptRanges", this._acceptRanges);
this._dirAllowed = getInitBoolean("dirAllowed", this._dirAllowed);
this._acceptRanges = this.getInitBoolean("acceptRanges", this._acceptRanges);
this._dirAllowed = this.getInitBoolean("dirAllowed", this._dirAllowed);
Resource.setDefaultUseCaches(false); // caching is handled internally (prevent double caching)
final String rb = getInitParameter("resourceBase");
final String rb = this.getInitParameter("resourceBase");
try {
if (rb != null) {
this._resourceBase = Resource.newResource(rb);
@@ -176,12 +176,12 @@ public class YaCyDefaultServlet extends HttpServlet {
if (ConcurrentLog.isFine("FILEHANDLER")) {
ConcurrentLog.fine("FILEHANDLER","YaCyDefaultServlet: resource base = " + this._resourceBase);
}
this.templateMethodCache = new ConcurrentHashMap<String, Method>();
this.templateMethodCache = new ConcurrentHashMap<>();
}
/* ------------------------------------------------------------ */
protected boolean getInitBoolean(final String name, final boolean dft) {
final String value = getInitParameter(name);
final String value = this.getInitParameter(name);
if (value == null || value.length() == 0) {
return dft;
}
@@ -243,7 +243,7 @@ public class YaCyDefaultServlet extends HttpServlet {
// Is this a Range request?
reqRanges = request.getHeaders(HeaderFramework.RANGE);
if (!hasDefinedRange(reqRanges)) {
if (!this.hasDefinedRange(reqRanges)) {
reqRanges = null;
}
}
@@ -261,7 +261,7 @@ public class YaCyDefaultServlet extends HttpServlet {
if (reqRanges == null && !endsWithSlash) {
final int p = pathInContext.lastIndexOf('.');
if (p >= 0) {
final Method rewriteMethod = rewriteMethod(pathInContext);
final Method rewriteMethod = this.rewriteMethod(pathInContext);
if (rewriteMethod != null) {
hasClass = true;
} else {
@@ -276,7 +276,7 @@ public class YaCyDefaultServlet extends HttpServlet {
}
// find resource
resource = getResource(pathInContext);
resource = this.getResource(pathInContext);
if (!hasClass && (resource == null || !resource.exists()) && !pathInContext.contains("..")) {
// try to get this in the alternative htDocsPath
@@ -304,10 +304,10 @@ public class YaCyDefaultServlet extends HttpServlet {
response.sendRedirect(response.encodeRedirectURL(URIUtil.addPaths(this._servletContext.getContextPath(), pathInContext)));
} else {
if (hasClass) { // this is a YaCy servlet, handle the template
handleTemplate(pathInfo, request, response);
this.handleTemplate(pathInfo, request, response);
} else {
if (included || passConditionalHeaders(request, response, resource)) {
sendData(request, response, included, resource, reqRanges);
if (included || this.passConditionalHeaders(request, response, resource)) {
this.sendData(request, response, included, resource, reqRanges);
}
}
}
@@ -332,7 +332,7 @@ public class YaCyDefaultServlet extends HttpServlet {
response.sendRedirect(response.encodeRedirectURL(buf.toString()));
}
} // else look for a welcome file
else if (null != (welcome = getWelcomeFile(pathInContext))) {
else if (null != (welcome = this.getWelcomeFile(pathInContext))) {
ConcurrentLog.fine("FILEHANDLER","welcome={}" + welcome);
// Forward to the index
@@ -345,8 +345,8 @@ public class YaCyDefaultServlet extends HttpServlet {
}
}
} else {
if (included || passConditionalHeaders(request, response, resource)) {
sendDirectory(request, response, resource, pathInContext);
if (included || this.passConditionalHeaders(request, response, resource)) {
this.sendDirectory(request, response, resource, pathInContext);
}
}
}
@@ -366,7 +366,7 @@ public class YaCyDefaultServlet extends HttpServlet {
@Override
protected void doPost(final HttpServletRequest request, final HttpServletResponse response)
throws ServletException, IOException {
doGet(request, response);
this.doGet(request, response);
}
/* ------------------------------------------------------------ */
@@ -399,7 +399,7 @@ public class YaCyDefaultServlet extends HttpServlet {
}
for (final String _welcome : this._welcomes) {
final String welcome_in_context = URIUtil.addPaths(pathInContext, _welcome);
final Resource welcome = getResource(welcome_in_context);
final Resource welcome = this.getResource(welcome_in_context);
if (welcome != null && welcome.exists()) {
return _welcome;
}
@@ -524,7 +524,7 @@ public class YaCyDefaultServlet extends HttpServlet {
if (include) {
resource.writeTo(out, 0, content_length);
} else {
writeHeaders(response, resource, content_length);
this.writeHeaders(response, resource, content_length);
resource.writeTo(out, 0, content_length);
}
} else {
@@ -533,7 +533,7 @@ public class YaCyDefaultServlet extends HttpServlet {
// if there are no satisfiable ranges, send 416 response
if (ranges == null || ranges.isEmpty()) {
writeHeaders(response, resource, content_length);
this.writeHeaders(response, resource, content_length);
response.setStatus(HttpServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE);
response.setHeader(HttpHeader.CONTENT_RANGE.asString(),
InclusiveByteRange.to416HeaderRangeString(content_length));
@@ -547,7 +547,7 @@ public class YaCyDefaultServlet extends HttpServlet {
if (ranges.size() == 1) {
final InclusiveByteRange singleSatisfiableRange = ranges.iterator().next();
final long singleLength = singleSatisfiableRange.getSize();
writeHeaders(response, resource, singleLength);
this.writeHeaders(response, resource, singleLength);
response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
response.setHeader(HttpHeader.CONTENT_RANGE.asString(),
singleSatisfiableRange.toHeaderRangeString(content_length));
@@ -560,7 +560,7 @@ public class YaCyDefaultServlet extends HttpServlet {
// 216 response which does not require an overall
// content-length header
//
writeHeaders(response, resource, -1);
this.writeHeaders(response, resource, -1);
final String mimetype = response.getContentType();
if (mimetype == null) {
ConcurrentLog.warn("FILEHANDLER","YaCyDefaultServlet: Unknown mimetype for " + request.getRequestURI());
@@ -660,7 +660,7 @@ public class YaCyDefaultServlet extends HttpServlet {
}
protected Object invokeServlet(final Method targetMethod, final RequestHeader request, final serverObjects args) throws IllegalArgumentException, IllegalAccessException, InvocationTargetException {
return targetMethod.invoke(null, new Object[]{request, args, Switchboard.getSwitchboard()}); // add switchboard
return targetMethod.invoke(null, request, args, Switchboard.getSwitchboard()); // add switchboard
}
/**
@@ -813,8 +813,8 @@ public class YaCyDefaultServlet extends HttpServlet {
localeSelection = lng;
}
}
final File targetLocalizedFile = getLocalizedFile(target, localeSelection);
final Method targetMethod = rewriteMethod(target);
final File targetLocalizedFile = this.getLocalizedFile(target, localeSelection);
final Method targetMethod = this.rewriteMethod(target);
final String targetExt = target.substring(target.lastIndexOf('.') + 1);
final long now = System.currentTimeMillis();
@@ -844,19 +844,19 @@ public class YaCyDefaultServlet extends HttpServlet {
// standard attributes are just pushed as string
args.put(argName, request.getParameter(argName));
}
final RequestHeader legacyRequestHeader = generateLegacyRequestHeader(request, target, targetExt);
final RequestHeader legacyRequestHeader = this.generateLegacyRequestHeader(request, target, targetExt);
// add multipart-form fields to parameter
if (ServletFileUpload.isMultipartContent(request)) {
parseMultipart(request, args);
this.parseMultipart(request, args);
}
// eof modification to read attribute
Object tmp;
try {
if (args.isEmpty()) {
// yacy servlets typically test for args != null (but not for args .isEmpty())
tmp = invokeServlet(targetMethod, legacyRequestHeader, null);
tmp = this.invokeServlet(targetMethod, legacyRequestHeader, null);
} else {
tmp = invokeServlet(targetMethod, legacyRequestHeader, args);
tmp = this.invokeServlet(targetMethod, legacyRequestHeader, args);
}
} catch(final InvocationTargetException e) {
if(e.getCause() instanceof InvalidURLLicenceException) {
@@ -929,7 +929,7 @@ public class YaCyDefaultServlet extends HttpServlet {
result = RasterPlotter.exportImage(bi, targetExt);
}
updateRespHeadersForImages(target, response);
this.updateRespHeadersForImages(target, response);
final String mimeType = Classification.ext2mime(targetExt, MimeTypes.Type.TEXT_HTML.asString());
response.setContentType(mimeType);
response.setContentLength(result.length());
@@ -942,9 +942,9 @@ public class YaCyDefaultServlet extends HttpServlet {
if (tmp instanceof InputStream) {
/* Images and favicons can also be written directly from an inputStream */
updateRespHeadersForImages(target, response);
this.updateRespHeadersForImages(target, response);
writeInputStream(response, targetExt, (InputStream)tmp);
this.writeInputStream(response, targetExt, (InputStream)tmp);
return;
}
@@ -1007,9 +1007,9 @@ public class YaCyDefaultServlet extends HttpServlet {
if (targetLocalizedFile.exists() && targetLocalizedFile.isFile() && targetLocalizedFile.canRead()) {
sb.setConfig(SwitchboardConstants.SERVER_SERVLETS_CALLED, appendPath(sb.getConfig(SwitchboardConstants.SERVER_SERVLETS_CALLED, ""), target));
sb.setConfig(SwitchboardConstants.SERVER_SERVLETS_CALLED, this.appendPath(sb.getConfig(SwitchboardConstants.SERVER_SERVLETS_CALLED, ""), target));
if (args != null && !args.isEmpty()) {
sb.setConfig("server.servlets.submitted", appendPath(sb.getConfig("server.servlets.submitted", ""), target));
sb.setConfig("server.servlets.submitted", this.appendPath(sb.getConfig("server.servlets.submitted", ""), target));
}
// add the application version, the uptime and the client name to every rewrite table
@@ -1071,7 +1071,7 @@ public class YaCyDefaultServlet extends HttpServlet {
TemplateEngine.writeTemplate(targetLocalizedFile.getName(), fis, bas, templatePatterns);
// handle SSI
parseSSI (bas.toByteArray(),request,response);
this.parseSSI (bas.toByteArray(),request,response);
} finally {
try {
fis.close();
@@ -1258,7 +1258,7 @@ public class YaCyDefaultServlet extends HttpServlet {
InputStream filecontent = null;
try {
filecontent = item.getInputStream();
files.put(new AbstractMap.SimpleEntry<String, byte[]>(item.getFieldName(), FileUtils.read(filecontent)));
files.put(new AbstractMap.SimpleEntry<>(item.getFieldName(), FileUtils.read(filecontent)));
} catch (final IOException e) {
ConcurrentLog.info("FILEHANDLER", e.getMessage());
} finally {
@@ -1271,7 +1271,7 @@ public class YaCyDefaultServlet extends HttpServlet {
final String n = job.getKey();
final byte[] v = job.getValue();
final String filename = args.get(n);
if (filename != null && filename.endsWith(".gz")) {
if (filename != null && (filename.endsWith(".gz") || filename.endsWith(".zip") || filename.endsWith(".warc") || filename.endsWith(".zim"))) {
// transform this value into base64
final String b64 = Base64Order.standardCoder.encode(v);
args.put(n + "$file", b64);
source/net/yacy/search/Switchboard.java
+23 -19
View File
@@ -2118,15 +2118,15 @@ public final class Switchboard extends serverSwitch {
return null;
}
public boolean processPack(final String s) {
final File infile = new File(this.packsLoadPath, s);
public boolean processPack(final String inFileName, String collectionName) {
final File infile = new File(this.packsLoadPath, inFileName);
if ( !infile.exists() || !infile.canWrite() || !infile.canRead() ) {
return false;
}
final File outfile = new File(this.packsLoadedPath, s);
final File outfile = new File(this.packsLoadedPath, inFileName);
//if (outfile.exists()) return false;
boolean moved = false;
if ( s.endsWith("xml.zip") ) {
if ( inFileName.endsWith("xml.zip") ) {
// open the zip file with all the xml files in it
ZipInputStream zis = null;
try {
@@ -2141,7 +2141,7 @@ public final class Switchboard extends serverSwitch {
baos.write(buffer, 0, size);
}
baos.flush();
this.processXMLPack(new ByteArrayInputStream(baos.toByteArray()), entry.getName());
this.processXMLPack(new ByteArrayInputStream(baos.toByteArray()), entry.getName(), collectionName);
baos.close();
if (this.shallTerminate()) break;
}
@@ -2154,9 +2154,9 @@ public final class Switchboard extends serverSwitch {
}
}
return moved;
} else if (s.endsWith(".warc") || s.endsWith(".warc.gz")) {
} else if (inFileName.endsWith(".warc") || inFileName.endsWith(".warc.gz")) {
try {
final WarcImporter wri = new WarcImporter(infile);
final WarcImporter wri = new WarcImporter(infile, null, collectionName);
wri.start();
try {
wri.join();
@@ -2168,9 +2168,9 @@ public final class Switchboard extends serverSwitch {
this.log.warn("IO Error processing warc file " + infile);
}
return moved;
} else if (s.endsWith(".zim")) {
} else if (inFileName.endsWith(".zim")) {
try {
final ZimImporter wri = new ZimImporter(infile.getAbsolutePath());
final ZimImporter wri = new ZimImporter(infile.getAbsolutePath(), null, collectionName);
wri.start();
try {
wri.join();
@@ -2183,16 +2183,16 @@ public final class Switchboard extends serverSwitch {
}
return moved;
} else if (
s.endsWith(".jsonl") || s.endsWith(".jsonl.gz") ||
s.endsWith(".jsonlist") || s.endsWith(".jsonlist.gz") ||
s.endsWith(".flatjson") || s.endsWith(".flatjson.gz")) {
inFileName.endsWith(".jsonl") || inFileName.endsWith(".jsonl.gz") ||
inFileName.endsWith(".jsonlist") || inFileName.endsWith(".jsonlist.gz") ||
inFileName.endsWith(".flatjson") || inFileName.endsWith(".flatjson.gz")) {
return this.processPackJson(infile, outfile);
}
InputStream is = null;
try {
is = new BufferedInputStream(new FileInputStream(infile));
if (s.endsWith(".gz")) is = new GZIPInputStream(is, 65535);
this.processXMLPack(is, infile.getName());
if (inFileName.endsWith(".gz")) is = new GZIPInputStream(is, 65535);
this.processXMLPack(is, infile.getName(), collectionName);
} catch (final IOException e ) {
ConcurrentLog.logException(e);
} finally {
@@ -2244,9 +2244,13 @@ public final class Switchboard extends serverSwitch {
return moved;
}
private void processXMLPack(final InputStream is, final String name) throws IOException {
private void processXMLPack(final InputStream is, final String name, String collection) throws IOException {
final int concurrency = Runtime.getRuntime().availableProcessors();
final CrawlProfile xmlProfile = (CrawlProfile) Switchboard.getSwitchboard().crawler.defaultPackProfile.clone();
xmlProfile.setCollections(collection);
xmlProfile.setHandle();
// start reader thread
final XMLPackReader reader = new XMLPackReader(is, 100, this.crawlStacker, this.index.fulltext().getDefaultConfiguration(), concurrency);
final Thread readerThread = new Thread(reader, name);
@@ -2305,10 +2309,10 @@ public final class Switchboard extends serverSwitch {
null,
"",
entry.getDate(),
Switchboard.this.crawler.defaultPackProfile.handle(),
xmlProfile.handle(),
0,
Switchboard.this.crawler.defaultPackProfile.timezoneOffset());
final Response response = new Response(request, null, null, Switchboard.this.crawler.defaultPackProfile, false, null);
xmlProfile.timezoneOffset());
final Response response = new Response(request, null, null, xmlProfile, false, null);
final IndexingQueueEntry queueEntry =
new IndexingQueueEntry(response, new Document[] {document}, null);
@@ -2401,7 +2405,7 @@ public final class Switchboard extends serverSwitch {
if (this.isPackFile(pack)) {
// read the pack file and store entry in index
if ( this.processPack(pack) ) {
if ( this.processPack(pack, "user") ) {
return true;
}
}