added collection attribute setting for zim and warc imports

Having the collection attribute for those imports enables to use those import methods for YaCy Pack generation which requires the collection name to produce a pack name. Fixed also file upload option through browser for both methods. For large zim/warc files you require large memory settings to be able to cache the uploaded file. You also need large memory requirements in your browser to upload gigabytes of files through multipart/form-data uploads.
2026-10-25 05:54:15 +00:00 · 2025-08-15 10:26:21 -06:00
1 parent a803c6063b
commit 4c953ee8fd
@@ -14,7 +14,7 @@
    #(import)#
    <p>No import thread is running, you can start a new thread here</p>
-    <form action="IndexImportWarc_p.html" method="get" accept-charset="UTF-8">
+    <form action="IndexImportWarc_p.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8">
      <!-- no post method here, we don't want to transmit the whole file, only the path-->
      <fieldset>
        <legend>Warc File Selection: select an warc file (which may be gz compressed)</legend>
@@ -30,6 +30,8 @@
          <dt class="TableCellDark"><label for="url">Url:</label></dt>
          <dd><input name="url" id="url" value="" size="75"/></dd>
          <dt></dt>
          <dt class="TableCellDark"><label for="collection">Collection:</label></dt>
          <dd><input name="collection" id="collection" type="text" value="user" size="75" /></dd>
          <dd><input name="submit" class="btn btn-primary" type="submit" value="Import Warc File" /></dd>
        </dl>
      </fieldset>
@@ -25,6 +25,8 @@
 		<dl>
          <dt class="TableCellDark"><label for="file">File:</label></dt>
          <dd><input name="file" id="file" type="file" value="" size="75" /></dd>
          <dt class="TableCellDark"><label for="collection">Collection:</label></dt>
          <dd><input name="collection" id="collection" type="text" value="user" size="75" /></dd>
          <dt></dt>
          <dd><input name="submit" class="btn btn-primary" type="submit" value="Import ZIM File" /></dd>
        </dl>
@@ -64,7 +64,7 @@ import net.yacy.server.serverObjects;
 /**
 *  this is a simple record structure that hold all properties of a single crawl start
 */
-public class CrawlProfile extends ConcurrentHashMap<String, String> implements Map<String, String> {
+public class CrawlProfile extends ConcurrentHashMap<String, String> implements Map<String, String>, Cloneable {
    private static final long serialVersionUID = 5527325718810703504L;
@@ -151,7 +151,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
        public String toString() {
            return this.key;
        }
-  }
+    }
    private Pattern crawlerurlmustmatch = null, crawlerurlmustnotmatch = null;
@@ -166,7 +166,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
    private Pattern indexurlmustmatch = null, indexurlmustnotmatch = null;
    private Pattern indexcontentmustmatch = null, indexcontentmustnotmatch = null;
-    /** Pattern on the media type documents must match before being indexed 
+    /** Pattern on the media type documents must match before being indexed
     * @see CollectionSchema#content_type */
    private Pattern indexMediaTypeMustMatch = null;
@@ -251,58 +251,58 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
            throw new NullPointerException("name must not be null or empty");
        }
        if (name.length() > 256) name = name.substring(256);
-        this.doms = new ConcurrentHashMap<String, AtomicInteger>();
+        this.doms = new ConcurrentHashMap<>();
        final String handle = Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(name + crawlerUrlMustMatch + depth + crawlerUrlMustNotMatch + domMaxPages + collections)).substring(0, Word.commonHashLength);
-        put(CrawlAttribute.HANDLE.key,           handle);
+        this.put(CrawlAttribute.HANDLE.key,           handle);
-        put(CrawlAttribute.NAME.key,             name);
+        this.put(CrawlAttribute.NAME.key,             name);
-        put(CrawlAttribute.AGENT_NAME.key, userAgentName);
+        this.put(CrawlAttribute.AGENT_NAME.key, userAgentName);
-        put(CrawlAttribute.CRAWLER_ALWAYS_CHECK_MEDIA_TYPE.key, true);
+        this.put(CrawlAttribute.CRAWLER_ALWAYS_CHECK_MEDIA_TYPE.key, true);
-        put(CrawlAttribute.CRAWLER_URL_MUSTMATCH.key,         (crawlerUrlMustMatch == null) ? CrawlProfile.MATCH_ALL_STRING : crawlerUrlMustMatch);
+        this.put(CrawlAttribute.CRAWLER_URL_MUSTMATCH.key,         (crawlerUrlMustMatch == null) ? CrawlProfile.MATCH_ALL_STRING : crawlerUrlMustMatch);
-        put(CrawlAttribute.CRAWLER_URL_MUSTNOTMATCH.key,      (crawlerUrlMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerUrlMustNotMatch);
+        this.put(CrawlAttribute.CRAWLER_URL_MUSTNOTMATCH.key,      (crawlerUrlMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerUrlMustNotMatch);
-        put(CrawlAttribute.CRAWLER_ORIGIN_URL_MUSTMATCH.key,  (crawlerUrlMustMatch == null) ? CrawlProfile.MATCH_ALL_STRING : crawlerUrlMustMatch);
+        this.put(CrawlAttribute.CRAWLER_ORIGIN_URL_MUSTMATCH.key,  (crawlerUrlMustMatch == null) ? CrawlProfile.MATCH_ALL_STRING : crawlerUrlMustMatch);
-        put(CrawlAttribute.CRAWLER_URL_MUSTNOTMATCH.key,      (crawlerUrlMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerUrlMustNotMatch);
+        this.put(CrawlAttribute.CRAWLER_URL_MUSTNOTMATCH.key,      (crawlerUrlMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerUrlMustNotMatch);
-        put(CrawlAttribute.CRAWLER_IP_MUSTMATCH.key,          (crawlerIpMustMatch == null) ? CrawlProfile.MATCH_ALL_STRING : crawlerIpMustMatch);
+        this.put(CrawlAttribute.CRAWLER_IP_MUSTMATCH.key,          (crawlerIpMustMatch == null) ? CrawlProfile.MATCH_ALL_STRING : crawlerIpMustMatch);
-        put(CrawlAttribute.CRAWLER_IP_MUSTNOTMATCH.key,       (crawlerIpMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerIpMustNotMatch);
+        this.put(CrawlAttribute.CRAWLER_IP_MUSTNOTMATCH.key,       (crawlerIpMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerIpMustNotMatch);
-        put(CrawlAttribute.CRAWLER_COUNTRY_MUSTMATCH.key,     (crawlerCountryMustMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerCountryMustMatch);
+        this.put(CrawlAttribute.CRAWLER_COUNTRY_MUSTMATCH.key,     (crawlerCountryMustMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerCountryMustMatch);
-        put(CrawlAttribute.CRAWLER_URL_NODEPTHLIMITMATCH.key, (crawlerNoDepthLimitMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerNoDepthLimitMatch);
+        this.put(CrawlAttribute.CRAWLER_URL_NODEPTHLIMITMATCH.key, (crawlerNoDepthLimitMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerNoDepthLimitMatch);
-        put(CrawlAttribute.INDEXING_URL_MUSTMATCH.key,        (indexUrlMustMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : indexUrlMustMatch);
+        this.put(CrawlAttribute.INDEXING_URL_MUSTMATCH.key,        (indexUrlMustMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : indexUrlMustMatch);
-        put(CrawlAttribute.INDEXING_URL_MUSTNOTMATCH.key,     (indexUrlMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : indexUrlMustNotMatch);
+        this.put(CrawlAttribute.INDEXING_URL_MUSTNOTMATCH.key,     (indexUrlMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : indexUrlMustNotMatch);
-        put(CrawlAttribute.INDEXING_CONTENT_MUSTMATCH.key,    (indexContentMustMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : indexContentMustMatch);
+        this.put(CrawlAttribute.INDEXING_CONTENT_MUSTMATCH.key,    (indexContentMustMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : indexContentMustMatch);
-        put(CrawlAttribute.INDEXING_CONTENT_MUSTNOTMATCH.key, (indexContentMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : indexContentMustNotMatch);
+        this.put(CrawlAttribute.INDEXING_CONTENT_MUSTNOTMATCH.key, (indexContentMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : indexContentMustNotMatch);
-        put(CrawlAttribute.DEPTH.key,                     depth);
+        this.put(CrawlAttribute.DEPTH.key,                     depth);
-        put(CrawlAttribute.DIRECT_DOC_BY_URL.key,         directDocByURL);
+        this.put(CrawlAttribute.DIRECT_DOC_BY_URL.key,         directDocByURL);
-        put(CrawlAttribute.RECRAWL_IF_OLDER.key,          recrawlIfOlder == null ? Long.MAX_VALUE : recrawlIfOlder.getTime());
+        this.put(CrawlAttribute.RECRAWL_IF_OLDER.key,          recrawlIfOlder == null ? Long.MAX_VALUE : recrawlIfOlder.getTime());
-        put(CrawlAttribute.DOM_MAX_PAGES.key,             domMaxPages);
+        this.put(CrawlAttribute.DOM_MAX_PAGES.key,             domMaxPages);
-        put(CrawlAttribute.CRAWLING_Q.key,                crawlingQ); // crawling of urls with '?'
+        this.put(CrawlAttribute.CRAWLING_Q.key,                crawlingQ); // crawling of urls with '?'
-        put(CrawlAttribute.FOLLOW_FRAMES.key,             followFrames); // load pages contained in frames or ifames
+        this.put(CrawlAttribute.FOLLOW_FRAMES.key,             followFrames); // load pages contained in frames or ifames
-        put(CrawlAttribute.OBEY_HTML_ROBOTS_NOINDEX.key,  obeyHtmlRobotsNoindex); // if false, then a meta robots tag containing 'noindex' is ignored
+        this.put(CrawlAttribute.OBEY_HTML_ROBOTS_NOINDEX.key,  obeyHtmlRobotsNoindex); // if false, then a meta robots tag containing 'noindex' is ignored
-        put(CrawlAttribute.OBEY_HTML_ROBOTS_NOFOLLOW.key, obeyHtmlRobotsNofollow);
+        this.put(CrawlAttribute.OBEY_HTML_ROBOTS_NOFOLLOW.key, obeyHtmlRobotsNofollow);
-        put(CrawlAttribute.INDEX_TEXT.key,                indexText);
+        this.put(CrawlAttribute.INDEX_TEXT.key,                indexText);
-        put(CrawlAttribute.INDEX_MEDIA.key,               indexMedia);
+        this.put(CrawlAttribute.INDEX_MEDIA.key,               indexMedia);
-        put(CrawlAttribute.STORE_HTCACHE.key,             storeHTCache);
+        this.put(CrawlAttribute.STORE_HTCACHE.key,             storeHTCache);
-        put(CrawlAttribute.REMOTE_INDEXING.key,           remoteIndexing);
+        this.put(CrawlAttribute.REMOTE_INDEXING.key,           remoteIndexing);
-        put(CrawlAttribute.SNAPSHOTS_MAXDEPTH.key,        snapshotsMaxDepth);
+        this.put(CrawlAttribute.SNAPSHOTS_MAXDEPTH.key,        snapshotsMaxDepth);
-        put(CrawlAttribute.SNAPSHOTS_LOADIMAGE.key,       snapshotsLoadImage);
+        this.put(CrawlAttribute.SNAPSHOTS_LOADIMAGE.key,       snapshotsLoadImage);
-        put(CrawlAttribute.SNAPSHOTS_REPLACEOLD.key,      snapshotsReplaceOld);
+        this.put(CrawlAttribute.SNAPSHOTS_REPLACEOLD.key,      snapshotsReplaceOld);
-        put(CrawlAttribute.SNAPSHOTS_MUSTNOTMATCH.key,    snapshotsMustnotmatch);
+        this.put(CrawlAttribute.SNAPSHOTS_MUSTNOTMATCH.key,    snapshotsMustnotmatch);
-        put(CrawlAttribute.CACHE_STRAGEGY.key,            cacheStrategy.toString());
+        this.put(CrawlAttribute.CACHE_STRAGEGY.key,            cacheStrategy.toString());
-        put(CrawlAttribute.COLLECTIONS.key,               CommonPattern.SPACE.matcher(collections.trim()).replaceAll(""));
+        this.put(CrawlAttribute.COLLECTIONS.key,               CommonPattern.SPACE.matcher(collections.trim()).replaceAll(""));
        // we transform the ignore_class_name and scraper information into a JSON Array
        this.defaultValency = defaultValency;
-        this.valencySwitchTagNames = valencySwitchTagNames == null ? new HashSet<String>() : valencySwitchTagNames;
+        this.valencySwitchTagNames = valencySwitchTagNames == null ? new HashSet<>() : valencySwitchTagNames;
        String jsonString = new JSONArray(valencySwitchTagNames).toString();
-        put(CrawlAttribute.DEFAULT_VALENCY.key, defaultValency.name());
+        this.put(CrawlAttribute.DEFAULT_VALENCY.key, defaultValency.name());
-        put(CrawlAttribute.VALENCY_SWITCH_TAG_NAMES.key, jsonString);
+        this.put(CrawlAttribute.VALENCY_SWITCH_TAG_NAMES.key, jsonString);
        this.scraper = scraper == null ? new VocabularyScraper() : scraper;
        jsonString = this.scraper.toString();
        assert jsonString != null && jsonString.length() > 0 && jsonString.charAt(0) == '{' : "jsonString = " + jsonString;
-        put(CrawlAttribute.SCRAPER.key, jsonString);
+        this.put(CrawlAttribute.SCRAPER.key, jsonString);
-        put(CrawlAttribute.TIMEZONEOFFSET.key, timezoneOffset);
+        this.put(CrawlAttribute.TIMEZONEOFFSET.key, timezoneOffset);
-        put(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTMATCH.key, CrawlProfile.MATCH_ALL_STRING);
+        this.put(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTMATCH.key, CrawlProfile.MATCH_ALL_STRING);
-        put(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTNOTMATCH.key, CrawlProfile.MATCH_NEVER_STRING);
+        this.put(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTNOTMATCH.key, CrawlProfile.MATCH_NEVER_STRING);
-        put(CrawlAttribute.INDEXING_SOLR_QUERY_MUSTMATCH.key, CrawlProfile.SOLR_MATCH_ALL_QUERY);
+        this.put(CrawlAttribute.INDEXING_SOLR_QUERY_MUSTMATCH.key, CrawlProfile.SOLR_MATCH_ALL_QUERY);
-        put(CrawlAttribute.INDEXING_SOLR_QUERY_MUSTNOTMATCH.key, CrawlProfile.SOLR_EMPTY_QUERY);
+        this.put(CrawlAttribute.INDEXING_SOLR_QUERY_MUSTNOTMATCH.key, CrawlProfile.SOLR_EMPTY_QUERY);
-        put(CrawlAttribute.NOINDEX_WHEN_CANONICAL_UNEQUAL_URL.key, noindexWhenCanonicalUnequalURL);
+        this.put(CrawlAttribute.NOINDEX_WHEN_CANONICAL_UNEQUAL_URL.key, noindexWhenCanonicalUnequalURL);
    }
    /**
@@ -311,9 +311,9 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
     */
    public CrawlProfile(final Map<String, String> ext) {
        super(ext == null ? 1 : ext.size());
-        if (ext != null) putAll(ext);
+        if (ext != null) this.putAll(ext);
-        this.doms = new ConcurrentHashMap<String, AtomicInteger>();
+        this.doms = new ConcurrentHashMap<>();
-        String defaultValency = ext.get(CrawlAttribute.DEFAULT_VALENCY.key);
+        final String defaultValency = ext.get(CrawlAttribute.DEFAULT_VALENCY.key);
        this.defaultValency = defaultValency == null || defaultValency.length() == 0 ? TagValency.EVAL : TagValency.valueOf(defaultValency);
        String jsonString = ext.get(CrawlAttribute.VALENCY_SWITCH_TAG_NAMES.key);
        JSONArray a;
@@ -327,10 +327,10 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
                a = new JSONArray();
            }
        }
-        this.valencySwitchTagNames = new HashSet<String>();
+        this.valencySwitchTagNames = new HashSet<>();
        for (int i = 0; i < a.length(); i++) try {
            this.valencySwitchTagNames.add(a.getString(i));
-        } catch (JSONException e) {}
+        } catch (final JSONException e) {}
        jsonString = ext.get(CrawlAttribute.SCRAPER.key);
        if (jsonString == null || jsonString.length() == 0) {
            this.scraper = new VocabularyScraper();
@@ -340,12 +340,19 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
                loadedScraper = new VocabularyScraper(jsonString);
            } catch(final JSONException e) {
                ConcurrentLog.logException(e);
-                loadedScraper = new VocabularyScraper();    
+                loadedScraper = new VocabularyScraper();
            }
            this.scraper = loadedScraper;
        }
    }
    @Override
    public Object clone() {
        final ConcurrentHashMap<String, String> m = new ConcurrentHashMap<>(this);
        final CrawlProfile cp = new CrawlProfile(m);
        return cp;
    }
    public TagValency defaultValency() {
        return this.defaultValency;
    }
@@ -389,10 +396,10 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
    }
    public ClientIdentification.Agent getAgent() {
-        String agentName = this.get(CrawlAttribute.AGENT_NAME.key);
+        final String agentName = this.get(CrawlAttribute.AGENT_NAME.key);
        return ClientIdentification.getAgent(agentName);
    }
-    
+
    public AtomicInteger getCount(final String domain) {
        if (domain == null) return new AtomicInteger(0); // in case of file indexing this is required
        AtomicInteger dp = this.doms.get(domain);
@@ -436,12 +443,24 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
     * @return handle of the profile
     */
    public String handle() {
-        final String r = get(CrawlAttribute.HANDLE.key);
+        final String r = this.get(CrawlAttribute.HANDLE.key);
        assert r != null;
        //if (r == null) return null;
        return r;
    }
    public void setHandle() {
        final String handle = Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(
                this.get(CrawlAttribute.NAME.key) +
                this.get(CrawlAttribute.CRAWLER_ORIGIN_URL_MUSTMATCH.key) +
                this.get(CrawlAttribute.DEPTH.key) +
                this.get(CrawlAttribute.CRAWLER_ORIGIN_URL_MUSTNOTMATCH.key) +
                this.domMaxPages() +
                this.get(CrawlAttribute.COLLECTIONS.key))).substring(0, Word.commonHashLength);
        assert handle != null && handle.length() == Word.commonHashLength : handle + " != " + Word.commonHashLength;
        this.put(CrawlAttribute.HANDLE.key, handle);
    }
    private Map<String, Pattern> cmap = null;
    /**
@@ -449,29 +468,33 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
     * @return a list of collection names
     */
    public Map<String, Pattern> collections() {
-        if (cmap != null) return cmap;
+        if (this.cmap != null) return this.cmap;
-        final String r = get(CrawlAttribute.COLLECTIONS.key);
+        final String r = this.get(CrawlAttribute.COLLECTIONS.key);
        this.cmap = collectionParser(r);
        return this.cmap;
    }
    public static Map<String, Pattern> collectionParser(String collectionString) {
-        if (collectionString == null || collectionString.length() == 0) return new HashMap<String, Pattern>();
+        if (collectionString == null || collectionString.length() == 0) return new HashMap<>();
-        String[] cs = CommonPattern.COMMA.split(collectionString);
+        final String[] cs = CommonPattern.COMMA.split(collectionString);
-        final Map<String, Pattern> cm = new LinkedHashMap<String, Pattern>();
+        final Map<String, Pattern> cm = new LinkedHashMap<>();
-        for (String c: cs) {
+        for (final String c: cs) {
-            int p = c.indexOf(':');
+            final int p = c.indexOf(':');
            if (p < 0) cm.put(c, QueryParams.catchall_pattern); else cm.put(c.substring(0, p), Pattern.compile(c.substring(p + 1)));
        }
        return cm;
    }
    public void setCollections(String collectionsList) {
        this.put(CrawlAttribute.COLLECTIONS.key, CommonPattern.SPACE.matcher(collectionsList.trim()).replaceAll(""));
    }
    /**
     * Gets the name of the CrawlProfile.
     * @return  name of the profile
     */
    public String name() {
-        final String r = get(CrawlAttribute.NAME.key);
+        final String r = this.get(CrawlAttribute.NAME.key);
        if (r == null) return "";
        return r;
    }
@@ -481,8 +504,8 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
     * @return the name of the collection if that is not "user" or the name() otherwise;
     */
    public String collectionName() {
-        final String r = get(CrawlAttribute.COLLECTIONS.key);
+        final String r = this.get(CrawlAttribute.COLLECTIONS.key);
-        return r == null || r.length() == 0 || "user".equals(r) ? name() : r;
+        return r == null || r.length() == 0 || "user".equals(r) ? this.name() : r;
    }
    /**
@@ -491,7 +514,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
     */
    public Pattern urlMustMatchPattern() {
        if (this.crawlerurlmustmatch == null) {
-            final String r = get(CrawlAttribute.CRAWLER_URL_MUSTMATCH.key);
+            final String r = this.get(CrawlAttribute.CRAWLER_URL_MUSTMATCH.key);
            try {
                this.crawlerurlmustmatch = (r == null || r.equals(CrawlProfile.MATCH_ALL_STRING)) ? CrawlProfile.MATCH_ALL_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
            } catch (final PatternSyntaxException e) { this.crawlerurlmustmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -505,11 +528,11 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
     * and to prevent exceeding the field size limit for
     * CollectionSchema.failreason_s (32k) when the pattern is present in a fail doc
     * added to the Solr index.
-     * 
+     *
     * @return the urlMustMatchPattern formatted as a String of limited size
     */
    public String formattedUrlMustMatchPattern() {
-        String patternStr = urlMustMatchPattern().toString();
+        String patternStr = this.urlMustMatchPattern().toString();
        if(patternStr.length() > 1000) {
            /* The pattern may be quite large when using the 'From Link-List of URL' crawl start point. */
            patternStr = patternStr.substring(0, Math.min(patternStr.length(), 1000)) + "...";
@@ -523,7 +546,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
     */
    public Pattern urlMustNotMatchPattern() {
        if (this.crawlerurlmustnotmatch == null) {
-            final String r = get(CrawlAttribute.CRAWLER_URL_MUSTNOTMATCH.key);
+            final String r = this.get(CrawlAttribute.CRAWLER_URL_MUSTNOTMATCH.key);
            try {
                this.crawlerurlmustnotmatch = (r == null || r.equals(CrawlProfile.MATCH_NEVER_STRING)) ? CrawlProfile.MATCH_NEVER_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
            } catch (final PatternSyntaxException e) { this.crawlerurlmustnotmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -533,7 +556,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
    /**
     * Get the pattern on the URL a document must match to allow adding its embedded links to the crawl stack
-     * 
+     *
     * @return a {@link Pattern} instance, defaulting to
     *         {@link CrawlProfile#MATCH_ALL_PATTERN} when the regular expression
     *         string is not set or its syntax is incorrect
@@ -541,7 +564,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
    public Pattern getCrawlerOriginUrlMustMatchPattern() {
        if (this.crawlerOriginUrlMustMatch == null) {
            /* Cache the compiled pattern for faster next calls */
-            final String patternStr = get(CrawlAttribute.CRAWLER_ORIGIN_URL_MUSTMATCH.key);
+            final String patternStr = this.get(CrawlAttribute.CRAWLER_ORIGIN_URL_MUSTMATCH.key);
            try {
                this.crawlerOriginUrlMustMatch = (patternStr == null
                        || patternStr.equals(CrawlProfile.MATCH_ALL_STRING)) ? CrawlProfile.MATCH_ALL_PATTERN
@@ -555,7 +578,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
    /**
     * Get the pattern on the URL a document must not match to allow adding its embedded links to the crawl stack
-     * 
+     *
     * @return a {@link Pattern} instance, defaulting to
     *         {@link CrawlProfile#MATCH_NEVER_PATTERN} when the regular expression
     *         string is not set or its syntax is incorrect
@@ -563,7 +586,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
    public Pattern getCrawlerOriginUrlMustNotMatchPattern() {
        if (this.crawlerOriginUrlMustNotMatch == null) {
            /* Cache the compiled pattern for faster next calls */
-            final String patternStr = get(CrawlAttribute.CRAWLER_ORIGIN_URL_MUSTNOTMATCH.key);
+            final String patternStr = this.get(CrawlAttribute.CRAWLER_ORIGIN_URL_MUSTNOTMATCH.key);
            try {
                this.crawlerOriginUrlMustNotMatch = (patternStr == null
                        || patternStr.equals(CrawlProfile.MATCH_NEVER_STRING)) ? CrawlProfile.MATCH_NEVER_PATTERN
@@ -581,7 +604,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
     */
    public Pattern ipMustMatchPattern() {
        if (this.crawleripmustmatch == null) {
-            final String r = get(CrawlAttribute.CRAWLER_IP_MUSTMATCH.key);
+            final String r = this.get(CrawlAttribute.CRAWLER_IP_MUSTMATCH.key);
            try {
                this.crawleripmustmatch = (r == null || r.equals(CrawlProfile.MATCH_ALL_STRING)) ? CrawlProfile.MATCH_ALL_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
            } catch (final PatternSyntaxException e) { this.crawleripmustmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -595,7 +618,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
     */
    public Pattern ipMustNotMatchPattern() {
        if (this.crawleripmustnotmatch == null) {
-            final String r = get(CrawlAttribute.CRAWLER_IP_MUSTNOTMATCH.key);
+            final String r = this.get(CrawlAttribute.CRAWLER_IP_MUSTNOTMATCH.key);
            try {
                this.crawleripmustnotmatch = (r == null || r.equals(CrawlProfile.MATCH_NEVER_STRING)) ? CrawlProfile.MATCH_NEVER_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
            } catch (final PatternSyntaxException e) { this.crawleripmustnotmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -608,7 +631,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
     * @return a list of country codes
     */
    public String[] countryMustMatchList() {
-        String countryMustMatch = get(CrawlAttribute.CRAWLER_COUNTRY_MUSTMATCH.key);
+        String countryMustMatch = this.get(CrawlAttribute.CRAWLER_COUNTRY_MUSTMATCH.key);
        if (countryMustMatch == null) countryMustMatch = CrawlProfile.MATCH_NEVER_STRING;
        if (countryMustMatch.isEmpty()) return new String[0];
        String[] list = CommonPattern.COMMA.split(countryMustMatch);
@@ -622,7 +645,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
     */
    public Pattern crawlerNoDepthLimitMatchPattern() {
        if (this.crawlernodepthlimitmatch == null) {
-            final String r = get(CrawlAttribute.CRAWLER_URL_NODEPTHLIMITMATCH.key);
+            final String r = this.get(CrawlAttribute.CRAWLER_URL_NODEPTHLIMITMATCH.key);
            try {
                this.crawlernodepthlimitmatch = (r == null || r.equals(CrawlProfile.MATCH_NEVER_STRING)) ? CrawlProfile.MATCH_NEVER_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
            } catch (final PatternSyntaxException e) { this.crawlernodepthlimitmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -636,7 +659,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
     */
    public Pattern indexUrlMustMatchPattern() {
        if (this.indexurlmustmatch == null) {
-            final String r = get(CrawlAttribute.INDEXING_URL_MUSTMATCH.key);
+            final String r = this.get(CrawlAttribute.INDEXING_URL_MUSTMATCH.key);
            try {
                this.indexurlmustmatch = (r == null || r.equals(CrawlProfile.MATCH_ALL_STRING)) ? CrawlProfile.MATCH_ALL_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
            } catch (final PatternSyntaxException e) { this.indexurlmustmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -650,7 +673,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
     */
    public Pattern indexUrlMustNotMatchPattern() {
        if (this.indexurlmustnotmatch == null) {
-            final String r = get(CrawlAttribute.INDEXING_URL_MUSTNOTMATCH.key);
+            final String r = this.get(CrawlAttribute.INDEXING_URL_MUSTNOTMATCH.key);
            try {
                this.indexurlmustnotmatch = (r == null || r.equals(CrawlProfile.MATCH_NEVER_STRING)) ? CrawlProfile.MATCH_NEVER_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
            } catch (final PatternSyntaxException e) { this.indexurlmustnotmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -664,7 +687,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
     */
    public Pattern indexContentMustMatchPattern() {
        if (this.indexcontentmustmatch == null) {
-            final String r = get(CrawlAttribute.INDEXING_CONTENT_MUSTMATCH.key);
+            final String r = this.get(CrawlAttribute.INDEXING_CONTENT_MUSTMATCH.key);
            try {
                this.indexcontentmustmatch = (r == null || r.equals(CrawlProfile.MATCH_ALL_STRING)) ? CrawlProfile.MATCH_ALL_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
            } catch (final PatternSyntaxException e) { this.indexcontentmustmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -678,7 +701,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
     */
    public Pattern indexContentMustNotMatchPattern() {
        if (this.indexcontentmustnotmatch == null) {
-            final String r = get(CrawlAttribute.INDEXING_CONTENT_MUSTNOTMATCH.key);
+            final String r = this.get(CrawlAttribute.INDEXING_CONTENT_MUSTNOTMATCH.key);
            try {
                this.indexcontentmustnotmatch = (r == null || r.equals(CrawlProfile.MATCH_NEVER_STRING)) ? CrawlProfile.MATCH_NEVER_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
            } catch (final PatternSyntaxException e) { this.indexcontentmustnotmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -688,7 +711,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
    /**
     * Get the Pattern on media type that documents must match in order to be indexed
-     * 
+     *
     * @return a {@link Pattern} instance, defaulting to
     *         {@link CrawlProfile#MATCH_ALL_PATTERN} when the regular expression
     *         string is not set or its syntax is incorrect
@@ -696,7 +719,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
    public Pattern getIndexMediaTypeMustMatchPattern() {
        if (this.indexMediaTypeMustMatch == null) {
            /* Cache the compiled pattern for faster next calls */
-            final String patternStr = get(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTMATCH.key);
+            final String patternStr = this.get(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTMATCH.key);
            try {
                this.indexMediaTypeMustMatch = (patternStr == null
                        || patternStr.equals(CrawlProfile.MATCH_ALL_STRING)) ? CrawlProfile.MATCH_ALL_PATTERN
@@ -710,7 +733,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
    /**
     * Get the Pattern on media type that documents must not match in order to be indexed
-     * 
+     *
     * @return a {@link Pattern} instance, defaulting to
     *         {@link CrawlProfile#MATCH_NEVER_PATTERN} when the regular expression
     *         string is not set or its syntax is incorrect
@@ -718,7 +741,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
    public Pattern getIndexMediaTypeMustNotMatchPattern() {
        if (this.indexMediaTypeMustNotMatch == null) {
            /* Cache the compiled pattern for faster next calls */
-            final String patternStr = get(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTNOTMATCH.key);
+            final String patternStr = this.get(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTNOTMATCH.key);
            try {
                this.indexMediaTypeMustNotMatch = (patternStr == null
                        || patternStr.equals(CrawlProfile.MATCH_NEVER_STRING)) ? CrawlProfile.MATCH_NEVER_PATTERN
@@ -736,7 +759,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
     * @return depth of crawl job
     */
    public int depth() {
-        final String r = get(CrawlAttribute.DEPTH.key);
+        final String r = this.get(CrawlAttribute.DEPTH.key);
        if (r == null) return 0;
        try {
            return Integer.parseInt(r);
@@ -751,7 +774,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
     *         be indexed as links (with metadata only on URL and not on content).
     */
    public boolean isIndexNonParseableUrls() {
-        final String r = get(CrawlAttribute.DIRECT_DOC_BY_URL.key);
+        final String r = this.get(CrawlAttribute.DIRECT_DOC_BY_URL.key);
        if (r == null) return false;
        return (r.equals(Boolean.TRUE.toString()));
    }
@@ -763,7 +786,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
     *         with an unknown or unsupported file extension.
     */
    public boolean isCrawlerAlwaysCheckMediaType() {
-        final String r = get(CrawlAttribute.CRAWLER_ALWAYS_CHECK_MEDIA_TYPE.key);
+        final String r = this.get(CrawlAttribute.CRAWLER_ALWAYS_CHECK_MEDIA_TYPE.key);
        if (r == null) {
            return false;
        }
@@ -771,7 +794,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
    }
    public CacheStrategy cacheStrategy() {
-        final String r = get(CrawlAttribute.CACHE_STRAGEGY.key);
+        final String r = this.get(CrawlAttribute.CACHE_STRAGEGY.key);
        if (r == null) return CacheStrategy.IFEXIST;
        try {
            return CacheStrategy.decode(Integer.parseInt(r));
@@ -782,7 +805,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
    }
    public void setCacheStrategy(final CacheStrategy newStrategy) {
-        put(CrawlAttribute.CACHE_STRAGEGY.key, newStrategy.toString());
+        this.put(CrawlAttribute.CACHE_STRAGEGY.key, newStrategy.toString());
    }
    /**
@@ -792,7 +815,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
    public long recrawlIfOlder() {
        // returns a long (millis) that is the minimum age that
        // an entry must have to be re-crawled
-        final String r = get(CrawlAttribute.RECRAWL_IF_OLDER.key);
+        final String r = this.get(CrawlAttribute.RECRAWL_IF_OLDER.key);
        if (r == null) return 0L;
        try {
            final long l = Long.parseLong(r);
@@ -806,7 +829,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
    public int domMaxPages() {
        // this is the maximum number of pages that are crawled for a single domain
        // if -1, this means no limit
-        final String r = get(CrawlAttribute.DOM_MAX_PAGES.key);
+        final String r = this.get(CrawlAttribute.DOM_MAX_PAGES.key);
        if (r == null) return Integer.MAX_VALUE;
        try {
            final int i = Integer.parseInt(r);
@@ -819,61 +842,61 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
    }
    public boolean crawlingQ() {
-        final String r = get(CrawlAttribute.CRAWLING_Q.key);
+        final String r = this.get(CrawlAttribute.CRAWLING_Q.key);
        if (r == null) return false;
        return (r.equals(Boolean.TRUE.toString()));
    }
    public boolean followFrames() {
-        final String r = get(CrawlAttribute.FOLLOW_FRAMES.key);
+        final String r = this.get(CrawlAttribute.FOLLOW_FRAMES.key);
        if (r == null) return false;
        return (r.equals(Boolean.TRUE.toString()));
    }
    public boolean obeyHtmlRobotsNoindex() {
-        final String r = get(CrawlAttribute.OBEY_HTML_ROBOTS_NOINDEX.key);
+        final String r = this.get(CrawlAttribute.OBEY_HTML_ROBOTS_NOINDEX.key);
        if (r == null) return false;
        return (r.equals(Boolean.TRUE.toString()));
    }
    public boolean obeyHtmlRobotsNofollow() {
-        final String r = get(CrawlAttribute.OBEY_HTML_ROBOTS_NOFOLLOW.key);
+        final String r = this.get(CrawlAttribute.OBEY_HTML_ROBOTS_NOFOLLOW.key);
        if (r == null) return false;
        return (r.equals(Boolean.TRUE.toString()));
    }
    public boolean indexText() {
-        final String r = get(CrawlAttribute.INDEX_TEXT.key);
+        final String r = this.get(CrawlAttribute.INDEX_TEXT.key);
        if (r == null) return true;
        return (r.equals(Boolean.TRUE.toString()));
    }
    public boolean indexMedia() {
-        final String r = get(CrawlAttribute.INDEX_MEDIA.key);
+        final String r = this.get(CrawlAttribute.INDEX_MEDIA.key);
        if (r == null) return true;
        return (r.equals(Boolean.TRUE.toString()));
    }
    public boolean noindexWhenCanonicalUnequalURL() {
-        final String r = get(CrawlAttribute.NOINDEX_WHEN_CANONICAL_UNEQUAL_URL.key);
+        final String r = this.get(CrawlAttribute.NOINDEX_WHEN_CANONICAL_UNEQUAL_URL.key);
        if (r == null) return true;
        return (r.equals(Boolean.TRUE.toString()));
    }
    public boolean storeHTCache() {
-        final String r = get(CrawlAttribute.STORE_HTCACHE.key);
+        final String r = this.get(CrawlAttribute.STORE_HTCACHE.key);
        if (r == null) return false;
        return (r.equals(Boolean.TRUE.toString()));
    }
    public boolean remoteIndexing() {
-        final String r = get(CrawlAttribute.REMOTE_INDEXING.key);
+        final String r = this.get(CrawlAttribute.REMOTE_INDEXING.key);
        if (r == null) return false;
        return (r.equals(Boolean.TRUE.toString()));
    }
    public int snapshotMaxdepth() {
-        final String r = get(CrawlAttribute.SNAPSHOTS_MAXDEPTH.key);
+        final String r = this.get(CrawlAttribute.SNAPSHOTS_MAXDEPTH.key);
        if (r == null) return -1;
        try {
            final int i = Integer.parseInt(r);
@@ -886,20 +909,20 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
    }
    public boolean snapshotLoadImage() {
-        final String r = get(CrawlAttribute.SNAPSHOTS_LOADIMAGE.key);
+        final String r = this.get(CrawlAttribute.SNAPSHOTS_LOADIMAGE.key);
        if (r == null) return false;
        return (r.equals(Boolean.TRUE.toString()));
    }
    public boolean snapshotReplaceold() {
-        final String r = get(CrawlAttribute.SNAPSHOTS_REPLACEOLD.key);
+        final String r = this.get(CrawlAttribute.SNAPSHOTS_REPLACEOLD.key);
        if (r == null) return false;
        return (r.equals(Boolean.TRUE.toString()));
    }
    public Pattern snapshotsMustnotmatch() {
        if (this.snapshotsMustnotmatch == null) {
-            final String r = get(CrawlAttribute.SNAPSHOTS_MUSTNOTMATCH.key);
+            final String r = this.get(CrawlAttribute.SNAPSHOTS_MUSTNOTMATCH.key);
            try {
                this.snapshotsMustnotmatch = (r == null || r.equals(CrawlProfile.MATCH_ALL_STRING)) ? CrawlProfile.MATCH_ALL_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
            } catch (final PatternSyntaxException e) { this.snapshotsMustnotmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -908,11 +931,11 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
    }
    public int timezoneOffset() {
-        final String timezoneOffset = get(CrawlAttribute.TIMEZONEOFFSET.key);
+        final String timezoneOffset = this.get(CrawlAttribute.TIMEZONEOFFSET.key);
        if (timezoneOffset == null) return 0;
        try {
            return Integer.parseInt(timezoneOffset);
-        } catch (NumberFormatException e) {
+        } catch (final NumberFormatException e) {
            return 0;
        }
    }
@@ -949,7 +972,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
    }
    public static String subpathFilter(final Collection<? extends MultiProtocolURL> urls) {
-        LinkedHashSet<String> filters = new LinkedHashSet<String>(); // first collect in a set to eliminate doubles
+        final LinkedHashSet<String> filters = new LinkedHashSet<>(); // first collect in a set to eliminate doubles
        for (final MultiProtocolURL url: urls) filters.add(mustMatchSubpath(url));
        final StringBuilder filter = new StringBuilder();
        for (final String urlfilter: filters) filter.append('|').append(urlfilter);
@@ -976,8 +999,8 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
            final boolean dark,
            final int count,
            final int domlistlength) {
-        boolean terminateButton = active && !CrawlSwitchboard.DEFAULT_PROFILES.contains(this.name());
+        final boolean terminateButton = active && !CrawlSwitchboard.DEFAULT_PROFILES.contains(this.name());
-        boolean deleteButton = !active;
+        final boolean deleteButton = !active;
        prop.put(CRAWL_PROFILE_PREFIX + count + "_status", terminateButton ? 1 : deleteButton ? 0 : 2);
        prop.put(CRAWL_PROFILE_PREFIX + count + "_terminateButton", terminateButton);
        prop.put(CRAWL_PROFILE_PREFIX + count + "_terminateButton_handle", this.handle());
@@ -1006,7 +1029,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
        prop.putXML(CRAWL_PROFILE_PREFIX + count + "_indexURLMustNotMatch", this.get(CrawlAttribute.INDEXING_URL_MUSTNOTMATCH.key));
        prop.putXML(CRAWL_PROFILE_PREFIX + count + "_indexContentMustMatch", this.get(CrawlAttribute.INDEXING_CONTENT_MUSTMATCH.key));
        prop.putXML(CRAWL_PROFILE_PREFIX + count + "_indexContentMustNotMatch", this.get(CrawlAttribute.INDEXING_CONTENT_MUSTNOTMATCH.key));
-        prop.put(CRAWL_PROFILE_PREFIX + count + "_" + CrawlAttribute.NOINDEX_WHEN_CANONICAL_UNEQUAL_URL, noindexWhenCanonicalUnequalURL() ? 1 : 0);
+        prop.put(CRAWL_PROFILE_PREFIX + count + "_" + CrawlAttribute.NOINDEX_WHEN_CANONICAL_UNEQUAL_URL, this.noindexWhenCanonicalUnequalURL() ? 1 : 0);
        prop.putXML(CRAWL_PROFILE_PREFIX + count + "_" + CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTMATCH.key, this.get(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTMATCH.key));
        prop.putXML(CRAWL_PROFILE_PREFIX + count + "_" + CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTNOTMATCH.key, this.get(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTNOTMATCH.key));
        prop.putXML(CRAWL_PROFILE_PREFIX + count + "_" + CrawlAttribute.INDEXING_SOLR_QUERY_MUSTMATCH.key, this.get(CrawlAttribute.INDEXING_SOLR_QUERY_MUSTMATCH.key));
@@ -1042,19 +1065,19 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
    public static void main(String[] args) {
        // test to convert the key set from set to string and back
-        Set<String> a = new HashSet<>();
+        final Set<String> a = new HashSet<>();
        a.add("eins"); a.add("zwei"); a.add("drei");
        JSONArray j = new JSONArray(a);
-        String s = j.toString();
+        final String s = j.toString();
        System.out.println(s);
-        JSONTokener o = new JSONTokener(s);
+        final JSONTokener o = new JSONTokener(s);
        try {
            j = new JSONArray(o);
            System.out.println(j);
-            Set<String> h = new HashSet<String>();
+            final Set<String> h = new HashSet<>();
            for (int i = 0; i < j.length(); i++) h.add(j.getString(i));
            System.out.println(h);
-        } catch (JSONException e) {
+        } catch (final JSONException e) {
            e.printStackTrace();
        }
    }
@@ -22,6 +22,7 @@
 */
 package net.yacy.document.importer;
 import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
@@ -44,6 +45,7 @@ import net.yacy.cora.protocol.RequestHeader;
 import net.yacy.cora.protocol.ResponseHeader;
 import net.yacy.cora.util.ByteBuffer;
 import net.yacy.cora.util.ConcurrentLog;
 import net.yacy.crawler.data.CrawlProfile;
 import net.yacy.crawler.retrieval.Request;
 import net.yacy.crawler.retrieval.Response;
 import net.yacy.document.TextParser;
@@ -83,23 +85,31 @@ public class WarcImporter extends Thread implements Importer {
    private final long sourceSize; // length of the input source (for statistic)
    private long consumed; // bytes consumed from input source (for statistic)
    private boolean abort = false; // flag to signal stop of import
    private String collection;
-    public WarcImporter(MultiProtocolURL url) throws IOException {
+    public WarcImporter(MultiProtocolURL url, String collection) throws IOException {
        super("WarcImporter - from InputStream");
        this.recordCnt = 0;
        this.sourceSize = -1;
        this.name = url.toNormalform(true);
        this.source = url.getInputStream(ClientIdentification.yacyInternetCrawlerAgent);
        if (this.name.endsWith(".gz")) this.source = new GZIPInputStream(this.source);
        this.collection = collection;
    }
-    public WarcImporter(File f) throws IOException {
+    public WarcImporter(File f, byte[] data, String collection) throws IOException {
       super("WarcImporter - from file " + f.getName());
       this.name = f.getName();
-       this.sourceSize = f.length();
+       if (!f.exists() && data != null) {
-       this.source = new FileInputStream(f);
+           this.sourceSize = data.length;
-       if (this.name.endsWith(".gz")) this.source = new GZIPInputStream(this.source);
+           this.source = new ByteArrayInputStream(data);
-    }
+       } else {
           this.sourceSize = f.length();
           this.source = new FileInputStream(f);
           if (this.name.endsWith(".gz")) this.source = new GZIPInputStream(this.source);
       }
       this.collection = collection;
   }
    /**
     * Reads a Warc file and adds all contained responses to the index.
@@ -115,6 +125,10 @@ public class WarcImporter extends Thread implements Importer {
        job = this;
        this.startTime = System.currentTimeMillis();
        final CrawlProfile warcProfile = (CrawlProfile) Switchboard.getSwitchboard().crawler.defaultPackProfile.clone();
        warcProfile.setCollections(this.collection);
        warcProfile.setHandle();
        WarcReader localwarcReader = WarcReaderFactory.getReader(f);
        WarcRecord wrec = localwarcReader.getNextRecord();
        while (wrec != null && !this.abort) {
@@ -165,15 +179,15 @@ public class WarcImporter extends Thread implements Importer {
                                    requestHeader.referer() == null ? null : requestHeader.referer().hash(),
                                    "warc",
                                    responseHeader.lastModified(),
-                                    Switchboard.getSwitchboard().crawler.defaultPackProfile.handle(),
+                                    warcProfile.handle(),
                                    0,
-                                    Switchboard.getSwitchboard().crawler.defaultPackProfile.timezoneOffset());
+                                    warcProfile.timezoneOffset());
                            final Response response = new Response(
                                    request,
                                    requestHeader,
                                    responseHeader,
-                                    Switchboard.getSwitchboard().crawler.defaultPackProfile,
+                                    warcProfile,
                                    false,
                                    content
                            );
@@ -37,6 +37,12 @@ import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
 import org.apache.commons.io.FileUtils;
 import org.openzim.ZIMFile;
 import org.openzim.ZIMReader;
 import org.openzim.ZIMReader.ArticleEntry;
 import org.openzim.ZIMReader.DirectoryEntry;
 import net.yacy.cora.document.encoding.ASCII;
 import net.yacy.cora.document.id.DigestURL;
 import net.yacy.cora.protocol.ClientIdentification;
@@ -44,16 +50,12 @@ import net.yacy.cora.protocol.HeaderFramework;
 import net.yacy.cora.protocol.RequestHeader;
 import net.yacy.cora.protocol.ResponseHeader;
 import net.yacy.cora.util.ConcurrentLog;
 import net.yacy.crawler.data.CrawlProfile;
 import net.yacy.crawler.retrieval.Request;
 import net.yacy.crawler.retrieval.Response;
 import net.yacy.document.TextParser;
 import net.yacy.search.Switchboard;
 import org.openzim.ZIMFile;
 import org.openzim.ZIMReader;
 import org.openzim.ZIMReader.ArticleEntry;
 import org.openzim.ZIMReader.DirectoryEntry;
 /**
 * ZIM importer
 * can import ZIM file i.e. from https://download.kiwix.org/zim/ or mirrors like https://ftp.fau.de/kiwix/zim/
@@ -69,10 +71,12 @@ public class ZimImporter extends Thread implements Importer {
    static public ZimImporter job;
-    private ZIMFile file;
+    private final ZIMFile file;
    private ZIMReader reader;
-    private String path; 
+    private final String path;
    private String guessedSource;
    private final byte[] data;
    private final String collection;
    private int recordCnt;
    private long startTime;
@@ -80,62 +84,75 @@ public class ZimImporter extends Thread implements Importer {
    private long consumed;
    private boolean abort = false;
-    public ZimImporter(String path) throws IOException {
+    public ZimImporter(String path, byte[] data, String collection) throws IOException {
       super("ZimImporter - from file " + path);
       this.path = path;
-       this.file = new ZIMFile(this.path); // this will read already some of the metadata and could consume some time
+       this.data = data;
-       this.sourceSize = this.file.length();
+       File zimFilePath = new File(path);
       if (!zimFilePath.exists() && this.data != null) {
           File tempFile = File.createTempFile(zimFilePath.getName().substring(0, zimFilePath.getName().length() - 4), "zim");
           FileUtils.writeByteArrayToFile(tempFile, data);
           this.file = new ZIMFile(tempFile.getPath());
       } else {
           this.file = new ZIMFile(this.path); // this will read already some of the metadata and could consume some time
       }
       this.sourceSize = data == null ? this.file.length() : data.length;
       this.collection = collection;
    }
    @Override
    public void run() {
        job = this;
        this.startTime = System.currentTimeMillis();
-        Switchboard sb = Switchboard.getSwitchboard();
+        final Switchboard sb = Switchboard.getSwitchboard();
        try {
            this.reader = new ZIMReader(this.file);
-            this.guessedSource = getSource(this.reader);
+            this.guessedSource = getSource(this.reader, this.file.getName());
-            Date guessedDate = getDate(this.reader);
+            final Date guessedDate = getDate(this.reader);
-            String dates = HeaderFramework.newRfc1123Format().format(guessedDate);
+            final String dates = HeaderFramework.newRfc1123Format().format(guessedDate);
            // verify the source
-            DirectoryEntry mainEntry = this.reader.getMainDirectoryEntry();
+            final DirectoryEntry mainEntry = this.reader.getMainDirectoryEntry();
-            DigestURL mainURL = guessURL(this.guessedSource, mainEntry);
+            final DigestURL mainURL = guessURL(this.guessedSource, mainEntry);
-            if (!mainURL.exists(ClientIdentification.browserAgent)) {
+            //if (!mainURL.exists(ClientIdentification.browserAgent)) {
-                sb.log.info("zim importer: file " + this.file.getName() + " failed main url existence test: " + mainURL);
+            //    sb.log.info("zim importer: file " + this.file.getName() + " failed main url existence test: " + mainURL);
-                return; 
+            //    return;
-            }
+            //}
            final CrawlProfile zimProfile = (CrawlProfile) Switchboard.getSwitchboard().crawler.defaultPackProfile.clone();
            zimProfile.setCollections(this.collection);
            zimProfile.setHandle();
            // read all documents
            for (int i = 0; i < this.file.header_entryCount; i++) {
                try {
                    if (this.abort) break;
-                    DirectoryEntry de = this.reader.getDirectoryInfo(i);
+                    final DirectoryEntry de = this.reader.getDirectoryInfo(i);
                    if (!(de instanceof ZIMReader.ArticleEntry)) continue;
-                    ArticleEntry ae = (ArticleEntry) de;
+                    final ArticleEntry ae = (ArticleEntry) de;
                    if (ae.namespace != 'C' && ae.namespace != 'A') continue;
-    
+
                    // check url
-                    DigestURL guessedUrl = guessURL(this.guessedSource, de);
+                    final DigestURL guessedUrl = guessURL(this.guessedSource, de);
-                    if (recordCnt < 10) {
+                    //if (this.recordCnt < 10) {
                        // critical test for the first 10 urls
-                        if (!guessedUrl.exists(ClientIdentification.browserAgent)) {
+                    //    if (!guessedUrl.exists(ClientIdentification.browserAgent)) {
-                            sb.log.info("zim importer: file " + this.file.getName() + " failed url " + recordCnt + " existence test: " + guessedUrl);
+                    //        sb.log.info("zim importer: file " + this.file.getName() + " failed url " + this.recordCnt + " existence test: " + guessedUrl);
-                            return; 
+                    //        return;
-                        }
+                    //    }
-                    }
+                    //}
-    
+
                    // check availability of text parser
-                    String mimeType = ae.getMimeType();
+                    final String mimeType = ae.getMimeType();
                    if (!mimeType.startsWith("text/") && !mimeType.equals("application/epub+zip")) continue; // in this import we want only text, not everything that is possible
                    if (TextParser.supportsMime(mimeType) != null) continue;
-    
+
                    // read the content
-                    byte[] b = this.reader.getArticleData(ae);
+                    final byte[] b = this.reader.getArticleData(ae);
-    
+
                    // create artificial request and response headers for the indexer
-                    RequestHeader requestHeader = new RequestHeader();
+                    final RequestHeader requestHeader = new RequestHeader();
-                    ResponseHeader responseHeader = new ResponseHeader(200);
+                    final ResponseHeader responseHeader = new ResponseHeader(200);
                    responseHeader.put(HeaderFramework.CONTENT_TYPE, de.getMimeType()); // very important to tell parser which kind of content
                    responseHeader.put(HeaderFramework.LAST_MODIFIED, dates); // put in the guessd date to have something that is not the current date
                    final Request request = new Request(
@@ -144,29 +161,29 @@ public class ZimImporter extends Thread implements Importer {
                            null, // referrerhash the hash of the referrer URL
                            de.title, // name the name of the document to crawl
                            null, // appdate the time when the url was first time appeared
-                            sb.crawler.defaultPackProfile.handle(),        // profileHandle the name of the prefetch profile. This must not be null!
+                            zimProfile.handle(),        // profileHandle the name of the prefetch profile. This must not be null!
                            0,    // depth the crawling depth of the entry
-                            sb.crawler.defaultPackProfile.timezoneOffset() // timezone offset
+                            zimProfile.timezoneOffset() // timezone offset
                    );
                    final Response response = new Response(
                            request,
                            requestHeader,
                            responseHeader,
-                            Switchboard.getSwitchboard().crawler.defaultPackProfile,
+                            zimProfile,
                            false,
                            b
                    );
-    
+
                    // throw this to the indexer
-                    String error = sb.toIndexer(response);
+                    final String error = sb.toIndexer(response);
                    if (error != null) ConcurrentLog.info("ZimImporter", "error parsing: " + error);
                    this.recordCnt++;
-                } catch (Exception e) {
+                } catch (final Exception e) {
                    // catch any error that could stop the importer
                    ConcurrentLog.info("ZimImporter", "error loading: " + e.getMessage());
                }
            }
-        } catch (IOException e) {
+        } catch (final IOException e) {
            ConcurrentLog.info("ZimImporter", "error reading: " + e.getMessage());
        }
        ConcurrentLog.info("ZimImporter", "Indexed " + this.recordCnt + " documents");
@@ -190,7 +207,7 @@ public class ZimImporter extends Thread implements Importer {
    @Override
    public int speed() {
        if (this.recordCnt == 0) return 0;
-        return (int) (this.recordCnt / Math.max(0L, runningTime() ));
+        return (int) (this.recordCnt / Math.max(0L, this.runningTime() ));
    }
    @Override
@@ -203,7 +220,7 @@ public class ZimImporter extends Thread implements Importer {
        if (this.consumed == 0) {
            return 0;
        }
-        long speed = this.consumed / runningTime();
+        final long speed = this.consumed / this.runningTime();
        return (this.sourceSize - this.consumed) / speed;
    }
@@ -217,11 +234,11 @@ public class ZimImporter extends Thread implements Importer {
            return null; // Handle null or empty input
        }
-        String[] parts = fileName.split("_");
+        final String[] parts = fileName.split("_");
        if (parts.length == 0) {
            return null;
        }
-        String firstPart = parts[0];
+        final String firstPart = parts[0];
        // Handling special cases where the domain name might not be obvious
        // These are based on your provided list and can be expanded as needed
@@ -293,8 +310,8 @@ public class ZimImporter extends Thread implements Importer {
        // Handling domain patterns
        if (firstPart.contains(".stackexchange.com")) {
            return firstPart;
-        } else if (firstPart.endsWith(".com") || firstPart.endsWith(".org") || firstPart.endsWith(".de") || 
+        } else if (firstPart.endsWith(".com") || firstPart.endsWith(".org") || firstPart.endsWith(".de") ||
-                   firstPart.endsWith(".fr") || firstPart.endsWith(".pt") || firstPart.endsWith(".it") || 
+                   firstPart.endsWith(".fr") || firstPart.endsWith(".pt") || firstPart.endsWith(".it") ||
                   firstPart.endsWith(".ja") || firstPart.endsWith(".es") || firstPart.endsWith(".eo")) {
            return firstPart;
        } else if (firstPart.contains("-")) {
@@ -303,7 +320,7 @@ public class ZimImporter extends Thread implements Importer {
        // Additional general domain extraction logic
        if (firstPart.contains(".")) {
-            int lastDotIndex = firstPart.lastIndexOf('.');
+            final int lastDotIndex = firstPart.lastIndexOf('.');
            if (lastDotIndex > 0 && lastDotIndex < firstPart.length() - 1) {
                // Extract up to the next character beyond the TLD, to support TLDs of variable length
                int endIndex = firstPart.indexOf('.', lastDotIndex + 1);
@@ -318,19 +335,20 @@ public class ZimImporter extends Thread implements Importer {
        return null;
    }
-    public static String getSource(ZIMReader r) throws IOException {
+    public static String getSource(ZIMReader r, String fileName) throws IOException {
        String source = r.getMetadata("Source");
        if (source != null) return source;
-        source = "https://" + guessDomainName(r.getZIMFile().getName()) + "/";
+        if (fileName == null || fileName.length() == 0) fileName = r.getZIMFile().getName();
        source = "https://" + guessDomainName(fileName) + "/";
        return source;
    }
    public static Date getDate(ZIMReader r) throws IOException {
-        String date = r.getMetadata("Date");
+        final String date = r.getMetadata("Date");
        if (date != null) try {
-            SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd", Locale.US);
+            final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd", Locale.US);
            return format.parse(date);
-        } catch (ParseException e) {}
+        } catch (final ParseException e) {}
        // failover situation: use file date
        return new Date(r.getZIMFile().lastModified());
    }
@@ -491,47 +509,47 @@ public class ZimImporter extends Thread implements Importer {
    };
    public static void main(String[] args) {
-        Set<String> skip = new HashSet<>();
+        final Set<String> skip = new HashSet<>();
-        for (String s: skip_files) skip.add(s);
+        for (final String s: skip_files) skip.add(s);
        // zim file import test
        // will test mostly if domain names are included in zim file urls
-        String zimFilesPath = args[0];
+        final String zimFilesPath = args[0];
-        File zimFiles = new File(zimFilesPath);
+        final File zimFiles = new File(zimFilesPath);
        // make ordered file list; order by file size (start with smallest)
-        String[] filelist = zimFiles.list();
+        final String[] filelist = zimFiles.list();
-        Map<Long, File> orderedFileMap = new TreeMap<>();
+        final Map<Long, File> orderedFileMap = new TreeMap<>();
        for (int i = 0; i < filelist.length; i++) {
            if (!filelist[i].endsWith(".zim")) continue;
-            File f = new File(zimFiles, filelist[i]);
+            final File f = new File(zimFiles, filelist[i]);
            orderedFileMap.put(f.length() * 1000 + i, f);
        }
-        Collection<File> orderedFiles = orderedFileMap.values();
+        final Collection<File> orderedFiles = orderedFileMap.values();
-        Set<String> files_ok = new LinkedHashSet<>();
+        final Set<String> files_ok = new LinkedHashSet<>();
-        Set<String> files_nok = new LinkedHashSet<>();
+        final Set<String> files_nok = new LinkedHashSet<>();
-        for (File f: orderedFiles) {
+        for (final File f: orderedFiles) {
            if (skip.contains(f.getName())) continue;
            try {
-                ZIMFile z = new ZIMFile(f.getAbsolutePath());
+                final ZIMFile z = new ZIMFile(f.getAbsolutePath());
-                ZIMReader r = new ZIMReader(z);
+                final ZIMReader r = new ZIMReader(z);
-                DirectoryEntry de = r.getMainDirectoryEntry();
+                final DirectoryEntry de = r.getMainDirectoryEntry();
                System.out.println("ZIM file:  " + f.getAbsolutePath());
-                for (String key: ZIMReader.METADATA_KEYS) {String s = r.getMetadata(key); if (s != null) System.out.println("Metadata " + key + ": " + s);};
+                for (final String key: ZIMReader.METADATA_KEYS) {final String s = r.getMetadata(key); if (s != null) System.out.println("Metadata " + key + ": " + s);}
                System.out.println("Namespace: " + de.namespace);
                System.out.println("Title:     " + de.title);
                System.out.println("URL:       " + de.url);
                System.out.println("Mime Type  " + de.getMimeType());
                System.out.println("guessed domain: " + guessDomainName(f.getName())); // uses a table and rules that deduces a source from the file name
-                String source = getSource(r);
+                final String source = getSource(r, null);
                System.out.println("guessed Source: " + source); // this uses metadata stored in the zim file
-                DigestURL mainURL = guessURL(source, de);
+                final DigestURL mainURL = guessURL(source, de);
                System.out.println("guessed main article: " + mainURL);
-                boolean ok = mainURL.exists(ClientIdentification.browserAgent);
+                final boolean ok = mainURL.exists(ClientIdentification.browserAgent);
                System.out.println("main article exists: " + ok);
                if (ok) files_ok.add(f.getName()); else files_nok.add(f.getName());
                System.out.println();
-            } catch (IOException e) {
+            } catch (final IOException e) {
                e.printStackTrace();
            }
        }
@@ -23,6 +23,7 @@ import java.io.IOException;
 import java.net.MalformedURLException;
 import net.yacy.cora.document.id.MultiProtocolURL;
 import net.yacy.cora.order.Base64Order;
 import net.yacy.cora.protocol.RequestHeader;
 import net.yacy.document.importer.WarcImporter;
 import net.yacy.server.serverObjects;
@@ -52,11 +53,14 @@ public class IndexImportWarc_p {
            if (post != null) {
                if (post.containsKey("file") || post.containsKey("url")) {
                    final String filename = post.get("file");
                    final String collection = post.get("collection", "user");
                    final String data64 = post.get("file$file", null); // file uploads are all base64-encoded in YaCyDefaultServlet.parseMultipart
                    final byte[] data = data64 == null ? null : Base64Order.standardCoder.decode(data64);
                    if (filename != null && filename.length() > 0) {
                        final File sourcefile = new File(filename);
-                        if (sourcefile.exists()) {
+                        if (sourcefile.exists() || data != null) {
                            try {
-                                final WarcImporter wi = new WarcImporter(sourcefile);
+                                final WarcImporter wi = new WarcImporter(sourcefile, data, collection);
                                wi.start();
                                prop.put("import_thread", "started");
                            } catch (final IOException ex) {
@@ -73,7 +77,7 @@ public class IndexImportWarc_p {
                        if (urlstr != null && urlstr.length() > 0) {
                            try {
                                final MultiProtocolURL url = new MultiProtocolURL(urlstr);
-                                final WarcImporter wi = new WarcImporter(url);
+                                final WarcImporter wi = new WarcImporter(url, collection);
                                wi.start();
                                prop.put("import_thread", "started");
                            } catch (final MalformedURLException ex) {
@@ -21,6 +21,7 @@ package net.yacy.htroot;
 import java.io.File;
 import java.io.IOException;
 import net.yacy.cora.order.Base64Order;
 import net.yacy.cora.protocol.RequestHeader;
 import net.yacy.document.importer.ZimImporter;
 import net.yacy.server.serverObjects;
@@ -29,16 +30,16 @@ import net.yacy.server.serverSwitch;
 public class IndexImportZim_p {
    public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader request, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
-    
+
        // read multipart data from post request
-        
+
        final serverObjects prop = new serverObjects();
-    
+
        if (ZimImporter.job != null && ZimImporter.job.isAlive()) {
            // one import is running, no option to insert anything
            prop.put("import", 1);
            prop.put("import_thread", "running");
-            prop.put("import_warcfile", ZimImporter.job.source());
+            prop.put("import_zimfile", ZimImporter.job.source());
            prop.put("import_count", ZimImporter.job.count());
            prop.put("import_speed", ZimImporter.job.speed());
            prop.put("import_runningHours", (ZimImporter.job.runningTime() / 60) / 60);
@@ -51,13 +52,17 @@ public class IndexImportZim_p {
        } else {
            prop.put("import", 0);
            if (post != null) {
                //for (final String s: post.keySet()) System.out.println("key: " + s); // print all post key attributes
                if (post.containsKey("file")) {
                    final String filename = post.get("file");
                    final String collection = post.get("collection", "user");
                    final String data64 = post.get("file$file", null); // file uploads are all base64-encoded in YaCyDefaultServlet.parseMultipart
                    final byte[] data = data64 == null ? null : Base64Order.standardCoder.decode(data64);
                    if (filename != null && filename.length() > 0) {
                        final File sourcefile = new File(filename);
-                        if (sourcefile.exists()) {
+                        if (data != null || sourcefile.exists()) {
                            try {
-                                final ZimImporter zi = new ZimImporter(sourcefile.getAbsolutePath());
+                                final ZimImporter zi = new ZimImporter(sourcefile.getAbsolutePath(), data, collection);
                                zi.start();
                                prop.put("import_thread", "started");
                            } catch (final IOException ex) {
@@ -70,7 +75,7 @@ public class IndexImportZim_p {
                            prop.put("import_thread", "Error: file not found [" + filename + "]");
                        }
                    }
-        
+
                    prop.put("import_count", 0);
                    prop.put("import_speed", 0);
                    prop.put("import_runningHours", 0);
@@ -147,7 +147,7 @@ public class YaCyDefaultServlet extends HttpServlet  {
        this._htDocsPath = sb.htDocsPath;
        this._htLocalePath = sb.getDataPath("locale.translated_html", "DATA/LOCALE/htroot");
-        this._servletContext = getServletContext();
+        this._servletContext = this.getServletContext();
        this._mimeTypes = new MimeTypes();
        final String tmpstr = this.getServletContext().getInitParameter("welcomeFile");
@@ -156,12 +156,12 @@ public class YaCyDefaultServlet extends HttpServlet  {
        } else {
            this._welcomes = new String[]{tmpstr,"index.html"};
        }
-        this._acceptRanges = getInitBoolean("acceptRanges", this._acceptRanges);
+        this._acceptRanges = this.getInitBoolean("acceptRanges", this._acceptRanges);
-        this._dirAllowed = getInitBoolean("dirAllowed", this._dirAllowed);
+        this._dirAllowed = this.getInitBoolean("dirAllowed", this._dirAllowed);
        Resource.setDefaultUseCaches(false); // caching is handled internally (prevent double caching)
-        final String rb = getInitParameter("resourceBase");
+        final String rb = this.getInitParameter("resourceBase");
        try {
            if (rb != null) {
                this._resourceBase = Resource.newResource(rb);
@@ -176,12 +176,12 @@ public class YaCyDefaultServlet extends HttpServlet  {
        if (ConcurrentLog.isFine("FILEHANDLER")) {
            ConcurrentLog.fine("FILEHANDLER","YaCyDefaultServlet: resource base = " + this._resourceBase);
        }
-        this.templateMethodCache = new ConcurrentHashMap<String, Method>();
+        this.templateMethodCache = new ConcurrentHashMap<>();
    }
    /* ------------------------------------------------------------ */
    protected boolean getInitBoolean(final String name, final boolean dft) {
-        final String value = getInitParameter(name);
+        final String value = this.getInitParameter(name);
        if (value == null || value.length() == 0) {
            return dft;
        }
@@ -243,7 +243,7 @@ public class YaCyDefaultServlet extends HttpServlet  {
            // Is this a Range request?
            reqRanges = request.getHeaders(HeaderFramework.RANGE);
-            if (!hasDefinedRange(reqRanges)) {
+            if (!this.hasDefinedRange(reqRanges)) {
                reqRanges = null;
            }
        }
@@ -261,7 +261,7 @@ public class YaCyDefaultServlet extends HttpServlet  {
            if (reqRanges == null && !endsWithSlash) {
                final int p = pathInContext.lastIndexOf('.');
                if (p >= 0) {
-                    final Method rewriteMethod = rewriteMethod(pathInContext);
+                    final Method rewriteMethod = this.rewriteMethod(pathInContext);
                    if (rewriteMethod != null) {
                        hasClass = true;
                    } else {
@@ -276,7 +276,7 @@ public class YaCyDefaultServlet extends HttpServlet  {
            }
            // find resource
-            resource = getResource(pathInContext);
+            resource = this.getResource(pathInContext);
            if (!hasClass && (resource == null || !resource.exists()) && !pathInContext.contains("..")) {
                // try to get this in the alternative htDocsPath
@@ -304,10 +304,10 @@ public class YaCyDefaultServlet extends HttpServlet  {
                    response.sendRedirect(response.encodeRedirectURL(URIUtil.addPaths(this._servletContext.getContextPath(), pathInContext)));
                } else {
                    if (hasClass) { // this is a YaCy servlet, handle the template
-                        handleTemplate(pathInfo, request, response);
+                        this.handleTemplate(pathInfo, request, response);
                    } else {
-                        if (included || passConditionalHeaders(request, response, resource)) {
+                        if (included || this.passConditionalHeaders(request, response, resource)) {
-                            sendData(request, response, included, resource, reqRanges);
+                            this.sendData(request, response, included, resource, reqRanges);
                        }
                    }
                }
@@ -332,7 +332,7 @@ public class YaCyDefaultServlet extends HttpServlet  {
                        response.sendRedirect(response.encodeRedirectURL(buf.toString()));
                    }
                } // else look for a welcome file
-                else if (null != (welcome = getWelcomeFile(pathInContext))) {
+                else if (null != (welcome = this.getWelcomeFile(pathInContext))) {
                    ConcurrentLog.fine("FILEHANDLER","welcome={}" + welcome);
                    // Forward to the index
@@ -345,8 +345,8 @@ public class YaCyDefaultServlet extends HttpServlet  {
                        }
                    }
                } else {
-                    if (included || passConditionalHeaders(request, response, resource)) {
+                    if (included || this.passConditionalHeaders(request, response, resource)) {
-                        sendDirectory(request, response, resource, pathInContext);
+                        this.sendDirectory(request, response, resource, pathInContext);
                    }
                }
            }
@@ -366,7 +366,7 @@ public class YaCyDefaultServlet extends HttpServlet  {
    @Override
    protected void doPost(final HttpServletRequest request, final HttpServletResponse response)
            throws ServletException, IOException {
-        doGet(request, response);
+        this.doGet(request, response);
    }
    /* ------------------------------------------------------------ */
@@ -399,7 +399,7 @@ public class YaCyDefaultServlet extends HttpServlet  {
        }
        for (final String _welcome : this._welcomes) {
            final String welcome_in_context = URIUtil.addPaths(pathInContext, _welcome);
-            final Resource welcome = getResource(welcome_in_context);
+            final Resource welcome = this.getResource(welcome_in_context);
            if (welcome != null && welcome.exists()) {
                return _welcome;
            }
@@ -524,7 +524,7 @@ public class YaCyDefaultServlet extends HttpServlet  {
            if (include) {
                resource.writeTo(out, 0, content_length);
            } else {
-                writeHeaders(response, resource, content_length);
+                this.writeHeaders(response, resource, content_length);
                resource.writeTo(out, 0, content_length);
            }
        } else {
@@ -533,7 +533,7 @@ public class YaCyDefaultServlet extends HttpServlet  {
            //  if there are no satisfiable ranges, send 416 response
            if (ranges == null || ranges.isEmpty()) {
-                writeHeaders(response, resource, content_length);
+                this.writeHeaders(response, resource, content_length);
                response.setStatus(HttpServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE);
                response.setHeader(HttpHeader.CONTENT_RANGE.asString(),
                        InclusiveByteRange.to416HeaderRangeString(content_length));
@@ -547,7 +547,7 @@ public class YaCyDefaultServlet extends HttpServlet  {
            if (ranges.size() == 1) {
                final InclusiveByteRange singleSatisfiableRange = ranges.iterator().next();
                final long singleLength = singleSatisfiableRange.getSize();
-                writeHeaders(response, resource, singleLength);
+                this.writeHeaders(response, resource, singleLength);
                response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
                response.setHeader(HttpHeader.CONTENT_RANGE.asString(),
                        singleSatisfiableRange.toHeaderRangeString(content_length));
@@ -560,7 +560,7 @@ public class YaCyDefaultServlet extends HttpServlet  {
            //  216 response which does not require an overall
            //  content-length header
            //
-            writeHeaders(response, resource, -1);
+            this.writeHeaders(response, resource, -1);
            final String mimetype = response.getContentType();
            if (mimetype == null) {
                ConcurrentLog.warn("FILEHANDLER","YaCyDefaultServlet: Unknown mimetype for " + request.getRequestURI());
@@ -660,7 +660,7 @@ public class YaCyDefaultServlet extends HttpServlet  {
    }
    protected Object invokeServlet(final Method targetMethod, final RequestHeader request, final serverObjects args) throws IllegalArgumentException, IllegalAccessException, InvocationTargetException {
-        return targetMethod.invoke(null, new Object[]{request, args, Switchboard.getSwitchboard()}); // add switchboard
+        return targetMethod.invoke(null, request, args, Switchboard.getSwitchboard()); // add switchboard
    }
    /**
@@ -813,8 +813,8 @@ public class YaCyDefaultServlet extends HttpServlet  {
                localeSelection = lng;
            }
        }
-        final File targetLocalizedFile = getLocalizedFile(target, localeSelection);
+        final File targetLocalizedFile = this.getLocalizedFile(target, localeSelection);
-        final Method targetMethod = rewriteMethod(target);
+        final Method targetMethod = this.rewriteMethod(target);
        final String targetExt = target.substring(target.lastIndexOf('.') + 1);
        final long now = System.currentTimeMillis();
@@ -844,19 +844,19 @@ public class YaCyDefaultServlet extends HttpServlet  {
                // standard attributes are just pushed as string
                args.put(argName, request.getParameter(argName));
            }
-            final RequestHeader legacyRequestHeader = generateLegacyRequestHeader(request, target, targetExt);
+            final RequestHeader legacyRequestHeader = this.generateLegacyRequestHeader(request, target, targetExt);
            // add multipart-form fields to parameter
            if (ServletFileUpload.isMultipartContent(request)) {
-                parseMultipart(request, args);
+                this.parseMultipart(request, args);
            }
            // eof modification to read attribute
            Object tmp;
            try {
                if (args.isEmpty()) {
                    // yacy servlets typically test for args != null (but not for args .isEmpty())
-                    tmp = invokeServlet(targetMethod, legacyRequestHeader, null);
+                    tmp = this.invokeServlet(targetMethod, legacyRequestHeader, null);
                } else {
-                    tmp = invokeServlet(targetMethod, legacyRequestHeader, args);
+                    tmp = this.invokeServlet(targetMethod, legacyRequestHeader, args);
                }
            } catch(final InvocationTargetException e) {
                if(e.getCause() instanceof InvalidURLLicenceException) {
@@ -929,7 +929,7 @@ public class YaCyDefaultServlet extends HttpServlet  {
                    result = RasterPlotter.exportImage(bi, targetExt);
                }
-                updateRespHeadersForImages(target, response);
+                this.updateRespHeadersForImages(target, response);
                final String mimeType = Classification.ext2mime(targetExt, MimeTypes.Type.TEXT_HTML.asString());
                response.setContentType(mimeType);
                response.setContentLength(result.length());
@@ -942,9 +942,9 @@ public class YaCyDefaultServlet extends HttpServlet  {
            if (tmp instanceof InputStream) {
                /* Images and favicons can also be written directly from an inputStream */
-                updateRespHeadersForImages(target, response);
+                this.updateRespHeadersForImages(target, response);
-                writeInputStream(response, targetExt, (InputStream)tmp);
+                this.writeInputStream(response, targetExt, (InputStream)tmp);
                return;
            }
@@ -1007,9 +1007,9 @@ public class YaCyDefaultServlet extends HttpServlet  {
            if (targetLocalizedFile.exists() && targetLocalizedFile.isFile() && targetLocalizedFile.canRead()) {
-                sb.setConfig(SwitchboardConstants.SERVER_SERVLETS_CALLED, appendPath(sb.getConfig(SwitchboardConstants.SERVER_SERVLETS_CALLED, ""), target));
+                sb.setConfig(SwitchboardConstants.SERVER_SERVLETS_CALLED, this.appendPath(sb.getConfig(SwitchboardConstants.SERVER_SERVLETS_CALLED, ""), target));
                if (args != null && !args.isEmpty()) {
-                    sb.setConfig("server.servlets.submitted", appendPath(sb.getConfig("server.servlets.submitted", ""), target));
+                    sb.setConfig("server.servlets.submitted", this.appendPath(sb.getConfig("server.servlets.submitted", ""), target));
                }
                // add the application version, the uptime and the client name to every rewrite table
@@ -1071,7 +1071,7 @@ public class YaCyDefaultServlet extends HttpServlet  {
                    TemplateEngine.writeTemplate(targetLocalizedFile.getName(), fis, bas, templatePatterns);
                    // handle SSI
-                    parseSSI (bas.toByteArray(),request,response);
+                    this.parseSSI (bas.toByteArray(),request,response);
                } finally {
                    try {
                        fis.close();
@@ -1258,7 +1258,7 @@ public class YaCyDefaultServlet extends HttpServlet  {
                    InputStream filecontent = null;
                    try {
                        filecontent = item.getInputStream();
-                        files.put(new AbstractMap.SimpleEntry<String, byte[]>(item.getFieldName(), FileUtils.read(filecontent)));
+                        files.put(new AbstractMap.SimpleEntry<>(item.getFieldName(), FileUtils.read(filecontent)));
                    } catch (final IOException e) {
                        ConcurrentLog.info("FILEHANDLER", e.getMessage());
                    } finally {
@@ -1271,7 +1271,7 @@ public class YaCyDefaultServlet extends HttpServlet  {
                    final String n = job.getKey();
                    final byte[] v = job.getValue();
                    final String filename = args.get(n);
-                    if (filename != null && filename.endsWith(".gz")) {
+                    if (filename != null && (filename.endsWith(".gz") || filename.endsWith(".zip") || filename.endsWith(".warc") || filename.endsWith(".zim"))) {
                        // transform this value into base64
                        final String b64 = Base64Order.standardCoder.encode(v);
                        args.put(n + "$file", b64);
@@ -2118,15 +2118,15 @@ public final class Switchboard extends serverSwitch {
        return null;
    }
-    public boolean processPack(final String s) {
+    public boolean processPack(final String inFileName, String collectionName) {
-        final File infile = new File(this.packsLoadPath, s);
+        final File infile = new File(this.packsLoadPath, inFileName);
        if ( !infile.exists() || !infile.canWrite() || !infile.canRead() ) {
            return false;
        }
-        final File outfile = new File(this.packsLoadedPath, s);
+        final File outfile = new File(this.packsLoadedPath, inFileName);
        //if (outfile.exists()) return false;
        boolean moved = false;
-        if ( s.endsWith("xml.zip") ) {
+        if ( inFileName.endsWith("xml.zip") ) {
            // open the zip file with all the xml files in it
            ZipInputStream zis = null;
            try {
@@ -2141,7 +2141,7 @@ public final class Switchboard extends serverSwitch {
                        baos.write(buffer, 0, size);
                    }
                    baos.flush();
-                    this.processXMLPack(new ByteArrayInputStream(baos.toByteArray()), entry.getName());
+                    this.processXMLPack(new ByteArrayInputStream(baos.toByteArray()), entry.getName(), collectionName);
                    baos.close();
                    if (this.shallTerminate()) break;
                }
@@ -2154,9 +2154,9 @@ public final class Switchboard extends serverSwitch {
                }
            }
            return moved;
-        } else if (s.endsWith(".warc") || s.endsWith(".warc.gz")) {
+        } else if (inFileName.endsWith(".warc") || inFileName.endsWith(".warc.gz")) {
            try {
-                final WarcImporter wri = new WarcImporter(infile);
+                final WarcImporter wri = new WarcImporter(infile, null, collectionName);
                wri.start();
                try {
                    wri.join();
@@ -2168,9 +2168,9 @@ public final class Switchboard extends serverSwitch {
                this.log.warn("IO Error processing warc file " + infile);
            }
            return moved;
-        } else if (s.endsWith(".zim")) {
+        } else if (inFileName.endsWith(".zim")) {
            try {
-                final ZimImporter wri = new ZimImporter(infile.getAbsolutePath());
+                final ZimImporter wri = new ZimImporter(infile.getAbsolutePath(), null, collectionName);
                wri.start();
                try {
                    wri.join();
@@ -2183,16 +2183,16 @@ public final class Switchboard extends serverSwitch {
            }
            return moved;
        } else if (
-                s.endsWith(".jsonl") || s.endsWith(".jsonl.gz") ||
+                inFileName.endsWith(".jsonl") || inFileName.endsWith(".jsonl.gz") ||
-                s.endsWith(".jsonlist") || s.endsWith(".jsonlist.gz") ||
+                inFileName.endsWith(".jsonlist") || inFileName.endsWith(".jsonlist.gz") ||
-                s.endsWith(".flatjson") || s.endsWith(".flatjson.gz")) {
+                inFileName.endsWith(".flatjson") || inFileName.endsWith(".flatjson.gz")) {
            return this.processPackJson(infile, outfile);
        }
        InputStream is = null;
        try {
            is = new BufferedInputStream(new FileInputStream(infile));
-            if (s.endsWith(".gz")) is = new GZIPInputStream(is, 65535);
+            if (inFileName.endsWith(".gz")) is = new GZIPInputStream(is, 65535);
-            this.processXMLPack(is, infile.getName());
+            this.processXMLPack(is, infile.getName(), collectionName);
        } catch (final IOException e ) {
            ConcurrentLog.logException(e);
        } finally {
@@ -2244,9 +2244,13 @@ public final class Switchboard extends serverSwitch {
        return moved;
    }
-    private void processXMLPack(final InputStream is, final String name) throws IOException {
+    private void processXMLPack(final InputStream is, final String name, String collection) throws IOException {
        final int concurrency = Runtime.getRuntime().availableProcessors();
        final CrawlProfile xmlProfile = (CrawlProfile) Switchboard.getSwitchboard().crawler.defaultPackProfile.clone();
        xmlProfile.setCollections(collection);
        xmlProfile.setHandle();
        // start reader thread
        final XMLPackReader reader = new XMLPackReader(is, 100, this.crawlStacker, this.index.fulltext().getDefaultConfiguration(), concurrency);
        final Thread readerThread = new Thread(reader, name);
@@ -2305,10 +2309,10 @@ public final class Switchboard extends serverSwitch {
                                            null,
                                            "",
                                            entry.getDate(),
-                                            Switchboard.this.crawler.defaultPackProfile.handle(),
+                                            xmlProfile.handle(),
                                            0,
-                                            Switchboard.this.crawler.defaultPackProfile.timezoneOffset());
+                                            xmlProfile.timezoneOffset());
-                            final Response response = new Response(request, null, null, Switchboard.this.crawler.defaultPackProfile, false, null);
+                            final Response response = new Response(request, null, null, xmlProfile, false, null);
                            final IndexingQueueEntry queueEntry =
                                    new IndexingQueueEntry(response, new Document[] {document}, null);
@@ -2401,7 +2405,7 @@ public final class Switchboard extends serverSwitch {
                    if (this.isPackFile(pack)) {
                        // read the pack file and store entry in index
-                        if ( this.processPack(pack) ) {
+                        if ( this.processPack(pack, "user") ) {
                            return true;
                        }
                    }