mirror of
https://github.com/yacy/yacy_search_server.git
synced 2026-10-25 05:54:15 +00:00
Michael Peter Christen
4c953ee8fd
Having the collection attribute for those imports enables to use those import methods for YaCy Pack generation which requires the collection name to produce a pack name. Fixed also file upload option through browser for both methods. For large zim/warc files you require large memory settings to be able to cache the uploaded file. You also need large memory requirements in your browser to upload gigabytes of files through multipart/form-data uploads.
105 lines
5.2 KiB
Java
105 lines
5.2 KiB
Java
// IndexImportWarc_p.java
|
|
// -------------------------
|
|
// (c) 2017 by reger24; https://github.com/reger24
|
|
//
|
|
// This program is free software; you can redistribute it and/or modify
|
|
// it under the terms of the GNU General Public License as published by
|
|
// the Free Software Foundation; either version 2 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU General Public License
|
|
// along with this program; if not, write to the Free Software
|
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
|
|
package net.yacy.htroot;
|
|
|
|
import java.io.File;
|
|
import java.io.IOException;
|
|
import java.net.MalformedURLException;
|
|
|
|
import net.yacy.cora.document.id.MultiProtocolURL;
|
|
import net.yacy.cora.order.Base64Order;
|
|
import net.yacy.cora.protocol.RequestHeader;
|
|
import net.yacy.document.importer.WarcImporter;
|
|
import net.yacy.server.serverObjects;
|
|
import net.yacy.server.serverSwitch;
|
|
|
|
public class IndexImportWarc_p {
|
|
|
|
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
|
|
final serverObjects prop = new serverObjects();
|
|
|
|
if (WarcImporter.job != null && WarcImporter.job.isAlive()) {
|
|
// one import is running, no option to insert anything
|
|
prop.put("import", 1);
|
|
prop.put("import_thread", "running");
|
|
prop.put("import_warcfile", WarcImporter.job.source());
|
|
prop.put("import_count", WarcImporter.job.count());
|
|
prop.put("import_speed", WarcImporter.job.speed());
|
|
prop.put("import_runningHours", (WarcImporter.job.runningTime() / 60) / 60);
|
|
prop.put("import_runningMinutes", (WarcImporter.job.runningTime() / 60) % 60);
|
|
prop.put("import_remainingHours", (WarcImporter.job.remainingTime() / 60) / 60);
|
|
prop.put("import_remainingMinutes", (WarcImporter.job.remainingTime() / 60) % 60);
|
|
if (post != null && post.containsKey("abort")) {
|
|
WarcImporter.job.quit();
|
|
}
|
|
} else {
|
|
prop.put("import", 0);
|
|
if (post != null) {
|
|
if (post.containsKey("file") || post.containsKey("url")) {
|
|
final String filename = post.get("file");
|
|
final String collection = post.get("collection", "user");
|
|
final String data64 = post.get("file$file", null); // file uploads are all base64-encoded in YaCyDefaultServlet.parseMultipart
|
|
final byte[] data = data64 == null ? null : Base64Order.standardCoder.decode(data64);
|
|
if (filename != null && filename.length() > 0) {
|
|
final File sourcefile = new File(filename);
|
|
if (sourcefile.exists() || data != null) {
|
|
try {
|
|
final WarcImporter wi = new WarcImporter(sourcefile, data, collection);
|
|
wi.start();
|
|
prop.put("import_thread", "started");
|
|
} catch (final IOException ex) {
|
|
prop.put("import_thread", "Error: file not found [" + filename + "]");
|
|
}
|
|
prop.put("import", 1);
|
|
prop.put("import_warcfile", filename);
|
|
} else {
|
|
prop.put("import_warcfile", "");
|
|
prop.put("import_thread", "Error: file not found [" + filename + "]");
|
|
}
|
|
} else {
|
|
final String urlstr = post.get("url");
|
|
if (urlstr != null && urlstr.length() > 0) {
|
|
try {
|
|
final MultiProtocolURL url = new MultiProtocolURL(urlstr);
|
|
final WarcImporter wi = new WarcImporter(url, collection);
|
|
wi.start();
|
|
prop.put("import_thread", "started");
|
|
} catch (final MalformedURLException ex) {
|
|
prop.put("import_thread", ex.getMessage());
|
|
} catch (final IOException ex) {
|
|
prop.put("import_thread", ex.getMessage());
|
|
}
|
|
prop.put("import", 1);
|
|
prop.put("import_warcfile", urlstr);
|
|
}
|
|
}
|
|
|
|
prop.put("import_count", 0);
|
|
prop.put("import_speed", 0);
|
|
prop.put("import_runningHours", 0);
|
|
prop.put("import_runningMinutes", 0);
|
|
prop.put("import_remainingHours", 0);
|
|
prop.put("import_remainingMinutes", 0);
|
|
}
|
|
}
|
|
}
|
|
return prop;
|
|
}
|
|
}
|