mirror of
https://github.com/yacy/yacy_search_server.git
synced 2026-10-25 05:54:15 +00:00
removed the snapshot feature and all dependencies from non-java code
This commit is contained in:
Michael Peter Christen committed
1 parent
9e5958ab0e
commit
35a1d85f8b
37 files changed
+11
-2544
No files matched your search
@@ -27,7 +27,7 @@ jobs:
|
||||
- name: Install build dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -yq ant git wkhtmltopdf imagemagick xvfb ghostscript
|
||||
sudo apt-get install -yq ant git
|
||||
|
||||
- name: Configure runtime settings
|
||||
run: |
|
||||
|
||||
@@ -896,10 +896,6 @@ crawler.latencyFactor = 0.5
|
||||
# defined here
|
||||
crawler.onDemandLimit = 1000
|
||||
|
||||
# The maximum time in seconds to wait for each wkhtmltopdf call when rendering PDF snapshots
|
||||
# Beyond that limit the process is killed
|
||||
snapshots.wkhtmltopdf.timeout = 30
|
||||
|
||||
# maximum size of indexing queue
|
||||
indexer.slots = 100
|
||||
|
||||
@@ -964,7 +960,6 @@ search.result.show.indexbrowser = true
|
||||
search.result.show.vocabulary = false
|
||||
# Set of comma separated vocabulary names not to be used as search results facets
|
||||
search.result.show.vocabulary.omit =
|
||||
search.result.show.snapshots = false
|
||||
# when true, display the raw ranking score value
|
||||
search.result.show.ranking = false
|
||||
|
||||
|
||||
@@ -32,11 +32,6 @@ RUN sed -i "/adminAccountBase64MD5=/c\adminAccountBase64MD5=MD5:8cffbc0d66567a09
|
||||
|
||||
## build final image
|
||||
FROM eclipse-temurin:24-jdk-noble AS app
|
||||
RUN for i in 1 2 3; do \
|
||||
apt-get update && break || sleep 5; \
|
||||
done && \
|
||||
apt-get install -y --no-install-recommends wkhtmltopdf xvfb ghostscript && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
LABEL maintainer="Michael Peter Christen <mc@yacy.net>"
|
||||
|
||||
|
||||
@@ -45,36 +45,6 @@ ENV ANT_HOME="/opt/ant"
|
||||
ENV PATH="/opt/ant/bin:${PATH}"
|
||||
RUN ant -f /opt/ant/fetch.xml -Ddest=system
|
||||
|
||||
# install the latest version dependencies of Yacy (copied from the official Dockerfile)
|
||||
RUN pacman -Sy imagemagick ruby qt5-tools qt5-doc gperf python xorg-server-xvfb ghostscript git qt5-svg qt5-xmlpatterns base-devel qt5-location qt5-sensors qt5-webchannel libwebp libxslt libxcomposite gst-plugins-base hyphen hyphen-en hyphen-de woff2 cmake --noconfirm && \
|
||||
pacman -Scc --noconfirm
|
||||
RUN useradd --no-create-home --shell=/bin/false build && \
|
||||
usermod -L build
|
||||
USER build
|
||||
WORKDIR /tmp
|
||||
|
||||
# buid package for qt5-webkit from AUR
|
||||
RUN git clone https://aur.archlinux.org/qt5-webkit.git && \
|
||||
cd qt5-webkit && \
|
||||
MAKEFLAGS="-j$(nproc)" makepkg --noconfirm && \
|
||||
cd ..
|
||||
USER root
|
||||
|
||||
# install the package
|
||||
RUN pacman -U --noconfirm /tmp/qt5-webkit/qt5-webkit*.pkg.tar.zst && \
|
||||
rm -rf /tmp/qt5-webkit
|
||||
USER build
|
||||
|
||||
# build package for wkhtmltopdf from AUR
|
||||
RUN git clone https://aur.archlinux.org/wkhtmltopdf.git && \
|
||||
cd wkhtmltopdf && \
|
||||
MAKEFLAGS="-j$(nproc)" makepkg --noconfirm --skippgpcheck && \
|
||||
cd ..
|
||||
USER root
|
||||
|
||||
# install the package
|
||||
RUN pacman -U --noconfirm /tmp/wkhtmltopdf/wkhtmltopdf*.pkg.tar.zst && \
|
||||
rm -rf /tmp/wkhtmltopdf
|
||||
WORKDIR /opt
|
||||
|
||||
# download the latest version of yacy
|
||||
|
||||
@@ -30,11 +30,6 @@ RUN sed -i "/adminAccountBase64MD5=/c\adminAccountBase64MD5=MD5:8cffbc0d66567a09
|
||||
|
||||
## build final image
|
||||
FROM arm64v8/openjdk:23-jdk-slim-bookworm AS app
|
||||
RUN for i in 1 2 3; do \
|
||||
apt-get update && break || sleep 5; \
|
||||
done && \
|
||||
apt-get install -y --no-install-recommends wkhtmltopdf xvfb ghostscript && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
LABEL maintainer="Michael Peter Christen <mc@yacy.net>"
|
||||
|
||||
|
||||
@@ -27,18 +27,9 @@ RUN sed -i "/adminAccountBase64MD5=/c\adminAccountBase64MD5=MD5:8cffbc0d66567a09
|
||||
sed -i "/server.https=false/c\server.https=true" /opt/yacy_search_server/defaults/yacy.init
|
||||
|
||||
## build final image
|
||||
FROM surnet/alpine-wkhtmltopdf:3.21.2-0.12.6-small AS wkhtmltopdf
|
||||
FROM eclipse-temurin:21-jre-alpine-3.21 AS app
|
||||
|
||||
RUN apk add --no-cache \
|
||||
imagemagick \
|
||||
xvfb \
|
||||
ghostscript \
|
||||
# Install dependencies for wkhtmltopdf
|
||||
libstdc++ \
|
||||
libx11 \
|
||||
libxrender \
|
||||
libxext \
|
||||
libssl3 \
|
||||
ca-certificates \
|
||||
fontconfig \
|
||||
@@ -57,9 +48,6 @@ RUN apk add --no-cache \
|
||||
&& rm -rf /tmp/* \
|
||||
&& apk del .build-deps
|
||||
|
||||
# Copy wkhtmltopdf files from docker-wkhtmltopdf image
|
||||
COPY --from=wkhtmltopdf /bin/wkhtmltopdf /bin/wkhtmltopdf
|
||||
|
||||
# copy YaCy to app image
|
||||
RUN addgroup yacy && adduser -S -G yacy -H -D yacy
|
||||
WORKDIR /opt
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
FROM arm32v7/openjdk:11-jdk as builder
|
||||
|
||||
# Install needed packages not in base image
|
||||
RUN apt-get update && apt-get install -yq curl wkhtmltopdf imagemagick xvfb ghostscript && \
|
||||
RUN apt-get update && apt-get install -yq curl && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# set current working dir & copy sources
|
||||
|
||||
@@ -295,7 +295,6 @@ var solr= $.getJSON("solr/collection1/select?q=*:*&start=0&rows=0&wt=json&facet=
|
||||
<td><span role="separator" aria-orientation="vertical"> | </span><a id="cacheLink" href="CacheResource_p.html" target="LayouTest">Cache</a></td>
|
||||
<td><span role="separator" aria-orientation="vertical"> | </span><a id="proxyLink" href="proxy.html" target="LayouTest">View via Proxy</a></td>
|
||||
<td><span role="separator" aria-orientation="vertical"> | </span><a id="browseIndexLink" href="IndexBrowser_p.html" target="LayouTest"><img src="env/grafics/minitree.png" width="15" height="8" title="Browse index" alt="Browse index"/></a></td>
|
||||
<td><span role="separator" aria-orientation="vertical"> | </span><a id="snapshotsLink" href="yacysearch.html" target="LayouTest">JPG Snapshot</a></td>
|
||||
#(search.result.show.ranking)#::<td><span role="separator" aria-orientation="vertical"> | </span><span title="Raw ranking score value">Ranking: 1.12195955E9</span>#(/search.result.show.ranking)#
|
||||
</tr>
|
||||
<tr>
|
||||
@@ -315,7 +314,6 @@ var solr= $.getJSON("solr/collection1/select?q=*:*&start=0&rows=0&wt=json&facet=
|
||||
</span></span>
|
||||
</td>
|
||||
<td style="text-align:center;"><input type="checkbox" name="search.result.show.indexbrowser" aria-labelledby="browseIndexLink" value="true" #(search.result.show.indexbrowser)#::checked="checked" #(/search.result.show.indexbrowser)# /></td>
|
||||
<td style="text-align:center;"><input type="checkbox" name="search.result.show.snapshots" aria-labelledby="snapshotsLink" value="true" #(search.result.show.snapshots)#::checked="checked" #(/search.result.show.snapshots)# /></td>
|
||||
#(search.result.show.ranking)#::<td style="text-align:center;">
|
||||
<span class="info" style="padding-left: 10px"><img src="env/grafics/i16.gif" alt="info"/><span>
|
||||
Ranking score value, mainly for debug/analysis purpose, configured in <a href="Settings_p.html?page=debug">Debug/Analysis Settings</a><br>
|
||||
|
||||
@@ -706,42 +706,6 @@
|
||||
</fieldset>
|
||||
#(/vocabularySelect)#
|
||||
|
||||
<fieldset>
|
||||
<legend>Snapshot Creation</legend>
|
||||
<dl>
|
||||
<dt><label>Max Depth for Snapshots</label></dt>
|
||||
<dd>
|
||||
<span class="info" style="float:right"><img src="env/grafics/i16.gif" width="16" height="16" alt="info"/><span style="right:0px;">
|
||||
Snapshots are xml metadata and pictures of web pages that can be created during crawling time.
|
||||
The xml data is stored in the same way as a Solr search result with one hit and the pictures will be stored as pdf into subdirectories
|
||||
of HTCACHE/snapshots/. From the pdfs the jpg thumbnails are computed. Snapshot generation can be controlled using a depth parameter; that
|
||||
means a snapshot is only be generated if the crawl depth of a document is smaller or equal to the given number here. If the number is set to -1,
|
||||
no snapshots are generated.
|
||||
</span></span>
|
||||
<input type="text" name="snapshotsMaxDepth" id="snapshotsMaxDepth" size="2" maxlength="2" value="#[snapshotsMaxDepth]#" />
|
||||
</dd>
|
||||
<dt><label>Multiple Snapshot Versions</label></dt>
|
||||
<dd>
|
||||
<input type="radio" name="snapshotsReplaceOld" value="on" checked="checked"/> replace old snapshots with new one
|
||||
<input type="radio" name="snapshotsReplaceOld" value="off" /> add new versions for each crawl
|
||||
</dd>
|
||||
<dt><label for="snapshotsMustnotmatch">must-not-match filter for snapshot generation</label></dt>
|
||||
<dd><input name="snapshotsMustnotmatch" id="snapshotsMustnotmatch" type="text" size="55" maxlength="100000" value="#[snapshotsMustnotmatch]#" /></dd>
|
||||
#(snapshotEnableImages)#
|
||||
<input type="hidden" name="snapshotsLoadImage" id="snapshotsLoadImage" value="false"/>
|
||||
<dt><label>Image Creation</label></dt>
|
||||
<dd>
|
||||
<div class="info">Only XML snapshots can be generated. as the <a href="https://wkhtmltopdf.org/" target="_blank">wkhtmltopdf</a> util is not found by YaCy on your system.
|
||||
It is required to generate PDF snapshots from crawled pages that can then be converted to images.</div>
|
||||
</dd>::
|
||||
<dt><label>Image Creation</label></dt>
|
||||
<dd>
|
||||
<input type="checkbox" name="snapshotsLoadImage" id="snapshotsLoadImage"#(snapshotsLoadImageChecked)#:: checked="checked"#(/snapshotsLoadImageChecked)#/>
|
||||
</dd>
|
||||
#(/snapshotEnableImages)#
|
||||
</dl>
|
||||
</fieldset>
|
||||
|
||||
<dl>
|
||||
<dt><input type="hidden" name="crawlingstart" value="1"/><input type="submit" value="Start New Crawl Job" class="btn btn-primary"/></dt><dd></dd>
|
||||
</dl>
|
||||
|
||||
Vendored
-8
@@ -957,14 +957,6 @@ div.BlogCommentPrevNext {
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
/* SnapshotImagesReponseWriter.java */
|
||||
body#SnapshotImagesReponseWriter img {
|
||||
word-break: break-all; /* prevent snapshot image alternate text (original URL) to overflow over image width */
|
||||
overflow: hidden; /* prevent snapshot image alternate text (original URL) to overflow over image height */
|
||||
display: inline-block; /* constant width and heigth even for snapshot image in error */
|
||||
background-color: #eeeeee; /* light background colour to distinguish snapshots images in error */
|
||||
}
|
||||
|
||||
/* other ------------------------------------*/
|
||||
ul#menu {
|
||||
padding:0;
|
||||
|
||||
@@ -52,7 +52,6 @@
|
||||
#(showProxy)#::<span role="separator" aria-orientation="vertical"> | </span><a href="proxy.html?url=#[link]#" target="_blank">View via proxy</a>#(/showProxy)#
|
||||
#(showIndexBrowser)#::<span role="separator" aria-orientation="vertical"> | </span><a href="IndexBrowser_p.html?path=#[link]#"><img src="env/grafics/minitree.png" width="15" height="8" title="Browse index" alt="Browse index"/></a>#(/showIndexBrowser)#
|
||||
#(showVocabulary)#::<br/>#{vocabulary}##[name]#:#[terms]# #{/vocabulary}##(/showVocabulary)#
|
||||
#(showSnapshots)#::<span role="separator" aria-orientation="vertical"> | </span><a href="#[link]#" target="_blank">#[extension]# Snapshot</a>#(/showSnapshots)#
|
||||
#(showRanking)#::<span role="separator" aria-orientation="vertical"> | </span><span title="Raw ranking score value">Ranking: #[ranking]#</span>#(/showRanking)#
|
||||
</p>
|
||||
</div>
|
||||
|
||||
@@ -591,7 +591,6 @@ Maximum range (in days)==Période maximale (en jours)
|
||||
Maximum days number in the histogram. Beware that a large value may trigger high CPU loads both on the server and on the browser with large result sets.==Nombre maximum de jours de l'histogramme. Veuillez noter qu'une valeur élevée peut déclencher des charges CPU importantes tant sur le serveur que sur le navigateur lorsqu'il y a un grand nombre de résultats.
|
||||
keyword subject keyword2 keyword3==mot-clé sujet mot-clé2 mot-clé3
|
||||
View via Proxy==Ouvrir via le proxy
|
||||
>JPG Snapshot<==>Capture JPG<
|
||||
"Raw ranking score value"=="Score de classement brut"
|
||||
Ranking: 1.12195955E9==Score: 1.12195955E9
|
||||
"Delete navigator"=="Supprimer le groupe"
|
||||
|
||||
@@ -779,7 +779,6 @@ Maximum Pages per Domain==ドメイン毎の最大のページ数
|
||||
>No Doubles<==>重複させない<
|
||||
>Re-load<==>再読み込み<
|
||||
>Document Cache<==>ドキュメントのキャッシュ<
|
||||
>Snapshot Creation<==>スナップショットの作成<
|
||||
>Index Attributes<==>索引の属性<
|
||||
"Start New Crawl Job"=="新たなクロールのジョブを開始する"
|
||||
#-----------------------------
|
||||
|
||||
@@ -1313,16 +1313,7 @@ never use the cache, all content from fresh internet source;==Никогда н
|
||||
use the cache if the cache exists and is fresh using the proxy-fresh rules;==использовать кэш, если кэш существует и это новое использование правил прокси
|
||||
use the cache if the cache exist. Do no check freshness. Otherwise use online source;==использовать кэш, если кэш существует. Не проверять на обновления. Иначе использовать интернет-источник;
|
||||
never go online, use all content from cache. If no cache exist, treat content as unavailable==всегда оффлайн, использовать весь контент из кэша. Если кэш не существует, то считать что контент недоступен.
|
||||
>Snapshot Creation<==>Создание снимка<
|
||||
Max Depth for Snapshots==Максимальная глубина снимка
|
||||
Multiple Snapshot Versions==Версии снимков
|
||||
replace old snapshots with new one==заменить старые снимки новыми
|
||||
add new versions for each crawl==добавить новые версии для каждого индексирования
|
||||
Snapshots are xml metadata and pictures of web pages that can be created during crawling time.==Снимки состоят их XML-данных и изображений вэб-страниц и могут быть созданы во время индексирования.
|
||||
The xml data is stored in the same way as a Solr search result with one hit and the pictures will be stored as pdf into subdirectories==XML-данные хранятся таким же образом, как и результаты поиска Solr; изображения сохраняются в формате pdf в директории
|
||||
of HTCACHE/snapshots/. From the pdfs the jpg thumbnails are computed. Snapshot generation can be controlled using a depth parameter; that==HTCACHE/snapshots/. Из pdf создаются эскизы в формате jpg. Создание снимков регулируется параметром "глубина снимка".
|
||||
means a snapshot is only be generated if the crawl depth of a document is smaller or equal to the given number here. If the number is set to -1,==Это означает, что снимки создаются только, если глубина индексирования документов низкая или равна заданному здесь значению. Если значение "-1",
|
||||
no snapshots are generated.==то снимки не создаются.
|
||||
>Crawler Filter<==>Фильтр индексатора<
|
||||
These are limitations on the crawl stacker. The filters will be applied before a web page is loaded.==Ограничения работы индексатора. Фильтры применяются до загрузки страницы.
|
||||
Crawling Depth<==Глубина индексирования<
|
||||
|
||||
@@ -908,7 +908,6 @@ Max. tags initially displayed==初始显示的最大标签数
|
||||
>Pictures<==>图片<
|
||||
>Cache<==>缓存<
|
||||
>View via Proxy<==>通过代理查看<
|
||||
>JPG Snapshot<==>JPG快照<
|
||||
For this option URL proxy must be enabled.==对于这个选项，必须启用URL代理。
|
||||
menu: System Administration > Advanced Settings==菜单：系统管理>高级设置
|
||||
Ranking score value, mainly for debug/analysis purpose, configured in <a href="Settings_p.html?page=debug">Debug/Analysis Settings</a>==排名分数值，主要用于调试/分析目的，在<a href="Settings_p.html?page=debug">调试/分析</a>设置中配置
|
||||
@@ -1465,20 +1464,8 @@ alternative user agents here which have different crawl timings and also identif
|
||||
>Enrich Vocabulary<==>丰富词汇<
|
||||
>Scraping Fields<==>刮领域<
|
||||
You can use class names to enrich the terms of a vocabulary based on the text content that appears on web pages. Please write the names of classes into the matrix.==你可以根据网页上显示的文本内容，使用类名丰富词汇表中的术语。请把类名写进表格。
|
||||
>Snapshot Creation==>创建快照
|
||||
>Max Depth for Snapshots<==>快照最大深度<
|
||||
Snapshots are xml metadata and pictures of web pages that can be created during crawling time.==快照是可以在爬取期间创建的xml元数据和网页图片。
|
||||
The xml data is stored in the same way as a Solr search result with one hit and the pictures will be stored as pdf into subdirectories==xml数据以与Solr搜索结果相同的方式存储，只需点击一次，图片将以pdf格式存储到HTCACHE/snapshots/的子目录中。
|
||||
of HTCACHE/snapshots/. From the pdfs the jpg thumbnails are computed. Snapshot generation can be controlled using a depth parameter; that==根据PDF计算jpg缩略图。可以使用深度参数控制快照生成；
|
||||
means a snapshot is only be generated if the crawl depth of a document is smaller or equal to the given number here. If the number is set to -1,==这意味着只有当文档的爬网深度小于或等于此处给定的数字时，才会生成快照。
|
||||
no snapshots are generated.==如果该数字设置为-1，则不会生成快照。
|
||||
>Multiple Snapshot Versions<==>多个快照版本<
|
||||
replace old snapshots with new one==用新快照代替老快照
|
||||
add new versions for each crawl==每次爬取添加新版本
|
||||
>must-not-match filter for snapshot generation<==>快照产生排除过滤器<
|
||||
>Image Creation<==>生成快照<
|
||||
Only XML snapshots can be generated. as the <a href="https://wkhtmltopdf.org/" target="_blank">wkhtmltopdf</a> util is not found by YaCy on your system.==只能生成XML快照。因为YaCy在你的系统上找不到<a href="https://wkhtmltopdf.org/" target="_blank">wkhtmltopdf</a>工具。
|
||||
It is required to generate PDF snapshots from crawled pages that can then be converted to images.==需要从爬取的页面中生成PDF快照，然后将其转换为图像。
|
||||
>Index Attributes==>索引属性
|
||||
>Indexing<==>创建索引<
|
||||
This enables indexing of the webpages the crawler will download. This should be switched on by default, unless you want to crawl only to fill the==这样就可以对爬虫将下载的网页进行索引。
|
||||
|
||||
+2
-9
@@ -15,10 +15,6 @@ icon: addon/YaCy256x256.png
|
||||
|
||||
apps:
|
||||
yacy-search:
|
||||
environment:
|
||||
# needed to run wkhtmltopdf from YaCy within the Snap for Snapshots generation
|
||||
# with wkhtmltopdf 0.12.2.4 included with Ubuntu 16.04 LTS (recommended minimum system to build Snaps in 2018)
|
||||
QT_QPA_PLATFORM_PLUGIN_PATH: "$SNAP/usr/lib/x86_64-linux-gnu/qt5/plugins/platforms"
|
||||
command: sh "$SNAP/yacy/addon/snap/snapStartYACY.sh"
|
||||
# Interfaces authorizations requirements
|
||||
# network-bind - to allow acting as an HTTP server and client
|
||||
@@ -26,8 +22,7 @@ apps:
|
||||
# home - not strictly necessary but can be convenient for some features to have read/write access to files outside the snap data folders, notably
|
||||
# - to index files in Intranet mode
|
||||
# - for index, dump, vocabulary import/export features
|
||||
# x11 - needed by wkhtmltopdf to generate Snapshots using WebKit engine - not necessary when Snapshots feature is not used
|
||||
plugs: [network-bind, desktop, home, x11]
|
||||
plugs: [network-bind, desktop, home]
|
||||
passwd: # set the YaCy administrator password
|
||||
command: sh "$SNAP/yacy/addon/snap/snapPasswd.sh"
|
||||
# Interfaces authorizations requirements
|
||||
@@ -43,9 +38,7 @@ parts:
|
||||
yacy-search:
|
||||
# Additional packages
|
||||
# curl : needed by the passwd.sh script to call a YaCy API when the server is running
|
||||
# wkhtmltopdf : needed by the snapshot generation feature.
|
||||
# Conversion from PDF snapshots to images is done using the pure java pdfbox library, as imagemagick is more difficult to propertly embed within a Snap
|
||||
stage-packages: [curl, wkhtmltopdf]
|
||||
stage-packages: [curl]
|
||||
plugin: ant
|
||||
ant-build-targets:
|
||||
- clean
|
||||
|
||||
@@ -1,205 +0,0 @@
|
||||
package net.yacy.cora.federate.solr.responsewriter;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Writer;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.lucene.document.Document;
|
||||
import org.apache.lucene.index.IndexableField;
|
||||
import org.apache.solr.client.solrj.response.QueryResponse;
|
||||
import org.apache.solr.common.SolrDocument;
|
||||
import org.apache.solr.common.SolrDocumentList;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.response.QueryResponseWriter;
|
||||
import org.apache.solr.response.ResultContext;
|
||||
import org.apache.solr.response.SolrQueryResponse;
|
||||
import org.apache.solr.search.DocIterator;
|
||||
import org.apache.solr.search.DocList;
|
||||
import org.apache.solr.search.SolrIndexSearcher;
|
||||
|
||||
import net.yacy.cora.util.CommonPattern;
|
||||
import net.yacy.search.schema.CollectionSchema;
|
||||
|
||||
/**
|
||||
* this writer is supposed to be used to generate iframes. It generates links for the /api/snapshot.jpg servlet.
|
||||
*/
|
||||
public class SnapshotImagesReponseWriter implements QueryResponseWriter, SolrjResponseWriter {
|
||||
|
||||
private static final Set<String> DEFAULT_FIELD_LIST = new HashSet<>();
|
||||
|
||||
static {
|
||||
DEFAULT_FIELD_LIST.add(CollectionSchema.id.getSolrFieldName());
|
||||
DEFAULT_FIELD_LIST.add(CollectionSchema.sku.getSolrFieldName());
|
||||
}
|
||||
|
||||
/** Default width for each snapshot image */
|
||||
private static final int DEFAULT_WIDTH = 256;
|
||||
|
||||
/** Default height for each snapshot image */
|
||||
private static final int DEFAULT_HEIGTH = 256;
|
||||
|
||||
public SnapshotImagesReponseWriter() {
|
||||
super();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getContentType(SolrQueryRequest arg0, SolrQueryResponse arg1) {
|
||||
return "text/html";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(@SuppressWarnings("rawtypes") NamedList n) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the root path as a relative URL prefix from the original servlet
|
||||
* request URI if provided in the Solr request context, otherwise return a
|
||||
* regular "/". Using relative URLs when possible makes deployment behind a
|
||||
* reverse proxy more reliable and convenient as no URL rewriting is needed.
|
||||
*
|
||||
* @param request the Solr request.
|
||||
* @return the root context path to use as a prefix for resources URLs such as
|
||||
* stylesheets
|
||||
*/
|
||||
private String getRootPath(final SolrQueryRequest request) {
|
||||
String rootPath = "/";
|
||||
if (request != null) {
|
||||
final Map<Object, Object> context = request.getContext();
|
||||
if (context != null) {
|
||||
final Object requestUriObj = context.get("requestURI");
|
||||
if (requestUriObj instanceof String) {
|
||||
String servletRequestUri = (String) requestUriObj;
|
||||
if (servletRequestUri.startsWith("/")) {
|
||||
servletRequestUri = servletRequestUri.substring(1);
|
||||
|
||||
final String[] pathParts = CommonPattern.SLASH.split(servletRequestUri);
|
||||
if (pathParts.length > 1) {
|
||||
final StringBuilder sb = new StringBuilder();
|
||||
for (int i = 1; i < pathParts.length; i++) {
|
||||
sb.append("../");
|
||||
}
|
||||
rootPath = sb.toString();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return rootPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Append the response HTML head to the writer.
|
||||
* @param writer an open output writer. Must not be null.
|
||||
* @param request the Solr request
|
||||
* @throws IOException when a write error occurred
|
||||
*/
|
||||
private void writeHtmlHead(final Writer writer, final SolrQueryRequest request) throws IOException {
|
||||
final String rootPath = getRootPath(request);
|
||||
|
||||
writer.write("<!DOCTYPE html>\n");
|
||||
writer.write("<html lang=\"en\">");
|
||||
writer.write("<head>\n");
|
||||
writer.write("<meta charset=\"UTF-8\">");
|
||||
writer.write("<title>Documents snapshots</title>\n");
|
||||
writer.write("<link rel=\"stylesheet\" type=\"text/css\" media=\"all\" href=\"" + rootPath + "env/base.css\" />\n");
|
||||
writer.write("<link rel=\"stylesheet\" type=\"text/css\" media=\"screen\" href=\"" + rootPath + "env/style.css\" />\n");
|
||||
writer.write("</head>\n");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(final Writer writer, final SolrQueryRequest request, final SolrQueryResponse rsp) throws IOException {
|
||||
final NamedList<?> values = rsp.getValues();
|
||||
assert values.get("responseHeader") != null;
|
||||
assert values.get("response") != null;
|
||||
|
||||
writeHtmlHead(writer, request);
|
||||
writer.write("<body id=\"SnapshotImagesReponseWriter\">\n");
|
||||
final SolrParams originalParams = request.getOriginalParams();
|
||||
|
||||
final int width = originalParams != null ? originalParams.getInt("width", DEFAULT_WIDTH) : DEFAULT_WIDTH;
|
||||
final int height = originalParams != null ? originalParams.getInt("height", DEFAULT_HEIGTH) : DEFAULT_HEIGTH;
|
||||
|
||||
final DocList response = ((ResultContext) values.get("response")).getDocList();
|
||||
final int sz = response.size();
|
||||
if (sz > 0) {
|
||||
final SolrIndexSearcher searcher = request.getSearcher();
|
||||
final DocIterator iterator = response.iterator();
|
||||
while (iterator.hasNext()) {
|
||||
final int id = iterator.nextDoc();
|
||||
final Document doc = searcher.doc(id, DEFAULT_FIELD_LIST);
|
||||
final IndexableField docId = doc.getField(CollectionSchema.id.getSolrFieldName());
|
||||
final IndexableField docSku = doc.getField(CollectionSchema.sku.getSolrFieldName());
|
||||
if(docId != null && docSku != null) {
|
||||
writeDoc(writer, width, height, docId.stringValue(), docSku.stringValue());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
writer.write("</body></html>\n");
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a document and append its representation to the output writer.
|
||||
* @param writer an open ouput writer. Must not be null.
|
||||
* @param width the width of the snapshot image to render
|
||||
* @param height the height of the snapshot image to render
|
||||
* @param docId the document id (URL hash).
|
||||
* @param docUrl the document URL.
|
||||
* @throws IOException when a write error occurred
|
||||
*/
|
||||
private void writeDoc(final Writer writer, final int width, final int height, final String docId, final String docUrl)
|
||||
throws IOException {
|
||||
if(docId != null && docUrl != null) {
|
||||
writer.write("<a href=\"");
|
||||
writer.write(docUrl);
|
||||
writer.write("\" class=\"forceNoExternalIcon\"><img width=\"");
|
||||
writer.write(String.valueOf(width));
|
||||
writer.write("\" height=\"");
|
||||
writer.write(String.valueOf(height));
|
||||
writer.write("\" src=\"/api/snapshot.jpg?urlhash=");
|
||||
writer.write(docId);
|
||||
writer.write("&width=");
|
||||
writer.write(String.valueOf(width));
|
||||
writer.write("&height=");
|
||||
writer.write(String.valueOf(height));
|
||||
writer.write("\" alt=\"");
|
||||
writer.write(docUrl);
|
||||
writer.write("\"></a>\n");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(final Writer writer, final SolrQueryRequest request, final String coreName,
|
||||
final QueryResponse rsp) throws IOException {
|
||||
|
||||
writeHtmlHead(writer, request);
|
||||
writer.write("<body id=\"SnapshotImagesReponseWriter\">\n");
|
||||
final SolrParams originalParams = request.getOriginalParams();
|
||||
|
||||
final int width = originalParams != null ? originalParams.getInt("width", DEFAULT_WIDTH) : DEFAULT_WIDTH;
|
||||
final int height = originalParams != null ? originalParams.getInt("height", DEFAULT_HEIGTH) : DEFAULT_HEIGTH;
|
||||
|
||||
final SolrDocumentList docList = rsp.getResults();
|
||||
final int sz = docList.size();
|
||||
if (sz > 0) {
|
||||
final Iterator<SolrDocument> iterator = docList.iterator();
|
||||
while (iterator.hasNext()) {
|
||||
final SolrDocument doc = iterator.next();
|
||||
final Object docId = doc.getFieldValue(CollectionSchema.id.getSolrFieldName());
|
||||
final Object docSku = doc.getFieldValue(CollectionSchema.sku.getSolrFieldName());
|
||||
if (docId != null && docSku != null) {
|
||||
writeDoc(writer, width, height, docId.toString(), docSku.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
writer.write("</body></html>\n");
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,581 +0,0 @@
|
||||
/**
|
||||
* Html2Image
|
||||
* Copyright 2014 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
||||
* First published 26.11.2014 on http://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.cora.util;
|
||||
|
||||
import java.awt.Container;
|
||||
import java.awt.Dimension;
|
||||
import java.awt.Graphics;
|
||||
import java.awt.Image;
|
||||
import java.awt.MediaTracker;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
import javax.swing.JEditorPane;
|
||||
import javax.swing.text.Document;
|
||||
import javax.swing.text.Element;
|
||||
import javax.swing.text.View;
|
||||
import javax.swing.text.ViewFactory;
|
||||
import javax.swing.text.html.HTMLDocument;
|
||||
import javax.swing.text.html.HTMLEditorKit;
|
||||
import javax.swing.text.html.ImageView;
|
||||
|
||||
import org.apache.pdfbox.Loader;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.rendering.ImageType;
|
||||
import org.apache.pdfbox.rendering.PDFRenderer;
|
||||
|
||||
import net.yacy.cora.document.id.MultiProtocolURL;
|
||||
import net.yacy.cora.protocol.ClientIdentification;
|
||||
import net.yacy.cora.protocol.Domains;
|
||||
import net.yacy.cora.protocol.http.HTTPClient;
|
||||
import net.yacy.document.ImageParser;
|
||||
import net.yacy.kelondro.util.FileUtils;
|
||||
import net.yacy.kelondro.util.OS;
|
||||
|
||||
/**
|
||||
* Convert html to an copy on disk-image in a other file format
|
||||
* currently (pdf and/or jpg)
|
||||
*/
|
||||
public class Html2Image {
|
||||
|
||||
// Mac
|
||||
/**
|
||||
* Path to wkhtmltopdf executable on Mac OS when installed using
|
||||
* wkhtmltox-n.n.n.macos-cocoa.pkg from https://wkhtmltopdf.org/downloads.html.
|
||||
* This can also be a path on Debian or another Gnu/Linux distribution.
|
||||
*/
|
||||
private final static File wkhtmltopdfMac = new File("/usr/local/bin/wkhtmltopdf");
|
||||
|
||||
// to install imagemagick, download from http://cactuslab.com/imagemagick/assets/ImageMagick-6.8.9-9.pkg.zip
|
||||
// the convert command from imagemagick needs ghostscript, if not present on older macs, download a version of gs from http://pages.uoregon.edu/koch/
|
||||
|
||||
private final static File convertMac1 = new File("/opt/local/bin/convert");
|
||||
private final static File convertMac2 = new File("/opt/ImageMagick/bin/convert");
|
||||
|
||||
/* Debian packages to install: apt-get install wkhtmltopdf imagemagick xvfb ghostscript
|
||||
The imagemagick policy at /etc should also be checked :
|
||||
if it contains a line such as <policy domain="coder" rights="none" pattern="PDF" /> it must be edited with rights="read" at minimum
|
||||
*/
|
||||
private final static File wkhtmltopdfDebian = new File("/usr/bin/wkhtmltopdf"); // there is no wkhtmltoimage, use convert to create images
|
||||
private final static File convertDebian = new File("/usr/bin/convert");
|
||||
|
||||
/**
|
||||
* Path to wkhtmltopdf executable on Windows, when installed with default
|
||||
* settings using wkhtmltox-n.n.n.msvc2015-win64.exe from
|
||||
* https://wkhtmltopdf.org/downloads.html
|
||||
*/
|
||||
private static final File WKHTMLTOPDF_WINDOWS = new File("C:\\Program Files\\wkhtmltopdf\\bin\\wkhtmltopdf.exe");
|
||||
|
||||
/**
|
||||
* Path to wkhtmltopdf executable on Windows, when installed with default
|
||||
* settings using wkhtmltox-n.n.n.msvc2015-win32.exe from
|
||||
* https://wkhtmltopdf.org/downloads.html
|
||||
*/
|
||||
private static final File WKHTMLTOPDF_WINDOWS_X86 = new File(
|
||||
"C:\\Program Files (x86)\\wkhtmltopdf\\bin\\wkhtmltopdf.exe");
|
||||
|
||||
/** Command to use when wkhtmltopdf is included in the system Path */
|
||||
private static final String WKHTMLTOPDF_COMMAND = "wkhtmltopdf";
|
||||
|
||||
/** Command to use when imagemagick convert is included in the system Path */
|
||||
private static final String CONVERT_COMMAND = "convert";
|
||||
|
||||
private static boolean usexvfb = false;
|
||||
|
||||
/**
|
||||
* @return when the wkhtmltopdf command is detected as available in the system
|
||||
*/
|
||||
public static boolean wkhtmltopdfAvailable() {
|
||||
/* Check wkhtmltopdf common installation paths and system Path */
|
||||
return wkhtmltopdfExecutable() != null || wkhtmltopdfAvailableInPath();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return a wkhtmltopdf executable file when one can be found, null otherwise
|
||||
*/
|
||||
private static File wkhtmltopdfExecutable() {
|
||||
File executable = null;
|
||||
if(OS.isWindows) {
|
||||
if(WKHTMLTOPDF_WINDOWS.exists()) {
|
||||
executable = WKHTMLTOPDF_WINDOWS;
|
||||
} else if(WKHTMLTOPDF_WINDOWS_X86.exists()) {
|
||||
executable = WKHTMLTOPDF_WINDOWS_X86;
|
||||
}
|
||||
} else {
|
||||
if(wkhtmltopdfMac.exists()) {
|
||||
executable = wkhtmltopdfMac;
|
||||
} else if(wkhtmltopdfDebian.exists()) {
|
||||
executable = wkhtmltopdfDebian;
|
||||
}
|
||||
}
|
||||
return executable;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return true when wkhtmltopdf is available in system path
|
||||
*/
|
||||
private static boolean wkhtmltopdfAvailableInPath() {
|
||||
boolean available = false;
|
||||
try {
|
||||
final Process p = Runtime.getRuntime().exec(WKHTMLTOPDF_COMMAND + " -V");
|
||||
available = p.waitFor(2, TimeUnit.SECONDS) && p.exitValue() == 0;
|
||||
} catch (final IOException e) {
|
||||
ConcurrentLog.fine("Html2Image", "wkhtmltopdf is not included in system path.");
|
||||
} catch (final InterruptedException e) {
|
||||
Thread.currentThread().interrupt(); // preserve thread interrupted state
|
||||
}
|
||||
return available;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return a imagemagick convert executable file when one can be found, null otherwise
|
||||
*/
|
||||
private static File convertExecutable() {
|
||||
File executable = null;
|
||||
if(!OS.isWindows) {
|
||||
if(convertMac1.exists()) {
|
||||
executable = convertMac1;
|
||||
} else if(convertMac2.exists()) {
|
||||
executable = convertMac2;
|
||||
} else if(convertDebian.exists()) {
|
||||
executable = convertDebian;
|
||||
}
|
||||
}
|
||||
return executable;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return when the imagemagick convert command is detected as available in the system
|
||||
*/
|
||||
public static boolean convertAvailable() {
|
||||
/* Check convert common installation paths and system Path */
|
||||
return convertExecutable() != null || convertAvailableInPath();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return when imagemagick convert is available in system path
|
||||
*/
|
||||
private static boolean convertAvailableInPath() {
|
||||
boolean available = false;
|
||||
if(!OS.isWindows) { // on MS Windows convert is a system tool to convert volumes from FAT to NTFS
|
||||
try {
|
||||
final Process p = Runtime.getRuntime().exec(CONVERT_COMMAND + " -version");
|
||||
available = p.waitFor(2, TimeUnit.SECONDS) && p.exitValue() == 0;
|
||||
} catch (final IOException e) {
|
||||
ConcurrentLog.fine("Html2Image", "convert is not included in system path.");
|
||||
} catch (final InterruptedException e) {
|
||||
Thread.currentThread().interrupt(); // preserve thread interrupted state
|
||||
}
|
||||
}
|
||||
return available;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the wkhtmltopdf external tool to fetch and render to PDF a web resource.
|
||||
* wKhtmltopdf may be called multiple times with various parameters flavors in
|
||||
* case of failure.
|
||||
*
|
||||
* @param url the URL of a web resource to fetch, render and convert to
|
||||
* a pdf file. Must not be null.
|
||||
* @param proxy the eventual proxy address to use. Can be null. Must be of
|
||||
* the form http://host:port; use YaCy here as proxy which is
|
||||
* mostly http://localhost:8090
|
||||
* @param destination the destination PDF file that should be written. Must not
|
||||
* be null.
|
||||
* @param maxSeconds the maximum time in seconds to wait for each wkhtmltopdf
|
||||
* call termination. Beyond this limit the process is killed.
|
||||
* @return true when the destination file was successfully written
|
||||
*/
|
||||
public static boolean writeWkhtmltopdf(final String url, final String proxy, final String userAgent, final String acceptLanguage, final File destination, final long maxSeconds) {
|
||||
boolean success = false;
|
||||
for (final boolean ignoreErrors: new boolean[]{false, true}) {
|
||||
success = writeWkhtmltopdfInternal(url, proxy, destination, userAgent, acceptLanguage, ignoreErrors, maxSeconds);
|
||||
if (success) break;
|
||||
if (!success && proxy != null) {
|
||||
ConcurrentLog.warn("Html2Image", "trying to load without proxy: " + url);
|
||||
success = writeWkhtmltopdfInternal(url, null, destination, userAgent, acceptLanguage, ignoreErrors, maxSeconds);
|
||||
if (success) break;
|
||||
}
|
||||
}
|
||||
if (success) {
|
||||
ConcurrentLog.info("Html2Image", "wrote " + destination.toString() + " for " + url);
|
||||
} else {
|
||||
ConcurrentLog.warn("Html2Image", "could not generate snapshot for " + url);
|
||||
}
|
||||
return success;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run wkhtmltopdf in a separate process to fetch and render to PDF a web
|
||||
* resource.
|
||||
*
|
||||
* @param url the URL of a web resource to fetch, render and convert to
|
||||
* a pdf file. Must not be null.
|
||||
* @param proxy the eventual proxy address to use. Can be null.
|
||||
* @param destination the destination PDF file that should be written. Must not
|
||||
* be null.
|
||||
* @param userAgent TODO: implement
|
||||
* @param acceptLanguage TODO: implement
|
||||
* @param ignoreErrors when true wkhtmltopdf is instructed to ignore load errors
|
||||
* @param maxSeconds the maximum time in seconds to wait for the wkhtmltopdf
|
||||
* dedicated process termination. Beyond this limit the
|
||||
* process is killed.
|
||||
* @return true when the destination file was successfully written
|
||||
*/
|
||||
private static boolean writeWkhtmltopdfInternal(final String url, final String proxy, final File destination,
|
||||
final String userAgent, final String acceptLanguage, final boolean ignoreErrors, final long maxSeconds) {
|
||||
final String wkhtmltopdfCmd;
|
||||
final File wkhtmltopdf = wkhtmltopdfExecutable();
|
||||
if(wkhtmltopdf != null) {
|
||||
wkhtmltopdfCmd = wkhtmltopdf.getAbsolutePath();
|
||||
} else if(wkhtmltopdfAvailableInPath()) {
|
||||
wkhtmltopdfCmd = WKHTMLTOPDF_COMMAND;
|
||||
} else {
|
||||
ConcurrentLog.warn("Html2Pdf", "Unable to locate wkhtmltopdf executable on this system!");
|
||||
return false;
|
||||
}
|
||||
String commandline =
|
||||
wkhtmltopdfCmd + " -q --title '" + url + "' " +
|
||||
//acceptLanguage == null ? "" : "--custom-header 'Accept-Language' '" + acceptLanguage + "' " +
|
||||
//(userAgent == null ? "" : "--custom-header \"User-Agent\" \"" + userAgent + "\" --custom-header-propagation ") +
|
||||
(proxy == null ? "" : "--proxy " + proxy + " ") +
|
||||
(ignoreErrors ? (OS.isMacArchitecture ? "--load-error-handling ignore " : "--ignore-load-errors ") : "") + // some versions do not have that flag and fail if attempting to use it...
|
||||
//"--footer-font-name 'Courier' --footer-font-size 9 --footer-left [webpage] --footer-right [date]/[time]([page]/[topage]) " +
|
||||
"--footer-left [webpage] --footer-right '[date]/[time]([page]/[topage])' --footer-font-size 7 " +
|
||||
url + " " + destination.getAbsolutePath();
|
||||
try {
|
||||
ConcurrentLog.info("Html2Pdf", "creating pdf from url " + url + " with command: " + commandline);
|
||||
if (!usexvfb && execWkhtmlToPdf(proxy, destination, commandline, maxSeconds)) {
|
||||
return true;
|
||||
}
|
||||
// if this fails, we should try to wrap the X server with a virtual screen using xvfb, this works on headless servers
|
||||
commandline = "xvfb-run -a " + commandline;
|
||||
return execWkhtmlToPdf(proxy, destination, commandline, maxSeconds);
|
||||
} catch (final IOException e) {
|
||||
ConcurrentLog.warn("Html2Pdf", "exception while creation of pdf with command: " + commandline, e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a wkhtmltopdf commandline in a separate process.
|
||||
*
|
||||
* @param proxy the eventual proxy address to use. Can be null.
|
||||
* @param destination the destination PDF file that should be written. Must not
|
||||
* be null.
|
||||
* @param commandline the wkhtmltopdf command line to execute. Must not be null.
|
||||
* @param maxSeconds the maximum time in seconds to wait for the process
|
||||
* termination. Beyond this limit the process is killed.
|
||||
* @return true when the destination file was successfully written
|
||||
* @throws IOException when an unexpected error occurred
|
||||
*/
|
||||
private static boolean execWkhtmlToPdf(final String proxy, final File destination, final String commandline, final long maxSeconds) throws IOException {
|
||||
final Process p = Runtime.getRuntime().exec(commandline);
|
||||
|
||||
try {
|
||||
p.waitFor(maxSeconds, TimeUnit.SECONDS);
|
||||
} catch (final InterruptedException e) {
|
||||
p.destroyForcibly();
|
||||
ConcurrentLog.warn("Html2Pdf", "Interrupted creation of pdf. Killing the process started with command : " + commandline);
|
||||
Thread.currentThread().interrupt(); // Keep the thread interrupted state
|
||||
return false;
|
||||
}
|
||||
if(p.isAlive()) {
|
||||
ConcurrentLog.warn("Html2Pdf", "Creation of pdf did not terminate within " + maxSeconds + " seconds. Killing the process started with command : " + commandline);
|
||||
p.destroyForcibly();
|
||||
return false;
|
||||
}
|
||||
if (p.exitValue() == 0 && destination.exists()) {
|
||||
return true;
|
||||
}
|
||||
final List<String> messages = OS.readStreams(p);
|
||||
ConcurrentLog.warn("Html2Image", "failed to create pdf " + (proxy == null ? "" : "using proxy " + proxy) + " with command : " + commandline);
|
||||
for (final String message : messages) {
|
||||
ConcurrentLog.warn("Html2Image", ">> " + message);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a pdf (first page) to an image. Proper values are i.e. width = 1024, height = 1024, density = 300, quality = 75
|
||||
* using internal pdf library or external command line tool on linux or mac
|
||||
* @param pdf input pdf file. Must not be null.
|
||||
* @param image output image file. Must not be null, and should end with ".jpg" or ".png".
|
||||
* @param width output width in pixels
|
||||
* @param height output height in pixels
|
||||
* @param density (dpi)
|
||||
* @param quality JPEG/PNG compression level
|
||||
* @return true when the ouput image file was successfully written.
|
||||
*/
|
||||
public static boolean pdf2image(final File pdf, final File image, final int width, final int height, final int density, final int quality) {
|
||||
/* Deduce the ouput image format from the file extension */
|
||||
String imageFormat = MultiProtocolURL.getFileExtension(image.getName());
|
||||
if(imageFormat.isEmpty()) {
|
||||
/* Use JPEG as a default fallback */
|
||||
imageFormat = "jpg";
|
||||
}
|
||||
String convertCmd = null;
|
||||
final File convert = convertExecutable();
|
||||
if(convert != null) {
|
||||
convertCmd = convert.getAbsolutePath();
|
||||
} else if(convertAvailableInPath()) {
|
||||
convertCmd = CONVERT_COMMAND;
|
||||
} else {
|
||||
ConcurrentLog.info("Html2Image", "Unable to locate convert executable on this system!");
|
||||
}
|
||||
|
||||
// convert pdf to jpg using internal pdfbox capability
|
||||
if (convertCmd == null) {
|
||||
try (final PDDocument pdoc = Loader.loadPDF(pdf);) {
|
||||
|
||||
final BufferedImage bi = new PDFRenderer(pdoc).renderImageWithDPI(0, density, ImageType.RGB);
|
||||
|
||||
return ImageIO.write(bi, imageFormat, image);
|
||||
|
||||
} catch (final IOException ex) {
|
||||
ConcurrentLog.warn("Html2Image", "Failed to create image with pdfbox"
|
||||
+ (ex.getMessage() != null ? " : " + ex.getMessage() : ""));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// convert using external command line utility
|
||||
try {
|
||||
// i.e. convert -density 300 -trim yacy.pdf[0] -trim -resize 1024x -crop x1024+0+0 -quality 75% yacy-convert-300.jpg
|
||||
// note: both -trim are necessary, otherwise it is trimmed only on one side. The [0] selects the first page of the pdf
|
||||
final String command = convertCmd + " -alpha remove -density " + density + " -trim " + pdf.getAbsolutePath() + "[0] -trim -resize " + width + "x -crop x" + height + "+0+0 -quality " + quality + "% " + image.getAbsolutePath();
|
||||
List<String> message = OS.execSynchronous(new String[] {
|
||||
convertCmd,
|
||||
"-alpha", "remove", "-density", Integer.toString(density),
|
||||
"-trim", pdf.getAbsolutePath() + "[0]", "-trim",
|
||||
"-resize", Integer.toString(width) + "x",
|
||||
"-crop", "x" + Integer.toString(height) + "+0+0",
|
||||
"-quality", Integer.toString(quality) + "%",
|
||||
image.getAbsolutePath()
|
||||
});
|
||||
if (image.exists()) return true;
|
||||
ConcurrentLog.warn("Html2Image", "failed to create image with command: " + command);
|
||||
for (final String m: message) ConcurrentLog.warn("Html2Image", ">> " + m);
|
||||
|
||||
// another try for mac: use Image Events using AppleScript in osacript commands...
|
||||
// the following command overwrites a pdf with an png, so we must make a copy first
|
||||
if (!OS.isMacArchitecture) return false;
|
||||
final File pngFile = new File(pdf.getAbsolutePath() + ".tmp.pdf");
|
||||
org.apache.commons.io.FileUtils.copyFile(pdf, pngFile);
|
||||
final String[] commandx = {"osascript",
|
||||
"-e", "set ImgFile to \"" + pngFile.getAbsolutePath() + "\"",
|
||||
"-e", "tell application \"Image Events\"",
|
||||
"-e", "set Img to open file ImgFile",
|
||||
"-e", "save Img as PNG",
|
||||
"-e", "end tell"};
|
||||
//ConcurrentLog.warn("Html2Image", "failed to create image with command: " + commandx);
|
||||
message = OS.execSynchronous(commandx);
|
||||
for (final String m: message) ConcurrentLog.warn("Html2Image", ">> " + m);
|
||||
// now we must read and convert this file to the target format with the target size 1024x1024
|
||||
try {
|
||||
final File newPngFile = new File(pngFile.getAbsolutePath() + ".png");
|
||||
pngFile.renameTo(newPngFile);
|
||||
final Image img = ImageParser.parse(pngFile.getAbsolutePath(), FileUtils.read(newPngFile));
|
||||
if(img == null) {
|
||||
/* Should not happen. If so, ImageParser.parse() should already have logged about the error */
|
||||
return false;
|
||||
}
|
||||
final Image scaled = img.getScaledInstance(width, height, Image.SCALE_AREA_AVERAGING);
|
||||
final MediaTracker mediaTracker = new MediaTracker(new Container());
|
||||
mediaTracker.addImage(scaled, 0);
|
||||
try {mediaTracker.waitForID(0);} catch (final InterruptedException e) {}
|
||||
// finally write the image
|
||||
final BufferedImage bi = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
|
||||
bi.createGraphics().drawImage(scaled, 0, 0, width, height, null);
|
||||
ImageIO.write(bi, imageFormat, image);
|
||||
newPngFile.delete();
|
||||
return image.exists();
|
||||
} catch (final IOException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
return false;
|
||||
}
|
||||
} catch (final IOException e) {
|
||||
e.printStackTrace();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* render a html page with a JEditorPane, which can do html up to html v 3.2. No CSS supported!
|
||||
* @param url
|
||||
* @param size
|
||||
* @throws IOException
|
||||
*/
|
||||
public static void writeSwingImage(final String url, final Dimension size, final File destination) throws IOException {
|
||||
|
||||
// set up a pane for rendering
|
||||
final JEditorPane htmlPane = new JEditorPane();
|
||||
htmlPane.setSize(size);
|
||||
htmlPane.setEditable(false);
|
||||
final HTMLEditorKit kit = new HTMLEditorKit() {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
@Override
|
||||
public Document createDefaultDocument() {
|
||||
final HTMLDocument doc = (HTMLDocument) super.createDefaultDocument();
|
||||
doc.setAsynchronousLoadPriority(-1);
|
||||
return doc;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ViewFactory getViewFactory() {
|
||||
return new HTMLFactory() {
|
||||
@Override
|
||||
public View create(final Element elem) {
|
||||
final View view = super.create(elem);
|
||||
if (view instanceof ImageView) {
|
||||
((ImageView) view).setLoadsSynchronously(true);
|
||||
}
|
||||
return view;
|
||||
}
|
||||
};
|
||||
}
|
||||
};
|
||||
htmlPane.setEditorKitForContentType("text/html", kit);
|
||||
htmlPane.setContentType("text/html");
|
||||
htmlPane.addPropertyChangeListener(evt -> {
|
||||
});
|
||||
|
||||
// load the page
|
||||
try {
|
||||
htmlPane.setPage(url);
|
||||
} catch (final IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
// render the page
|
||||
final Dimension prefSize = htmlPane.getPreferredSize();
|
||||
final BufferedImage img = new BufferedImage(prefSize.width, htmlPane.getPreferredSize().height, BufferedImage.TYPE_INT_ARGB);
|
||||
final Graphics graphics = img.getGraphics();
|
||||
htmlPane.setSize(prefSize);
|
||||
htmlPane.paint(graphics);
|
||||
ImageIO.write(img, destination.getName().endsWith("jpg") ? "jpg" : "png", destination);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test PDF or image snapshot generation for a given URL.
|
||||
* @param args main arguments list:
|
||||
* <ol>
|
||||
* <li>Source remote URL (required)</li>
|
||||
* <li>Target local file path (required)</li>
|
||||
* <li>Snapshot generation method identifier (optional) :
|
||||
* <ul>
|
||||
* <li>"wkhtmltopdf" (default): generate a PDF snapshot using external wkhtmltopdf tool.</li>
|
||||
* <li>"swing" : use JRE provided Swing to generate a jpg or png image snapshot.</li>
|
||||
* </ul>
|
||||
* </li>
|
||||
* </ol>
|
||||
*/
|
||||
public static void main(final String[] args) {
|
||||
final String usageMessage = "Usage : java " + Html2Image.class.getName()
|
||||
+ " <url> <target-file[.pdf|.jpg|.png]> [wkhtmltopdf|swing]";
|
||||
int exitStatus = 0;
|
||||
try {
|
||||
if (args.length < 2) {
|
||||
System.out.println("Missing required parameter(s).");
|
||||
System.out.println(usageMessage);
|
||||
exitStatus = 1;
|
||||
return;
|
||||
}
|
||||
final String targetPath = args[1];
|
||||
if (args.length < 3 || "wkhtmltopdf".equals(args[2])) {
|
||||
if(Html2Image.wkhtmltopdfAvailable()) {
|
||||
final File targetPdfFile;
|
||||
if(targetPath.endsWith(".jpg") || targetPath.endsWith(".png")) {
|
||||
targetPdfFile = new File(targetPath.substring(0, targetPath.length() - 4) + ".pdf");
|
||||
} else if(targetPath.endsWith(".pdf")) {
|
||||
targetPdfFile = new File(targetPath);
|
||||
} else {
|
||||
System.out.println("Unsupported output format");
|
||||
System.out.println(usageMessage);
|
||||
exitStatus = 1;
|
||||
return;
|
||||
}
|
||||
if(Html2Image.writeWkhtmltopdf(args[0], null, ClientIdentification.yacyInternetCrawlerAgent.userAgent(),
|
||||
"en-us,en;q=0.5", targetPdfFile, 30)) {
|
||||
if(targetPath.endsWith(".jpg") || targetPath.endsWith(".png")) {
|
||||
if(Html2Image.pdf2image(targetPdfFile, new File(targetPath), 1024, 1024, 300, 75)) {
|
||||
ConcurrentLog.info("Html2Image", "wrote " + targetPath + " converted from " + targetPdfFile);
|
||||
} else {
|
||||
exitStatus = 1;
|
||||
return;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
exitStatus = 1;
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
System.out.println("Unable to locate wkhtmltopdf executable on this system!");
|
||||
exitStatus = 1;
|
||||
return;
|
||||
}
|
||||
} else if ("swing".equals(args[2])) {
|
||||
if(targetPath.endsWith(".pdf")) {
|
||||
System.out.println("Pdf output format is not supported with swing method.");
|
||||
exitStatus = 1;
|
||||
return;
|
||||
}
|
||||
if(!targetPath.endsWith(".jpg") && !targetPath.endsWith(".png")) {
|
||||
System.out.println("Unsupported output format");
|
||||
System.out.println(usageMessage);
|
||||
exitStatus = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
Html2Image.writeSwingImage(args[0], new Dimension(1200, 2000), new File(targetPath));
|
||||
} catch (final IOException e) {
|
||||
e.printStackTrace();
|
||||
exitStatus = 1;
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
System.out.println("Unknown method : please specify either wkhtmltopdf or swing.");
|
||||
exitStatus = 1;
|
||||
return;
|
||||
}
|
||||
} finally {
|
||||
/* Shutdown running threads */
|
||||
Domains.close();
|
||||
try {
|
||||
HTTPClient.closeConnectionManager();
|
||||
} catch (final InterruptedException e) {
|
||||
Thread.currentThread().interrupt(); // restore interrupted state
|
||||
}
|
||||
ConcurrentLog.shutdown();
|
||||
if(exitStatus != 0) {
|
||||
System.exit(exitStatus);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -304,8 +304,6 @@ public final class CrawlSwitchboard {
|
||||
sb.getConfigBool(SwitchboardConstants.AUTOCRAWL_INDEX_MEDIA, true),
|
||||
false,
|
||||
false,
|
||||
-1,
|
||||
false, true, CrawlProfile.MATCH_NEVER_STRING,
|
||||
CacheStrategy.NOCACHE,
|
||||
"robot_" + CRAWL_PROFILE_AUTOCRAWL_DEEP,
|
||||
ClientIdentification.yacyInternetCrawlerAgentName,
|
||||
@@ -339,8 +337,6 @@ public final class CrawlSwitchboard {
|
||||
sb.getConfigBool(SwitchboardConstants.AUTOCRAWL_INDEX_MEDIA, true),
|
||||
false,
|
||||
false,
|
||||
-1,
|
||||
false, true, CrawlProfile.MATCH_NEVER_STRING,
|
||||
CacheStrategy.NOCACHE,
|
||||
"robot_" + CRAWL_PROFILE_AUTOCRAWL_SHALLOW,
|
||||
ClientIdentification.yacyInternetCrawlerAgentName,
|
||||
@@ -374,7 +370,6 @@ public final class CrawlSwitchboard {
|
||||
sb.getConfigBool(SwitchboardConstants.PROXY_INDEXING_LOCAL_MEDIA, true),
|
||||
true,
|
||||
sb.getConfigBool(SwitchboardConstants.PROXY_INDEXING_REMOTE, false),
|
||||
-1, false, true, CrawlProfile.MATCH_NEVER_STRING,
|
||||
CacheStrategy.IFFRESH,
|
||||
"robot_" + CRAWL_PROFILE_PROXY,
|
||||
ClientIdentification.yacyProxyAgentName,
|
||||
@@ -408,7 +403,6 @@ public final class CrawlSwitchboard {
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
-1, false, true, CrawlProfile.MATCH_NEVER_STRING,
|
||||
CacheStrategy.IFFRESH,
|
||||
"robot_" + CRAWL_PROFILE_REMOTE,
|
||||
ClientIdentification.yacyInternetCrawlerAgentName,
|
||||
@@ -442,7 +436,6 @@ public final class CrawlSwitchboard {
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
-1, false, true, CrawlProfile.MATCH_NEVER_STRING,
|
||||
CacheStrategy.IFEXIST,
|
||||
"robot_" + CRAWL_PROFILE_SNIPPET_LOCAL_TEXT,
|
||||
ClientIdentification.yacyIntranetCrawlerAgentName,
|
||||
@@ -476,7 +469,6 @@ public final class CrawlSwitchboard {
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
-1, false, true, CrawlProfile.MATCH_NEVER_STRING,
|
||||
CacheStrategy.IFEXIST,
|
||||
"robot_" + CRAWL_PROFILE_SNIPPET_GLOBAL_TEXT,
|
||||
ClientIdentification.yacyIntranetCrawlerAgentName,
|
||||
@@ -518,7 +510,6 @@ public final class CrawlSwitchboard {
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
-1, false, true, CrawlProfile.MATCH_NEVER_STRING,
|
||||
CacheStrategy.IFEXIST,
|
||||
"robot_" + CRAWL_PROFILE_GREEDY_LEARNING_TEXT,
|
||||
ClientIdentification.browserAgentName,
|
||||
@@ -552,7 +543,6 @@ public final class CrawlSwitchboard {
|
||||
false, // indexMedia
|
||||
true,
|
||||
false,
|
||||
-1, false, true, CrawlProfile.MATCH_NEVER_STRING,
|
||||
CacheStrategy.IFEXIST,
|
||||
"robot_" + CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA,
|
||||
ClientIdentification.yacyIntranetCrawlerAgentName,
|
||||
@@ -586,7 +576,6 @@ public final class CrawlSwitchboard {
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
-1, false, true, CrawlProfile.MATCH_NEVER_STRING,
|
||||
CacheStrategy.IFEXIST,
|
||||
"robot_" + CRAWL_PROFILE_SNIPPET_GLOBAL_MEDIA,
|
||||
ClientIdentification.yacyIntranetCrawlerAgentName,
|
||||
@@ -620,7 +609,6 @@ public final class CrawlSwitchboard {
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
-1, false, true, CrawlProfile.MATCH_NEVER_STRING,
|
||||
CacheStrategy.NOCACHE,
|
||||
"robot_" + CRAWL_PROFILE_PACKS,
|
||||
ClientIdentification.yacyIntranetCrawlerAgentName,
|
||||
@@ -657,7 +645,6 @@ public final class CrawlSwitchboard {
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
-1, false, true, CrawlProfile.MATCH_NEVER_STRING,
|
||||
CacheStrategy.NOCACHE,
|
||||
collection,
|
||||
ClientIdentification.yacyIntranetCrawlerAgentName,
|
||||
|
||||
@@ -355,7 +355,7 @@ public class RecrawlBusyThread extends AbstractBusyThread {
|
||||
false, //noindexWhenCanonicalUnequalURL
|
||||
0, false, CrawlProfile.getRecrawlDate(CrawlSwitchboard.CRAWL_PROFILE_RECRAWL_JOB_RECRAWL_CYCLE), -1,
|
||||
true, true, true, false, // crawlingQ, followFrames, obeyHtmlRobotsNoindex, obeyHtmlRobotsNofollow,
|
||||
true, true, true, false, -1, false, true, CrawlProfile.MATCH_NEVER_STRING, CacheStrategy.IFFRESH,
|
||||
true, true, true, false, CacheStrategy.IFFRESH,
|
||||
"robot_" + CrawlSwitchboard.CRAWL_PROFILE_RECRAWL_JOB,
|
||||
ClientIdentification.yacyInternetCrawlerAgentName,
|
||||
TagValency.EVAL, null, null, 0);
|
||||
|
||||
@@ -120,10 +120,6 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
|
||||
STORE_HTCACHE ("storeHTCache", false, CrawlAttribute.BOOLEAN, "Store in HTCache"),
|
||||
CACHE_STRAGEGY ("cacheStrategy", false, CrawlAttribute.STRING, "Cache Strategy (NOCACHE,IFFRESH,IFEXIST,CACHEONLY)"),
|
||||
AGENT_NAME ("agentName", false, CrawlAttribute.STRING, "User Agent Profile Name"),
|
||||
SNAPSHOTS_MAXDEPTH ("snapshotsMaxDepth", false, CrawlAttribute.INTEGER, "Max Depth for Snapshots"),
|
||||
SNAPSHOTS_REPLACEOLD ("snapshotsReplaceOld", false, CrawlAttribute.BOOLEAN, "Multiple Snapshot Versions - replace old with new"),
|
||||
SNAPSHOTS_MUSTNOTMATCH ("snapshotsMustnotmatch", false, CrawlAttribute.STRING, "must-not-match filter for snapshot generation"),
|
||||
SNAPSHOTS_LOADIMAGE ("snapshotsLoadImage", false, CrawlAttribute.BOOLEAN, "Flag for Snapshot image generation"),
|
||||
REMOTE_INDEXING ("remoteIndexing", false, CrawlAttribute.BOOLEAN, "Remote Indexing (only for p2p networks)"),
|
||||
INDEX_TEXT ("indexText", false, CrawlAttribute.BOOLEAN, "Index Text"),
|
||||
INDEX_MEDIA ("indexMedia", false, CrawlAttribute.BOOLEAN, "Index Media"),
|
||||
@@ -174,7 +170,6 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
|
||||
* @see CollectionSchema#content_type */
|
||||
private Pattern indexMediaTypeMustNotMatch = null;
|
||||
|
||||
private Pattern snapshotsMustnotmatch = null;
|
||||
|
||||
private final Map<String, AtomicInteger> doms;
|
||||
private final TagValency defaultValency;
|
||||
@@ -204,10 +199,6 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
|
||||
* @param indexMedia true if media content of URL shall be indexed
|
||||
* @param storeHTCache true if content chall be kept in cache after indexing
|
||||
* @param remoteIndexing true if part of the crawl job shall be distributed
|
||||
* @param snapshotsMaxDepth if the current crawl depth is equal or below that given depth, a snapshot is generated
|
||||
* @param snapshotsLoadImage true if graphical (== pdf) shapshots shall be made
|
||||
* @param snapshotsReplaceOld true if snapshots shall not be historized
|
||||
* @param snapshotsMustnotmatch a regular expression; if it matches on the url, the snapshot is not generated
|
||||
* @param xsstopw true if static stop words shall be ignored
|
||||
* @param xdstopw true if dynamic stop words shall be ignored
|
||||
* @param xpstopw true if parent stop words shall be ignored
|
||||
@@ -235,10 +226,6 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
|
||||
final boolean indexMedia,
|
||||
final boolean storeHTCache,
|
||||
final boolean remoteIndexing,
|
||||
final int snapshotsMaxDepth,
|
||||
final boolean snapshotsLoadImage,
|
||||
final boolean snapshotsReplaceOld,
|
||||
final String snapshotsMustnotmatch,
|
||||
final CacheStrategy cacheStrategy,
|
||||
final String collections,
|
||||
final String userAgentName,
|
||||
@@ -281,10 +268,6 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
|
||||
this.put(CrawlAttribute.INDEX_MEDIA.key, indexMedia);
|
||||
this.put(CrawlAttribute.STORE_HTCACHE.key, storeHTCache);
|
||||
this.put(CrawlAttribute.REMOTE_INDEXING.key, remoteIndexing);
|
||||
this.put(CrawlAttribute.SNAPSHOTS_MAXDEPTH.key, snapshotsMaxDepth);
|
||||
this.put(CrawlAttribute.SNAPSHOTS_LOADIMAGE.key, snapshotsLoadImage);
|
||||
this.put(CrawlAttribute.SNAPSHOTS_REPLACEOLD.key, snapshotsReplaceOld);
|
||||
this.put(CrawlAttribute.SNAPSHOTS_MUSTNOTMATCH.key, snapshotsMustnotmatch);
|
||||
this.put(CrawlAttribute.CACHE_STRAGEGY.key, cacheStrategy.toString());
|
||||
this.put(CrawlAttribute.COLLECTIONS.key, CommonPattern.SPACE.matcher(collections.trim()).replaceAll(""));
|
||||
// we transform the ignore_class_name and scraper information into a JSON Array
|
||||
@@ -895,41 +878,6 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
|
||||
return (r.equals(Boolean.TRUE.toString()));
|
||||
}
|
||||
|
||||
public int snapshotMaxdepth() {
|
||||
final String r = this.get(CrawlAttribute.SNAPSHOTS_MAXDEPTH.key);
|
||||
if (r == null) return -1;
|
||||
try {
|
||||
final int i = Integer.parseInt(r);
|
||||
if (i < 0) return -1;
|
||||
return i;
|
||||
} catch (final NumberFormatException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
public boolean snapshotLoadImage() {
|
||||
final String r = this.get(CrawlAttribute.SNAPSHOTS_LOADIMAGE.key);
|
||||
if (r == null) return false;
|
||||
return (r.equals(Boolean.TRUE.toString()));
|
||||
}
|
||||
|
||||
public boolean snapshotReplaceold() {
|
||||
final String r = this.get(CrawlAttribute.SNAPSHOTS_REPLACEOLD.key);
|
||||
if (r == null) return false;
|
||||
return (r.equals(Boolean.TRUE.toString()));
|
||||
}
|
||||
|
||||
public Pattern snapshotsMustnotmatch() {
|
||||
if (this.snapshotsMustnotmatch == null) {
|
||||
final String r = this.get(CrawlAttribute.SNAPSHOTS_MUSTNOTMATCH.key);
|
||||
try {
|
||||
this.snapshotsMustnotmatch = (r == null || r.equals(CrawlProfile.MATCH_ALL_STRING)) ? CrawlProfile.MATCH_ALL_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
|
||||
} catch (final PatternSyntaxException e) { this.snapshotsMustnotmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
|
||||
}
|
||||
return this.snapshotsMustnotmatch;
|
||||
}
|
||||
|
||||
public int timezoneOffset() {
|
||||
final String timezoneOffset = this.get(CrawlAttribute.TIMEZONEOFFSET.key);
|
||||
if (timezoneOffset == null) return 0;
|
||||
@@ -1038,10 +986,6 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
|
||||
prop.put(CRAWL_PROFILE_PREFIX + count + "_storeHTCache", this.storeHTCache() ? 1 : 0);
|
||||
prop.putXML(CRAWL_PROFILE_PREFIX + count + "_cacheStrategy", this.get(CrawlAttribute.CACHE_STRAGEGY.key));
|
||||
prop.putXML(CRAWL_PROFILE_PREFIX + count + "_agentName", this.get(CrawlAttribute.AGENT_NAME.key));
|
||||
prop.putXML(CRAWL_PROFILE_PREFIX + count + "_" + CrawlAttribute.SNAPSHOTS_MAXDEPTH.key, this.get(CrawlAttribute.SNAPSHOTS_MAXDEPTH.key));
|
||||
prop.putXML(CRAWL_PROFILE_PREFIX + count + "_" + CrawlAttribute.SNAPSHOTS_REPLACEOLD.key, this.get(CrawlAttribute.SNAPSHOTS_REPLACEOLD.key));
|
||||
prop.putXML(CRAWL_PROFILE_PREFIX + count + "_" + CrawlAttribute.SNAPSHOTS_MUSTNOTMATCH.key, this.get(CrawlAttribute.SNAPSHOTS_MUSTNOTMATCH.key));
|
||||
prop.putXML(CRAWL_PROFILE_PREFIX + count + "_" + CrawlAttribute.SNAPSHOTS_LOADIMAGE.key, this.get(CrawlAttribute.SNAPSHOTS_LOADIMAGE.key));
|
||||
prop.put(CRAWL_PROFILE_PREFIX + count + "_remoteIndexing", this.remoteIndexing() ? 1 : 0);
|
||||
prop.put(CRAWL_PROFILE_PREFIX + count + "_indexText", this.indexText() ? 1 : 0);
|
||||
prop.put(CRAWL_PROFILE_PREFIX + count + "_indexMedia", this.indexMedia() ? 1 : 0);
|
||||
|
||||
@@ -1,516 +0,0 @@
|
||||
/**
|
||||
* DocumentImage
|
||||
* SPDX-FileCopyrightText: 2014 Michael Peter Christen <mc@yacy.net)>
|
||||
* SPDX-License-Identifier: GPL-2.0-or-later
|
||||
* First released 29.11.2014 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.crawler.data;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.net.URLEncoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.text.ParseException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.TreeMap;
|
||||
import java.util.TreeSet;
|
||||
|
||||
import org.apache.solr.common.SolrDocument;
|
||||
|
||||
import net.yacy.cora.date.GenericFormatter;
|
||||
import net.yacy.cora.document.encoding.ASCII;
|
||||
import net.yacy.cora.document.id.DigestURL;
|
||||
import net.yacy.cora.util.ConcurrentLog;
|
||||
import net.yacy.search.index.Fulltext;
|
||||
import net.yacy.search.schema.CollectionSchema;
|
||||
|
||||
/**
|
||||
* This class hosts document snapshots.
|
||||
*
|
||||
* The storage is organized in the following hierarchy:
|
||||
* - in the root path are subpaths for each host:port
|
||||
* - in the host:port path are subpaths for the crawl depth, two digits length
|
||||
* - in the crawl depth path are subpaths for the first two charaters of the url-hash, called shard
|
||||
* - in the shard path are files, named with <urlhash>'.'<date>.<ext>
|
||||
* .. where the <date> has the form "yyyyMMdd" and ext may be one of {pdf,jpg,png,xml,json}.
|
||||
* The pdf is created with wxhtmltopdf, jpg/png is created with convert
|
||||
* and the xml/json is an extract from solr.
|
||||
*
|
||||
* The construction of the file name with the date allows to make several copies of the same document
|
||||
* for different snapshot-times. The usage of the crawl depth makes it easier to extract a specific part
|
||||
* of the domain.
|
||||
*/
|
||||
public class Snapshots {
|
||||
|
||||
private File storageLocation;
|
||||
|
||||
private Map<String, TreeMap<Integer, TreeSet<String>>> directory; // a TreeMap for each domain (host.port) where the key is the depth and the value is a Set containing a key/urlhash id to get all files into a specific order to provide a recent view on the documents
|
||||
|
||||
public Snapshots(final File location) {
|
||||
this.storageLocation = location;
|
||||
this.storageLocation.mkdirs();
|
||||
// scan the location to fill the directory
|
||||
this.directory = new HashMap<>();
|
||||
for (String hostport: location.list()) {
|
||||
TreeMap<Integer, TreeSet<String>> domaindepth = new TreeMap<>();
|
||||
this.directory.put(hostport, domaindepth);
|
||||
File domaindir = new File(location, hostport);
|
||||
if (domaindir.isDirectory()) domainscan: for (String depth: domaindir.list()) {
|
||||
TreeSet<String> dateid = new TreeSet<>();
|
||||
Integer depthi = -1;
|
||||
try {
|
||||
depthi = Integer.parseInt(depth);
|
||||
} catch (NumberFormatException e) {
|
||||
continue domainscan;
|
||||
}
|
||||
domaindepth.put(depthi, dateid);
|
||||
File sharddir = new File(domaindir, depth);
|
||||
if (sharddir.isDirectory()) for (String shard: sharddir.list()) {
|
||||
File snapshotdir = new File(sharddir, shard);
|
||||
if (snapshotdir.isDirectory()) {
|
||||
for (String snapshotfile: snapshotdir.list()) {
|
||||
if (snapshotfile.endsWith(".xml")) {
|
||||
String s = snapshotfile.substring(0, snapshotfile.length() - 4);
|
||||
int p = s.indexOf('.');
|
||||
assert p == 12;
|
||||
if (p > 0) {
|
||||
String key = s.substring(p + 1) + '.' + s.substring(0, p);
|
||||
dateid.add(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (dateid.size() == 0) domaindepth.remove(depthi);
|
||||
}
|
||||
if (domaindepth.size() == 0) this.directory.remove(hostport);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* get the number of entries in the snapshot directory
|
||||
* @return the total number of different documents
|
||||
*/
|
||||
public int size() {
|
||||
int c = 0;
|
||||
for (Map<Integer, TreeSet<String>> m: directory.values()) {
|
||||
for (TreeSet<String> n: m.values()) {
|
||||
c += n.size();
|
||||
}
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
/**
|
||||
* get a list of <host>.<port> names in the snapshot directory
|
||||
* @return
|
||||
*/
|
||||
public Set<String> listHosts() {
|
||||
return directory.keySet();
|
||||
}
|
||||
|
||||
public final class Revisions {
|
||||
public final int depth;
|
||||
public final Date[] dates;
|
||||
public final String urlhash;
|
||||
public final String url;
|
||||
public final File[] pathtoxml;
|
||||
public Revisions(final String hostport, final int depth, final String datehash) {
|
||||
this.depth = depth;
|
||||
int p = datehash.indexOf('.');
|
||||
this.dates = new Date[1];
|
||||
String datestring = datehash.substring(0, p);
|
||||
this.dates[0] = parseDate(datestring);
|
||||
this.urlhash = datehash.substring(p + 1);
|
||||
this.pathtoxml = new File[1];
|
||||
this.pathtoxml[0] = new File(pathToShard(hostport, urlhash, depth), this.urlhash + "." + datestring + ".xml");
|
||||
String u = null;
|
||||
if (this.pathtoxml[0].exists()) {
|
||||
BufferedReader reader = null;
|
||||
try {
|
||||
reader = new BufferedReader(new InputStreamReader(new FileInputStream(this.pathtoxml[0])));
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
if (line.startsWith("<str name=\"sku\">")) {
|
||||
u = line.substring(16, line.length() - 6);
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
ConcurrentLog.warn("SNAPSHOTS", "Error while reading file " + this.pathtoxml[0]);
|
||||
} finally {
|
||||
if(reader != null) {
|
||||
try {
|
||||
reader.close();
|
||||
} catch (IOException ignored) {
|
||||
ConcurrentLog.warn("SNAPSHOTS", "Could not close input stream on file " + this.pathtoxml[0]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
this.url = u;
|
||||
}
|
||||
}
|
||||
|
||||
public Revisions getRevisions(String urlhash) {
|
||||
if (urlhash == null || urlhash.length() == 0) return null;
|
||||
// search for the hash, we must iterate through all entries
|
||||
for (Map.Entry<String, TreeMap<Integer, TreeSet<String>>> hostportDomaindepth: this.directory.entrySet()) {
|
||||
String hostport = hostportDomaindepth.getKey();
|
||||
for (Map.Entry<Integer, TreeSet<String>> depthDateHash: hostportDomaindepth.getValue().entrySet()) {
|
||||
int depth = depthDateHash.getKey();
|
||||
for (String dateHash: depthDateHash.getValue()) {
|
||||
if (dateHash.endsWith(urlhash)) {
|
||||
return new Revisions(hostport, depth, dateHash);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* list the snapshots for a given host name
|
||||
* @param hostport the <host>.<port> identifier for the domain (with the same format as applied by the Snapshots.pathToHostPortDir() function)
|
||||
* @param depth restrict the result to the given depth or if depth == -1 do not restrict to a depth
|
||||
* @return a map with a set for each depth in the domain of the host name
|
||||
*/
|
||||
public TreeMap<Integer, Collection<Revisions>> listIDs(final String hostport, final int depth) {
|
||||
TreeMap<Integer, Collection<Revisions>> result = new TreeMap<>();
|
||||
TreeMap<Integer, TreeSet<String>> list = directory.get(hostport);
|
||||
if (list != null) {
|
||||
for (Map.Entry<Integer, TreeSet<String>> entry: list.entrySet()) {
|
||||
if (depth != -1 && entry.getKey() != depth) continue;
|
||||
Collection<Revisions> r = new ArrayList<>(entry.getValue().size());
|
||||
for (String datehash: entry.getValue()) {
|
||||
r.add(new Revisions(hostport, entry.getKey(), datehash));
|
||||
}
|
||||
result.put(entry.getKey(), r);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* get the number of snapshots for the given host name
|
||||
* @param hostport the <host>.<port> identifier for the domain
|
||||
* @param depth restrict the result to the given depth or if depth == -1 do not restrict to a depth
|
||||
* @return a count, the total number of documents for the domain and depth
|
||||
*/
|
||||
public int listIDsSize(final String hostport, final int depth) {
|
||||
int count = 0;
|
||||
TreeMap<Integer, TreeSet<String>> list = directory.get(hostport);
|
||||
if (list != null) {
|
||||
for (Map.Entry<Integer, TreeSet<String>> entry: list.entrySet()) {
|
||||
if (depth != -1 && entry.getKey() != depth) continue;
|
||||
count += entry.getValue().size();
|
||||
}
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the path of a snapshot. This does not create the snapshot, only gives a path.
|
||||
* Also, the path to the storage location is not created.
|
||||
* @param url
|
||||
* @param ext
|
||||
* @param depth
|
||||
* @param date
|
||||
* @return a file to the snapshot
|
||||
*/
|
||||
public File definePath(final DigestURL url, final int depth, final Date date, final String ext) {
|
||||
String id = ASCII.String(url.hash());
|
||||
String ds = GenericFormatter.SHORT_MINUTE_FORMATTER.format(date);
|
||||
return new File(pathToShard(url, depth), id + "." + ds + "." + ext);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write information about the storage of a snapshot to the Snapshot-internal index.
|
||||
* The actual writing of files to the target directory must be done elsewehre, this method does not store the snapshot files.
|
||||
* @param url
|
||||
* @param depth
|
||||
* @param date
|
||||
*/
|
||||
public void announceStorage(final DigestURL url, final int depth, final Date date) {
|
||||
String id = ASCII.String(url.hash());
|
||||
String ds = GenericFormatter.SHORT_MINUTE_FORMATTER.format(date);
|
||||
String pathToHostPortDir = pathToHostPortDir(url.getHost(), url.getPort());
|
||||
TreeMap<Integer, TreeSet<String>> domaindepth = this.directory.get(pathToHostPortDir);
|
||||
if (domaindepth == null) {domaindepth = new TreeMap<Integer, TreeSet<String>>(); this.directory.put(pathToHostPortDir(url.getHost(), url.getPort()), domaindepth);}
|
||||
TreeSet<String> dateid = domaindepth.get(depth);
|
||||
if (dateid == null) {dateid = new TreeSet<String>(); domaindepth.put(depth, dateid);}
|
||||
dateid.add(ds + '.' + id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete information about the storage of a snapshot to the Snapshot-internal index.
|
||||
* The actual deletion of files in the target directory must be done elsewhere, this method does not store the snapshot files.
|
||||
* @param url
|
||||
* @param depth
|
||||
*/
|
||||
public Set<Date> announceDeletion(final DigestURL url, final int depth) {
|
||||
HashSet<Date> dates = new HashSet<>();
|
||||
String id = ASCII.String(url.hash());
|
||||
String pathToHostPortDir = pathToHostPortDir(url.getHost(), url.getPort());
|
||||
TreeMap<Integer, TreeSet<String>> domaindepth = this.directory.get(pathToHostPortDir);
|
||||
if (domaindepth == null) return dates;
|
||||
TreeSet<String> dateid = domaindepth.get(depth);
|
||||
if (dateid == null) return dates;
|
||||
Iterator<String> i = dateid.iterator();
|
||||
while (i.hasNext()) {
|
||||
String dis = i.next();
|
||||
if (dis.endsWith("." + id)) {
|
||||
String d = dis.substring(0, dis.length() - id.length() - 1);
|
||||
Date date = parseDate(d);
|
||||
if (date != null) dates.add(date);
|
||||
i.remove();
|
||||
}
|
||||
}
|
||||
if (dateid.size() == 0) domaindepth.remove(depth);
|
||||
if (domaindepth.size() == 0) this.directory.remove(pathToHostPortDir);
|
||||
return dates;
|
||||
}
|
||||
|
||||
/**
|
||||
* Order enum class for the select method
|
||||
*/
|
||||
public static enum Order {
|
||||
ANY, OLDESTFIRST, LATESTFIRST;
|
||||
}
|
||||
|
||||
/**
|
||||
* select a set of urlhashes from the snapshot directory. The selection either ordered
|
||||
* by generation date (upwards == OLDESTFIRST or downwards == LATESTFIRST) or with any
|
||||
* order. The result set can be selected either with a given host or a depth
|
||||
* @param host selected host or null for all hosts
|
||||
* @param depth selected depth or null for all depths
|
||||
* @param order Order.ANY, Order.OLDESTFIRST or Order.LATESTFIRST
|
||||
* @param maxcount the maximum number of hosthashes. If unlimited, submit Integer.MAX_VALUE
|
||||
* @return a map of hosthashes with the associated creation date
|
||||
*/
|
||||
public LinkedHashMap<String, Revisions> select(final String host, final Integer depth, final Order order, int maxcount) {
|
||||
TreeMap<String, String[]> dateIdResult = new TreeMap<>();
|
||||
if (host == null && depth == null) {
|
||||
loop: for (Map.Entry<String, TreeMap<Integer, TreeSet<String>>> hostportDepths: this.directory.entrySet()) {
|
||||
for (Map.Entry<Integer, TreeSet<String>> depthIds: hostportDepths.getValue().entrySet()) {
|
||||
for (String id: depthIds.getValue()) {
|
||||
dateIdResult.put(id, new String[]{hostportDepths.getKey(), Integer.toString(depthIds.getKey())});
|
||||
if (order == Order.ANY && dateIdResult.size() >= maxcount) break loop;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (host == null && depth != null) {
|
||||
loop: for (Map.Entry<String, TreeMap<Integer, TreeSet<String>>> hostportDepths: this.directory.entrySet()) {
|
||||
TreeSet<String> ids = hostportDepths.getValue().get(depth);
|
||||
if (ids != null) for (String id: ids) {
|
||||
dateIdResult.put(id, new String[]{hostportDepths.getKey(), Integer.toString(depth)});
|
||||
if (order == Order.ANY && dateIdResult.size() >= maxcount) break loop;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (host != null && depth == null) {
|
||||
String hostport = pathToHostPortDir(host, 80);
|
||||
TreeMap<Integer, TreeSet<String>> depthIdsMap = this.directory.get(hostport);
|
||||
if(depthIdsMap == null && isIpv6AddrHost(host)) {
|
||||
/* If the host is a raw IPV6 address, we check also if a snapshot was recorded with the old format (without percent-encoding) */
|
||||
hostport = pathToHostPortDir(host, 80, false);
|
||||
depthIdsMap = this.directory.get(hostport);
|
||||
}
|
||||
if (depthIdsMap != null) {
|
||||
loop: for (Map.Entry<Integer, TreeSet<String>> depthIds: depthIdsMap.entrySet()) {
|
||||
for (String id: depthIds.getValue()) {
|
||||
dateIdResult.put(id, new String[]{hostport, Integer.toString(depthIds.getKey())});
|
||||
if (order == Order.ANY && dateIdResult.size() >= maxcount) break loop;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (host != null && depth != null) {
|
||||
String hostport = pathToHostPortDir(host, 80);
|
||||
TreeMap<Integer, TreeSet<String>> domaindepth = this.directory.get(hostport);
|
||||
if(domaindepth == null && isIpv6AddrHost(host)) {
|
||||
/* If the host is a raw IPV6 address, we check also if a snapshot was recorded with the old format (without percent-encoding) */
|
||||
hostport = pathToHostPortDir(host, 80, false);
|
||||
domaindepth = this.directory.get(hostport);
|
||||
}
|
||||
if (domaindepth != null) {
|
||||
TreeSet<String> ids = domaindepth.get(depth);
|
||||
if (ids != null) loop: for (String id: ids) {
|
||||
dateIdResult.put(id, new String[]{hostport, Integer.toString(depth)});
|
||||
if (order == Order.ANY && dateIdResult.size() >= maxcount) break loop;
|
||||
}
|
||||
}
|
||||
}
|
||||
LinkedHashMap<String, Revisions> result = new LinkedHashMap<>();
|
||||
Iterator<Map.Entry<String, String[]>> i = order == Order.LATESTFIRST ? dateIdResult.descendingMap().entrySet().iterator() : dateIdResult.entrySet().iterator();
|
||||
while (i.hasNext() && result.size() < maxcount) {
|
||||
Map.Entry<String, String[]> entry = i.next();
|
||||
String datehash = entry.getKey();
|
||||
int p = datehash.indexOf('.');
|
||||
assert p >= 0;
|
||||
Revisions r = new Revisions(entry.getValue()[0], Integer.parseInt(entry.getValue()[1]), datehash);
|
||||
result.put(datehash.substring(p + 1), r);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static Date parseDate(String d) {
|
||||
try {
|
||||
return GenericFormatter.SHORT_MINUTE_FORMATTER.parse(d, 0).getTime();
|
||||
} catch (ParseException e) {
|
||||
try {
|
||||
return GenericFormatter.SHORT_DAY_FORMATTER.parse(d, 0).getTime();
|
||||
} catch (ParseException ee) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* get the depth to a document, helper method for definePath to determine the depth value
|
||||
* @param url
|
||||
* @param fulltext
|
||||
* @return the crawldepth of the document
|
||||
*/
|
||||
public int getDepth(final DigestURL url, final Fulltext fulltext) {
|
||||
Integer depth = null;
|
||||
if (fulltext.getDefaultConfiguration().contains(CollectionSchema.crawldepth_i)) {
|
||||
try {
|
||||
SolrDocument doc = fulltext.getDefaultConnector().getDocumentById(ASCII.String(url.hash()), CollectionSchema.crawldepth_i.getSolrFieldName());
|
||||
if (doc != null) {
|
||||
depth = (Integer) doc.getFieldValue(CollectionSchema.crawldepth_i.getSolrFieldName());
|
||||
}
|
||||
} catch (IOException e) {
|
||||
}
|
||||
}
|
||||
return depth == null ? 0 : depth;
|
||||
}
|
||||
|
||||
/**
|
||||
* for a given url, get all paths for storage locations.
|
||||
* The locations are all for the single url but may represent different storage times.
|
||||
* This method is inefficient because it tests all different depths, it would be better to use
|
||||
* findPaths/3 with a given depth.
|
||||
* @param url
|
||||
* @param ext
|
||||
* @return a set of files for snapshots of the url
|
||||
*/
|
||||
public Collection<File> findPaths(final DigestURL url, final String ext) {
|
||||
for (int i = 0; i < 100; i++) {
|
||||
Collection<File> paths = findPaths(url, i, ext);
|
||||
if (paths.size() > 0) return paths;
|
||||
}
|
||||
return new ArrayList<>(0);
|
||||
}
|
||||
|
||||
// pathtoxml = <storageLocation>/<host>.<port>/<depth>/<shard>/<urlhash>.<date>.xml
|
||||
|
||||
/**
|
||||
* for a given url, get all paths for storage locations.
|
||||
* The locations are all for the single url but may represent different storage times.
|
||||
* @param url
|
||||
* @param ext required extension or null if the extension must not be checked
|
||||
* @param depth
|
||||
* @return a set of files for snapshots of the url
|
||||
*/
|
||||
public Collection<File> findPaths(final DigestURL url, final int depth, final String ext) {
|
||||
String id = ASCII.String(url.hash());
|
||||
File pathToShard = pathToShard(url, depth);
|
||||
if(!pathToShard.exists() && isIpv6AddrHost(url.getHost())) {
|
||||
/* If the host is a raw IPV6 address, we check also if a snapshot was recorded with the old format (without percent-encoding) */
|
||||
pathToShard = pathToShard(pathToHostPortDir(url.getHost(), url.getPort(), false), ASCII.String(url.hash()), depth);
|
||||
}
|
||||
String[] list = pathToShard.exists() && pathToShard.isDirectory() ? pathToShard.list() : null; // may be null if path does not exist
|
||||
ArrayList<File> paths = new ArrayList<>();
|
||||
if (list != null) {
|
||||
for (String f: list) {
|
||||
if (f.startsWith(id) && (ext == null || f.endsWith(ext))) paths.add(new File(pathToShard, f));
|
||||
}
|
||||
}
|
||||
return paths;
|
||||
}
|
||||
|
||||
private File pathToShard(final DigestURL url, final int depth) {
|
||||
return pathToShard(pathToHostPortDir(url.getHost(), url.getPort()), ASCII.String(url.hash()), depth);
|
||||
}
|
||||
|
||||
private File pathToShard(final String hostport, final String urlhash, final int depth) {
|
||||
File pathToHostDir = new File(storageLocation, hostport);
|
||||
File pathToDepthDir = new File(pathToHostDir, pathToDepthDir(depth));
|
||||
File pathToShard = new File(pathToDepthDir, pathToShard(urlhash));
|
||||
return pathToShard;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param host a domain name or IP address
|
||||
* @return true when the host string is a raw IPV6 address (with square brackets)
|
||||
*/
|
||||
private boolean isIpv6AddrHost(final String host) {
|
||||
return (host != null && host.startsWith("[") && host.endsWith("]") && host.contains(":"));
|
||||
}
|
||||
|
||||
/**
|
||||
* @param host a domain name or IP address
|
||||
* @param port a port number
|
||||
* @return a representation of the host and port encoding IPV6 addresses for better support accross file systems (notably FAT or NTFS)
|
||||
*/
|
||||
private String pathToHostPortDir(final String host, final int port) {
|
||||
return pathToHostPortDir(host, port, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param host a domain name or IP address
|
||||
* @param port a port number
|
||||
* @param encodeIpv6 when true, encode the host for better support accross file systems (notably FAT or NTFS)
|
||||
* @return a representation of the host and port
|
||||
*/
|
||||
private String pathToHostPortDir(final String host, final int port, final boolean encodeIpv6) {
|
||||
String encodedHost = host;
|
||||
if(encodeIpv6 && isIpv6AddrHost(host)) {
|
||||
/* Percent-encode the host name when it is an IPV6 address, as the ':' character is illegal in a file name on MS Windows FAT32 and NTFS file systems */
|
||||
try {
|
||||
encodedHost = URLEncoder.encode(host, StandardCharsets.UTF_8.name());
|
||||
} catch (final UnsupportedEncodingException e) {
|
||||
/* This should not happen has UTF-8 encoding support is required for any JVM implementation */
|
||||
}
|
||||
}
|
||||
return encodedHost + "." + port;
|
||||
}
|
||||
|
||||
private String pathToDepthDir(final int depth) {
|
||||
return depth < 10 ? "0" + depth : Integer.toString(depth);
|
||||
}
|
||||
|
||||
private String pathToShard(final String urlhash) {
|
||||
return urlhash.substring(0, 2);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,403 +0,0 @@
|
||||
/**
|
||||
* Transactions
|
||||
* SPDX-FileCopyrightText: 2014 Michael Peter Christen <mc@yacy.net)>
|
||||
* SPDX-License-Identifier: GPL-2.0-or-later
|
||||
* First released 08.12.2014 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.crawler.data;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.net.MalformedURLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.TreeMap;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import org.apache.solr.common.SolrInputDocument;
|
||||
|
||||
import net.yacy.cora.document.id.DigestURL;
|
||||
import net.yacy.cora.federate.solr.responsewriter.EnhancedXMLResponseWriter;
|
||||
import net.yacy.cora.protocol.ClientIdentification;
|
||||
import net.yacy.cora.util.ConcurrentLog;
|
||||
import net.yacy.cora.util.Html2Image;
|
||||
import net.yacy.crawler.data.Snapshots.Order;
|
||||
import net.yacy.crawler.data.Snapshots.Revisions;
|
||||
import net.yacy.search.schema.CollectionSchema;
|
||||
|
||||
/**
|
||||
* This is a static class holding one or several Snapshot directories
|
||||
* Transacted snapshots are moved from the inventory snapshot directory to the archive snapshot directory.
|
||||
*
|
||||
*/
|
||||
public class Transactions {
|
||||
|
||||
private final static String XML_PREFIX = "<response>\n<!--\n";
|
||||
private final static char[] WHITESPACE = new char[132];
|
||||
//private final static int WHITESPACE_START = XML_PREFIX.length();
|
||||
//private final static int WHITESPACE_LENGTH = WHITESPACE.length;
|
||||
private static File transactionDir = null, inventoryDir = null, archiveDir = null;
|
||||
private static Snapshots inventory = null, archive = null;
|
||||
private static ExecutorService executor = Executors.newCachedThreadPool();
|
||||
private static AtomicInteger executorRunning = new AtomicInteger(0);
|
||||
|
||||
/** the maximum to wait for each wkhtmltopdf call when rendering PDF snapshots */
|
||||
private static long wkhtmltopdfTimeout = 30;
|
||||
|
||||
static {
|
||||
for (int i = 0; i < WHITESPACE.length; i++) WHITESPACE[i] = 32;
|
||||
}
|
||||
|
||||
public static enum State {
|
||||
INVENTORY("inventory"), ARCHIVE("archive"), ANY(null);
|
||||
public String dirname;
|
||||
State(String dirname) {
|
||||
this.dirname = dirname;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param dir the parent directory of inventory and archive snapshots.
|
||||
* @param wkhtmltopdfSecondsTimeout the maximum to wait for each wkhtmltopdf call when rendering PDF snapshots
|
||||
*/
|
||||
public static void init(final File dir, final long wkhtmltopdfSecondsTimeout) {
|
||||
transactionDir = dir;
|
||||
transactionDir.mkdirs();
|
||||
inventoryDir = new File(transactionDir, State.INVENTORY.dirname);
|
||||
inventory = new Snapshots(inventoryDir);
|
||||
archiveDir = new File(transactionDir, State.ARCHIVE.dirname);
|
||||
archive = new Snapshots(archiveDir);
|
||||
wkhtmltopdfTimeout = wkhtmltopdfSecondsTimeout;
|
||||
}
|
||||
|
||||
public static synchronized void migrateIPV6Snapshots() {
|
||||
executor.shutdown();
|
||||
try {
|
||||
executor.awaitTermination(10, TimeUnit.SECONDS);
|
||||
} catch (final InterruptedException e) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* get the number of entries for each of the transaction states
|
||||
* @return the total number of different documents for each transaction state
|
||||
*/
|
||||
public static Map<String, Integer> sizes() {
|
||||
HashMap<String, Integer> m = new HashMap<>();
|
||||
m.put(State.INVENTORY.name(), inventory.size());
|
||||
m.put(State.ARCHIVE.name(), archive.size());
|
||||
return m;
|
||||
}
|
||||
|
||||
public static Revisions getRevisions(final State state, final String urlhash) {
|
||||
switch (state) {
|
||||
case INVENTORY : return inventory.getRevisions(urlhash);
|
||||
case ARCHIVE : return archive.getRevisions(urlhash);
|
||||
default : Revisions a = inventory.getRevisions(urlhash); return a == null ? archive.getRevisions(urlhash) : a;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* get a list of <host>.<port> names in the snapshot directory
|
||||
* @return the list of the given state. if the state is ALL or unknown, all lists are combined
|
||||
*/
|
||||
public static Set<String> listHosts(final State state) {
|
||||
switch (state) {
|
||||
case INVENTORY : return inventory.listHosts();
|
||||
case ARCHIVE : return archive.listHosts();
|
||||
default : Set<String> a = inventory.listHosts(); a.addAll(archive.listHosts()); return a;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* list the snapshots for a given host name
|
||||
* @param hostport the <host>.<port> identifier for the domain (with the same format as applied by the Snapshots.pathToHostPortDir() function).
|
||||
* @param depth restrict the result to the given depth or if depth == -1 do not restrict to a depth
|
||||
* @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
|
||||
* @return a map with a set for each depth in the domain of the host name
|
||||
*/
|
||||
public static TreeMap<Integer, Collection<Revisions>> listIDs(final String hostport, final int depth, final State state) {
|
||||
switch (state) {
|
||||
case INVENTORY : return inventory.listIDs(hostport, depth);
|
||||
case ARCHIVE : return archive.listIDs(hostport, depth);
|
||||
default : TreeMap<Integer, Collection<Revisions>> a = inventory.listIDs(hostport, depth); a.putAll(archive.listIDs(hostport, depth)); return a;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* get the number of snapshots for the given host name
|
||||
* @param hostport the <host>.<port> identifier for the domain
|
||||
* @param depth restrict the result to the given depth or if depth == -1 do not restrict to a depth
|
||||
* @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
|
||||
* @return a count, the total number of documents for the domain and depth
|
||||
*/
|
||||
public static int listIDsSize(final String hostport, final int depth, final State state) {
|
||||
switch (state) {
|
||||
case INVENTORY : return inventory.listIDsSize(hostport, depth);
|
||||
case ARCHIVE : return archive.listIDsSize(hostport, depth);
|
||||
default : return inventory.listIDsSize(hostport, depth) + archive.listIDsSize(hostport, depth);
|
||||
}
|
||||
}
|
||||
|
||||
public static boolean store(final SolrInputDocument doc, final boolean concurrency, final boolean loadImage, final boolean replaceOld, final String proxy, final String acceptLanguage) {
|
||||
|
||||
// GET METADATA FROM DOC
|
||||
final String urls = (String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName());
|
||||
final Date date = (Date) doc.getFieldValue(CollectionSchema.load_date_dt.getSolrFieldName());
|
||||
final Integer o_depth = (Integer) doc.getFieldValue(CollectionSchema.crawldepth_i.getSolrFieldName()); // may return null
|
||||
final int depth = o_depth == null ? 0 : o_depth.intValue();
|
||||
|
||||
DigestURL url;
|
||||
try {
|
||||
url = new DigestURL(urls);
|
||||
} catch (MalformedURLException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
return false;
|
||||
}
|
||||
|
||||
boolean success = loadImage ? store(url, date, depth, concurrency, replaceOld, proxy, acceptLanguage) : true;
|
||||
if (success) {
|
||||
// STORE METADATA FOR THE IMAGE
|
||||
File metadataPath = Transactions.definePath(url, depth, date, "xml", Transactions.State.INVENTORY);
|
||||
metadataPath.getParentFile().mkdirs();
|
||||
if (doc != null) {
|
||||
try (
|
||||
/* Resources automatically closed by this try-with-resources statement */
|
||||
final FileOutputStream fos = new FileOutputStream(metadataPath);
|
||||
final OutputStreamWriter osw = new OutputStreamWriter(fos);
|
||||
) {
|
||||
osw.write(XML_PREFIX);
|
||||
osw.write(WHITESPACE); osw.write("\n-->\n"); // placeholder for transaction information properties (a hack to attach metadata to metadata)
|
||||
osw.write("<result name=\"response\" numFound=\"1\" start=\"0\">\n");
|
||||
EnhancedXMLResponseWriter.writeDoc(osw, doc);
|
||||
osw.write("</result>\n");
|
||||
osw.write("</response>\n");
|
||||
} catch (IOException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
success = false;
|
||||
}
|
||||
if(success) {
|
||||
Transactions.announceStorage(url, depth, date, State.INVENTORY);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return success;
|
||||
}
|
||||
|
||||
|
||||
public static boolean store(final DigestURL url, final Date date, final int depth, final boolean concurrency, final boolean replaceOld, final String proxy, final String acceptLanguage) {
|
||||
|
||||
// CLEAN UP OLD DATA (if wanted)
|
||||
Collection<File> oldPaths = Transactions.findPaths(url, depth, null, Transactions.State.INVENTORY);
|
||||
if (replaceOld && oldPaths != null) {
|
||||
for (File oldPath: oldPaths) {
|
||||
oldPath.delete();
|
||||
}
|
||||
}
|
||||
|
||||
// STORE METADATA FOR THE IMAGE
|
||||
File metadataPath = Transactions.definePath(url, depth, date, "xml", Transactions.State.INVENTORY);
|
||||
metadataPath.getParentFile().mkdirs();
|
||||
boolean success = true;
|
||||
|
||||
// STORE AN IMAGE
|
||||
final String urls = url.toNormalform(true);
|
||||
final File pdfPath = Transactions.definePath(url, depth, date, "pdf", Transactions.State.INVENTORY);
|
||||
if (concurrency && executorRunning.intValue() < Runtime.getRuntime().availableProcessors()) {
|
||||
Thread t = new Thread("Transactions.store"){
|
||||
@Override
|
||||
public void run() {
|
||||
executorRunning.incrementAndGet();
|
||||
try {
|
||||
Html2Image.writeWkhtmltopdf(urls, proxy, ClientIdentification.browserAgent.userAgent(), acceptLanguage, pdfPath, wkhtmltopdfTimeout);
|
||||
} catch (Throwable e) {} finally {
|
||||
executorRunning.decrementAndGet();
|
||||
}
|
||||
}
|
||||
};
|
||||
executor.execute(t);
|
||||
} else {
|
||||
success = Html2Image.writeWkhtmltopdf(urls, proxy, ClientIdentification.browserAgent.userAgent(), acceptLanguage, pdfPath, wkhtmltopdfTimeout);
|
||||
}
|
||||
|
||||
return success;
|
||||
}
|
||||
|
||||
/**
|
||||
* Announce the commit of a snapshot: this will move all data for the given urlhash from the inventory to the archive
|
||||
* The index within the snapshot management will update also.
|
||||
* @param urlhash
|
||||
* @return a revision object from the moved document if the commit has succeeded, null if something went wrong
|
||||
*/
|
||||
public static Revisions commit(String urlhash) {
|
||||
return transact(urlhash, State.INVENTORY, State.ARCHIVE);
|
||||
}
|
||||
|
||||
/**
|
||||
* Announce the rollback of a snapshot: this will move all data for the given urlhash from the archive to the inventory
|
||||
* The index within the snapshot management will update also.
|
||||
* @param urlhash
|
||||
* @return a revision object from the moved document if the commit has succeeded, null if something went wrong
|
||||
*/
|
||||
public static Revisions rollback(String urlhash) {
|
||||
return transact(urlhash, State.ARCHIVE, State.INVENTORY);
|
||||
}
|
||||
|
||||
private static Revisions transact(final String urlhash, final State from, final State to) {
|
||||
Revisions r = Transactions.getRevisions(from, urlhash);
|
||||
if (r == null) return null;
|
||||
// we take all pathtoxml and move that to archive
|
||||
for (File f: r.pathtoxml) {
|
||||
String name = f.getName();
|
||||
String nameStub = name.substring(0, name.length() - 4);
|
||||
File sourceParent = f.getParentFile();
|
||||
File targetParent = new File(sourceParent.getAbsolutePath().replace("/" + from.dirname + "/", "/" + to.dirname + "/"));
|
||||
targetParent.mkdirs();
|
||||
// list all files in the parent directory
|
||||
for (String a: sourceParent.list()) {
|
||||
if (a.startsWith(nameStub)) {
|
||||
new File(sourceParent, a).renameTo(new File(targetParent, a));
|
||||
}
|
||||
}
|
||||
// delete empty directories
|
||||
while (sourceParent.list().length == 0) {
|
||||
sourceParent.delete();
|
||||
sourceParent = sourceParent.getParentFile();
|
||||
}
|
||||
}
|
||||
// announce the movement
|
||||
DigestURL durl;
|
||||
try {
|
||||
durl = new DigestURL(r.url);
|
||||
Transactions.announceDeletion(durl, r.depth, from);
|
||||
Transactions.announceStorage(durl, r.depth, r.dates[0], to);
|
||||
return r;
|
||||
} catch (MalformedURLException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* select a set of urlhashes from the snapshot directory. The selection either ordered
|
||||
* by generation date (upwards == OLDESTFIRST or downwards == LATESTFIRST) or with any
|
||||
* order. The result set can be selected either with a given host or a depth
|
||||
* @param host selected host or null for all hosts
|
||||
* @param depth selected depth or null for all depths
|
||||
* @param order Order.ANY, Order.OLDESTFIRST or Order.LATESTFIRST
|
||||
* @param maxcount the maximum number of hosthashes. If unlimited, submit Integer.MAX_VALUE
|
||||
* @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
|
||||
* @return a map of hosthashes with the associated creation date
|
||||
*/
|
||||
public static LinkedHashMap<String, Revisions> select(String host, Integer depth, final Order order, int maxcount, State state) {
|
||||
LinkedHashMap<String, Revisions> result = new LinkedHashMap<>();
|
||||
if (state == State.INVENTORY || state == State.ANY) result.putAll(inventory.select(host, depth, order, maxcount));
|
||||
if (state == State.ARCHIVE || state == State.ANY) result.putAll(archive.select(host, depth, order, maxcount));
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the path of a snapshot. This does not create the snapshot, only gives a path.
|
||||
* Also, the path to the storage location is not created.
|
||||
* @param url
|
||||
* @param depth
|
||||
* @param date
|
||||
* @param ext
|
||||
* @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
|
||||
* @return a file to the snapshot
|
||||
*/
|
||||
public static File definePath(final DigestURL url, final int depth, final Date date, final String ext, State state) {
|
||||
if (state == State.ANY) throw new RuntimeException("definePath must be selected with INVENTORY or ARCHIVE state");
|
||||
if (state == State.INVENTORY) return inventory.definePath(url, depth, date, ext);
|
||||
if (state == State.ARCHIVE) return archive.definePath(url, depth, date, ext);
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write information about the storage of a snapshot to the Snapshot-internal index.
|
||||
* The actual writing of files to the target directory must be done elsewehre, this method does not store the snapshot files.
|
||||
* @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
|
||||
* @param url
|
||||
* @param depth
|
||||
* @param date
|
||||
*/
|
||||
public static void announceStorage(final DigestURL url, final int depth, final Date date, State state) {
|
||||
if (state == State.INVENTORY || state == State.ANY) inventory.announceStorage(url, depth, date);
|
||||
if (state == State.ARCHIVE || state == State.ANY) archive.announceStorage(url, depth, date);
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete information about the storage of a snapshot to the Snapshot-internal index.
|
||||
* The actual deletion of files in the target directory must be done elsewehre, this method does not store the snapshot files.
|
||||
* @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
|
||||
* @param url
|
||||
* @param depth
|
||||
*/
|
||||
public static void announceDeletion(final DigestURL url, final int depth, final State state) {
|
||||
if (state == State.INVENTORY || state == State.ANY) inventory.announceDeletion(url, depth);
|
||||
if (state == State.ARCHIVE || state == State.ANY) archive.announceDeletion(url, depth);
|
||||
}
|
||||
|
||||
/**
|
||||
* for a given url, get all paths for storage locations.
|
||||
* The locations are all for the single url but may represent different storage times.
|
||||
* This method is inefficient because it tests all different depths, it would be better to use
|
||||
* findPaths/3 with a given depth.
|
||||
* @param url
|
||||
* @param ext required extension or null if the extension must not be checked
|
||||
* @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
|
||||
* @return a set of files for snapshots of the url
|
||||
*/
|
||||
public static Collection<File> findPaths(final DigestURL url, final String ext, State state) {
|
||||
Collection<File> result = new ArrayList<>();
|
||||
if (state == State.INVENTORY || state == State.ANY) result.addAll(inventory.findPaths(url, ext));
|
||||
if (state == State.ARCHIVE || state == State.ANY) result.addAll(archive.findPaths(url, ext));
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* for a given url, get all paths for storage locations.
|
||||
* The locations are all for the single url but may represent different storage times.
|
||||
* @param url
|
||||
* @param ext required extension or null if the extension must not be checked
|
||||
* @param depth
|
||||
* @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
|
||||
* @return a set of files for snapshots of the url
|
||||
*/
|
||||
public static Collection<File> findPaths(final DigestURL url, final int depth, final String ext, State state) {
|
||||
Collection<File> result = new ArrayList<>();
|
||||
if (state == State.INVENTORY || state == State.ANY) result.addAll(inventory.findPaths(url, depth, ext));
|
||||
if (state == State.ARCHIVE || state == State.ANY) result.addAll(archive.findPaths(url, depth, ext));
|
||||
return result;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -33,7 +33,6 @@ import java.io.File;
|
||||
import java.io.FileReader;
|
||||
import java.io.FileWriter;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Date;
|
||||
|
||||
@@ -47,10 +46,6 @@ import net.yacy.kelondro.util.FileUtils;
|
||||
|
||||
public class psParser extends AbstractParser implements Parser {
|
||||
|
||||
private final static Object modeScan = new Object();
|
||||
private static boolean modeScanDone = false;
|
||||
private static String parserMode = "java";
|
||||
|
||||
public psParser() {
|
||||
super("PostScript Document Parser");
|
||||
this.SUPPORTED_EXTENSIONS.add("ps");
|
||||
@@ -59,33 +54,8 @@ public class psParser extends AbstractParser implements Parser {
|
||||
this.SUPPORTED_MIME_TYPES.add("application/x-postscript");
|
||||
this.SUPPORTED_MIME_TYPES.add("application/x-ps");
|
||||
this.SUPPORTED_MIME_TYPES.add("application/x-postscript-not-eps");
|
||||
if (!modeScanDone) synchronized (modeScan) {
|
||||
if (testForPs2Ascii()) parserMode = "ps2ascii";
|
||||
else parserMode = "java";
|
||||
modeScanDone = true;
|
||||
}
|
||||
}
|
||||
|
||||
private boolean testForPs2Ascii() {
|
||||
try {
|
||||
String procOutputLine = null;
|
||||
final StringBuilder procOutput = new StringBuilder(80);
|
||||
|
||||
final Process ps2asciiProc = Runtime.getRuntime().exec(new String[]{"ps2ascii", "--version"});
|
||||
final BufferedReader stdOut = new BufferedReader(new InputStreamReader(ps2asciiProc.getInputStream()));
|
||||
while ((procOutputLine = stdOut.readLine()) != null) {
|
||||
procOutput.append(procOutputLine).append(", ");
|
||||
}
|
||||
stdOut.close();
|
||||
final int returnCode = ps2asciiProc.waitFor();
|
||||
return (returnCode == 0);
|
||||
} catch (final Exception e) {
|
||||
if (AbstractParser.log != null) AbstractParser.log.info("ps2ascii not found. Switching to java parser mode.");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private Document[] parse(final DigestURL location, final String mimeType, @SuppressWarnings("unused") final String charset, final File sourceFile) throws Parser.Failure, InterruptedException {
|
||||
|
||||
File outputFile = null;
|
||||
@@ -93,12 +63,7 @@ public class psParser extends AbstractParser implements Parser {
|
||||
// creating a temp file for the output
|
||||
outputFile = FileUtils.createTempFile(this.getClass(), "ascii.txt");
|
||||
|
||||
// decide with parser mode to use
|
||||
if (parserMode.equals("ps2ascii")) {
|
||||
parseUsingPS2ascii(sourceFile,outputFile);
|
||||
} else {
|
||||
parseUsingJava(sourceFile,outputFile);
|
||||
}
|
||||
parseUsingJava(sourceFile,outputFile);
|
||||
|
||||
// return result
|
||||
final Document[] docs = new Document[]{new Document(
|
||||
@@ -221,41 +186,6 @@ public class psParser extends AbstractParser implements Parser {
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* This function requires the ghostscript-library
|
||||
* @param inputFile
|
||||
* @param outputFile
|
||||
* @throws Exception
|
||||
*/
|
||||
private void parseUsingPS2ascii(final File inputFile, final File outputFile) throws Exception {
|
||||
int execCode = 0;
|
||||
StringBuilder procErr = null;
|
||||
try {
|
||||
String procOutputLine;
|
||||
final StringBuilder procOut = new StringBuilder();
|
||||
procErr = new StringBuilder();
|
||||
|
||||
final Process ps2asciiProc = Runtime.getRuntime().exec(new String[]{"ps2ascii", inputFile.getAbsolutePath(),outputFile.getAbsolutePath()});
|
||||
final BufferedReader stdOut = new BufferedReader(new InputStreamReader(ps2asciiProc.getInputStream()));
|
||||
final BufferedReader stdErr = new BufferedReader(new InputStreamReader(ps2asciiProc.getErrorStream()));
|
||||
while ((procOutputLine = stdOut.readLine()) != null) {
|
||||
procOut.append(procOutputLine);
|
||||
}
|
||||
stdOut.close();
|
||||
while ((procOutputLine = stdErr.readLine()) != null) {
|
||||
procErr.append(procOutputLine);
|
||||
}
|
||||
stdErr.close();
|
||||
execCode = ps2asciiProc.waitFor();
|
||||
} catch (final Exception e) {
|
||||
final String errorMsg = "Unable to convert ps to ascii. " + e.getMessage();
|
||||
AbstractParser.log.severe(errorMsg);
|
||||
throw new Exception(errorMsg);
|
||||
}
|
||||
|
||||
if (execCode != 0) throw new Exception("Unable to convert ps to ascii. ps2ascii returned statuscode " + execCode + "\n" + procErr.toString());
|
||||
}
|
||||
|
||||
@Override
|
||||
public Document[] parse(
|
||||
final DigestURL location,
|
||||
|
||||
@@ -100,7 +100,6 @@ public class ConfigSearchPage_p {
|
||||
sb.setConfig("search.result.show.cache", post.getBoolean("search.result.show.cache"));
|
||||
sb.setConfig("search.result.show.proxy", post.getBoolean("search.result.show.proxy"));
|
||||
sb.setConfig("search.result.show.indexbrowser", post.getBoolean("search.result.show.indexbrowser"));
|
||||
sb.setConfig("search.result.show.snapshots", post.getBoolean("search.result.show.snapshots"));
|
||||
|
||||
// construct navigation String
|
||||
final Set<String> navConfigs = new HashSet<>();
|
||||
@@ -190,7 +189,6 @@ public class ConfigSearchPage_p {
|
||||
sb.setConfig("search.result.show.cache", config.getProperty("search.result.show.cache","true"));
|
||||
sb.setConfig("search.result.show.proxy", config.getProperty("search.result.show.proxy","false"));
|
||||
sb.setConfig("search.result.show.indexbrowser", config.getProperty("search.result.show.indexbrowser","true"));
|
||||
sb.setConfig("search.result.show.snapshots", config.getProperty("search.result.show.snapshots","true"));
|
||||
sb.setConfig(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT,
|
||||
config.getProperty(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT,
|
||||
String.valueOf(QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT)));
|
||||
@@ -254,7 +252,6 @@ public class ConfigSearchPage_p {
|
||||
prop.put("search.result.show.cache", sb.getConfigBool("search.result.show.cache", false) ? 1 : 0);
|
||||
prop.put("search.result.show.proxy", sb.getConfigBool("search.result.show.proxy", false) ? 1 : 0);
|
||||
prop.put("search.result.show.indexbrowser", sb.getConfigBool("search.result.show.indexbrowser", false) ? 1 : 0);
|
||||
prop.put("search.result.show.snapshots", sb.getConfigBool("search.result.show.snapshots", false) ? 1 : 0);
|
||||
prop.put("search.result.show.ranking", sb.getConfigBool(SwitchboardConstants.SEARCH_RESULT_SHOW_RANKING, SwitchboardConstants.SEARCH_RESULT_SHOW_RANKING_DEFAULT) ? 1 : 0);
|
||||
|
||||
final Set<String> navConfigs = sb.getConfigSet("search.navigation");
|
||||
|
||||
@@ -36,7 +36,6 @@ import net.yacy.cora.federate.solr.instance.EmbeddedInstance;
|
||||
import net.yacy.cora.lod.vocabulary.Tagging;
|
||||
import net.yacy.cora.protocol.ClientIdentification;
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import net.yacy.cora.util.Html2Image;
|
||||
import net.yacy.crawler.data.CrawlProfile;
|
||||
import net.yacy.crawler.data.CrawlProfile.CrawlAttribute;
|
||||
import net.yacy.document.LibraryProvider;
|
||||
@@ -633,18 +632,6 @@ public class CrawlStartExpert {
|
||||
prop.put("vocabularySelect_vocabularyset", count);
|
||||
}
|
||||
|
||||
// ---------- Snapshot generation
|
||||
final boolean wkhtmltopdfAvailable = Html2Image.wkhtmltopdfAvailable();
|
||||
//boolean convertAvailable = Html2Image.convertAvailable();
|
||||
prop.put("snapshotsMaxDepth", post == null ? "-1" : post.get("snapshotsMaxDepth", "-1"));
|
||||
prop.put("snapshotsMustnotmatch", post == null ? "" : post.get("snapshotsMustnotmatch", ""));
|
||||
if (wkhtmltopdfAvailable) {
|
||||
prop.put("snapshotEnableImages", 1);
|
||||
prop.put("snapshotEnableImages_snapshotsLoadImageChecked", post == null ? 1 : post.getBoolean("snapshotsLoadImage") ? 1 : 0);
|
||||
} else {
|
||||
prop.put("snapshotEnableImages", 0);
|
||||
}
|
||||
|
||||
// ---------- Index Administration
|
||||
// Do Local Indexing
|
||||
if (post == null) {
|
||||
|
||||
@@ -480,12 +480,6 @@ public class Crawler_p {
|
||||
}
|
||||
}
|
||||
|
||||
final String snapshotsMaxDepthString = post.get("snapshotsMaxDepth", "-1");
|
||||
final int snapshotsMaxDepth = Integer.parseInt(snapshotsMaxDepthString);
|
||||
final boolean snapshotsLoadImage = post.getBoolean("snapshotsLoadImage");
|
||||
final boolean snapshotsReplaceOld = post.getBoolean("snapshotsReplaceOld");
|
||||
final String snapshotsMustnotmatch = post.get("snapshotsMustnotmatch", "");
|
||||
|
||||
final String valency_switch_tag_names_s = post.get("valency_switch_tag_names");
|
||||
final Set<String> valency_switch_tag_names = new HashSet<>();
|
||||
if (valency_switch_tag_names_s != null) {
|
||||
@@ -626,10 +620,6 @@ public class Crawler_p {
|
||||
indexMedia,
|
||||
storeHTCache,
|
||||
crawlOrder,
|
||||
snapshotsMaxDepth,
|
||||
snapshotsLoadImage,
|
||||
snapshotsReplaceOld,
|
||||
snapshotsMustnotmatch,
|
||||
cachePolicy,
|
||||
collection,
|
||||
agentName,
|
||||
|
||||
@@ -160,7 +160,6 @@ public class QuickCrawlLink_p {
|
||||
obeyHtmlRobotsNoindex, obeyHtmlRobotsNofollow,
|
||||
indexText, indexMedia,
|
||||
storeHTCache, remoteIndexing,
|
||||
-1, false, true, CrawlProfile.MATCH_NEVER_STRING,
|
||||
CacheStrategy.IFFRESH,
|
||||
collection,
|
||||
ClientIdentification.yacyIntranetCrawlerAgentName,
|
||||
|
||||
@@ -1,382 +0,0 @@
|
||||
/**
|
||||
* snapshot
|
||||
* Copyright 2014 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 02.12.2014 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.htroot.api;
|
||||
|
||||
import java.awt.Container;
|
||||
import java.awt.Image;
|
||||
import java.awt.MediaTracker;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.util.Collection;
|
||||
import java.util.Date;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import org.apache.http.HttpStatus;
|
||||
import org.apache.solr.common.SolrDocument;
|
||||
import org.apache.solr.common.SolrInputDocument;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
|
||||
import net.yacy.cora.document.encoding.ASCII;
|
||||
import net.yacy.cora.document.encoding.UTF8;
|
||||
import net.yacy.cora.document.feed.RSSFeed;
|
||||
import net.yacy.cora.document.feed.RSSMessage;
|
||||
import net.yacy.cora.document.id.DigestURL;
|
||||
import net.yacy.cora.protocol.HeaderFramework;
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import net.yacy.cora.util.ConcurrentLog;
|
||||
import net.yacy.cora.util.Html2Image;
|
||||
import net.yacy.crawler.data.Snapshots;
|
||||
import net.yacy.crawler.data.Snapshots.Revisions;
|
||||
import net.yacy.crawler.data.Transactions;
|
||||
import net.yacy.document.ImageParser;
|
||||
import net.yacy.http.servlets.TemplateMissingParameterException;
|
||||
import net.yacy.http.servlets.TemplateProcessingException;
|
||||
import net.yacy.kelondro.util.FileUtils;
|
||||
import net.yacy.peers.graphics.EncodedImage;
|
||||
import net.yacy.search.Switchboard;
|
||||
import net.yacy.search.SwitchboardConstants;
|
||||
import net.yacy.server.serverObjects;
|
||||
import net.yacy.server.serverSwitch;
|
||||
|
||||
public class snapshot {
|
||||
|
||||
//width = 1024, height = 1024, density = 300, quality = 75
|
||||
private final static int DEFAULT_WIDTH = 1024;
|
||||
private final static int DEFAULT_HEIGHT = 1024;
|
||||
private final static int DEFAULT_DENSITY = 300;
|
||||
private final static int DEFAULT_QUALITY = 75;
|
||||
private final static String DEFAULT_EXT = "jpg";
|
||||
|
||||
public static Object respond(final RequestHeader header, serverObjects post, final serverSwitch env) {
|
||||
final Switchboard sb = (Switchboard) env;
|
||||
|
||||
final serverObjects defaultResponse = new serverObjects();
|
||||
|
||||
|
||||
final boolean authenticated = sb.adminAuthenticated(header) >= 2;
|
||||
final String ext = header.get(HeaderFramework.CONNECTION_PROP_EXT, "");
|
||||
|
||||
if(ext.isEmpty()) {
|
||||
throw new TemplateProcessingException("Missing extension. Try with rss, xml, json, pdf, png or jpg." + ext,
|
||||
HttpStatus.SC_BAD_REQUEST);
|
||||
}
|
||||
|
||||
|
||||
if (ext.equals("rss")) {
|
||||
// create a report about the content of the snapshot directory
|
||||
if (!authenticated) {
|
||||
defaultResponse.authenticationRequired();
|
||||
return defaultResponse;
|
||||
}
|
||||
final int maxcount = post == null ? 10 : post.getInt("maxcount", 10);
|
||||
final int depthx = post == null ? -1 : post.getInt("depth", -1);
|
||||
final Integer depth = depthx == -1 ? null : depthx;
|
||||
final String orderx = post == null ? "ANY" : post.get("order", "ANY");
|
||||
final Snapshots.Order order = Snapshots.Order.valueOf(orderx);
|
||||
final String statex = post == null ? Transactions.State.INVENTORY.name() : post.get("state", Transactions.State.INVENTORY.name());
|
||||
final Transactions.State state = Transactions.State.valueOf(statex);
|
||||
final String host = post == null ? null : post.get("host");
|
||||
final Map<String, Revisions> iddate = Transactions.select(host, depth, order, maxcount, state);
|
||||
// now select the URL from the index for these ids in iddate and make an RSS feed
|
||||
final RSSFeed rssfeed = new RSSFeed(Integer.MAX_VALUE);
|
||||
rssfeed.setChannel(new RSSMessage("Snapshot list for host = " + host + ", depth = " + depth + ", order = " + order + ", maxcount = " + maxcount, "", ""));
|
||||
for (final Map.Entry<String, Revisions> e: iddate.entrySet()) {
|
||||
try {
|
||||
final String u = e.getValue().url == null ? sb.index.fulltext().getURL(e.getKey()) : e.getValue().url;
|
||||
if (u == null) continue;
|
||||
final RSSMessage message = new RSSMessage(u, "", new DigestURL(u), e.getKey());
|
||||
message.setPubDate(e.getValue().dates[0]);
|
||||
rssfeed.addMessage(message);
|
||||
} catch (final IOException ee) {
|
||||
ConcurrentLog.logException(ee);
|
||||
}
|
||||
}
|
||||
final byte[] rssBinary = UTF8.getBytes(rssfeed.toString());
|
||||
return new ByteArrayInputStream(rssBinary);
|
||||
}
|
||||
|
||||
// for the following methods we (mostly) need an url or a url hash
|
||||
if (post == null) post = new serverObjects();
|
||||
final boolean xml = ext.equals("xml");
|
||||
final boolean pdf = ext.equals("pdf");
|
||||
if (pdf && !authenticated) {
|
||||
defaultResponse.authenticationRequired();
|
||||
return defaultResponse;
|
||||
}
|
||||
final boolean pngjpg = ext.equals("png") || ext.equals(DEFAULT_EXT);
|
||||
String urlhash = post.get("urlhash", "");
|
||||
final String url = post.get("url", "");
|
||||
DigestURL durl = null;
|
||||
if (urlhash.length() == 0 && url.length() > 0) {
|
||||
try {
|
||||
durl = new DigestURL(url);
|
||||
urlhash = ASCII.String(durl.hash());
|
||||
} catch (final MalformedURLException e) {
|
||||
}
|
||||
}
|
||||
if (durl == null && urlhash.length() > 0) {
|
||||
try {
|
||||
final String u = sb.index.fulltext().getURL(urlhash);
|
||||
durl = u == null ? null : new DigestURL(u);
|
||||
} catch (final IOException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
}
|
||||
}
|
||||
|
||||
if (ext.equals("json")) {
|
||||
// command interface: view and change a transaction state, get metadata about transactions in the past
|
||||
final String command = post.get("command", "metadata");
|
||||
final String statename = post.get("state");
|
||||
final JSONObject result = new JSONObject();
|
||||
try {
|
||||
if (command.equals("status")) {
|
||||
// return a status of the transaction archive
|
||||
final JSONObject sizes = new JSONObject();
|
||||
for (final Map.Entry<String, Integer> state: Transactions.sizes().entrySet()) sizes.put(state.getKey(), state.getValue());
|
||||
result.put("size", sizes);
|
||||
} else if (command.equals("list")) {
|
||||
if (!authenticated) {
|
||||
defaultResponse.authenticationRequired();
|
||||
return defaultResponse;
|
||||
}
|
||||
// return a status of the transaction archive
|
||||
final String host = post.get("host");
|
||||
final String depth = post.get("depth");
|
||||
final int depthi = depth == null ? -1 : Integer.parseInt(depth);
|
||||
for (final Transactions.State state: statename == null ?
|
||||
new Transactions.State[]{Transactions.State.INVENTORY, Transactions.State.ARCHIVE} :
|
||||
new Transactions.State[]{Transactions.State.valueOf(statename)}) {
|
||||
if (host == null) {
|
||||
final JSONObject hostCountInventory = new JSONObject();
|
||||
for (final String h: Transactions.listHosts(state)) {
|
||||
final int size = Transactions.listIDsSize(h, depthi, state);
|
||||
if (size > 0) hostCountInventory.put(h, size);
|
||||
}
|
||||
result.put("count." + state.name(), hostCountInventory);
|
||||
} else {
|
||||
final TreeMap<Integer, Collection<Revisions>> ids = Transactions.listIDs(host, depthi, state);
|
||||
if (ids == null) {
|
||||
result.put("result", "fail");
|
||||
result.put("comment", "no entries for host " + host + " found");
|
||||
} else {
|
||||
for (final Map.Entry<Integer, Collection<Revisions>> entry: ids.entrySet()) {
|
||||
for (final Revisions r: entry.getValue()) {
|
||||
try {
|
||||
final JSONObject metadata = new JSONObject();
|
||||
final String u = r.url != null ? r.url : sb.index.fulltext().getURL(r.urlhash);
|
||||
metadata.put("url", u == null ? "unknown" : u);
|
||||
metadata.put("dates", r.dates);
|
||||
assert r.depth == entry.getKey().intValue();
|
||||
metadata.put("depth", entry.getKey().intValue());
|
||||
result.put(r.urlhash, metadata);
|
||||
} catch (final IOException e) {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (command.equals("commit")) {
|
||||
if (!authenticated) {
|
||||
defaultResponse.authenticationRequired();
|
||||
return defaultResponse;
|
||||
}
|
||||
final Revisions r = Transactions.commit(urlhash);
|
||||
if (r != null) {
|
||||
result.put("result", "success");
|
||||
result.put("depth", r.depth);
|
||||
result.put("url", r.url);
|
||||
result.put("dates", r.dates);
|
||||
} else {
|
||||
result.put("result", "fail");
|
||||
}
|
||||
result.put("urlhash", urlhash);
|
||||
} else if (command.equals("rollback")) {
|
||||
if (!authenticated) {
|
||||
defaultResponse.authenticationRequired();
|
||||
return defaultResponse;
|
||||
}
|
||||
final Revisions r = Transactions.rollback(urlhash);
|
||||
if (r != null) {
|
||||
result.put("result", "success");
|
||||
result.put("depth", r.depth);
|
||||
result.put("url", r.url);
|
||||
result.put("dates", r.dates);
|
||||
} else {
|
||||
result.put("result", "fail");
|
||||
}
|
||||
result.put("urlhash", urlhash);
|
||||
} else if (command.equals("metadata")) {
|
||||
try {
|
||||
Revisions r;
|
||||
Transactions.State state = statename == null || statename.length() == 0 ? null : Transactions.State.valueOf(statename);
|
||||
if (state == null) {
|
||||
r = Transactions.getRevisions(Transactions.State.INVENTORY, urlhash);
|
||||
if (r != null) state = Transactions.State.INVENTORY;
|
||||
r = Transactions.getRevisions(Transactions.State.ARCHIVE, urlhash);
|
||||
if (r != null) state = Transactions.State.ARCHIVE;
|
||||
} else {
|
||||
r = Transactions.getRevisions(state, urlhash);
|
||||
}
|
||||
if (r != null) {
|
||||
final JSONObject metadata = new JSONObject();
|
||||
final String u = r.url != null ? r.url : sb.index.fulltext().getURL(r.urlhash);
|
||||
metadata.put("url", u == null ? "unknown" : u);
|
||||
metadata.put("dates", r.dates);
|
||||
metadata.put("depth", r.depth);
|
||||
metadata.put("state", state.name());
|
||||
result.put(r.urlhash, metadata);
|
||||
}
|
||||
} catch (IOException |IllegalArgumentException e) {}
|
||||
}
|
||||
} catch (final JSONException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
}
|
||||
String json = result.toString();
|
||||
if (post.containsKey("callback")) json = post.get("callback") + "([" + json + "]);";
|
||||
return new ByteArrayInputStream(UTF8.getBytes(json));
|
||||
}
|
||||
|
||||
// for the following methods we always need the durl to fetch data
|
||||
if (durl == null) {
|
||||
throw new TemplateMissingParameterException("Missing valid url or urlhash parameter");
|
||||
}
|
||||
|
||||
if (xml) {
|
||||
final Collection<File> xmlSnapshots = Transactions.findPaths(durl, "xml", Transactions.State.ANY);
|
||||
File xmlFile = null;
|
||||
if (xmlSnapshots.isEmpty()) {
|
||||
throw new TemplateProcessingException("Could not find the xml snapshot file.", HttpStatus.SC_NOT_FOUND);
|
||||
}
|
||||
xmlFile = xmlSnapshots.iterator().next();
|
||||
try {
|
||||
final byte[] xmlBinary = FileUtils.read(xmlFile);
|
||||
return new ByteArrayInputStream(xmlBinary);
|
||||
} catch (final IOException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
throw new TemplateProcessingException("Could not read the xml snapshot file.");
|
||||
}
|
||||
}
|
||||
|
||||
if (pdf || pngjpg) {
|
||||
Collection<File> pdfSnapshots = Transactions.findPaths(durl, "pdf", Transactions.State.INVENTORY);
|
||||
File pdfFile = null;
|
||||
if (pdfSnapshots.isEmpty()) {
|
||||
// if the client is authenticated, we create the pdf on the fly!
|
||||
if (!authenticated) {
|
||||
throw new TemplateProcessingException(
|
||||
"Could not find the pdf snapshot file. You must be authenticated to generate one on the fly.",
|
||||
HttpStatus.SC_NOT_FOUND);
|
||||
}
|
||||
final SolrDocument sd = sb.index.fulltext().getMetadata(durl.hash());
|
||||
boolean success = false;
|
||||
if (sd == null) {
|
||||
success = Transactions.store(durl, new Date(), 99, false, true, sb.getConfigBool(SwitchboardConstants.PROXY_TRANSPARENT_PROXY, false) ? "http://127.0.0.1:" + sb.getConfigInt(SwitchboardConstants.SERVER_PORT, 8090) : null, sb.getConfig("crawler.http.acceptLanguage", null));
|
||||
} else {
|
||||
final SolrInputDocument sid = sb.index.fulltext().getDefaultConfiguration().toSolrInputDocument(sd);
|
||||
success = Transactions.store(sid, false, true, true, sb.getConfigBool(SwitchboardConstants.PROXY_TRANSPARENT_PROXY, false) ? "http://127.0.0.1:" + sb.getConfigInt(SwitchboardConstants.SERVER_PORT, 8090) : null, sb.getConfig("crawler.http.acceptLanguage", null));
|
||||
}
|
||||
if (success) {
|
||||
pdfSnapshots = Transactions.findPaths(durl, "pdf", Transactions.State.ANY);
|
||||
if (!pdfSnapshots.isEmpty()) {
|
||||
pdfFile = pdfSnapshots.iterator().next();
|
||||
}
|
||||
}
|
||||
} else {
|
||||
pdfFile = pdfSnapshots.iterator().next();
|
||||
}
|
||||
if (pdfFile == null) {
|
||||
throw new TemplateProcessingException(
|
||||
"Could not find the pdf snapshot file and could not generate one on the fly.",
|
||||
HttpStatus.SC_NOT_FOUND);
|
||||
}
|
||||
if (pdf) {
|
||||
try {
|
||||
final byte[] pdfBinary = FileUtils.read(pdfFile);
|
||||
return new ByteArrayInputStream(pdfBinary);
|
||||
} catch (final IOException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
throw new TemplateProcessingException("Could not read the pdf snapshot file.");
|
||||
}
|
||||
}
|
||||
|
||||
if (pngjpg) {
|
||||
final int width = Math.min(post.getInt("width", DEFAULT_WIDTH), DEFAULT_WIDTH);
|
||||
final int height = Math.min(post.getInt("height", DEFAULT_HEIGHT), DEFAULT_HEIGHT);
|
||||
String imageFileStub = pdfFile.getAbsolutePath(); imageFileStub = imageFileStub.substring(0, imageFileStub.length() - 3); // cut off extension
|
||||
final File imageFile = new File(imageFileStub + DEFAULT_WIDTH + "." + DEFAULT_HEIGHT + "." + ext);
|
||||
if (!imageFile.exists() && authenticated) {
|
||||
if(!Html2Image.pdf2image(pdfFile, imageFile, DEFAULT_WIDTH, DEFAULT_HEIGHT, DEFAULT_DENSITY, DEFAULT_QUALITY)) {
|
||||
throw new TemplateProcessingException(
|
||||
"Could not generate the " + ext + " image snapshot file.");
|
||||
}
|
||||
}
|
||||
if (!imageFile.exists()) {
|
||||
throw new TemplateProcessingException(
|
||||
"Could not find the " + ext
|
||||
+ " image snapshot file. You must be authenticated to generate one on the fly.",
|
||||
HttpStatus.SC_NOT_FOUND);
|
||||
}
|
||||
if (width == DEFAULT_WIDTH && height == DEFAULT_HEIGHT) {
|
||||
try {
|
||||
final byte[] imageBinary = FileUtils.read(imageFile);
|
||||
return new ByteArrayInputStream(imageBinary);
|
||||
} catch (final IOException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
throw new TemplateProcessingException("Could not read the " + ext + " image snapshot file.");
|
||||
}
|
||||
}
|
||||
// lets read the file and scale
|
||||
Image image;
|
||||
try {
|
||||
image = ImageParser.parse(imageFile.getAbsolutePath(), FileUtils.read(imageFile));
|
||||
if(image == null) {
|
||||
throw new TemplateProcessingException("Could not parse the " + ext + " image snapshot file.");
|
||||
}
|
||||
final Image scaled = image.getScaledInstance(width, height, Image.SCALE_AREA_AVERAGING);
|
||||
final MediaTracker mediaTracker = new MediaTracker(new Container());
|
||||
mediaTracker.addImage(scaled, 0);
|
||||
try {mediaTracker.waitForID(0);} catch (final InterruptedException e) {}
|
||||
|
||||
/*
|
||||
* Ensure there is no alpha component on the ouput image, as it is pointless
|
||||
* here and it is not well supported by the JPEGImageWriter from OpenJDK
|
||||
*/
|
||||
final BufferedImage scaledBufferedImg = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
|
||||
scaledBufferedImg.createGraphics().drawImage(scaled, 0, 0, width, height, null);
|
||||
return new EncodedImage(scaledBufferedImg, ext, true);
|
||||
} catch (final IOException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
throw new TemplateProcessingException("Could not scale the " + ext + " image snapshot file.");
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
throw new TemplateProcessingException(
|
||||
"Unsupported extension : " + ext + ". Try with rss, xml, json, pdf, png or jpg.",
|
||||
HttpStatus.SC_BAD_REQUEST);
|
||||
}
|
||||
}
|
||||
@@ -27,7 +27,6 @@
|
||||
package net.yacy.htroot;
|
||||
|
||||
import java.awt.Dimension;
|
||||
import java.io.File;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URLEncoder;
|
||||
@@ -36,7 +35,6 @@ import java.util.Collection;
|
||||
import java.util.Comparator;
|
||||
import java.util.Date;
|
||||
import java.util.Iterator;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
import java.util.StringTokenizer;
|
||||
import java.util.TreeSet;
|
||||
@@ -55,8 +53,6 @@ import net.yacy.cora.protocol.RequestHeader.FileType;
|
||||
import net.yacy.cora.util.ConcurrentLog;
|
||||
import net.yacy.cora.util.Memory;
|
||||
import net.yacy.crawler.data.Cache;
|
||||
import net.yacy.crawler.data.Transactions;
|
||||
import net.yacy.crawler.data.Transactions.State;
|
||||
import net.yacy.crawler.retrieval.Response;
|
||||
import net.yacy.data.URLLicense;
|
||||
import net.yacy.data.UserDB;
|
||||
@@ -291,7 +287,6 @@ public class yacysearchitem {
|
||||
final Date[] events = result.events();
|
||||
final boolean showEvent = events != null && events.length > 0 && sb.getConfig("search.navigation", "").indexOf("date",0) >= 0;
|
||||
prop.put("content_showEvent", showEvent ? 1 : 0);
|
||||
final Collection<File> snapshotPaths = sb.getConfigBool("search.result.show.snapshots", true) ? Transactions.findPaths(result.url(), null, State.ANY) : null;
|
||||
if (fileType == FileType.HTML) { // html template specific settings
|
||||
final boolean showKeywords = (sb.getConfigBool(SwitchboardConstants.SEARCH_RESULT_SHOW_KEYWORDS,
|
||||
SwitchboardConstants.SEARCH_RESULT_SHOW_KEYWORDS_DEFAULT) && !result.dc_subject().isEmpty());
|
||||
@@ -305,7 +300,6 @@ public class yacysearchitem {
|
||||
prop.put("content_showCache", sb.getConfigBool("search.result.show.cache", true) && Cache.has(resultURL.hash()) ? 1 : 0);
|
||||
prop.put("content_showProxy", sb.getConfigBool("search.result.show.proxy", true) && sb.getConfigBool("proxyURL", false) ? 1 : 0);
|
||||
prop.put("content_showIndexBrowser", sb.getConfigBool("search.result.show.indexbrowser", true) ? 1 : 0);
|
||||
prop.put("content_showSnapshots", snapshotPaths != null && snapshotPaths.size() > 0 && sb.getConfigBool("search.result.show.snapshots", true) ? 1 : 0);
|
||||
prop.put("content_showVocabulary", sb.getConfigBool("search.result.show.vocabulary", true) ? 1 : 0);
|
||||
prop.put("content_showRanking", sb.getConfigBool("search.result.show.ranking", false) ? 1 : 0);
|
||||
|
||||
@@ -376,29 +370,6 @@ public class yacysearchitem {
|
||||
prop.put("content_showVocabulary_vocabulary", 0);
|
||||
prop.put("content_showVocabulary", 0);
|
||||
}
|
||||
if (snapshotPaths != null && snapshotPaths.size() > 0) {
|
||||
/* Only add a link to the eventual snapshot file in the format it is stored (no resource fetching and conversion here) */
|
||||
String selectedExt = null, ext;
|
||||
for(final File snapshot : snapshotPaths) {
|
||||
ext = MultiProtocolURL.getFileExtension(snapshot.getName());
|
||||
if("jpg".equals(ext) || "png".equals(ext)) {
|
||||
/* Prefer snapshots in jpeg or png format */
|
||||
selectedExt = ext;
|
||||
break;
|
||||
} else if("pdf".equals(ext)) {
|
||||
selectedExt = ext;
|
||||
} else if("xml".equals(ext) && selectedExt == null) {
|
||||
/* Use the XML metadata snapshot in last resort */
|
||||
selectedExt = ext;
|
||||
}
|
||||
}
|
||||
if(selectedExt != null) {
|
||||
prop.putHTML("content_showSnapshots_extension", selectedExt.toUpperCase(Locale.ROOT));
|
||||
prop.putHTML("content_showSnapshots_link", "api/snapshot." + selectedExt + "?url=" + resultURL);
|
||||
} else {
|
||||
prop.put("content_showSnapshots", 0);
|
||||
}
|
||||
}
|
||||
prop.put("content_showRanking_ranking", Float.toString(result.score()));
|
||||
prop.put("content_ranking", Float.toString(result.score()));
|
||||
}
|
||||
|
||||
@@ -45,7 +45,6 @@ import net.yacy.cora.federate.solr.responsewriter.EnhancedXMLResponseWriter;
|
||||
import net.yacy.cora.federate.solr.responsewriter.GrepHTMLResponseWriter;
|
||||
import net.yacy.cora.federate.solr.responsewriter.HTMLResponseWriter;
|
||||
import net.yacy.cora.federate.solr.responsewriter.OpensearchResponseWriter;
|
||||
import net.yacy.cora.federate.solr.responsewriter.SnapshotImagesReponseWriter;
|
||||
import net.yacy.cora.federate.solr.responsewriter.SolrjResponseWriter;
|
||||
import net.yacy.cora.federate.solr.responsewriter.YJsonResponseWriter;
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
@@ -109,7 +108,6 @@ public class SolrSelectServlet extends HttpServlet {
|
||||
RESPONSE_WRITER.put("xslt", xsltWriter); // try i.e. http://localhost:8090/solr/select?q=*:*&start=0&rows=10&wt=xslt&tr=json.xsl
|
||||
RESPONSE_WRITER.put("exml", new EnhancedXMLResponseWriter());
|
||||
RESPONSE_WRITER.put("html", new HTMLResponseWriter());
|
||||
RESPONSE_WRITER.put("snapshots", new SnapshotImagesReponseWriter());
|
||||
RESPONSE_WRITER.put("grephtml", new GrepHTMLResponseWriter());
|
||||
RESPONSE_WRITER.put("rss", opensearchResponseWriter); //try http://localhost:8090/solr/select?wt=rss&q=olympia&hl=true&hl.fl=text_t,h1,h2
|
||||
RESPONSE_WRITER.put("opensearch", opensearchResponseWriter); //try http://localhost:8090/solr/select?wt=rss&q=olympia&hl=true&hl.fl=text_t,h1,h2
|
||||
|
||||
@@ -148,7 +148,6 @@ import net.yacy.crawler.data.NoticedURL.StackType;
|
||||
import net.yacy.crawler.data.ResultImages;
|
||||
import net.yacy.crawler.data.ResultURLs;
|
||||
import net.yacy.crawler.data.ResultURLs.EventOrigin;
|
||||
import net.yacy.crawler.data.Transactions;
|
||||
import net.yacy.crawler.retrieval.Request;
|
||||
import net.yacy.crawler.retrieval.Response;
|
||||
import net.yacy.crawler.robots.RobotsTxt;
|
||||
@@ -789,9 +788,6 @@ public final class Switchboard extends serverSwitch {
|
||||
SwitchboardConstants.HTCACHE_SYNC_LOCK_TIMEOUT_DEFAULT),
|
||||
this.getConfigInt(SwitchboardConstants.HTCACHE_COMPRESSION_LEVEL,
|
||||
SwitchboardConstants.HTCACHE_COMPRESSION_LEVEL_DEFAULT));
|
||||
final File transactiondir = new File(this.htCachePath, "snapshots");
|
||||
Transactions.init(transactiondir, this.getConfigLong(SwitchboardConstants.SNAPSHOTS_WKHTMLTOPDF_TIMEOUT,
|
||||
SwitchboardConstants.SNAPSHOTS_WKHTMLTOPDF_TIMEOUT_DEFAULT));
|
||||
|
||||
// create the packs directories
|
||||
this.packsHoldPath = this.getDataPath(SwitchboardConstants.PACKS_HOLD_PATH, SwitchboardConstants.PACKS_HOLD_PATH_DEFAULT);
|
||||
@@ -3455,9 +3451,7 @@ public final class Switchboard extends serverSwitch {
|
||||
condenser,
|
||||
searchEvent,
|
||||
sourceName,
|
||||
this.getConfigBool(SwitchboardConstants.NETWORK_UNIT_DHT, false),
|
||||
this.getConfigBool(SwitchboardConstants.PROXY_TRANSPARENT_PROXY, false) ? "http://127.0.0.1:" + sb.getConfigInt(SwitchboardConstants.SERVER_PORT, 8090) : null,
|
||||
this.getConfig("crawler.http.acceptLanguage", null));
|
||||
this.getConfigBool(SwitchboardConstants.NETWORK_UNIT_DHT, false));
|
||||
final RSSFeed feed =
|
||||
EventChannel.channels(queueEntry.initiator() == null
|
||||
? EventChannel.PROXY
|
||||
|
||||
@@ -360,12 +360,6 @@ public final class SwitchboardConstants {
|
||||
public static final String CRAWLER_USER_AGENT_MINIMUMDELTA = "crawler.userAgent.minimumdelta";
|
||||
public static final String CRAWLER_USER_AGENT_CLIENTTIMEOUT = "crawler.userAgent.clienttimeout";
|
||||
|
||||
/** Key of the setting controlling the maximum time to wait for each wkhtmltopdf call when rendering PDF snapshots */
|
||||
public static final String SNAPSHOTS_WKHTMLTOPDF_TIMEOUT = "snapshots.wkhtmltopdf.timeout";
|
||||
|
||||
/** Default maximum time in seconds to wait for each wkhtmltopdf call when rendering PDF snapshots*/
|
||||
public static final long SNAPSHOTS_WKHTMLTOPDF_TIMEOUT_DEFAULT = 30;
|
||||
|
||||
/* --- debug flags --- */
|
||||
|
||||
/** when set to true : do not use the local dht/rwi index (which is not done if we do remote searches) */
|
||||
|
||||
@@ -193,9 +193,7 @@ public class DocumentIndex extends Segment {
|
||||
condenser,
|
||||
null,
|
||||
DocumentIndex.class.getName() + ".add",
|
||||
false,
|
||||
null,
|
||||
null);
|
||||
false);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
@@ -63,12 +63,10 @@ import net.yacy.cora.util.ConcurrentLog;
|
||||
import net.yacy.cora.util.LookAheadIterator;
|
||||
import net.yacy.cora.util.SpaceExceededException;
|
||||
import net.yacy.crawler.data.CrawlProfile;
|
||||
import net.yacy.crawler.data.Transactions;
|
||||
import net.yacy.crawler.retrieval.Response;
|
||||
import net.yacy.document.Condenser;
|
||||
import net.yacy.document.Document;
|
||||
import net.yacy.document.Parser;
|
||||
import net.yacy.document.parser.htmlParser;
|
||||
import net.yacy.kelondro.data.citation.CitationReference;
|
||||
import net.yacy.kelondro.data.citation.CitationReferenceFactory;
|
||||
import net.yacy.kelondro.data.word.Word;
|
||||
@@ -596,9 +594,7 @@ public class Segment {
|
||||
final Condenser condenser,
|
||||
final SearchEvent searchEvent,
|
||||
final String sourceName, // contains the crawl profile hash if this comes from a web crawl
|
||||
final boolean storeToRWI,
|
||||
final String proxy,
|
||||
final String acceptLanguage
|
||||
final boolean storeToRWI
|
||||
) {
|
||||
final CollectionConfiguration collectionConfig = this.fulltext.getDefaultConfiguration();
|
||||
final String language = votedLanguage(url, url.toNormalform(true), document, condenser); // identification of the language
|
||||
@@ -608,7 +604,7 @@ public class Segment {
|
||||
this.fulltext().useWebgraph() ? this.fulltext.getWebgraphConfiguration() : null, sourceName);
|
||||
|
||||
return storeDocument(url, crawlProfile, responseHeader, document, vector, language, condenser,
|
||||
searchEvent, sourceName, storeToRWI, proxy, acceptLanguage);
|
||||
searchEvent, sourceName, storeToRWI);
|
||||
}
|
||||
|
||||
public SolrInputDocument storeDocument(
|
||||
@@ -621,9 +617,7 @@ public class Segment {
|
||||
final Condenser condenser,
|
||||
final SearchEvent searchEvent,
|
||||
final String sourceName, // contains the crawl profile hash if this comes from a web crawl
|
||||
final boolean storeToRWI,
|
||||
final String proxy,
|
||||
final String acceptLanguage
|
||||
final boolean storeToRWI
|
||||
) {
|
||||
final long startTime = System.currentTimeMillis();
|
||||
|
||||
@@ -647,21 +641,6 @@ public class Segment {
|
||||
// ENRICH DOCUMENT WITH RANKING INFORMATION
|
||||
this.fulltext.getDefaultConfiguration().postprocessing_references(this.getReferenceReportCache(), vector, url, null);
|
||||
|
||||
// CREATE SNAPSHOT
|
||||
if ((url.getProtocol().equals("http") || url.getProtocol().equals("https")) &&
|
||||
crawlProfile != null && document.getDepth() <= crawlProfile.snapshotMaxdepth() &&
|
||||
!crawlProfile.snapshotsMustnotmatch().matcher(urlNormalform).matches()) {
|
||||
// load pdf in case that is wanted. This can later be used to compute a web page preview in the search results
|
||||
Parser p = document.getParserObject();
|
||||
boolean mimesupported = false;
|
||||
if (p instanceof htmlParser)
|
||||
mimesupported = ((htmlParser)p).supportedMimeTypes().contains(document.dc_format());
|
||||
|
||||
if (mimesupported)
|
||||
// STORE IMAGE AND METADATA
|
||||
Transactions.store(vector, true, crawlProfile.snapshotLoadImage(), crawlProfile.snapshotReplaceold(), proxy, acceptLanguage);
|
||||
}
|
||||
|
||||
// STORE TO SOLR
|
||||
this.putDocument(vector);
|
||||
List<SolrInputDocument> webgraph = vector.getWebgraphDocuments();
|
||||
|
||||
@@ -1,78 +0,0 @@
|
||||
// Html2ImageTest.java
|
||||
// Copyright 2016,2017 by reger; https://github.com/reger24 luccioman; https://github.com/luccioman
|
||||
//
|
||||
// This is a part of YaCy, a peer-to-peer based web search engine
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package net.yacy.cora.util;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FilenameFilter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import net.yacy.utils.translation.ExtensionsFileFilter;
|
||||
import org.junit.Test;
|
||||
import static org.junit.Assert.*;
|
||||
import org.junit.BeforeClass;
|
||||
|
||||
|
||||
public class Html2ImageTest {
|
||||
|
||||
@BeforeClass
|
||||
public static void setUpClass() throws Exception {
|
||||
// make sure DATA exists
|
||||
File f = new File("test/DATA");
|
||||
if (!f.exists()) f.mkdir();
|
||||
}
|
||||
|
||||
/**
|
||||
* Test of pdf2image method, of class Html2Image.
|
||||
*/
|
||||
@Test
|
||||
public void testPdf2image() {
|
||||
// collect pdf filenames in test directory
|
||||
File pd = new File("test/parsertest");
|
||||
List<String> extensions = new ArrayList<>();
|
||||
extensions.add("pdf");
|
||||
FilenameFilter fileFilter = new ExtensionsFileFilter(extensions);
|
||||
String[] pdffiles = pd.list(fileFilter);
|
||||
assertTrue("no pdf files in test/parsertest directory", pdffiles.length > 0);
|
||||
|
||||
for (String pdffilename : pdffiles) {
|
||||
File pdffile = new File(pd, pdffilename);
|
||||
File jpgfile = new File("test/DATA", pdffilename + ".jpg");
|
||||
if (jpgfile.exists()) {
|
||||
jpgfile.delete();
|
||||
}
|
||||
assertTrue(Html2Image.pdf2image(pdffile, jpgfile, 1024, 1024, 300, 75));
|
||||
assertTrue(jpgfile.exists());
|
||||
assertTrue(jpgfile.length() > 0);
|
||||
System.out.println("Test image file successfully written : " + jpgfile.getAbsolutePath());
|
||||
|
||||
final File pngFile = new File("test/DATA", pdffilename + ".png");
|
||||
if (pngFile.exists()) {
|
||||
pngFile.delete();
|
||||
}
|
||||
assertTrue(Html2Image.pdf2image(pdffile, pngFile, 1024, 1024, 300, 75));
|
||||
assertTrue(pngFile.exists());
|
||||
assertTrue(pngFile.length() > 0);
|
||||
System.out.println("Test image file successfully written : " + pngFile.getAbsolutePath());
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -8,7 +8,6 @@ sudo pacman -Sy --noconfirm --needed base-devel git python pyalpm
|
||||
git clone https://github.com/actionless/pikaur.git
|
||||
python3 pikaur/pikaur.py -S --noconfirm pikaur devtools python-pysocks python-defusedxml
|
||||
pikaur -S --noconfirm jdk-openjdk ant
|
||||
pikaur -S --noconfirm imagemagick ruby qt5-tools qt5-doc gperf python xorg-server-xvfb ghostscript git qt5-svg qt5-xmlpatterns base-devel qt5-location qt5-sensors qt5-webchannel libwebp libxslt libxcomposite gst-plugins-base hyphen hyphen-en hyphen-de woff2 cmake qt5-webkit wkhtmltopdf
|
||||
|
||||
cd /opt
|
||||
git clone --depth 1 https://github.com/yacy/yacy_search_server.git
|
||||
|
||||
Reference in new issue
Block a user