diff --git a/source/net/yacy/http/CrashProtectionHandler.java b/source/net/yacy/http/CrashProtectionHandler.java index 12ae12c5c..1cbf27bcd 100644 --- a/source/net/yacy/http/CrashProtectionHandler.java +++ b/source/net/yacy/http/CrashProtectionHandler.java @@ -15,6 +15,16 @@ import org.eclipse.jetty.server.handler.HandlerWrapper; import net.yacy.cora.util.ConcurrentLog; +/** + * Last-resort exception barrier wrapped around the complete handler chain. + * + * Note for servlet container migration: this must stay a container level + * handler (it can not become a servlet filter): inside the servlet context a + * filter would catch servlet exceptions before the containers error dispatch + * and thereby replace the YaCyErrorHandler error page with a plain text stack + * trace. Its purpose is to catch failures outside the servlet context, e.g. + * in the transparent proxy handlers. + */ public class CrashProtectionHandler extends HandlerWrapper implements Handler, HandlerContainer { public CrashProtectionHandler() { diff --git a/source/net/yacy/http/InetPathAccessHandler.java b/source/net/yacy/http/InetPathAccessHandler.java index f49145cbe..84a3ddc4a 100644 --- a/source/net/yacy/http/InetPathAccessHandler.java +++ b/source/net/yacy/http/InetPathAccessHandler.java @@ -38,6 +38,9 @@ import org.eclipse.jetty.util.component.DumpableCollection; * previously available in the deprecated IPAccessHandler. *

* + * Note for servlet container migration: the InetAccessHandler of Jetty 10 and + * later supports path patterns natively ("addr|path" syntax), this class can + * then be removed. */ public class InetPathAccessHandler extends InetAccessHandler { diff --git a/source/net/yacy/http/Jetty9HttpServerImpl.java b/source/net/yacy/http/Jetty9HttpServerImpl.java index fea7e30a8..5bf942e06 100644 --- a/source/net/yacy/http/Jetty9HttpServerImpl.java +++ b/source/net/yacy/http/Jetty9HttpServerImpl.java @@ -24,14 +24,18 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.net.InetAddress; +import java.net.InetSocketAddress; import java.security.KeyStore; +import java.util.EnumSet; import java.util.StringTokenizer; import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.SSLContext; +import javax.servlet.DispatcherType; import org.eclipse.jetty.http.HttpMethod; import org.eclipse.jetty.http.HttpVersion; +import org.eclipse.jetty.io.Connection; import org.eclipse.jetty.server.Connector; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.HttpConfiguration; @@ -46,13 +50,16 @@ import org.eclipse.jetty.server.handler.DefaultHandler; import org.eclipse.jetty.server.handler.HandlerList; import org.eclipse.jetty.server.handler.InetAccessHandler; import org.eclipse.jetty.server.handler.gzip.GzipHandler; +import org.eclipse.jetty.servlet.FilterHolder; import org.eclipse.jetty.servlet.ServletHolder; import org.eclipse.jetty.util.log.Log; import org.eclipse.jetty.util.resource.Resource; import org.eclipse.jetty.util.ssl.SslContextFactory; import org.eclipse.jetty.webapp.WebAppContext; +import net.yacy.cora.protocol.ConnectionInfo; import net.yacy.cora.util.ConcurrentLog; +import net.yacy.http.servlets.MonitorFilter; import net.yacy.http.servlets.YaCyDefaultServlet; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; @@ -74,6 +81,21 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer { this.server = new Server(); + // remove the ConnectionInfo tracking entry (added per request by the MonitorFilter) + // when the tcp connection closes; added as bean to each connector below + final Connection.Listener connectionCloseMonitor = new Connection.Listener() { + @Override + public void onOpened(final Connection connection) { + } + @Override + public void onClosed(final Connection connection) { + final InetSocketAddress remote = connection.getEndPoint().getRemoteAddress(); + if (remote != null) { + ConnectionInfo.removeServerConnection(MonitorFilter.connectionId(remote.getAddress().getHostAddress(), remote.getPort())); + } + } + }; + final int cores = Runtime.getRuntime().availableProcessors(); final int acceptors = Math.max(1, Math.min(4, cores/2)); // original: Math.max(1, Math.min(4,cores/8)); @@ -86,7 +108,7 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer { connector.setName("httpd-" + host + ":" + Integer.toString(port)); connector.setIdleTimeout(9000); // timout in ms when no bytes send / received connector.setAcceptQueueSize(128); - + connector.addBean(connectionCloseMonitor); this.server.addConnector(connector); @@ -113,6 +135,7 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer { sslConnector.setPort(sslport); sslConnector.setName("ssld:" + Integer.toString(sslport)); // name must start with ssl (for withSSL() to work correctly) sslConnector.setIdleTimeout(9000); // timout in ms when no bytes send / received + sslConnector.addBean(connectionCloseMonitor); this.server.addConnector(sslConnector); ConcurrentLog.info("SERVER", "SSL support initialized successfully on port " + sslport); @@ -156,6 +179,12 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer { //sholder.setInitParameter("welcomeFile", "index.html"); // default is index.html, welcome.html htrootContext.addServlet(sholder, "/*"); + // as fundamental component this filter is hardcoded too: it feeds the + // Connections_p.html monitoring and rejects requests above the connection limit + final FilterHolder monitorFilter = new FilterHolder(MonitorFilter.class); + monitorFilter.setAsyncSupported(true); + htrootContext.addFilter(monitorFilter, "/*", EnumSet.of(DispatcherType.REQUEST)); + final GzipHandler gzipHandler = new GzipHandler(); /* * Decompression of incoming requests body is required for index distribution @@ -198,9 +227,9 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer { if (sb.getConfigBool(SwitchboardConstants.PROXY_TRANSPARENT_PROXY, false)) { // Proxyhandlers are only needed if feature activated (save resources if not used) ConcurrentLog.info("SERVER", "load Jetty handler for transparent proxy"); - handlers.setHandlers(new Handler[]{new MonitorHandler(), domainHandler, new ProxyCacheHandler(), new ProxyHandler()}); + handlers.setHandlers(new Handler[]{domainHandler, new ProxyCacheHandler(), new ProxyHandler()}); } else { - handlers.setHandlers(new Handler[]{new MonitorHandler(), domainHandler}); + handlers.setHandlers(new Handler[]{domainHandler}); } // context handler for dispatcher and security (hint: dispatcher requires a context) final ContextHandler context = new ContextHandler(); diff --git a/source/net/yacy/http/MonitorHandler.java b/source/net/yacy/http/MonitorHandler.java deleted file mode 100644 index d859a5c3d..000000000 --- a/source/net/yacy/http/MonitorHandler.java +++ /dev/null @@ -1,81 +0,0 @@ -/** - * MonitorHandler - * Copyright 2014 by Sebastian Gaebel - * First released 15.05.2014 at https://yacy.net - * - * $LastChangedDate$ - * $LastChangedRevision$ - * $LastChangedBy$ - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program in the file lgpl21.txt - * If not, see . - */ - -package net.yacy.http; - -import java.io.IOException; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import net.yacy.cora.protocol.ConnectionInfo; -import net.yacy.cora.protocol.Domains; -import net.yacy.cora.protocol.RequestHeader; - -import org.eclipse.jetty.io.Connection; -import org.eclipse.jetty.server.Request; -import org.eclipse.jetty.server.handler.AbstractHandler; - -public class MonitorHandler extends AbstractHandler { - - private final Connection.Listener remover = new Connection.Listener() { - - @Override - public void onClosed(Connection c) { - ConnectionInfo.removeServerConnection(c.hashCode()); - } - - @Override - public void onOpened(Connection c) { - } - }; - - @Override - public void handle(String target, Request baseRequest, HttpServletRequest request, - HttpServletResponse response) throws IOException, ServletException { - - final Connection connection = baseRequest.getHttpChannel().getEndPoint().getConnection(); - final ConnectionInfo info = new ConnectionInfo( - baseRequest.getScheme(), - RequestHeader.client(baseRequest) + ":" + baseRequest.getRemotePort(), - baseRequest.getMethod() + " " + baseRequest.getHttpURI().getPathQuery(), - connection.hashCode(), - baseRequest.getTimeStamp(), - -1); - - if (ConnectionInfo.getServerConnections().contains(info)) { - ConnectionInfo.removeServerConnection(info); - } else { - connection.addListener(remover); - } - ConnectionInfo.addServerConnection(info); - - if (ConnectionInfo.isServerCountReached()) { - if (Domains.isLocal(baseRequest.getRemoteAddr(), baseRequest.getRemoteInetSocketAddress().getAddress())) return; - response.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE,"max. server connections reached (increase /PerformanceQueues_p.html -> httpd Session Pool)."); - baseRequest.setHandled(true); - } - } -} diff --git a/source/net/yacy/http/YacyDomainHandler.java b/source/net/yacy/http/YacyDomainHandler.java index e91048d8f..747b0fe5d 100644 --- a/source/net/yacy/http/YacyDomainHandler.java +++ b/source/net/yacy/http/YacyDomainHandler.java @@ -46,6 +46,12 @@ import org.eclipse.jetty.server.handler.AbstractHandler; /** * handling of request to virtual ".yacy" domain determines public adress from * seedlist and forwards modified/wrapped request to it + * + * Note for servlet container migration: this must stay a container level + * handler (it can not become a servlet filter), because it cooperates with the + * ProxyHandler chain: the re-dispatched request with the rewritten (remote) + * host is picked up and forwarded to the peer by the transparent proxy + * handlers, before the local servlet context would handle it. */ public class YacyDomainHandler extends AbstractHandler implements Handler { diff --git a/source/net/yacy/http/servlets/MonitorFilter.java b/source/net/yacy/http/servlets/MonitorFilter.java new file mode 100644 index 000000000..23b5affb4 --- /dev/null +++ b/source/net/yacy/http/servlets/MonitorFilter.java @@ -0,0 +1,99 @@ +/** + * MonitorFilter + * Copyright 2014 by Sebastian Gaebel + * First released 15.05.2014 at https://yacy.net + * + * $LastChangedDate$ + * $LastChangedRevision$ + * $LastChangedBy$ + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see . + */ + +package net.yacy.http.servlets; + +import java.io.IOException; + +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import net.yacy.cora.protocol.ConnectionInfo; +import net.yacy.cora.protocol.Domains; +import net.yacy.cora.protocol.RequestHeader; + +/** + * Records incoming server requests into {@link ConnectionInfo} (displayed on + * Connections_p.html) and rejects remote requests with http status 503 when + * the configured maximum number of server connections is reached. + * + * This is a plain servlet filter (former Jetty handler MonitorHandler); the + * tracking entry of a connection is removed on connection close by a + * servlet-container specific listener, see Jetty9HttpServerImpl. + */ +public class MonitorFilter implements Filter { + + @Override + public void init(FilterConfig filterConfig) throws ServletException { + } + + @Override + public void destroy() { + } + + /** + * @return the ConnectionInfo id of the tcp connection identified by client address and port + */ + public static int connectionId(String remoteAddr, final int remotePort) { + // the servlet API reports IPv6 addresses in bracketed form ("[::1]"), the + // connection endpoint in plain form: normalize to the plain form + if (remoteAddr.startsWith("[") && remoteAddr.endsWith("]")) { + remoteAddr = remoteAddr.substring(1, remoteAddr.length() - 1); + } + return (remoteAddr + ":" + remotePort).hashCode(); + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException { + + final HttpServletRequest hrequest = (HttpServletRequest) request; + final String query = hrequest.getQueryString(); + final ConnectionInfo info = new ConnectionInfo( + hrequest.getScheme(), + RequestHeader.client(hrequest) + ":" + hrequest.getRemotePort(), + hrequest.getMethod() + " " + hrequest.getRequestURI() + (query == null ? "" : "?" + query), + connectionId(hrequest.getRemoteAddr(), hrequest.getRemotePort()), + System.currentTimeMillis(), + -1); + + // a keep-alive connection reuses the id: remove a previous entry to show the latest request + ConnectionInfo.removeServerConnection(info); + ConnectionInfo.addServerConnection(info); + + if (ConnectionInfo.isServerCountReached() + && !Domains.isLocal(hrequest.getRemoteAddr(), null)) { + ((HttpServletResponse) response).sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE, + "max. server connections reached (increase /PerformanceQueues_p.html -> httpd Session Pool)."); + return; + } + chain.doFilter(request, response); + } +}