jody/yacy_search_server
1
0 Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2026-10-25 05:54:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

isolate Solr 9.0 Jetty client dependencies in a relocated bridge

Browse Source
This commit is contained in:
Michael Peter Christen committed 2026-07-11 23:40:15 +02:00
1 parent edeb6162c4
commit bfddfe5e7e
20 files changed
+1118 -152
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.classpath
+10
View File
@@ -12,5 +12,15 @@
<attribute name="module" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="lib" path="lib/solr9-bridge-solr-core-9.0.0.jar"/>
<classpathentry kind="lib" path="lib/solr9-bridge-solr-solrj-9.0.0.jar"/>
<classpathentry kind="lib" path="lib/solr9-bridge-solr-scripting-9.0.0.jar"/>
<classpathentry kind="lib" path="lib/solr9-bridge-jetty-client-9.4.58.v20250814.jar"/>
<classpathentry kind="lib" path="lib/solr9-bridge-jetty-http-9.4.58.v20250814.jar"/>
<classpathentry kind="lib" path="lib/solr9-bridge-jetty-io-9.4.58.v20250814.jar"/>
<classpathentry kind="lib" path="lib/solr9-bridge-jetty-util-9.4.58.v20250814.jar"/>
<classpathentry kind="lib" path="lib/solr9-bridge-http2-client-9.4.58.v20250814.jar"/>
<classpathentry kind="lib" path="lib/solr9-bridge-http2-common-9.4.58.v20250814.jar"/>
<classpathentry kind="lib" path="lib/solr9-bridge-http2-http-client-transport-9.4.58.v20250814.jar"/>
<classpathentry kind="output" path="gen"/>
</classpath>
.externalToolBuilders/solr9-bridge.launch
+10
View File
@@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<launchConfiguration type="org.eclipse.ant.AntBuilderLaunchConfigurationType">
<booleanAttribute key="org.eclipse.ant.ui.ATTR_TARGETS_UPDATED" value="true"/>
<stringAttribute key="org.eclipse.ant.ui.ATTR_ANT_FULL_TARGETS" value="build-solr9-bridge,"/>
<stringAttribute key="org.eclipse.ant.ui.ATTR_ANT_MANUAL_TARGETS" value="build-solr9-bridge,"/>
<booleanAttribute key="org.eclipse.ui.externaltools.ATTR_TRIGGERS_CONFIGURED" value="true"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_RUN_BUILD_KINDS" value="full,"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_LOCATION" value="${workspace_loc:/yacy/build.xml}"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_WORKING_DIRECTORY" value="${workspace_loc:/yacy}"/>
</launchConfiguration>
.project
+10
View File
@@ -10,6 +10,16 @@
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.ui.externaltools.ExternalToolBuilder</name>
<triggers>full,</triggers>
<arguments>
<dictionary>
<key>LaunchConfigHandle</key>
<value>&lt;project&gt;/.externalToolBuilders/solr9-bridge.launch</value>
</dictionary>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
JETTY12_MIGRATION.md
+203
View File
@@ -0,0 +1,203 @@
# Jetty 12 Migration Contract
## Scope
YaCy will migrate its embedded HTTP server from Jetty 9 to Jetty 12 while
remaining on Java 17 and keeping the existing Servlet 4.0 `javax.servlet`
source API. Jetty 12's EE8 environment provides this compatibility layer; the
migration does not require converting YaCy to `jakarta.servlet`.
This contract covers the dependency boundary only. It does not yet migrate the
Jetty-specific adapters under `source/net/yacy/http`.
## Solr Boundary
YaCy embeds Solr 9 but does not directly use Solr's embedded `JettySolrRunner`
or its Jetty-based HTTP/2 clients. Remote Solr access is implemented with
`HttpSolrClient` and `ConcurrentUpdateSolrClient`, both configured with Apache
HttpClient in `RemoteInstance`.
The three Solr artifacts in `ivy.xml` therefore belong to the private
`solr9-bridge` configuration and explicitly exclude the `org.eclipse.jetty`
and `org.eclipse.jetty.http2` families. They are retrieved below
`build/solr9-bridge/input`, not directly into the runtime `lib` directory.
Solr 9.0 nevertheless has an eager internal dependency that cannot be removed:
`CoreContainer` constructs both `HttpShardHandlerFactory` and
`UpdateShardHandler`, which create `Http2SolrClient` instances during startup
even in YaCy's standalone embedded configuration. The build therefore rewrites
Solr's Jetty references and the minimal Jetty 9 client/HTTP2 implementation to
the private `net.yacy.solr9.jetty` package. The generated artifacts have the
`solr9-bridge-` prefix under `lib/`; unrelocated Solr and HTTP2 jars are removed
from that directory before compilation.
Run the boundary guard after resolving dependencies:
```sh
ant clean compile
test/jetty-solr-dependency-guard.sh
```
## Current Jetty 9 Roots
The direct Jetty dependencies reflect APIs imported by YaCy source code.
Dependencies needed only by another Jetty module remain transitive.
| Responsibility | Direct Jetty 9 artifact |
| --- | --- |
| HTTP primitives | `jetty-http` |
| Connection and output APIs | `jetty-io` |
| Server and handlers | `jetty-server` |
| Utility, resource, and TLS APIs | `jetty-util` |
| CONNECT and proxy handlers | `jetty-proxy` |
| Authentication and constraints | `jetty-security` |
| Servlet container | `jetty-servlet` |
| QoS and servlet helpers | `jetty-servlets` |
| Web application context | `jetty-webapp` |
The public `jetty-client` is owned transitively by `jetty-proxy`. YaCy has no
direct client API imports. `jetty-deploy` and `jetty-jmx` are not part of the
current root graph. HTTP/2 exists only inside the relocated Solr island, not as
an embedded-server feature.
## Jetty 12 EE8 Target Graph
The minimal target keeps Jetty Core separate from the EE8 Servlet layer.
| Current responsibility | Jetty 12 target family |
| --- | --- |
| HTTP, IO, server, utilities | Jetty Core `jetty-http`, `jetty-io`, `jetty-server`, `jetty-util` |
| Proxy handlers | Jetty Core `jetty-proxy` and its client transitive |
| Servlet container | `org.eclipse.jetty.ee8:jetty-ee8-servlet` |
| Servlet helpers and QoS | `org.eclipse.jetty.ee8:jetty-ee8-servlets` |
| Web application context | `org.eclipse.jetty.ee8:jetty-ee8-webapp` |
| Servlet authentication | `org.eclipse.jetty.ee8:jetty-ee8-security` plus Jetty Core security transitives |
| Servlet API | Servlet 4.0 in the `javax.servlet` namespace, aligned with the EE8 environment |
Do not add Jetty deploy, JMX, server-side HTTP/2, or Jakarta EE modules unless
a concrete YaCy feature requires them and its contract is verified separately.
The private Solr 9 HTTP/2 island above is the only current exception.
### Solr 9 Isolation Gate (P1)
Jetty 9 client classes and Jetty 12 server classes cannot safely share the
same application classloader because they use overlapping
`org.eclipse.jetty.*` packages with incompatible APIs.
The P1 investigation rules out the two initially attractive shortcuts:
1. **A Solr upgrade is not part of this migration.** YaCy remains on Solr
9.0.0. A later Solr upgrade may provide another migration option, but it is
deliberately not a P1 implementation path. A comparison with Solr 9.10.1
also showed that upgrading within the Solr 9 line would not remove the
boundary: it selects Jetty 10.0.26 and still eagerly constructs
`Http2SolrClient` instances.
2. **Whole-Solr classloader isolation is not a minimal boundary for YaCy.**
Solr API objects are part of the application boundary: 88 files under
`source/` and `test/` currently import `org.apache.solr` types. A child
classloader would either create incompatible class identities or require a
broad new facade and data conversion layer.
Solr 10 is also outside this migration contract: although it moves to Jetty
12, the Solr 10 server requires Java 21 and uses the Jakarta Servlet namespace.
P1 implements a **relocated Solr 9.0.0 Jetty client bridge**. The bridge keeps
the public `org.apache.solr.*` classes visible to YaCy while
rewriting Solr 9.0.0's internal `org.eclipse.jetty.*` references and the
required Jetty 9 client/HTTP2 implementation into a private package. It is
built reproducibly by Ant; no edited jar is stored in the repository. P1 does
not change the Solr or Lucene versions.
P1 is complete only when a bridge proof passes all of these checks:
1. only the selected embedded-server Jetty line uses the public
`org.eclipse.jetty` package;
2. no unrelocated Jetty 9/10 class is packaged by the bridge;
3. `EmbeddedSolrConnectorTest` starts and closes a `CoreContainer`;
4. an embedded update followed by a query succeeds;
5. the bridge dependency set and relocation rules are generated by the build;
6. the bridge can be removed without changing YaCy's Solr-facing source API.
The integrated bridge passes `EmbeddedSolrConnectorTest` (`OK (4 tests)`) with
the original Solr and HTTP2 jars removed. It also passes when
Jetty 12.1.11 Core client, HTTP, IO, proxy, security, server, and utility jars
are present in the same application classpath. The reduced private island is:
- Solr Core, SolrJ, and Solr Scripting 9.0.0 with only their Jetty references
rewritten;
- Jetty 9.4 client, HTTP, IO, and utility;
- Jetty 9.4 HTTP/2 client, common, and HTTP-client transport.
Jetty server, servlet, security, proxy, webapp, and XML are not included in
the Solr island. The proof can be repeated with:
```sh
ant clean compile
test/solr9-jetty-bridge-spike.sh
```
Set `JETTY12_CLASSPATH` to a colon-separated set of resolved Jetty 12 jars to
repeat the coexistence variant. The script compiles only its focused test into
a temporary directory; the bridge itself is already produced by `ant compile`.
All six P1 checks pass. The Solr isolation gate is therefore closed for the
Jetty 12 server migration. Reconsidering Solr remains a separate future
decision, not an automatic part of this migration.
The optional version comparison that established the limitation of the Solr 9
line can be repeated without changing YaCy's production dependencies with:
```sh
javap -classpath ~/.ivy2/cache/org.apache.solr/solr-core/jars/solr-core-9.10.1.jar \
-private -c org.apache.solr.update.UpdateShardHandler
javap -classpath ~/.ivy2/cache/org.apache.solr/solr-core/jars/solr-core-9.10.1.jar \
-private -c org.apache.solr.handler.component.HttpShardHandlerFactory
```
## Completion Gates
The dependency phase is complete when all of these checks pass with one public
Jetty version on the resolved classpath and the private Solr island:
1. `ant clean compile`
2. `test/jetty-solr-dependency-guard.sh`
3. Startup with embedded Solr enabled
4. A proven resolution for the Solr 9 isolation gate
5. A remote Solr request through the Apache-based client
6. Proxy traffic including CONNECT
7. `test/jetty-smoke-test.sh`
The following implementation phase may then replace `Jetty9HttpServerImpl` and
the remaining Jetty adapter APIs without changing the Solr dependency graph.
## P2.1 Helper Removal
`YaCyDefaultServlet` no longer imports Jetty HTTP header/method constants,
MIME lookup, URI joining, writer adaptation, inclusive byte ranges, multipart
output, or `Resource`. The small operations use Servlet/JDK APIs or local
implementations. Static resources are exposed through the container-neutral
`ServletResource` interface; `Jetty9ServletResource` is the only Jetty 9
adapter for the existing resource behavior.
`YaCyQoSFilter` and `YaCyDigestCredential` remain explicit container adapters
rather than being replaced by simplified local implementations that could
change request priority or authentication behavior.
## P2.2 Authentication And Access Rules
The request-level administrator decision is represented by
`AdminAccessPolicy`: public access, the configured localhost bypass, or the
administrator role. It combines the existing pure `AdminSecurity` checks
without depending on Servlet or Jetty APIs. In particular, the localhost
without account option and the localhost-only stored-hash authentication used
by `bin/apicall.sh` remain supported.
`AdminAuthenticationContext` carries the true socket peer IP only for the
duration of the current authentication call. `YaCySecurityHandler` publishes
and clears that context, and `YaCyDigestCredential` only adapts Jetty's BASIC
and DIGEST credential objects to the container-neutral password check.
The portable address/path syntax of `serverClient` is represented by
`InetPathAccessRule`. `InetPathAccessHandler` remains the Jetty 9 matcher
adapter; Jetty 12 can consume the normalized `address|path` rules with its
native path-aware access handler.
build.xml
+35 -1
View File
@@ -38,6 +38,10 @@
<property name="lib" location="lib"/>
<property name="libt" location="libt"/>
<property name="build" location="build/classes/java/main"/> <!-- reuse Gradle build path -->
<property name="solr9.bridge.build" location="build/solr9-bridge"/>
<property name="solr9.bridge.input" location="${solr9.bridge.build}/input"/>
<property name="solr9.bridge.tool" location="${solr9.bridge.build}/tool"/>
<property name="solr9.bridge.tool.libs" location="${solr9.bridge.build}/tool-libs"/>
<property name="javadoc" location="javadoc"/>
<property name="htroot" location="htroot"/>
<property name="test" location="test/java"/>
@@ -91,6 +95,8 @@
-->
<ivy:retrieve conf="compile" pathid="compile.path" pattern="${ivy.lib.dir}/[artifact]-[revision].[ext]" />
<ivy:retrieve conf="test" pathid="test.path" pattern="${libt}/[artifact]-[revision].[ext]" />
<ivy:retrieve conf="solr9-bridge" pathid="solr9.bridge.input.path" pattern="${solr9.bridge.input}/[artifact]-[revision].[ext]" />
<ivy:retrieve conf="solr9-bridge-tool" pathid="solr9.bridge.tool.path" pattern="${solr9.bridge.tool.libs}/[artifact]-[revision].[ext]" />
<property name="target-resolve-already-run" value="true" />
</target>
@@ -171,9 +177,36 @@
<pathelement location="${build}" />
<pathelement location="${lib}/J7Zip-modified-1.0.2.jar" />
<path refid="compile.path" />
<fileset dir="${lib}" includes="solr9-bridge-*.jar" />
</path>
<target name="compile-core" depends="init,resolve" description="compile YaCy core">
<target name="build-solr9-bridge" depends="init,resolve" description="build the private Solr 9.0 Jetty client island">
<delete dir="${solr9.bridge.tool}" failonerror="false"/>
<delete failonerror="false">
<fileset dir="${lib}" includes="solr9-bridge-*.jar,solr-core-9.0.0.jar,solr-solrj-9.0.0.jar,solr-scripting-9.0.0.jar,http2-*.jar" />
</delete>
<mkdir dir="${solr9.bridge.tool}"/>
<javac srcdir="test/jetty" destdir="${solr9.bridge.tool}"
includes="RelocateJettyPackages.java" includeantruntime="false"
release="${javacRelease}" encoding="UTF-8">
<classpath refid="solr9.bridge.tool.path"/>
</javac>
<path id="solr9.bridge.runtime.path">
<pathelement location="${solr9.bridge.tool}"/>
<path refid="solr9.bridge.tool.path"/>
</path>
<apply executable="${java.home}/bin/java" parallel="false" failonerror="true" dest="${lib}">
<arg value="-cp"/>
<arg pathref="solr9.bridge.runtime.path"/>
<arg value="net.yacy.test.jetty.RelocateJettyPackages"/>
<srcfile/>
<targetfile/>
<fileset dir="${solr9.bridge.input}" includes="*.jar"/>
<mapper type="glob" from="*" to="solr9-bridge-*"/>
</apply>
</target>
<target name="compile-core" depends="init,resolve,build-solr9-bridge" description="compile YaCy core">
<!-- compile the core sources -->
<echo message="project.class.path: ${toString:project.class.path}" />
<javac srcdir="${src}/" destdir="${build}"
@@ -417,6 +450,7 @@
<fileset dir="." includes="TEST-*" />
</delete>
<delete dir="test/DATA" failonerror="false"/>
<delete dir="${solr9.bridge.build}" failonerror="false"/>
</target>
<!-- run YaCy (needed for NetBeans4) -->
ivy.xml
+36 -11
View File
@@ -4,6 +4,8 @@
<!-- https://wrongnotes.blogspot.com/2014/02/simplest-explanation-of-ivy.html -->
<conf name="compile" visibility="public"/>
<conf name="test" visibility="public"/>
<conf name="solr9-bridge" visibility="private"/>
<conf name="solr9-bridge-tool" visibility="private"/>
</configurations>
<dependencies defaultconf="compile->default">
@@ -64,17 +66,31 @@
<dependency org="org.apache.poi" name="poi" rev="5.5.1" />
<dependency org="org.apache.poi" name="poi-scratchpad" rev="5.5.1" />
<dependency org="org.apache.solr" name="solr-core" rev="9.0.0" conf="compile->master"/>
<dependency org="org.apache.solr" name="solr-scripting" rev="9.0.0" conf="compile->master"/>
<dependency org="org.apache.solr" name="solr-solrj" rev="9.0.0" conf="compile->master" />
<!--
Solr is embedded and its Jetty-backed runner and HTTP/2 clients are not used directly by YaCy.
Keep Solr's Jetty 9 runtime family out of the embedded-server classpath so Jetty can
be migrated independently. The master configuration already omits transitives; the
explicit excludes preserve that boundary if the configuration mapping changes later.
-->
<dependency org="org.apache.solr" name="solr-core" rev="9.0.0" conf="solr9-bridge->master">
<exclude org="org.eclipse.jetty" module="*"/>
<exclude org="org.eclipse.jetty.http2" module="*"/>
</dependency>
<dependency org="org.apache.solr" name="solr-scripting" rev="9.0.0" conf="solr9-bridge->master">
<exclude org="org.eclipse.jetty" module="*"/>
<exclude org="org.eclipse.jetty.http2" module="*"/>
</dependency>
<dependency org="org.apache.solr" name="solr-solrj" rev="9.0.0" conf="solr9-bridge->master">
<exclude org="org.eclipse.jetty" module="*"/>
<exclude org="org.eclipse.jetty.http2" module="*"/>
</dependency>
<dependency org="org.apache.zookeeper" name="zookeeper" rev="3.9.5" conf="compile->master" />
<dependency org="org.bitlet" name="weupnp" rev="0.1.4" />
<dependency org="org.bouncycastle" name="bcmail-jdk18on" rev="1.84" />
<dependency org="com.fasterxml.woodstox" name="woodstox-core" rev="7.2.1" />
<dependency org="org.eclipse.jetty" name="jetty-client" rev="9.4.58.v20250814" />
<dependency org="org.eclipse.jetty" name="jetty-deploy" rev="9.4.58.v20250814" conf="compile->master" />
<dependency org="org.eclipse.jetty" name="jetty-jmx" rev="9.4.58.v20250814" conf="compile->master"/>
<dependency org="org.eclipse.jetty" name="jetty-http" rev="9.4.58.v20250814"/>
<dependency org="org.eclipse.jetty" name="jetty-http" rev="9.4.58.v20250814" conf="compile->default;solr9-bridge->master"/>
<dependency org="org.eclipse.jetty" name="jetty-io" rev="9.4.58.v20250814" conf="compile->default;solr9-bridge->master"/>
<dependency org="org.eclipse.jetty" name="jetty-client" rev="9.4.58.v20250814" conf="solr9-bridge->master"/>
<dependency org="org.eclipse.jetty" name="jetty-proxy" rev="9.4.58.v20250814"/>
<dependency org="org.eclipse.jetty" name="jetty-security" rev="9.4.58.v20250814"/>
<dependency org="org.eclipse.jetty" name="jetty-server" rev="9.4.58.v20250814"/>
@@ -82,11 +98,17 @@
<dependency org="org.eclipse.jetty" name="jetty-servlet" rev="9.4.58.v20250814">
<exclude module="jetty-util-ajax" />
</dependency>
<dependency org="org.eclipse.jetty" name="jetty-util" rev="9.4.58.v20250814" />
<dependency org="org.eclipse.jetty" name="jetty-util" rev="9.4.58.v20250814" conf="compile->default;solr9-bridge->master" />
<dependency org="org.eclipse.jetty" name="jetty-webapp" rev="9.4.58.v20250814" />
<dependency org="org.eclipse.jetty.http2" name="http2-client" rev="9.4.58.v20250814" conf="compile->master"/>
<dependency org="org.eclipse.jetty.http2" name="http2-common" rev="9.4.58.v20250814" conf="compile->master"/>
<dependency org="org.eclipse.jetty.http2" name="http2-http-client-transport" rev="9.4.58.v20250814" conf="compile->master"/>
<!--
Temporary Solr 9.0 runtime bridge: CoreContainer eagerly constructs an
Http2SolrClient for its shard/update handlers even in standalone mode.
YaCy source must not use these clients. Remove this Jetty 9 island only
after the relocated Solr 9.0 bridge replaces these unrelocated jars.
-->
<dependency org="org.eclipse.jetty.http2" name="http2-client" rev="9.4.58.v20250814" conf="solr9-bridge->master"/>
<dependency org="org.eclipse.jetty.http2" name="http2-common" rev="9.4.58.v20250814" conf="solr9-bridge->master"/>
<dependency org="org.eclipse.jetty.http2" name="http2-http-client-transport" rev="9.4.58.v20250814" conf="solr9-bridge->master"/>
<dependency org="org.jsoup" name="jsoup" rev="1.22.2" />
<dependency org="org.jwat" name="jwat-warc" rev="1.2.1">
<exclude module="bcprov-jdk15to18" />
@@ -104,6 +126,9 @@
<dependency org="junit" name="junit" rev="4.13.2" conf="test->default"/>
<dependency org="org.hamcrest" name="hamcrest-core" rev="3.0" conf="test->default"/>
<!-- Build-time bytecode relocation for the private Solr 9.0 Jetty client island. -->
<dependency org="org.ow2.asm" name="asm" rev="9.9" conf="solr9-bridge-tool->master"/>
<dependency org="org.ow2.asm" name="asm-commons" rev="9.9" conf="solr9-bridge-tool->master"/>
<dependency org="com.github.luben" name="zstd-jni" rev="1.5.7-11"></dependency>
<dependency org="ai.djl" name ="api" rev="0.36.0"></dependency>
</dependencies>
source/net/yacy/http/AdminAccessPolicy.java
+60
View File
@@ -0,0 +1,60 @@
package net.yacy.http;
import java.net.MalformedURLException;
import net.yacy.cora.document.id.MultiProtocolURL;
/** Container-neutral policy for administrator access to a request path. */
public final class AdminAccessPolicy {
public enum Decision {
PUBLIC,
LOCAL_BYPASS,
ADMIN_REQUIRED
}
private final boolean protectAllPages;
private final boolean privateRobinsonMode;
private final boolean publicSearchPage;
private final boolean allowLocalhostWithoutLogin;
private final String adminUser;
private final String adminHash;
public AdminAccessPolicy(final boolean protectAllPages, final boolean privateRobinsonMode,
final boolean publicSearchPage, final boolean allowLocalhostWithoutLogin,
final String adminUser, final String adminHash) {
this.protectAllPages = protectAllPages;
this.privateRobinsonMode = privateRobinsonMode;
this.publicSearchPage = publicSearchPage;
this.allowLocalhostWithoutLogin = allowLocalhostWithoutLogin;
this.adminUser = adminUser;
this.adminHash = adminHash;
}
public Decision decide(final String path, final String socketPeerIp, final String referer,
final String authorizationHeader) {
if (!AdminSecurity.isProtectedPath(path, this.protectAllPages,
this.privateRobinsonMode, this.publicSearchPage)) {
return Decision.PUBLIC;
}
if (AdminSecurity.isLocalhostAccess(socketPeerIp, refererHost(referer))) {
if (this.allowLocalhostWithoutLogin || AdminSecurity.checkLocalhostLazyAuth(
authorizationHeader, this.adminUser, this.adminHash)) {
return Decision.LOCAL_BYPASS;
}
}
return Decision.ADMIN_REQUIRED;
}
private static String refererHost(final String referer) {
if (referer == null || referer.isEmpty()) {
return null;
}
try {
return new MultiProtocolURL(referer).getHost();
} catch (final MalformedURLException e) {
return null;
}
}
}
source/net/yacy/http/AdminAuthenticationContext.java
+25
View File
@@ -0,0 +1,25 @@
package net.yacy.http;
import net.yacy.cora.protocol.Domains;
/** Request-bound facts needed while the container verifies admin credentials. */
public final class AdminAuthenticationContext {
private static final ThreadLocal<String> SOCKET_PEER_IP = new ThreadLocal<>();
private AdminAuthenticationContext() {
}
public static void setSocketPeerIp(final String ip) {
SOCKET_PEER_IP.set(ip);
}
public static void clear() {
SOCKET_PEER_IP.remove();
}
public static boolean isLocalhostRequest() {
final String ip = SOCKET_PEER_IP.get();
return ip != null && Domains.isLocalhost(ip);
}
}
source/net/yacy/http/InetPathAccessHandler.java
+7 -16
View File
@@ -98,23 +98,14 @@ public class InetPathAccessHandler extends InetAccessHandler {
*/
protected void addPattern(final String pattern, final PathMappings<InetAddressSet> pathMappings)
throws IllegalArgumentException {
if (pattern != null && !pattern.isEmpty()) {
final int idx = pattern.indexOf('|');
final String addr = idx > 0 ? pattern.substring(0, idx) : pattern;
final String path = (idx > 0 && (pattern.length() > idx + 1)) ? pattern.substring(idx + 1) : "/*";
if (!addr.isEmpty()) {
final PathSpec pathSpec = PathSpec.from(path);
InetAddressSet addresses = pathMappings.get(pathSpec);
if (addresses == null) {
addresses = new InetAddressSet();
pathMappings.put(pathSpec, addresses);
}
addresses.add(addr);
}
final InetPathAccessRule rule = InetPathAccessRule.parse(pattern);
final PathSpec pathSpec = PathSpec.from(rule.pathPattern());
InetAddressSet addresses = pathMappings.get(pathSpec);
if (addresses == null) {
addresses = new InetAddressSet();
pathMappings.put(pathSpec, addresses);
}
addresses.add(rule.addressPattern());
}
/**
source/net/yacy/http/InetPathAccessRule.java
+42
View File
@@ -0,0 +1,42 @@
package net.yacy.http;
/** Container-neutral representation of a server-client address/path rule. */
public final class InetPathAccessRule {
private static final String DEFAULT_PATH = "/*";
private final String addressPattern;
private final String pathPattern;
private InetPathAccessRule(final String addressPattern, final String pathPattern) {
this.addressPattern = addressPattern;
this.pathPattern = pathPattern;
}
public static InetPathAccessRule parse(final String pattern) {
if (pattern == null || pattern.isEmpty()) {
throw new IllegalArgumentException("Access rule must not be empty");
}
final int separator = pattern.indexOf('|');
final String address = separator > 0 ? pattern.substring(0, separator) : pattern;
final String path = separator > 0 && pattern.length() > separator + 1
? pattern.substring(separator + 1)
: DEFAULT_PATH;
if (address.isEmpty()) {
throw new IllegalArgumentException("Access rule has no address: " + pattern);
}
return new InetPathAccessRule(address, path);
}
public String addressPattern() {
return this.addressPattern;
}
public String pathPattern() {
return this.pathPattern;
}
public String asJettyPattern() {
return this.addressPattern + '|' + this.pathPattern;
}
}
source/net/yacy/http/YaCyDigestCredential.java
+2 -43
View File
@@ -24,7 +24,6 @@
package net.yacy.http;
import net.yacy.cora.protocol.Domains;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
@@ -46,43 +45,6 @@ public class YaCyDigestCredential extends Credential {
private static final long serialVersionUID = -3527894085562480001L;
/**
* True socket peer IP of the request currently being authenticated on this thread.
* <p>
* Jetty's {@link Credential#check(Object)} API does not hand over the request, so the
* credential can not tell on its own whether a request comes from localhost. The
* {@link YaCySecurityHandler} therefore publishes the request's socket peer IP here for
* the duration of the request (set at the start of handling, cleared in a finally block).
* This replaces the former global "recent localhost access" timestamp, which was a
* process-wide value not bound to the request being checked.
*/
private static final ThreadLocal<String> REQUEST_CLIENT_IP = new ThreadLocal<String>();
/**
* Publish the socket peer IP of the request being authenticated on the current thread.
* Must be paired with {@link #clearRequestClientIP()} in a finally block.
* @param ip the true socket peer IP (never an X-Real-IP derived address)
*/
public static void setRequestClientIP(final String ip) {
REQUEST_CLIENT_IP.set(ip);
}
/**
* Remove the request client IP published for the current thread.
*/
public static void clearRequestClientIP() {
REQUEST_CLIENT_IP.remove();
}
/**
* @return true when the request currently authenticated on this thread comes from
* localhost. Fails closed (returns false) when no request IP was published.
*/
private static boolean isRequestFromLocalhost() {
final String ip = REQUEST_CLIENT_IP.get();
return ip != null && Domains.isLocalhost(ip);
}
private String hash; // remember password hash, either MD5(Base64(user:pwd)) or with encryption prefix "MD5:" + MD5(user:realm:pwd)
private String foruser; // remember the user as YaCy credential is username:pwd (not just pwd)
private Credential c;
@@ -108,14 +70,11 @@ public class YaCyDigestCredential extends Credential {
//
// We must therefore know whether THIS request comes from localhost. Jetty's
// Credential.check() is not given the request, so YaCySecurityHandler publishes
// the request's true socket peer IP into REQUEST_CLIENT_IP for the duration of the
// request (see the detailed rationale on YaCySecurityHandler.handle()); we read it
// back here. Using the socket peer - not the spoofable X-Real-IP header - keeps
// this exception restricted to genuine local callers.
// the true socket peer IP through AdminAuthenticationContext for this request.
return AdminSecurity.checkAdminPassword(this.foruser, this.hash,
sb.getConfig(SwitchboardConstants.ADMIN_REALM, ""),
sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"),
isRequestFromLocalhost(),
AdminAuthenticationContext.isLocalhostRequest(),
(String) credentials);
}
throw new UnsupportedOperationException();
source/net/yacy/http/YaCySecurityHandler.java
+13 -25
View File
@@ -25,13 +25,11 @@
package net.yacy.http;
import java.io.IOException;
import java.net.MalformedURLException;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
@@ -47,7 +45,7 @@ import org.eclipse.jetty.server.Request;
* and updates AccessTracker
*
* This is a thin adapter to the servlet container: the decision logic is in
* the container neutral {@link AdminSecurity}.
* the container-neutral {@link AdminAccessPolicy} and {@link AdminSecurity}.
*/
public class YaCySecurityHandler extends ConstraintSecurityHandler {
@@ -82,11 +80,11 @@ public class YaCySecurityHandler extends ConstraintSecurityHandler {
public void handle(final String pathInContext, final Request baseRequest,
final HttpServletRequest request, final HttpServletResponse response)
throws IOException, ServletException {
YaCyDigestCredential.setRequestClientIP(baseRequest.getRemoteAddr());
AdminAuthenticationContext.setSocketPeerIp(baseRequest.getRemoteAddr());
try {
super.handle(pathInContext, baseRequest, request, response);
} finally {
YaCyDigestCredential.clearRequestClientIP();
AdminAuthenticationContext.clear();
}
}
@@ -109,30 +107,20 @@ public class YaCySecurityHandler extends ConstraintSecurityHandler {
final String remoteip = request.getRemoteAddr();
serverAccessTracker.track(remoteip, pathInContext);
final boolean protectedPage = AdminSecurity.isProtectedPath(pathInContext,
final AdminAccessPolicy policy = new AdminAccessPolicy(
sb.getConfigBool(SwitchboardConstants.ADMIN_ACCOUNT_All_PAGES, false),
sb.isRobinsonMode() && !sb.isPublicRobinson(),
sb.getConfigBool(SwitchboardConstants.PUBLIC_SEARCHPAGE, true));
if (!protectedPage) {
sb.getConfigBool(SwitchboardConstants.PUBLIC_SEARCHPAGE, true),
sb.getConfigBool(SwitchboardConstants.ADMIN_ACCOUNT_FOR_LOCALHOST, false),
sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"),
sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""));
final AdminAccessPolicy.Decision decision = policy.decide(pathInContext, remoteip,
request.getHeader(RequestHeader.REFERER), request.getHeader(RequestHeader.AUTHORIZATION));
if (decision == AdminAccessPolicy.Decision.PUBLIC) {
return super.prepareConstraintInfo(pathInContext, request);
}
String refererHost;
try {
refererHost = new MultiProtocolURL(request.getHeader(RequestHeader.REFERER)).getHost();
} catch (MalformedURLException e) {
refererHost = null;
}
if (AdminSecurity.isLocalhostAccess(remoteip, refererHost)) {
if (sb.getConfigBool(SwitchboardConstants.ADMIN_ACCOUNT_FOR_LOCALHOST, false)) {
return null;
}
// last chance to authorize using the admin from localhost
if (AdminSecurity.checkLocalhostLazyAuth(request.getHeader(RequestHeader.AUTHORIZATION),
sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"),
sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""))) {
return null;
}
if (decision == AdminAccessPolicy.Decision.LOCAL_BYPASS) {
return null;
}
RoleInfo roleinfo = new RoleInfo();
roleinfo.setChecked(true); // RoleInfo.setChecked() : in Jetty this means - marked to have any security constraint
source/net/yacy/http/servlets/Jetty9ServletResource.java
+99
View File
@@ -0,0 +1,99 @@
package net.yacy.http.servlets;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL;
import org.eclipse.jetty.util.resource.Resource;
/** Jetty 9 adapter for the static-resource operations used by YaCy. */
final class Jetty9ServletResource implements ServletResource {
private final Resource delegate;
private Jetty9ServletResource(final Resource delegate) {
this.delegate = delegate;
}
static void disableDefaultCaches() {
Resource.setDefaultUseCaches(false);
}
static ServletResource from(final String location) throws IOException {
return wrap(Resource.newResource(location));
}
static ServletResource from(final File file) throws IOException {
return wrap(Resource.newResource(file));
}
static ServletResource from(final URL url) throws IOException {
return wrap(Resource.newResource(url));
}
private static ServletResource wrap(final Resource resource) {
return resource == null ? null : new Jetty9ServletResource(resource);
}
@Override
public ServletResource addPath(final String path) throws IOException {
return wrap(this.delegate.addPath(path));
}
@Override
public boolean exists() {
return this.delegate.exists();
}
@Override
public boolean isDirectory() {
return this.delegate.isDirectory();
}
@Override
public long lastModified() {
return this.delegate.lastModified();
}
@Override
public long length() {
return this.delegate.length();
}
@Override
public String getName() {
return this.delegate.getName();
}
@Override
public File getFile() throws IOException {
return this.delegate.getFile();
}
@Override
public InputStream getInputStream() throws IOException {
return this.delegate.getInputStream();
}
@Override
public String getListHTML(final String base, final boolean parent, final String query) throws IOException {
return this.delegate.getListHTML(base, parent, query);
}
@Override
public void writeTo(final OutputStream output, final long start, final long count) throws IOException {
this.delegate.writeTo(output, start, count);
}
@Override
public void close() {
this.delegate.close();
}
@Override
public String toString() {
return this.delegate.toString();
}
}
source/net/yacy/http/servlets/ServletResource.java
+33
View File
@@ -0,0 +1,33 @@
package net.yacy.http.servlets;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.File;
/** Container-neutral view of a static resource served by YaCy. */
public interface ServletResource extends AutoCloseable {
ServletResource addPath(String path) throws IOException;
boolean exists();
boolean isDirectory();
long lastModified();
long length();
String getName();
File getFile() throws IOException;
InputStream getInputStream() throws IOException;
String getListHTML(String base, boolean parent, String query) throws IOException;
void writeTo(OutputStream output, long start, long count) throws IOException;
@Override
void close();
}
source/net/yacy/http/servlets/YaCyDefaultServlet.java
+229 -56
View File
@@ -28,7 +28,9 @@ import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStream;
import java.io.Writer;
import java.io.FilterOutputStream;
import java.lang.reflect.Array;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
@@ -57,16 +59,7 @@ import org.apache.commons.fileupload.FileItem;
import org.apache.commons.fileupload.FileItemFactory;
import org.apache.commons.fileupload.disk.DiskFileItemFactory;
import org.apache.commons.fileupload.servlet.ServletFileUpload;
import org.eclipse.jetty.http.HttpHeader;
import org.eclipse.jetty.http.HttpMethod;
import org.eclipse.jetty.http.MimeTypes;
import org.eclipse.jetty.io.WriterOutputStream;
import org.eclipse.jetty.server.InclusiveByteRange;
import org.eclipse.jetty.util.MultiPartOutputStream;
import org.eclipse.jetty.util.URIUtil;
import org.eclipse.jetty.util.resource.Resource;
import com.google.common.net.HttpHeaders;
import com.google.common.net.HttpHeaders;
import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.analysis.Classification;
@@ -127,10 +120,18 @@ public class YaCyDefaultServlet extends HttpServlet {
private static final long serialVersionUID = 4900000000000001110L;
protected ServletContext _servletContext;
protected boolean _acceptRanges = true;
protected boolean _dirAllowed = true;
protected Resource _resourceBase;
protected MimeTypes _mimeTypes;
protected String[] _welcomes;
protected boolean _dirAllowed = true;
protected ServletResource _resourceBase;
protected String[] _welcomes;
private static final String METHOD_HEAD = "HEAD";
private static final String METHOD_POST = "POST";
private static final String HEADER_CONTENT_RANGE = "Content-Range";
private static final String HEADER_IF_MODIFIED_SINCE = "If-Modified-Since";
private static final String HEADER_IF_UNMODIFIED_SINCE = "If-Unmodified-Since";
private static final String HEADER_REQUEST_RANGE = "Request-Range";
private static final String MIME_TEXT_HTML = "text/html";
private static final String MIME_TEXT_HTML_UTF8 = "text/html;charset=utf-8";
protected File _htLocalePath;
protected File _htDocsPath;
@@ -149,8 +150,7 @@ public class YaCyDefaultServlet extends HttpServlet {
this._servletContext = this.getServletContext();
this._mimeTypes = new MimeTypes();
final String tmpstr = this.getServletContext().getInitParameter("welcomeFile");
final String tmpstr = this.getServletContext().getInitParameter("welcomeFile");
if (tmpstr == null) {
this._welcomes = HTTPDFileHandler.defaultFiles;
} else {
@@ -159,14 +159,14 @@ public class YaCyDefaultServlet extends HttpServlet {
this._acceptRanges = this.getInitBoolean("acceptRanges", this._acceptRanges);
this._dirAllowed = this.getInitBoolean("dirAllowed", this._dirAllowed);
Resource.setDefaultUseCaches(false); // caching is handled internally (prevent double caching)
Jetty9ServletResource.disableDefaultCaches(); // caching is handled internally (prevent double caching)
final String rb = this.getInitParameter("resourceBase");
try {
if (rb != null) {
this._resourceBase = Resource.newResource(rb);
this._resourceBase = Jetty9ServletResource.from(rb);
} else {
this._resourceBase = Resource.newResource(sb.getConfig(SwitchboardConstants.HTROOT_PATH, SwitchboardConstants.HTROOT_PATH_DEFAULT)); //default
this._resourceBase = Jetty9ServletResource.from(sb.getConfig(SwitchboardConstants.HTROOT_PATH, SwitchboardConstants.HTROOT_PATH_DEFAULT)); //default
}
} catch (final IOException e) {
ConcurrentLog.severe("FILEHANDLER", "event=http.resource subsystem=http result=missing-resource-base reason=" + e.getMessage());
@@ -201,14 +201,14 @@ public class YaCyDefaultServlet extends HttpServlet {
* @param pathInContext The path to find a resource for.
* @return The resource to serve.
*/
public Resource getResource(final String pathInContext) {
Resource r = null;
public ServletResource getResource(final String pathInContext) {
ServletResource r = null;
try {
if (this._resourceBase != null) {
r = this._resourceBase.addPath(pathInContext);
} else {
final URL u = this._servletContext.getResource(pathInContext);
r = Resource.newResource(u);
r = Jetty9ServletResource.from(u);
}
if (ConcurrentLog.isFine("FILEHANDLER")) {
@@ -249,10 +249,10 @@ public class YaCyDefaultServlet extends HttpServlet {
}
String pathInContext = pathInfo == null ? "/" : pathInfo; // this is the path of the resource in _resourceBase (= path within htroot respective htDocs)
final boolean endsWithSlash = pathInContext.endsWith(URIUtil.SLASH);
final boolean endsWithSlash = pathInContext.endsWith("/");
// Find the resource
Resource resource = null;
ServletResource resource = null;
try {
@@ -266,7 +266,7 @@ public class YaCyDefaultServlet extends HttpServlet {
hasClass = true;
} else {
final String pathofClass = pathInContext.substring(0, p) + ".class";
final Resource classresource = this._resourceBase.addPath(pathofClass);
final ServletResource classresource = this._resourceBase.addPath(pathofClass);
// Does a class resource exist?
if (classresource != null && classresource.exists() && !classresource.isDirectory()) {
hasClass = true;
@@ -281,7 +281,7 @@ public class YaCyDefaultServlet extends HttpServlet {
if (!hasClass && (resource == null || !resource.exists()) && !pathInContext.contains("..")) {
// try to get this in the alternative htDocsPath
if (resource != null) resource.close();
resource = Resource.newResource(new File(this._htDocsPath, pathInContext));
resource = Jetty9ServletResource.from(new File(this._htDocsPath, pathInContext));
}
if (ConcurrentLog.isFine("FILEHANDLER")) {
@@ -301,7 +301,7 @@ public class YaCyDefaultServlet extends HttpServlet {
if (q != null && q.length() != 0) {
pathInContext += "?" + q;
}
response.sendRedirect(response.encodeRedirectURL(URIUtil.addPaths(this._servletContext.getContextPath(), pathInContext)));
response.sendRedirect(response.encodeRedirectURL(addPaths(this._servletContext.getContextPath(), pathInContext)));
} else {
if (hasClass) { // this is a YaCy servlet, handle the template
this.handleTemplate(pathInfo, request, response);
@@ -379,7 +379,7 @@ public class YaCyDefaultServlet extends HttpServlet {
private boolean shouldWrapBody(final HttpServletRequest request) {
final String method = request.getMethod();
if (method == null || !HttpMethod.POST.asString().equalsIgnoreCase(method)) {
if (method == null || !METHOD_POST.equalsIgnoreCase(method)) {
return false;
}
final String contentType = request.getContentType();
@@ -498,8 +498,8 @@ public class YaCyDefaultServlet extends HttpServlet {
return null;
}
for (final String _welcome : this._welcomes) {
final String welcome_in_context = URIUtil.addPaths(pathInContext, _welcome);
final Resource welcome = this.getResource(welcome_in_context);
final String welcome_in_context = addPaths(pathInContext, _welcome);
final ServletResource welcome = this.getResource(welcome_in_context);
if (welcome != null && welcome.exists()) {
return _welcome;
}
@@ -510,15 +510,15 @@ public class YaCyDefaultServlet extends HttpServlet {
/* Check modification date headers.
* send a 304 response instead of content if not modified since
*/
protected boolean passConditionalHeaders(final HttpServletRequest request, final HttpServletResponse response, final Resource resource)
protected boolean passConditionalHeaders(final HttpServletRequest request, final HttpServletResponse response, final ServletResource resource)
throws IOException {
try {
if (!request.getMethod().equals(HttpMethod.HEAD.asString())) {
final String ifms = request.getHeader(HttpHeader.IF_MODIFIED_SINCE.asString());
if (!request.getMethod().equals(METHOD_HEAD)) {
final String ifms = request.getHeader(HEADER_IF_MODIFIED_SINCE);
if (ifms != null) {
final long ifmsl = request.getDateHeader(HttpHeader.IF_MODIFIED_SINCE.asString());
final long ifmsl = request.getDateHeader(HEADER_IF_MODIFIED_SINCE);
if (ifmsl != -1) {
if (resource.lastModified() / 1000 <= ifmsl / 1000) {
response.reset();
@@ -530,7 +530,7 @@ public class YaCyDefaultServlet extends HttpServlet {
}
// Parse the if[un]modified dates and compare to resource
final long date = request.getDateHeader(HttpHeader.IF_UNMODIFIED_SINCE.asString());
final long date = request.getDateHeader(HEADER_IF_UNMODIFIED_SINCE);
if (date != -1) {
if (resource.lastModified() / 1000 > date / 1000) {
@@ -552,7 +552,7 @@ public class YaCyDefaultServlet extends HttpServlet {
/* ------------------------------------------------------------------- */
protected void sendDirectory(final HttpServletRequest request,
final HttpServletResponse response,
final Resource resource,
final ServletResource resource,
final String pathInContext)
throws IOException {
if (!this._dirAllowed) {
@@ -560,7 +560,7 @@ public class YaCyDefaultServlet extends HttpServlet {
return;
}
final String base = URIUtil.addEncodedPaths(request.getRequestURI(), URIUtil.SLASH);
final String base = addPaths(request.getRequestURI(), "/");
final String dir = resource.getListHTML(base, pathInContext.length() > 1, request.getQueryString());
if (dir == null) {
@@ -569,7 +569,7 @@ public class YaCyDefaultServlet extends HttpServlet {
}
final byte[] data = dir.getBytes(StandardCharsets.UTF_8);
response.setContentType(MimeTypes.Type.TEXT_HTML_UTF_8.asString());
response.setContentType(MIME_TEXT_HTML_UTF8);
response.setContentLength(data.length);
response.setHeader(HeaderFramework.CACHE_CONTROL, "no-cache, no-store");
response.setDateHeader(HeaderFramework.EXPIRES, System.currentTimeMillis() + 10000); // consider that directories are not modified that often
@@ -591,7 +591,7 @@ public class YaCyDefaultServlet extends HttpServlet {
protected void sendData(final HttpServletRequest request,
final HttpServletResponse response,
final boolean include,
final Resource resource,
final ServletResource resource,
final Enumeration<String> reqRanges)
throws IOException {
@@ -602,7 +602,7 @@ public class YaCyDefaultServlet extends HttpServlet {
try {
out = response.getOutputStream();
} catch (final IllegalStateException e) {
out = new WriterOutputStream(response.getWriter());
out = new CharacterOutputStream(response.getWriter());
}
// remove the last-modified field since caching otherwise does not work
@@ -629,14 +629,14 @@ public class YaCyDefaultServlet extends HttpServlet {
}
} else {
// Parse the satisfiable ranges
final List<InclusiveByteRange> ranges = InclusiveByteRange.satisfiableRanges(reqRanges, content_length);
final List<HttpByteRange> ranges = HttpByteRange.satisfiableRanges(reqRanges, content_length);
// if there are no satisfiable ranges, send 416 response
if (ranges == null || ranges.isEmpty()) {
this.writeHeaders(response, resource, content_length);
response.setStatus(HttpServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE);
response.setHeader(HttpHeader.CONTENT_RANGE.asString(),
InclusiveByteRange.to416HeaderRangeString(content_length));
response.setHeader(HEADER_CONTENT_RANGE,
HttpByteRange.to416HeaderRangeString(content_length));
resource.writeTo(out, 0, content_length);
out.close();
return;
@@ -645,11 +645,11 @@ public class YaCyDefaultServlet extends HttpServlet {
// if there is only a single valid range (must be satisfiable
// since were here now), send that range with a 216 response
if (ranges.size() == 1) {
final InclusiveByteRange singleSatisfiableRange = ranges.iterator().next();
final HttpByteRange singleSatisfiableRange = ranges.iterator().next();
final long singleLength = singleSatisfiableRange.getSize();
this.writeHeaders(response, resource, singleLength);
response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
response.setHeader(HttpHeader.CONTENT_RANGE.asString(),
response.setHeader(HEADER_CONTENT_RANGE,
singleSatisfiableRange.toHeaderRangeString(content_length));
resource.writeTo(out, singleSatisfiableRange.getFirst(), singleLength);
out.close();
@@ -665,14 +665,14 @@ public class YaCyDefaultServlet extends HttpServlet {
if (mimetype == null) {
ConcurrentLog.warn("FILEHANDLER","YaCyDefaultServlet: Unknown mimetype for " + request.getRequestURI());
}
final MultiPartOutputStream multi = new MultiPartOutputStream(out);
final MultipartByteRangeOutputStream multi = new MultipartByteRangeOutputStream(out);
response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
// If the request has a "Request-Range" header then we need to
// send an old style multipart/x-byteranges Content-Type. This
// keeps Netscape and acrobat happy. This is what Apache does.
String ctp;
if (request.getHeader(HttpHeader.REQUEST_RANGE.asString()) != null) {
if (request.getHeader(HEADER_REQUEST_RANGE) != null) {
ctp = "multipart/x-byteranges; boundary=";
} else {
ctp = "multipart/byteranges; boundary=";
@@ -686,7 +686,7 @@ public class YaCyDefaultServlet extends HttpServlet {
int length = 0;
final String[] header = new String[ranges.size()];
for (int i = 0; i < ranges.size(); i++) {
final InclusiveByteRange ibr = ranges.get(i);
final HttpByteRange ibr = ranges.get(i);
header[i] = ibr.toHeaderRangeString(content_length);
length +=
((i > 0) ? 2 : 0)
@@ -700,7 +700,7 @@ public class YaCyDefaultServlet extends HttpServlet {
response.setContentLength(length);
for (int i = 0; i < ranges.size(); i++) {
final InclusiveByteRange ibr = ranges.get(i);
final HttpByteRange ibr = ranges.get(i);
multi.startPart(mimetype, new String[]{HeaderFramework.CONTENT_RANGE + ": " + header[i]});
final long start = ibr.getFirst();
@@ -731,10 +731,10 @@ public class YaCyDefaultServlet extends HttpServlet {
}
/* ------------------------------------------------------------ */
protected void writeHeaders(final HttpServletResponse response, final Resource resource, final long count) {
protected void writeHeaders(final HttpServletResponse response, final ServletResource resource, final long count) {
if (response.getContentType() == null) {
final String extensionmime;
if ((extensionmime = this._mimeTypes.getMimeByExtension(resource.getName())) != null) {
if ((extensionmime = this._servletContext.getMimeType(resource.getName())) != null) {
response.setContentType(extensionmime);
}
}
@@ -1035,7 +1035,7 @@ public class YaCyDefaultServlet extends HttpServlet {
}
this.updateRespHeadersForImages(target, response);
final String mimeType = Classification.ext2mime(targetExt, MimeTypes.Type.TEXT_HTML.asString());
final String mimeType = Classification.ext2mime(targetExt, MIME_TEXT_HTML);
response.setContentType(mimeType);
response.setContentLength(result.length());
response.setStatus(HttpServletResponse.SC_OK);
@@ -1150,7 +1150,7 @@ public class YaCyDefaultServlet extends HttpServlet {
templatePatterns.put(SwitchboardConstants.GREETING_IMAGE_ALT, sb.getConfig(SwitchboardConstants.GREETING_IMAGE_ALT, ""));
templatePatterns.put("clientlanguage", localeSelection);
final String mimeType = Classification.ext2mime(targetExt, MimeTypes.Type.TEXT_HTML.asString());
final String mimeType = Classification.ext2mime(targetExt, MIME_TEXT_HTML);
InputStream fis;
final long fileSize = targetLocalizedFile.length();
@@ -1239,7 +1239,7 @@ public class YaCyDefaultServlet extends HttpServlet {
*/
private void writeInputStream(final HttpServletResponse response, final String targetExt, final InputStream inStream)
throws IOException {
final String mimeType = Classification.ext2mime(targetExt, MimeTypes.Type.TEXT_HTML.asString());
final String mimeType = Classification.ext2mime(targetExt, MIME_TEXT_HTML);
response.setContentType(mimeType);
response.setStatus(HttpServletResponse.SC_OK);
final byte[] buffer = new byte[4096];
@@ -1372,4 +1372,177 @@ public class YaCyDefaultServlet extends HttpServlet {
}
}
/** Join two servlet paths without depending on a container URI utility. */
private static String addPaths(final String first, final String second) {
if (first == null || first.isEmpty()) {
return second == null ? "" : second;
}
if (second == null || second.isEmpty()) {
return first;
}
final boolean firstEndsWithSlash = first.charAt(first.length() - 1) == '/';
final boolean secondStartsWithSlash = second.charAt(0) == '/';
if (firstEndsWithSlash && secondStartsWithSlash) {
return first + second.substring(1);
}
if (!firstEndsWithSlash && !secondStartsWithSlash) {
return first + '/' + second;
}
return first + second;
}
/** Equivalent of Jetty's WriterOutputStream default-charset behavior. */
private static final class CharacterOutputStream extends OutputStream {
private final Writer writer;
private final byte[] singleByte = new byte[1];
private CharacterOutputStream(final Writer writer) {
this.writer = writer;
}
@Override
public void write(final int value) throws IOException {
this.singleByte[0] = (byte) value;
this.write(this.singleByte);
}
@Override
public void write(final byte[] data, final int offset, final int length) throws IOException {
this.writer.write(new String(data, offset, length));
}
@Override
public void flush() throws IOException {
this.writer.flush();
}
@Override
public void close() throws IOException {
this.writer.close();
}
}
/** Inclusive byte range parsed from an HTTP Range header. */
private static final class HttpByteRange {
private final long first;
private final long last;
private HttpByteRange(final long first, final long last) {
this.first = first;
this.last = last;
}
private long getFirst() {
return this.first;
}
private long getLast() {
return this.last;
}
private long getSize() {
return this.last - this.first + 1;
}
private String toHeaderRangeString(final long size) {
return "bytes " + this.first + '-' + this.last + '/' + size;
}
private static String to416HeaderRangeString(final long size) {
return "bytes */" + size;
}
private static List<HttpByteRange> satisfiableRanges(
final Enumeration<String> headers, final long size) {
final List<HttpByteRange> ranges = new java.util.ArrayList<>();
while (headers != null && headers.hasMoreElements()) {
final String header = headers.nextElement();
if (header == null || !header.regionMatches(true, 0, "bytes=", 0, 6)) {
continue;
}
final String[] specifications = header.substring(6).split(",");
for (final String specification : specifications) {
final HttpByteRange range = parse(specification.trim(), size);
if (range != null) {
ranges.add(range);
}
}
}
return ranges;
}
private static HttpByteRange parse(final String specification, final long size) {
final int separator = specification.indexOf('-');
if (separator < 0 || size <= 0) {
return null;
}
try {
if (separator == 0) {
final long suffixLength = Long.parseLong(specification.substring(1).trim());
if (suffixLength <= 0) {
return null;
}
return new HttpByteRange(Math.max(0, size - suffixLength), size - 1);
}
final long first = Long.parseLong(specification.substring(0, separator).trim());
if (first < 0 || first >= size) {
return null;
}
final String lastText = specification.substring(separator + 1).trim();
final long last = lastText.isEmpty()
? size - 1
: Math.min(Long.parseLong(lastText), size - 1);
return last < first ? null : new HttpByteRange(first, last);
} catch (final NumberFormatException e) {
return null;
}
}
}
/** Minimal multipart writer for HTTP byte-range responses. */
private static final class MultipartByteRangeOutputStream extends FilterOutputStream {
private static final byte[] CRLF = {'\r', '\n'};
private final String boundary = "yacy-" + java.util.UUID.randomUUID().toString();
private boolean firstPart = true;
private MultipartByteRangeOutputStream(final OutputStream output) {
super(output);
}
private String getBoundary() {
return this.boundary;
}
private void startPart(final String contentType, final String[] headers) throws IOException {
if (!this.firstPart) {
this.out.write(CRLF);
}
this.firstPart = false;
this.writeAscii("--" + this.boundary + "\r\n");
if (contentType != null) {
this.writeAscii(HeaderFramework.CONTENT_TYPE + ": " + contentType + "\r\n");
}
if (headers != null) {
for (final String header : headers) {
this.writeAscii(header + "\r\n");
}
}
this.out.write(CRLF);
}
@Override
public void close() throws IOException {
this.out.write(CRLF);
this.writeAscii("--" + this.boundary + "--\r\n");
super.close();
}
private void writeAscii(final String value) throws IOException {
this.out.write(value.getBytes(StandardCharsets.ISO_8859_1));
}
}
}
test/java/net/yacy/http/AdminSecurityTest.java
+37
View File
@@ -106,4 +106,41 @@ public class AdminSecurityTest {
// a request from localhost referred by a remote page is not a localhost access
Assert.assertFalse(AdminSecurity.isLocalhostAccess("127.0.0.1", "example.org"));
}
/** Test the complete request-level policy used by the container adapter. */
@Test
public void testAdminAccessPolicy() {
final String user = "admin";
final String hash = AdminSecurity.calcHash(user + ":secret");
final AdminAccessPolicy localAllowed = new AdminAccessPolicy(
false, false, true, true, user, hash);
Assert.assertEquals(AdminAccessPolicy.Decision.PUBLIC,
localAllowed.decide("/index.html", "192.0.2.1", null, null));
Assert.assertEquals(AdminAccessPolicy.Decision.ADMIN_REQUIRED,
localAllowed.decide("/Settings_p.html", "192.0.2.1", null, null));
Assert.assertEquals(AdminAccessPolicy.Decision.LOCAL_BYPASS,
localAllowed.decide("/Settings_p.html", "127.0.0.1", null, null));
Assert.assertEquals(AdminAccessPolicy.Decision.ADMIN_REQUIRED,
localAllowed.decide("/Settings_p.html", "127.0.0.1", "https://example.org/", null));
final AdminAccessPolicy loginRequired = new AdminAccessPolicy(
false, false, true, false, user, hash);
Assert.assertEquals(AdminAccessPolicy.Decision.ADMIN_REQUIRED,
loginRequired.decide("/Settings_p.html", "127.0.0.1", null, null));
final String lazyAuth = "Basic " + Base64Order.standardCoder.encodeString(user + ":" + hash);
Assert.assertEquals(AdminAccessPolicy.Decision.LOCAL_BYPASS,
loginRequired.decide("/Settings_p.html", "127.0.0.1", null, lazyAuth));
}
/** The credential context is request-bound and fails closed after cleanup. */
@Test
public void testAdminAuthenticationContext() {
AdminAuthenticationContext.clear();
Assert.assertFalse(AdminAuthenticationContext.isLocalhostRequest());
AdminAuthenticationContext.setSocketPeerIp("127.0.0.1");
Assert.assertTrue(AdminAuthenticationContext.isLocalhostRequest());
AdminAuthenticationContext.clear();
Assert.assertFalse(AdminAuthenticationContext.isLocalhostRequest());
}
}
test/java/net/yacy/http/InetPathAccessHandlerTest.java
+12
View File
@@ -32,6 +32,18 @@ import org.junit.Test;
*/
public class InetPathAccessHandlerTest {
@Test
public void testPortableRuleParsing() {
final InetPathAccessRule addressOnly = InetPathAccessRule.parse("192.168.1.0/24");
Assert.assertEquals("192.168.1.0/24", addressOnly.addressPattern());
Assert.assertEquals("/*", addressOnly.pathPattern());
final InetPathAccessRule addressAndPath = InetPathAccessRule.parse("127.0.0.1|/api/*");
Assert.assertEquals("127.0.0.1", addressAndPath.addressPattern());
Assert.assertEquals("/api/*", addressAndPath.pathPattern());
Assert.assertEquals("127.0.0.1|/api/*", addressAndPath.asJettyPattern());
}
/**
* Check the handler allow the given ip/path pairs.
*
test/jetty-solr-dependency-guard.sh
Executable
+81
View File
@@ -0,0 +1,81 @@
#!/usr/bin/env sh
# Guard the classpath boundary needed to migrate YaCy's embedded server from
# Jetty 9 to Jetty 12 while Solr 9's Jetty client is relocated into a private
# package and kept out of YaCy source code.
set -eu
repo_root=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)
cd "$repo_root"
fail() {
echo "FAIL: $*" >&2
exit 1
}
if grep -R -n -E \
--include='*.java' \
'(Http2SolrClient|CloudHttp2SolrClient|ConcurrentUpdateHttp2SolrClient|LBHttp2SolrClient|JettySolrRunner)' \
source >/dev/null 2>&1; then
grep -R -n -E \
--include='*.java' \
'(Http2SolrClient|CloudHttp2SolrClient|ConcurrentUpdateHttp2SolrClient|LBHttp2SolrClient|JettySolrRunner)' \
source >&2
fail "YaCy source must not use Solr's Jetty-backed clients or runner"
fi
if grep -E \
'name="(jetty-deploy|jetty-jmx)"' \
ivy.xml >/dev/null 2>&1; then
fail "ivy.xml contains a forbidden direct Jetty dependency"
fi
grep -E 'name="jetty-client".*conf="solr9-bridge->master"' ivy.xml >/dev/null 2>&1 || \
fail "jetty-client must only be a direct input of the Solr 9 bridge"
grep -E 'org="org.eclipse.jetty" name="jetty-io"' ivy.xml >/dev/null 2>&1 || \
fail "jetty-io must be an explicit dependency because YaCy imports its API"
for artifact in http2-client http2-common http2-http-client-transport; do
grep -E "org=\"org.eclipse.jetty.http2\" name=\"$artifact\".*conf=\"solr9-bridge->master\"" ivy.xml >/dev/null 2>&1 || \
fail "$artifact must only be an input of the Solr 9 bridge"
done
if [ -d lib ]; then
for pattern in \
'jetty-deploy-*.jar' \
'jetty-jmx-*.jar' \
'solr-core-*.jar' \
'solr-solrj-*.jar' \
'solr-scripting-*.jar' \
'http2-*.jar'; do
for artifact in lib/$pattern; do
[ -e "$artifact" ] || continue
fail "forbidden resolved artifact: $artifact"
done
done
for artifact in \
solr-core-9.0.0 \
solr-solrj-9.0.0 \
solr-scripting-9.0.0 \
jetty-client-9.4.58.v20250814 \
jetty-http-9.4.58.v20250814 \
jetty-io-9.4.58.v20250814 \
jetty-util-9.4.58.v20250814 \
http2-client-9.4.58.v20250814 \
http2-common-9.4.58.v20250814 \
http2-http-client-transport-9.4.58.v20250814; do
jar="lib/solr9-bridge-$artifact.jar"
[ -f "$jar" ] || fail "missing generated bridge artifact: $jar"
if jar tf "$jar" | grep '^org/eclipse/jetty/' >/dev/null 2>&1; then
fail "unrelocated Jetty class in $jar"
fi
if zipgrep -a -E 'org(/|\.)eclipse(/|\.)jetty' "$jar" >/dev/null 2>&1; then
fail "unrelocated Jetty reference in $jar"
fi
done
fi
echo "PASS: Solr 9 uses only the relocated Jetty client island."
test/jetty/RelocateJettyPackages.java
+135
View File
@@ -0,0 +1,135 @@
/*
* Proof-of-concept bytecode relocator for the Jetty 12 migration.
*
* This is deliberately a small build tool, not YaCy runtime code. It keeps
* Solr's public packages unchanged while moving its private Jetty 9 linkage
* below net.yacy.solr9.jetty.
*/
package net.yacy.test.jetty;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Enumeration;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import java.util.jar.JarOutputStream;
import org.objectweb.asm.ClassReader;
import org.objectweb.asm.ClassWriter;
import org.objectweb.asm.Opcodes;
import org.objectweb.asm.commons.ClassRemapper;
import org.objectweb.asm.commons.Remapper;
public final class RelocateJettyPackages {
private static final String SOURCE_INTERNAL = "org/eclipse/jetty";
private static final String TARGET_INTERNAL = "net/yacy/solr9/jetty";
private static final String SOURCE_BINARY = "org.eclipse.jetty";
private static final String TARGET_BINARY = "net.yacy.solr9.jetty";
private RelocateJettyPackages() {
}
public static void main(final String[] args) throws IOException {
if (args.length != 2) {
throw new IllegalArgumentException("usage: RelocateJettyPackages INPUT.jar OUTPUT.jar");
}
relocate(Path.of(args[0]), Path.of(args[1]));
}
private static void relocate(final Path input, final Path output) throws IOException {
Files.createDirectories(output.toAbsolutePath().getParent());
final Remapper remapper = new JettyRemapper();
try (JarFile source = new JarFile(input.toFile());
JarOutputStream target = new JarOutputStream(Files.newOutputStream(output))) {
final Enumeration<JarEntry> entries = source.entries();
while (entries.hasMoreElements()) {
final JarEntry entry = entries.nextElement();
if (entry.isDirectory() || isSignature(entry.getName())) {
continue;
}
final String outputName = relocateEntryName(entry.getName());
final JarEntry outputEntry = new JarEntry(outputName);
outputEntry.setTime(entry.getTime());
target.putNextEntry(outputEntry);
try (InputStream stream = source.getInputStream(entry)) {
final byte[] content = readAll(stream);
if (entry.getName().endsWith(".class")) {
target.write(relocateClass(content, remapper));
} else if (entry.getName().startsWith("META-INF/services/")) {
target.write(new String(content, StandardCharsets.UTF_8)
.replace(SOURCE_BINARY, TARGET_BINARY)
.getBytes(StandardCharsets.UTF_8));
} else {
target.write(content);
}
}
target.closeEntry();
}
}
}
private static final class JettyRemapper extends Remapper {
private JettyRemapper() {
super(Opcodes.ASM9);
}
@Override
public String map(final String internalName) {
return replacePrefix(internalName, SOURCE_INTERNAL, TARGET_INTERNAL);
}
@Override
public Object mapValue(final Object value) {
if (value instanceof String) {
final String text = (String) value;
return text.replace(SOURCE_BINARY, TARGET_BINARY)
.replace(SOURCE_INTERNAL, TARGET_INTERNAL);
}
return super.mapValue(value);
}
}
private static byte[] relocateClass(final byte[] content, final Remapper remapper) {
final ClassReader reader = new ClassReader(content);
final ClassWriter writer = new ClassWriter(0);
reader.accept(new ClassRemapper(writer, remapper), 0);
return writer.toByteArray();
}
private static boolean isSignature(final String name) {
final String upper = name.toUpperCase(java.util.Locale.ROOT);
return upper.startsWith("META-INF/")
&& (upper.endsWith(".SF") || upper.endsWith(".RSA") || upper.endsWith(".DSA"));
}
private static String replacePrefix(final String value, final String source, final String target) {
if (value.equals(source)) {
return target;
}
if (value.startsWith(source + "/")) {
return target + value.substring(source.length());
}
return value;
}
private static String relocateEntryName(final String name) {
final String internalName = replacePrefix(name, SOURCE_INTERNAL, TARGET_INTERNAL);
if (internalName.startsWith("META-INF/services/")) {
return internalName.replace(SOURCE_BINARY, TARGET_BINARY);
}
return internalName;
}
private static byte[] readAll(final InputStream stream) throws IOException {
final ByteArrayOutputStream output = new ByteArrayOutputStream();
stream.transferTo(output);
return output.toByteArray();
}
}
test/solr9-jetty-bridge-spike.sh
Executable
+39
View File
@@ -0,0 +1,39 @@
#!/usr/bin/env sh
# Exercise the generated Solr 9.0 bridge. Run `ant compile` first. An optional
# JETTY12_CLASSPATH verifies coexistence with already resolved Jetty 12 jars.
set -eu
repo_root=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)
cd "$repo_root"
fail() {
echo "FAIL: $*" >&2
exit 1
}
[ -d build/classes/java/main ] || fail "missing compiled YaCy classes; run ant compile first"
test/jetty-solr-dependency-guard.sh
work=$(mktemp -d "${TMPDIR:-/tmp}/yacy-solr9-bridge-test.XXXXXX")
trap 'rm -rf "$work"' EXIT HUP INT TERM
classpath=build/classes/java/main
for archive in lib/*.jar libt/*.jar; do
case $(basename "$archive") in
yacycore.jar) continue ;;
esac
classpath="$classpath:$archive"
done
if [ -n "${JETTY12_CLASSPATH:-}" ]; then
classpath="$classpath:$JETTY12_CLASSPATH"
fi
javac --release 17 -cp "$classpath" -d "$work" \
test/java/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnectorTest.java
java -cp "$work:$classpath" org.junit.runner.JUnitCore \
net.yacy.cora.federate.solr.connector.EmbeddedSolrConnectorTest
echo "PASS: integrated Solr 9.0 bridge starts, updates, queries and closes."