Merge pull request #778 from songproducer/fix/mcp-content-type-text

Fix MCP search tool returning non-standard content type 'json'

source/net/yacy/htroot/yacysearch.java

@@ -33,6 +33,7 @@ import static net.yacy.repository.BlacklistHelper.addBlacklistEntry;
 
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
+import javax.servlet.http.Cookie;
 import java.net.IDN;
 import java.net.MalformedURLException;
 import java.net.URLEncoder;
@@ -167,6 +168,22 @@ public class yacysearch {
         boolean global = post == null || (!post.get("resource-switch", post.get("resource", "global")).equals("local") && p2pmode);
         final boolean stealthmode = p2pmode && !global;
 
+        // Read persisted language preference from cookie so it is available in both the
+        // empty-search (early return) and full-search paths below.
+        String cookieLanguage = null;
+        final Cookie[] requestCookies = header.getCookies();
+        if (requestCookies != null) {
+            for (final Cookie c : requestCookies) {
+                if ("yacy-language".equals(c.getName())) {
+                    final String v = c.getValue();
+                    if (v != null && v.startsWith("lang_") && ISO639.exists(v.substring(5))) {
+                        cookieLanguage = v.substring(5);
+                    }
+                    break;
+                }
+            }
+        }
+
         if ( post == null || indexSegment == null || env == null || !searchAllowed ) {
             if (indexSegment == null) ConcurrentLog.info("yacysearch", "indexSegment == null");
             // we create empty entries for template strings
@@ -205,6 +222,7 @@ public class yacysearch {
             prop.put("rss_queryenc", "");
             prop.put("meanCount", 5);
             prop.put("eventID",""); // mandatory parameter for yacysearchtrailer/yacysearchitem includes
+            prop.put("languageSel", cookieLanguage != null ? "lang_" + cookieLanguage : "");
             return prop;
         }
 
@@ -616,25 +634,42 @@ public class yacysearch {
 
             if (urlmask == null || urlmask.isEmpty()) urlmask = ".*"; //if no urlmask was given
 
-            // read the language from the language-restrict option 'lr'
-            // if no one is given, use the user agent or the system language as default
-            String language = (post == null) ? null : post.get("lr");
-            if (language != null && language.startsWith("lang_") ) {
-                language = language.substring(5);
+            // read the language from the language-restrict option 'lr',
+            // then from a persisted cookie preference, then from Accept-Language header
+            final boolean languageFromParam = post.containsKey("lr"); // lr was explicitly submitted in this request
+            final String lrParam = languageFromParam ? post.get("lr") : null;
+            String language;
+            if (lrParam != null && lrParam.startsWith("lang_") && ISO639.exists(lrParam.substring(5))) {
+                language = lrParam.substring(5);
                 if (modifier.language == null) modifier.language = language;
-            }
-            if (language == null || !ISO639.exists(language) ) {
-                // find out language of the user by reading of the user-agent string
+            } else if (!languageFromParam && cookieLanguage != null) {
+                // no lr param submitted this request — use saved cookie preference
+                language = cookieLanguage;
+                if (modifier.language == null) modifier.language = language;
+            } else {
+                // lr was submitted as empty/invalid (clear preference) or no cookie exists
                 String agent = header.get(HeaderFramework.ACCEPT_LANGUAGE);
-                if ( agent == null ) {
+                if (agent == null) {
                     agent = System.getProperty("user.language");
                 }
                 language = (agent == null) ? "en" : ISO639.userAgentLanguageDetection(agent);
-                if ( language == null ) {
+                if (language == null) {
                     language = "en";
                 }
             }
 
+            // persist language preference when user explicitly submits a change via the lr param
+            prop.put("languageSel", modifier.language != null ? "lang_" + modifier.language : "");
+            if (languageFromParam) {
+                final ResponseHeader outgoingHeader = prop.getOutgoingHeader();
+                if (modifier.language != null) {
+                    outgoingHeader.setCookie("yacy-language", "lang_" + modifier.language, 365 * 24 * 60 * 60, "/", null, false);
+                } else {
+                    outgoingHeader.setCookie("yacy-language", "", 0, "/", null, false);
+                }
+                prop.setOutgoingHeader(outgoingHeader);
+            }
+
             // the query
             final QueryGoal qg = new QueryGoal(querystring.trim());
             final int maxDistance = (querystring.indexOf('"', 0) >= 0) ? qg.getIncludeHashes().size() - 1 : Integer.MAX_VALUE;

source/net/yacy/http/servlets/MCPSearchServlet.java

@@ -326,8 +326,8 @@ public class MCPSearchServlet extends HttpServlet {
             payload.put("results", results);
 
             final JSONObject contentItem = new JSONObject(true);
-            contentItem.put("type", "json");
-            contentItem.put("json", payload);
+            contentItem.put("type", "text");
+            contentItem.put("text", payload.toString());
 
             final JSONArray content = new JSONArray();
             content.put(contentItem);

source/net/yacy/search/query/SearchEvent.java

@@ -993,6 +993,16 @@ public final class SearchEvent implements ScoreMapUpdatesListener {
             this.remote_solr_peerCount.incrementAndGet();
         }
 
+        // Compute the max Solr score in this peer's result batch so we can normalize
+        // individual scores into [0, 1] before inserting into the shared priority queue.
+        // Without this, peers with higher absolute Solr scores always dominate peers
+        // with lower absolute scores, producing biased cross-node result ordering.
+        float maxSolrScore = 0.0f;
+        for (final URIMetadataNode node : nodeList) {
+            final Float s = (Float) node.getFieldValue("score");
+            if (s != null && s > maxSolrScore) maxSolrScore = s;
+        }
+
         long timer = System.currentTimeMillis();
 
         // normalize entries
@@ -1127,7 +1137,7 @@ public final class SearchEvent implements ScoreMapUpdatesListener {
                         // so far Solr score is used (with abitrary factor to get value similar to rwi ranking values)
                         final Float scorex = (Float) iEntry.getFieldValue("score"); // this is a special field containing the ranking score of a Solr search result
                         if (scorex != null && scorex > 0)
-                            score = (long) ((1000000.0f * scorex) - iEntry.urllength()); // we modify the score here since the solr score is equal in many cases and then the order would simply depend on the url hash which would be silly
+                            score = (long) ((1000000.0f * (maxSolrScore > 0.0f ? scorex / maxSolrScore : scorex)) - iEntry.urllength()); // normalize to [0,1] across this peer's batch so scores are comparable across peers
                         else
                             score = this.order.cardinal(iEntry);
                         this.nodeStack.put(new ReverseElement<>(iEntry, score)); // inserts the element and removes the worst (which is smallest)